From 2a00d910df014f6675aa58e80a8be875b7ca8f82 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Thu, 20 Aug 2026 21:10:11 +0200 Subject: [PATCH] feat: govern autonomous campaign delivery schedules --- README.md | 14 + .../backend/campaign/scheduling.py | 673 +++++++++++++++++- src/govoplan_campaign/backend/db/models.py | 26 + .../backend/documentation.py | 28 +- src/govoplan_campaign/backend/manifest.py | 6 +- ...0a1_v0121_autonomous_campaign_schedules.py | 100 +++ src/govoplan_campaign/backend/retention.py | 23 +- .../backend/routes/schedules.py | 54 ++ src/govoplan_campaign/backend/schemas.py | 9 + tests/test_campaign_scheduling.py | 292 +++++++- tests/test_retention.py | 36 + webui/src/api/campaigns.ts | 11 +- .../campaigns/CampaignOverviewPage.tsx | 37 +- .../campaign-lifecycle-ui-structure.test.mjs | 7 + 14 files changed, 1251 insertions(+), 65 deletions(-) create mode 100644 src/govoplan_campaign/backend/migrations/versions/b6c7d8e9f0a1_v0121_autonomous_campaign_schedules.py diff --git a/README.md b/README.md index 6ed2fd5..1eca8f7 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,20 @@ Files, Mail, Distribution Lists, Templates, Postbox, and Calendar are optional m Hybrid delivery never treats an opt-in as an implicit duplicate-send instruction. The Campaign author selects one primary route per recipient and may select a supported fallback. A fallback runs only after the first channel rejects before acceptance; accepted or outcome-unknown effects stop cross-channel retry. Printable output is generated once during build, optionally persisted through Files, reviewed with the exact Campaign version, and accepted idempotently per recipient job during delivery. +Recurring schedules have two immutable modes. Manual mode remains the default +and prepares independent drafts without Mail. Autonomous mode is explicit and +Mail-only: it seals an already built and explicitly approved execution snapshot, +rechecks approval, policy, credential/transport revision, live SMTP health, +recipient and attachment evidence before each occurrence, and submits one +Mail-owned durable command per frozen message. Occurrence-scoped idempotency is +allocated before delivery. Accepted and outcome-unknown effects are never +retried automatically; uncertain or systemic failures pause the schedule, +notify its accountable operator, and retain non-secret recovery evidence. +Generated EML retention excludes source versions while an autonomous schedule +has a remaining occurrence, including while it is paused; once the schedule +finishes, already accepted Mail commands retain their own encrypted payload and +evidence under Mail policy. + Public campaign, version, job, and report responses expose business data and delivery evidence, but never process-local paths, storage-backend keys, or worker claim tokens. Operational troubleshooting uses the dedicated job diff --git a/src/govoplan_campaign/backend/campaign/scheduling.py b/src/govoplan_campaign/backend/campaign/scheduling.py index 6105e9c..4d242e4 100644 --- a/src/govoplan_campaign/backend/campaign/scheduling.py +++ b/src/govoplan_campaign/backend/campaign/scheduling.py @@ -6,6 +6,8 @@ import hashlib import json from collections.abc import Mapping from datetime import UTC, datetime, timedelta +from email import policy +from email.parser import BytesParser from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from sqlalchemy.orm import Session @@ -13,19 +15,35 @@ from sqlalchemy.orm import Session from govoplan_campaign.backend.campaign.copying import campaign_copy_configuration from govoplan_campaign.backend.db.models import ( Campaign, + CampaignJob, CampaignSchedule, CampaignScheduleOccurrence, CampaignShare, CampaignVersion, + JobBuildStatus, ) +from govoplan_campaign.backend.approval_gate import ( + assert_campaign_approval, + campaign_approval_gate, +) +from govoplan_campaign.backend.campaign.models import DeliveryChannelPolicy +from govoplan_campaign.backend.integrations import mail_integration from govoplan_campaign.backend.persistence.campaigns import ( create_campaign_version_from_json, ) +from govoplan_campaign.backend.sending.execution import ensure_execution_snapshot +from govoplan_campaign.backend.sending.jobs import ( + _from_header_from_job, + _send_job_delivery_context, + _single_job_validation_allowed, + _synchronous_smtp_batch_manager, +) from govoplan_core.audit.logging import audit_event RECURRENCE_KINDS = frozenset({"once", "daily", "weekly", "monthly"}) SCHEDULE_SOURCE_SCHEMA = "govoplan.campaign.schedule-source.v1" +SCHEDULE_DELIVERY_MODES = frozenset({"manual", "autonomous"}) def canonical_configuration_hash(value: Mapping[str, object]) -> str: @@ -96,6 +114,11 @@ def dispatch_due_campaign_schedules( limit: int = 50, ) -> dict[str, object]: observed_at = _as_utc(now or datetime.now(UTC)) + refreshed = refresh_autonomous_schedule_outcomes( + session, + tenant_id=tenant_id, + now=observed_at, + ) query = session.query(CampaignSchedule).filter( CampaignSchedule.active.is_(True), CampaignSchedule.next_fire_at.is_not(None), @@ -112,39 +135,95 @@ def dispatch_due_campaign_schedules( result: dict[str, object] = { "selected": len(schedules), "prepared": 0, + "autonomous_prepared": 0, "failed": 0, "completed": 0, "coalesced": 0, + "duplicates": 0, + "deferred": 0, "campaign_ids": [], "operator_actions": [], + "refreshed": refreshed, } for schedule in schedules: scheduled_for = _as_utc(schedule.next_fire_at or observed_at) + if schedule.delivery_mode == "autonomous" and _has_open_occurrence( + session, schedule_id=schedule.id + ): + result["deferred"] = int(result["deferred"]) + 1 + continue try: with session.begin_nested(): - campaign, version, skipped = _prepare_occurrence( - session, - schedule=schedule, - scheduled_for=scheduled_for, - observed_at=observed_at, - ) + if schedule.delivery_mode == "autonomous": + _occurrence, skipped = _prepare_autonomous_occurrence( + session, + schedule=schedule, + scheduled_for=scheduled_for, + observed_at=observed_at, + ) + campaign_id = schedule.campaign_id + result["autonomous_prepared"] = ( + int(result["autonomous_prepared"]) + 1 + ) + else: + campaign, _version, skipped = _prepare_occurrence( + session, + schedule=schedule, + scheduled_for=scheduled_for, + observed_at=observed_at, + ) + campaign_id = campaign.id result["prepared"] = int(result["prepared"]) + 1 result["coalesced"] = int(result["coalesced"]) + skipped - result["campaign_ids"].append(campaign.id) # type: ignore[union-attr] + result["campaign_ids"].append(campaign_id) # type: ignore[union-attr] if not schedule.active: result["completed"] = int(result["completed"]) + 1 except Exception as exc: # noqa: BLE001 - persist bounded operator evidence + session.expire_all() + recorded = ( + session.query(CampaignScheduleOccurrence) + .filter( + CampaignScheduleOccurrence.schedule_id == schedule.id, + CampaignScheduleOccurrence.scheduled_for == scheduled_for, + ) + .one_or_none() + ) + if recorded is not None: + result["duplicates"] = int(result["duplicates"]) + 1 + if ( + schedule.active + and schedule.next_fire_at is not None + and _as_utc(schedule.next_fire_at) == scheduled_for + and recorded.status not in {"failed", "uncertain"} + ): + _advance_schedule( + session, + schedule=schedule, + occurrence=recorded, + scheduled_for=scheduled_for, + observed_at=observed_at, + sequence=schedule.occurrence_count + 1, + ) + continue session.add( CampaignScheduleOccurrence( tenant_id=schedule.tenant_id, schedule_id=schedule.id, scheduled_for=scheduled_for, status="failed", + idempotency_key=_occurrence_idempotency_key( + schedule.id, scheduled_for + ), error=str(exc)[:4000], + recovery_state="failed", + evidence={"delivery_mode": schedule.delivery_mode}, + last_checked_at=observed_at, ) ) schedule.active = False schedule.last_error = str(exc)[:4000] + schedule.last_outcome = "failed" + schedule.last_recovery_state = "operator_required" schedule.resource_revision += 1 session.add(schedule) result["failed"] = int(result["failed"]) + 1 @@ -153,8 +232,14 @@ def dispatch_due_campaign_schedules( "schedule_id": schedule.id, "campaign_id": schedule.campaign_id, "reason": "draft_preparation_failed", + "delivery_mode": schedule.delivery_mode, } ) + _notify_schedule_operator( + session, + schedule=schedule, + reason="policy_or_systemic_preflight_failed", + ) session.flush() return result @@ -234,41 +319,26 @@ def _prepare_occurrence( schedule_id=schedule.id, scheduled_for=scheduled_for, status="prepared", + idempotency_key=_occurrence_idempotency_key(schedule.id, scheduled_for), generated_campaign_id=generated_campaign.id, generated_version_id=generated_version.id, + recovery_state="none", + evidence={"delivery_mode": "manual"}, + last_checked_at=observed_at, ) session.add(occurrence) - schedule.occurrence_count = sequence - schedule.last_fired_at = scheduled_for + session.flush() schedule.last_campaign_id = generated_campaign.id - schedule.last_error = None - - next_fire = next_schedule_fire( - scheduled_for, - recurrence_kind=schedule.recurrence_kind, - interval_count=schedule.interval_count, - timezone_name=schedule.timezone, + schedule.last_outcome = "prepared" + schedule.last_recovery_state = "none" + coalesced = _advance_schedule( + session, + schedule=schedule, + occurrence=occurrence, + scheduled_for=scheduled_for, + observed_at=observed_at, + sequence=sequence, ) - coalesced = 0 - while next_fire is not None and next_fire <= observed_at: - next_fire = next_schedule_fire( - next_fire, - recurrence_kind=schedule.recurrence_kind, - interval_count=schedule.interval_count, - timezone_name=schedule.timezone, - ) - coalesced += 1 - if ( - next_fire is None - or sequence >= schedule.max_occurrences - or (schedule.ends_at is not None and next_fire > _as_utc(schedule.ends_at)) - ): - schedule.active = False - schedule.next_fire_at = None - else: - schedule.next_fire_at = next_fire - schedule.resource_revision += 1 - session.add(schedule) audit_event( session, tenant_id=schedule.tenant_id, @@ -291,6 +361,535 @@ def _prepare_occurrence( return generated_campaign, generated_version, coalesced +def validate_autonomous_schedule_source( + session: Session, + *, + campaign: Campaign, + version: CampaignVersion, +) -> dict[str, object]: + """Validate the exact immutable execution that an autonomous schedule reuses.""" + + gate = campaign_approval_gate(version) + if gate is None: + raise RuntimeError( + "Autonomous delivery requires an explicit Approval request for the built source version." + ) + assert_campaign_approval(session, tenant_id=campaign.tenant_id, version=version) + snapshot = ensure_execution_snapshot(session, version) + snapshot_hash = str(version.execution_snapshot_hash or "") + if len(snapshot_hash) != 64: + raise RuntimeError("The approved Campaign execution snapshot is incomplete.") + jobs = _autonomous_source_jobs( + session, + tenant_id=campaign.tenant_id, + campaign_id=campaign.id, + version=version, + ) + mail = mail_integration() + if not mail.durable_delivery_available: + raise RuntimeError( + "Autonomous delivery requires Mail's durable delivery-command outbox." + ) + if not snapshot.mail_profile_id or not snapshot.smtp_transport_revision: + raise RuntimeError( + "The approved Campaign execution has no immutable Mail transport evidence." + ) + summary = mail.campaign_profile_delivery_summary( + session, + tenant_id=campaign.tenant_id, + campaign_id=campaign.id, + profile_id=snapshot.mail_profile_id, + smtp_server_id=snapshot.smtp_server_id, + smtp_credential_id=snapshot.smtp_credential_id, + ) + if not summary.get("smtp_available"): + raise RuntimeError("The approved Campaign Mail transport is unavailable.") + if summary.get("smtp_transport_revision") != snapshot.smtp_transport_revision: + raise RuntimeError( + "The Campaign Mail transport changed after approval; rebuild and approve a new source version." + ) + return { + "execution_snapshot_hash": snapshot_hash, + "approval_request_id": str(gate.get("request_id") or ""), + "approval_subject_digest": str(gate.get("subject_digest") or ""), + "job_count": len(jobs), + "job_manifest_sha256": canonical_configuration_hash( + {"jobs": [{"id": job.id, "eml_sha256": job.eml_sha256} for job in jobs]} + ), + } + + +def _autonomous_source_jobs( + session: Session, + *, + tenant_id: str, + campaign_id: str, + version: CampaignVersion, +) -> list[CampaignJob]: + jobs = ( + session.query(CampaignJob) + .filter( + CampaignJob.tenant_id == tenant_id, + CampaignJob.campaign_id == campaign_id, + CampaignJob.campaign_version_id == version.id, + ) + .order_by(CampaignJob.entry_index.asc(), CampaignJob.id.asc()) + .all() + ) + if not jobs: + raise RuntimeError( + "Autonomous delivery requires a built source version with recipient jobs." + ) + for job in jobs: + if job.build_status != JobBuildStatus.BUILT.value: + raise RuntimeError( + "Autonomous delivery requires every source message to be built." + ) + if not _single_job_validation_allowed(version, job, include_warnings=True): + raise RuntimeError( + "Autonomous delivery requires every source message to pass its reviewed recipient and attachment gates." + ) + if DeliveryChannelPolicy(job.delivery_channel_policy) != DeliveryChannelPolicy.MAIL: + raise RuntimeError( + "Autonomous schedules currently support Mail-only delivery; use manual mode for hybrid, Postbox, or print delivery." + ) + return jobs + + +def _prepare_autonomous_occurrence( + session: Session, + *, + schedule: CampaignSchedule, + scheduled_for: datetime, + observed_at: datetime, +) -> tuple[CampaignScheduleOccurrence, int]: + existing = ( + session.query(CampaignScheduleOccurrence) + .filter( + CampaignScheduleOccurrence.schedule_id == schedule.id, + CampaignScheduleOccurrence.scheduled_for == scheduled_for, + ) + .one_or_none() + ) + if existing is not None: + raise RuntimeError("Campaign schedule occurrence was already recorded") + if canonical_configuration_hash(schedule.source_snapshot) != schedule.source_snapshot_hash: + raise RuntimeError("Campaign schedule source snapshot integrity check failed") + campaign = session.get(Campaign, schedule.campaign_id) + version = session.get(CampaignVersion, schedule.source_version_id) + if campaign is None or campaign.tenant_id != schedule.tenant_id: + raise RuntimeError("Campaign schedule source is no longer available") + if version is None or version.campaign_id != campaign.id: + raise RuntimeError("Campaign schedule source version is no longer available") + validation = validate_autonomous_schedule_source( + session, + campaign=campaign, + version=version, + ) + if ( + not schedule.approved_execution_snapshot_hash + or validation["execution_snapshot_hash"] + != schedule.approved_execution_snapshot_hash + ): + raise RuntimeError( + "The approved Campaign execution changed after the autonomous schedule was created." + ) + jobs = _autonomous_source_jobs( + session, + tenant_id=schedule.tenant_id, + campaign_id=campaign.id, + version=version, + ) + occurrence_key = _occurrence_idempotency_key(schedule.id, scheduled_for) + occurrence = CampaignScheduleOccurrence( + tenant_id=schedule.tenant_id, + schedule_id=schedule.id, + scheduled_for=scheduled_for, + status="preparing", + idempotency_key=occurrence_key, + recovery_state="prepared", + evidence={ + "delivery_mode": "autonomous", + "source_campaign_id": campaign.id, + "source_version_id": version.id, + "source_snapshot_hash": schedule.source_snapshot_hash, + **validation, + }, + last_checked_at=observed_at, + ) + session.add(occurrence) + session.flush() + + contexts = {job.id: _send_job_delivery_context(session, job) for job in jobs} + with _synchronous_smtp_batch_manager(session, jobs=jobs, contexts=contexts): + pass + + mail = mail_integration() + commands: list[dict[str, object]] = [] + for job in jobs: + context = contexts[job.id] + if context.envelope_from is None or not context.envelope_recipients: + raise RuntimeError("A frozen Campaign message has no delivery envelope.") + message = BytesParser(policy=policy.default).parsebytes(context.message_bytes) + commands.append( + mail.submit_delivery_command( + session, + tenant_id=schedule.tenant_id, + command_type="campaign_schedule_occurrence", + source_module="campaigns", + source_resource_type="campaign", + source_resource_id=campaign.id, + source_version_id=version.id, + idempotency_key=f"{occurrence_key}:{job.id}", + profile_id=context.snapshot.mail_profile_id, + message_bytes=context.message_bytes, + envelope_from=context.envelope_from, + envelope_recipients=context.envelope_recipients, + from_header=_from_header_from_job(job) or str(message.get("From") or ""), + expected_smtp_transport_revision=( + context.snapshot.smtp_transport_revision or "" + ), + smtp_server_id=context.snapshot.smtp_server_id, + smtp_credential_id=context.snapshot.smtp_credential_id, + created_by_user_id=schedule.created_by_user_id, + ) + ) + occurrence.delivery_command_ids = [str(item["id"]) for item in commands] + occurrence.status = "prepared" + occurrence.recovery_state = "pending" + occurrence.evidence = { + **occurrence.evidence, + "command_count": len(commands), + "duplicate_command_count": sum(bool(item.get("duplicate")) for item in commands), + "command_status_counts": _status_counts(commands), + } + occurrence.last_checked_at = observed_at + sequence = schedule.occurrence_count + 1 + schedule.last_campaign_id = campaign.id + schedule.last_outcome = "prepared" + schedule.last_recovery_state = "pending" + coalesced = _advance_schedule( + session, + schedule=schedule, + occurrence=occurrence, + scheduled_for=scheduled_for, + observed_at=observed_at, + sequence=sequence, + ) + audit_event( + session, + tenant_id=schedule.tenant_id, + user_id=schedule.created_by_user_id, + action="campaign.schedule.delivery_prepared", + object_type="campaign_schedule_occurrence", + object_id=occurrence.id, + details={ + "schedule_id": schedule.id, + "campaign_id": campaign.id, + "source_version_id": version.id, + "scheduled_for": scheduled_for.isoformat(), + "occurrence_idempotency_key": occurrence_key, + "delivery_command_count": len(commands), + "execution_snapshot_hash": validation["execution_snapshot_hash"], + "approval_request_id": validation["approval_request_id"], + "coalesced_missed_intervals": coalesced, + }, + commit=False, + ) + return occurrence, coalesced + + +def _advance_schedule( + session: Session, + *, + schedule: CampaignSchedule, + occurrence: CampaignScheduleOccurrence, + scheduled_for: datetime, + observed_at: datetime, + sequence: int, +) -> int: + schedule.occurrence_count = sequence + schedule.last_fired_at = scheduled_for + schedule.last_error = None + next_fire = next_schedule_fire( + scheduled_for, + recurrence_kind=schedule.recurrence_kind, + interval_count=schedule.interval_count, + timezone_name=schedule.timezone, + ) + coalesced = 0 + while next_fire is not None and next_fire <= observed_at: + session.add( + CampaignScheduleOccurrence( + tenant_id=schedule.tenant_id, + schedule_id=schedule.id, + scheduled_for=next_fire, + status="superseded", + idempotency_key=_occurrence_idempotency_key(schedule.id, next_fire), + recovery_state="superseded", + evidence={ + "delivery_mode": schedule.delivery_mode, + "reason": "coalesced_missed_interval", + "superseded_by_occurrence_id": occurrence.id, + }, + last_checked_at=observed_at, + ) + ) + next_fire = next_schedule_fire( + next_fire, + recurrence_kind=schedule.recurrence_kind, + interval_count=schedule.interval_count, + timezone_name=schedule.timezone, + ) + coalesced += 1 + if ( + next_fire is None + or sequence >= schedule.max_occurrences + or (schedule.ends_at is not None and next_fire > _as_utc(schedule.ends_at)) + ): + schedule.active = False + schedule.next_fire_at = None + else: + schedule.next_fire_at = next_fire + schedule.resource_revision += 1 + session.add(schedule) + return coalesced + + +def refresh_autonomous_schedule_outcomes( + session: Session, + *, + tenant_id: str | None = None, + now: datetime | None = None, +) -> dict[str, int]: + observed_at = _as_utc(now or datetime.now(UTC)) + query = session.query(CampaignScheduleOccurrence).filter( + CampaignScheduleOccurrence.status.in_(("prepared", "uncertain")), + ) + if tenant_id is not None: + query = query.filter(CampaignScheduleOccurrence.tenant_id == tenant_id) + counts = { + "checked": 0, + "accepted": 0, + "uncertain": 0, + "failed": 0, + "skipped": 0, + } + mail = mail_integration() + if not mail.durable_delivery_available: + for occurrence in query.order_by( + CampaignScheduleOccurrence.created_at + ).limit(250): + if not occurrence.delivery_command_ids: + continue + counts["checked"] += 1 + counts["uncertain"] += 1 + _mark_occurrence_uncertain( + session, + occurrence=occurrence, + observed_at=observed_at, + reason="mail_delivery_outbox_unavailable", + ) + return counts + for occurrence in query.order_by(CampaignScheduleOccurrence.created_at).limit(250): + if not occurrence.delivery_command_ids: + continue + summaries: list[dict[str, object]] = [] + try: + summaries = [ + mail.delivery_command_summary( + session, + tenant_id=occurrence.tenant_id, + command_id=command_id, + ) + for command_id in occurrence.delivery_command_ids + ] + except Exception: + counts["checked"] += 1 + counts["uncertain"] += 1 + _mark_occurrence_uncertain( + session, + occurrence=occurrence, + observed_at=observed_at, + reason="mail_delivery_status_unavailable", + ) + continue + counts["checked"] += 1 + outcome, recovery_state = _aggregate_command_outcome(summaries) + previous_outcome = occurrence.status + previous_recovery_state = occurrence.recovery_state + occurrence.status = outcome + occurrence.recovery_state = recovery_state + occurrence.last_checked_at = observed_at + occurrence.evidence = { + **(occurrence.evidence or {}), + "command_status_counts": _status_counts(summaries), + "accepted_recipient_count": sum( + int(item.get("accepted_count") or 0) for item in summaries + ), + "refused_recipient_count": sum( + int(item.get("refused_count") or 0) for item in summaries + ), + "failure_codes": sorted( + { + str(item["failure_code"]) + for item in summaries + if item.get("failure_code") + } + ), + } + schedule = session.get(CampaignSchedule, occurrence.schedule_id) + if schedule is not None: + schedule.last_outcome = outcome + schedule.last_recovery_state = recovery_state + transitioned_to_operator_required = ( + outcome in {"uncertain", "failed"} + and ( + previous_outcome != outcome + or previous_recovery_state != recovery_state + or schedule.active + ) + ) + if transitioned_to_operator_required: + schedule.active = False + schedule.last_error = ( + "Autonomous delivery needs operator review; automatic recurrence is paused." + ) + schedule.resource_revision += 1 + _notify_schedule_operator( + session, + schedule=schedule, + reason=f"delivery_{outcome}", + ) + session.add(schedule) + session.add(occurrence) + if outcome in counts: + counts[outcome] += 1 + return counts + + +def _mark_occurrence_uncertain( + session: Session, + *, + occurrence: CampaignScheduleOccurrence, + observed_at: datetime, + reason: str, +) -> None: + previous_outcome = occurrence.status + previous_recovery_state = occurrence.recovery_state + occurrence.status = "uncertain" + occurrence.recovery_state = "operator_required" + occurrence.last_checked_at = observed_at + occurrence.evidence = { + **(occurrence.evidence or {}), + "recovery_reason": reason, + } + schedule = session.get(CampaignSchedule, occurrence.schedule_id) + if schedule is not None: + transitioned = ( + previous_outcome != "uncertain" + or previous_recovery_state != "operator_required" + or schedule.active + ) + schedule.active = False + schedule.last_outcome = "uncertain" + schedule.last_recovery_state = "operator_required" + schedule.last_error = ( + "Autonomous delivery status is unavailable; automatic recurrence is paused." + ) + if transitioned: + schedule.resource_revision += 1 + _notify_schedule_operator( + session, + schedule=schedule, + reason=reason, + ) + session.add(schedule) + session.add(occurrence) + + +def _has_open_occurrence(session: Session, *, schedule_id: str) -> bool: + rows = ( + session.query(CampaignScheduleOccurrence.delivery_command_ids) + .filter( + CampaignScheduleOccurrence.schedule_id == schedule_id, + CampaignScheduleOccurrence.status == "prepared", + ) + .limit(1000) + .all() + ) + return any(bool(command_ids) for (command_ids,) in rows) + + +def _aggregate_command_outcome( + summaries: list[dict[str, object]], +) -> tuple[str, str]: + statuses = {str(item.get("status") or "") for item in summaries} + if statuses and statuses <= {"accepted", "reconciled_accepted"}: + return "accepted", "complete" + if statuses and statuses <= {"reconciled_not_accepted"}: + return "skipped", "reconciled" + if statuses & {"outcome_unknown", "in_progress"}: + return "uncertain", "operator_required" + if statuses & {"permanent_failure", "partially_refused", "reconciled_not_accepted"}: + return "failed", "operator_required" + return "prepared", "pending" + + +def _status_counts(items: list[dict[str, object]]) -> dict[str, int]: + result: dict[str, int] = {} + for item in items: + status = str(item.get("status") or "unknown") + result[status] = result.get(status, 0) + 1 + return result + + +def _occurrence_idempotency_key(schedule_id: str, scheduled_for: datetime) -> str: + return f"campaign-schedule:{schedule_id}:{_as_utc(scheduled_for).isoformat()}" + + +def _notify_schedule_operator( + session: Session, + *, + schedule: CampaignSchedule, + reason: str, +) -> None: + from govoplan_core.core.notifications import ( + NotificationDispatchRequest, + notification_dispatch_provider, + ) + from govoplan_campaign.backend.runtime import get_registry + + provider = notification_dispatch_provider(get_registry()) + if provider is None: + return + try: + provider.enqueue_notification( + session, + NotificationDispatchRequest( + tenant_id=schedule.tenant_id, + source_module="campaigns", + source_resource_type="campaign_schedule", + source_resource_id=schedule.id, + event_kind="campaign.schedule.operator_required", + channel="inbox", + recipient_type="user" if schedule.created_by_user_id else None, + recipient_id=schedule.created_by_user_id, + subject=f"Campaign schedule paused: {schedule.name}", + body_text=( + "Autonomous Campaign delivery was paused before another occurrence. " + "Review its recovery evidence before resuming." + ), + action_url=f"/campaigns/{schedule.campaign_id}", + priority=2, + payload={"schedule_id": schedule.id, "reason": reason}, + ), + enqueue_delivery=False, + ) + except Exception: + return + + def _copy_snapshot_shares( session: Session, *, @@ -360,4 +959,6 @@ __all__ = [ "canonical_configuration_hash", "dispatch_due_campaign_schedules", "next_schedule_fire", + "refresh_autonomous_schedule_outcomes", + "validate_autonomous_schedule_source", ] diff --git a/src/govoplan_campaign/backend/db/models.py b/src/govoplan_campaign/backend/db/models.py index c27d5f7..c8edd01 100644 --- a/src/govoplan_campaign/backend/db/models.py +++ b/src/govoplan_campaign/backend/db/models.py @@ -192,6 +192,12 @@ class CampaignSchedule(Base, TimestampMixin): index=True, ) name: Mapped[str] = mapped_column(String(255), nullable=False) + delivery_mode: Mapped[str] = mapped_column( + String(20), + default="manual", + nullable=False, + index=True, + ) recurrence_kind: Mapped[str] = mapped_column( String(20), default="once", @@ -210,6 +216,9 @@ class CampaignSchedule(Base, TimestampMixin): copy_options: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) source_snapshot: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=False) source_snapshot_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + approved_execution_snapshot_hash: Mapped[str | None] = mapped_column( + String(64), nullable=True, index=True + ) source_base_path: Mapped[str | None] = mapped_column(String(1000), nullable=True) last_fired_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) last_campaign_id: Mapped[str | None] = mapped_column( @@ -218,6 +227,10 @@ class CampaignSchedule(Base, TimestampMixin): index=True, ) last_error: Mapped[str | None] = mapped_column(Text, nullable=True) + last_outcome: Mapped[str | None] = mapped_column(String(30), nullable=True) + last_recovery_state: Mapped[str | None] = mapped_column( + String(30), nullable=True + ) class CampaignScheduleOccurrence(Base, TimestampMixin): @@ -240,6 +253,9 @@ class CampaignScheduleOccurrence(Base, TimestampMixin): ) scheduled_for: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) status: Mapped[str] = mapped_column(String(30), default="preparing", nullable=False, index=True) + idempotency_key: Mapped[str | None] = mapped_column( + String(200), nullable=True, index=True + ) generated_campaign_id: Mapped[str | None] = mapped_column( ForeignKey("campaigns.id", ondelete="SET NULL"), nullable=True, @@ -251,6 +267,16 @@ class CampaignScheduleOccurrence(Base, TimestampMixin): index=True, ) error: Mapped[str | None] = mapped_column(Text, nullable=True) + delivery_command_ids: Mapped[list[str]] = mapped_column( + JSON, default=list, nullable=False + ) + recovery_state: Mapped[str] = mapped_column( + String(30), default="none", nullable=False, index=True + ) + evidence: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict, nullable=False) + last_checked_at: Mapped[datetime | None] = mapped_column( + DateTime(timezone=True), nullable=True + ) class RecipientImportMappingProfile(Base, TimestampMixin): diff --git a/src/govoplan_campaign/backend/documentation.py b/src/govoplan_campaign/backend/documentation.py index 117d458..2ce40a5 100644 --- a/src/govoplan_campaign/backend/documentation.py +++ b/src/govoplan_campaign/backend/documentation.py @@ -236,9 +236,9 @@ CAMPAIGN_USER_DOCUMENTATION = ( ), _workflow_topic( topic_id="campaigns.workflow.schedule-drafts", - title="Schedule bounded recurring campaign drafts", - summary="Prepare fresh campaign drafts at a future time without bypassing validation, review, approval, or delivery controls.", - body="A Campaign schedule stores an integrity-sealed snapshot of the selected version, campaign policy, Mail profile policy, and optional shares, plus a bounded one-time, daily, weekly, or monthly recurrence. Each due occurrence creates a separately owned draft and an occurrence record. Missed intervals are coalesced instead of producing a catch-up storm. A schedule never validates, approves, queues, retries, or sends a campaign, and a preparation failure pauses it for operator review. Pause and resume reject stale browser state.", + title="Schedule bounded manual or autonomous campaigns", + summary="Prepare fresh drafts or explicitly opt in to governed delivery of an exact approved build.", + body="Every schedule is bounded, timezone-aware, and fixed to either manual or autonomous mode. Manual mode stores an integrity-sealed configuration snapshot and prepares a separately owned draft per due occurrence without requiring Mail. Autonomous mode never rebuilds or silently changes approved content: it requires a built Mail-only source version with an explicit valid Approval request, seals its execution-snapshot hash, and rechecks approval, policy, credential selection, SMTP transport revision and live transport health, recipient gates, attachment evidence, and snapshot integrity before every occurrence. It then creates one Mail-owned durable command per frozen message with occurrence-scoped idempotency before delivery. Mail never automatically retries accepted or outcome-unknown effects. Campaign records prepared, accepted, uncertain, failed, skipped, and superseded recovery evidence; uncertain, policy, configuration, and systemic failures pause the recurrence and notify the accountable operator. Missed intervals are coalesced instead of causing a catch-up storm, and pause/resume rejects stale browser state.", order=34, audience=("campaign_manager", "campaign_author", "operator"), required_scopes=("campaigns:campaign:read", "campaigns:campaign:copy", "campaigns:campaign:schedule"), @@ -249,24 +249,26 @@ CAMPAIGN_USER_DOCUMENTATION = ( "Choose the exact campaign version whose configuration should seed future drafts.", "Recipient data and active shares require their corresponding read or share authority.", "A worker and scheduler process must be running for automatic due-time preparation.", + "Autonomous mode additionally requires campaigns:campaign:queue, campaigns:campaign:send, mail:profile:use, Mail's durable outbox, and an explicitly approved built source version.", ), steps=( "Open the campaign overview and choose Schedule.", "Set the first occurrence, timezone, recurrence, and bounded maximum occurrence count.", - "Select which configuration domains may be copied and create the schedule.", - "Review each generated draft independently before validating, building, approving, and sending it.", - "Pause the schedule when the approved plan changes; a failed occurrence is paused automatically and remains visible as evidence.", + "Choose manual draft preparation or autonomous approved delivery; mode cannot be changed in place.", + "For manual mode, select which configuration domains may be copied and review each generated draft independently.", + "For autonomous mode, confirm that the selected version is built, Mail-only, and explicitly approved; the API rejects missing or stale evidence.", + "Review next occurrence, last outcome, and recovery state. Resolve uncertain or failed Mail commands explicitly before creating or resuming a replacement schedule.", ), - outcome="A bounded sequence of independent campaign drafts with durable schedule and occurrence evidence.", - verification="The Schedules section shows the next occurrence and generated count; each prepared occurrence links to a distinct draft with no delivery jobs or outcomes.", + outcome="A bounded sequence of manual drafts or at-most-once autonomous Mail commands with durable occurrence and recovery evidence.", + verification="The Schedules section shows mode, next occurrence, last outcome, recovery state, and any automatic pause; manual occurrences link to distinct drafts while autonomous occurrences retain Mail command identifiers and non-secret outcome totals.", related_topic_ids=("campaigns.workflow.copy-campaign", "campaigns.workflow.prepare-validate-and-build"), translations={ "de": { - "title": "Begrenzte wiederkehrende Kampagnenentwürfe planen", - "summary": "Künftige Kampagnenentwürfe vorbereiten, ohne Validierung, Prüfung, Freigabe oder Versandkontrollen zu umgehen.", - "body": "Ein Kampagnenzeitplan speichert einen integritätsgesicherten Stand der ausgewählten Version, Kampagnenrichtlinie, Mail-Profilrichtlinie und optionalen Freigaben sowie eine begrenzte einmalige, tägliche, wöchentliche oder monatliche Wiederholung. Jede fällige Ausführung erzeugt einen eigenständigen Entwurf und einen Ausführungsnachweis. Verpasste Intervalle werden zusammengefasst, statt unkontrolliert nachgeholt zu werden. Der Zeitplan validiert, genehmigt, startet, wiederholt oder versendet niemals eine Kampagne; ein Fehler pausiert ihn zur betrieblichen Prüfung. Pausieren und Fortsetzen weisen veraltete Browserstände zurück.", - "outcome": "Eine begrenzte Folge eigenständiger Kampagnenentwürfe mit dauerhaftem Zeitplan- und Ausführungsnachweis.", - "verification": "Der Abschnitt Zeitpläne zeigt die nächste Ausführung und die Zahl erzeugter Entwürfe; jede Ausführung verweist auf einen eigenen Entwurf ohne Versandaufträge oder Ergebnisse.", + "title": "Begrenzte manuelle oder autonome Kampagnen planen", + "summary": "Neue Entwürfe vorbereiten oder den Versand eines exakt freigegebenen Builds ausdrücklich autonom ausführen.", + "body": "Jeder Zeitplan ist begrenzt, zeitzonenfest und dauerhaft manuell oder autonom. Der manuelle Modus erzeugt eigenständige Entwürfe und funktioniert ohne Mail. Der autonome Modus verlangt eine gebaute, ausschließlich per Mail versendete und ausdrücklich freigegebene Quellversion. Vor jeder Ausführung werden Freigabe, Richtlinie, Zugangsdatenauswahl, Transportrevision und -erreichbarkeit, Empfänger, Anlagen und Snapshot-Integrität erneut geprüft. Pro eingefrorener Nachricht entsteht vor dem Versand ein dauerhafter Mail-Auftrag mit ausführungsspezifischem Idempotenzschlüssel. Angenommene oder unklare Ergebnisse werden nie automatisch wiederholt; unklare oder systemische Fehler pausieren den Zeitplan und benachrichtigen Verantwortliche. Verpasste Intervalle werden zusammengefasst und als Nachweis erhalten.", + "outcome": "Eine begrenzte Folge manueller Entwürfe oder höchstens einmal angenommener autonomer Mail-Aufträge mit dauerhaftem Wiederherstellungsnachweis.", + "verification": "Der Abschnitt Zeitpläne zeigt Modus, nächste Ausführung, letztes Ergebnis, Wiederherstellungsstatus und automatische Pausen.", } }, ), diff --git a/src/govoplan_campaign/backend/manifest.py b/src/govoplan_campaign/backend/manifest.py index feaf07e..ec12a0f 100644 --- a/src/govoplan_campaign/backend/manifest.py +++ b/src/govoplan_campaign/backend/manifest.py @@ -113,8 +113,8 @@ PERMISSIONS = ( ), _permission( "campaigns:campaign:schedule", - "Schedule campaign drafts", - "Prepare fresh campaign drafts at a governed time or bounded recurrence.", + "Schedule campaigns", + "Prepare manual drafts or opt in to approved autonomous Mail delivery at a governed time or bounded recurrence.", "Campaigns", ), _permission( @@ -401,7 +401,7 @@ manifest = ModuleManifest( provides_interfaces=( ModuleInterfaceProvider(name="campaigns.access", version="0.1.6"), ModuleInterfaceProvider(name="campaigns.delivery_tasks", version="0.1.6"), - ModuleInterfaceProvider(name="campaigns.schedules", version="0.1.0"), + ModuleInterfaceProvider(name="campaigns.schedules", version="0.2.0"), ModuleInterfaceProvider(name="campaigns.mail_policy_context", version="0.1.6"), ModuleInterfaceProvider(name="campaigns.policy_context", version="0.1.6"), ModuleInterfaceProvider(name="campaigns.retention", version="0.1.6"), diff --git a/src/govoplan_campaign/backend/migrations/versions/b6c7d8e9f0a1_v0121_autonomous_campaign_schedules.py b/src/govoplan_campaign/backend/migrations/versions/b6c7d8e9f0a1_v0121_autonomous_campaign_schedules.py new file mode 100644 index 0000000..5decbfb --- /dev/null +++ b/src/govoplan_campaign/backend/migrations/versions/b6c7d8e9f0a1_v0121_autonomous_campaign_schedules.py @@ -0,0 +1,100 @@ +"""add governed autonomous Campaign schedule evidence + +revision = "b6c7d8e9f0a1" +down_revision = "a5b6c7d8e9f0" +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + + +revision = "b6c7d8e9f0a1" +down_revision = "a5b6c7d8e9f0" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "campaign_schedules", + sa.Column("delivery_mode", sa.String(length=20), nullable=False, server_default="manual"), + ) + op.create_index( + "ix_campaign_schedules_delivery_mode", + "campaign_schedules", + ["delivery_mode"], + ) + op.add_column( + "campaign_schedules", + sa.Column("approved_execution_snapshot_hash", sa.String(length=64), nullable=True), + ) + op.create_index( + "ix_campaign_schedules_approved_execution_snapshot_hash", + "campaign_schedules", + ["approved_execution_snapshot_hash"], + ) + op.add_column( + "campaign_schedules", + sa.Column("last_outcome", sa.String(length=30), nullable=True), + ) + op.add_column( + "campaign_schedules", + sa.Column("last_recovery_state", sa.String(length=30), nullable=True), + ) + op.add_column( + "campaign_schedule_occurrences", + sa.Column("idempotency_key", sa.String(length=200), nullable=True), + ) + op.create_index( + "ix_campaign_schedule_occurrences_idempotency_key", + "campaign_schedule_occurrences", + ["idempotency_key"], + ) + op.add_column( + "campaign_schedule_occurrences", + sa.Column("delivery_command_ids", sa.JSON(), nullable=False, server_default="[]"), + ) + op.add_column( + "campaign_schedule_occurrences", + sa.Column("recovery_state", sa.String(length=30), nullable=False, server_default="none"), + ) + op.create_index( + "ix_campaign_schedule_occurrences_recovery_state", + "campaign_schedule_occurrences", + ["recovery_state"], + ) + op.add_column( + "campaign_schedule_occurrences", + sa.Column("evidence", sa.JSON(), nullable=False, server_default="{}"), + ) + op.add_column( + "campaign_schedule_occurrences", + sa.Column("last_checked_at", sa.DateTime(timezone=True), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("campaign_schedule_occurrences", "last_checked_at") + op.drop_column("campaign_schedule_occurrences", "evidence") + op.drop_index( + "ix_campaign_schedule_occurrences_recovery_state", + table_name="campaign_schedule_occurrences", + ) + op.drop_column("campaign_schedule_occurrences", "recovery_state") + op.drop_column("campaign_schedule_occurrences", "delivery_command_ids") + op.drop_index( + "ix_campaign_schedule_occurrences_idempotency_key", + table_name="campaign_schedule_occurrences", + ) + op.drop_column("campaign_schedule_occurrences", "idempotency_key") + op.drop_column("campaign_schedules", "last_recovery_state") + op.drop_column("campaign_schedules", "last_outcome") + op.drop_index( + "ix_campaign_schedules_approved_execution_snapshot_hash", + table_name="campaign_schedules", + ) + op.drop_column("campaign_schedules", "approved_execution_snapshot_hash") + op.drop_index("ix_campaign_schedules_delivery_mode", table_name="campaign_schedules") + op.drop_column("campaign_schedules", "delivery_mode") diff --git a/src/govoplan_campaign/backend/retention.py b/src/govoplan_campaign/backend/retention.py index 1f3600d..e2d54ec 100644 --- a/src/govoplan_campaign/backend/retention.py +++ b/src/govoplan_campaign/backend/retention.py @@ -31,7 +31,13 @@ from govoplan_core.core.object_storage import ( from govoplan_core.core.runtime_coordination import process_runtime_identity from govoplan_core.db.session import get_database from govoplan_core.settings import settings as core_settings -from govoplan_campaign.backend.db.models import CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus +from govoplan_campaign.backend.db.models import ( + CampaignJob, + CampaignSchedule, + CampaignVersion, + JobImapStatus, + JobQueueStatus, +) from govoplan_campaign.backend.runtime import get_settings FINAL_VERSION_STATES = { @@ -351,6 +357,18 @@ def _apply_eml_retention( "delete_failed": 0, "recovery_blocked": 0, "skipped_not_final": 0, + "skipped_schedule_source": 0, + } + protected_source_versions = { + str(version_id) + for (version_id,) in ( + session.query(CampaignSchedule.source_version_id) + .filter( + CampaignSchedule.delivery_mode == "autonomous", + CampaignSchedule.next_fire_at.is_not(None), + ) + .all() + ) } jobs = ( session.query(CampaignJob) @@ -359,6 +377,9 @@ def _apply_eml_retention( .all() ) for job in jobs: + if getattr(job, "campaign_version_id", None) in protected_source_versions: + result["skipped_schedule_source"] += 1 + continue policy = policy_for_campaign_id(job.campaign_id) cutoff = _cutoff(policy.generated_eml_retention_days, now=now) if not _is_before_cutoff(job.updated_at, cutoff): diff --git a/src/govoplan_campaign/backend/routes/schedules.py b/src/govoplan_campaign/backend/routes/schedules.py index d51262f..e2bfd9d 100644 --- a/src/govoplan_campaign/backend/routes/schedules.py +++ b/src/govoplan_campaign/backend/routes/schedules.py @@ -9,6 +9,7 @@ from sqlalchemy.orm import Session from govoplan_campaign.backend.campaign.scheduling import ( campaign_schedule_source_snapshot, canonical_configuration_hash, + validate_autonomous_schedule_source, ) from govoplan_campaign.backend.db.models import ( CampaignSchedule, @@ -80,6 +81,10 @@ def create_campaign_schedule( _require_permission(principal, "campaigns:recipient:read") if payload.include_shares: _require_permission(principal, "campaigns:campaign:share") + if payload.delivery_mode == "autonomous": + _require_permission(principal, "campaigns:campaign:queue") + _require_permission(principal, "campaigns:campaign:send") + _require_permission(principal, "mail:profile:use") source_version = ( session.query(CampaignVersion) .filter( @@ -129,12 +134,27 @@ def create_campaign_schedule( for item in source_shares ], ) + autonomous_evidence: dict[str, object] | None = None + if payload.delivery_mode == "autonomous": + try: + autonomous_evidence = validate_autonomous_schedule_source( + session, + campaign=campaign, + version=source_version, + ) + except (RuntimeError, ValueError) as exc: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=str(exc), + ) from exc + snapshot["autonomous_delivery"] = autonomous_evidence schedule = CampaignSchedule( tenant_id=principal.tenant_id, campaign_id=campaign.id, source_version_id=source_version.id, created_by_user_id=principal.user.id, name=payload.name.strip(), + delivery_mode=payload.delivery_mode, recurrence_kind=payload.recurrence_kind, interval_count=payload.interval_count, timezone=payload.timezone, @@ -151,6 +171,11 @@ def create_campaign_schedule( }, source_snapshot=snapshot, source_snapshot_hash=canonical_configuration_hash(snapshot), + approved_execution_snapshot_hash=( + str(autonomous_evidence["execution_snapshot_hash"]) + if autonomous_evidence is not None + else None + ), source_base_path=source_version.source_base_path, ) session.add(schedule) @@ -169,7 +194,19 @@ def create_campaign_schedule( "starts_at": schedule.starts_at.isoformat(), "ends_at": schedule.ends_at.isoformat() if schedule.ends_at else None, "max_occurrences": schedule.max_occurrences, + "delivery_mode": schedule.delivery_mode, "delivery_started": False, + "autonomous_delivery_opted_in": ( + schedule.delivery_mode == "autonomous" + ), + "approved_execution_snapshot_hash": ( + schedule.approved_execution_snapshot_hash + ), + "approval_request_id": ( + autonomous_evidence.get("approval_request_id") + if autonomous_evidence is not None + else None + ), }, commit=True, ) @@ -206,6 +243,23 @@ def set_campaign_schedule_state( status_code=status.HTTP_409_CONFLICT, detail="A completed campaign schedule cannot be resumed.", ) + if payload.active: + unresolved = ( + session.query(CampaignScheduleOccurrence.id) + .filter( + CampaignScheduleOccurrence.schedule_id == schedule.id, + CampaignScheduleOccurrence.status == "uncertain", + ) + .first() + ) + if unresolved is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=( + "Reconcile the autonomous delivery outcome in Mail before " + "resuming this schedule." + ), + ) schedule.active = payload.active schedule.last_error = None if payload.active else schedule.last_error schedule.resource_revision += 1 diff --git a/src/govoplan_campaign/backend/schemas.py b/src/govoplan_campaign/backend/schemas.py index 1baa60c..0e6ad67 100644 --- a/src/govoplan_campaign/backend/schemas.py +++ b/src/govoplan_campaign/backend/schemas.py @@ -64,6 +64,7 @@ class CampaignScheduleCreateRequest(BaseModel): source_version_id: str = Field(min_length=1, max_length=36) name: str = Field(min_length=1, max_length=255) + delivery_mode: Literal["manual", "autonomous"] = "manual" recurrence_kind: Literal["once", "daily", "weekly", "monthly"] = "once" interval_count: int = Field(default=1, ge=1, le=365) timezone: str = Field(default="UTC", min_length=1, max_length=100) @@ -107,9 +108,14 @@ class CampaignScheduleOccurrenceResponse(BaseModel): schedule_id: str scheduled_for: datetime status: str + idempotency_key: str | None = None generated_campaign_id: str | None = None generated_version_id: str | None = None error: str | None = None + delivery_command_ids: list[str] = Field(default_factory=list) + recovery_state: str = "none" + evidence: dict[str, object] = Field(default_factory=dict) + last_checked_at: datetime | None = None created_at: datetime @@ -120,6 +126,7 @@ class CampaignScheduleResponse(BaseModel): campaign_id: str source_version_id: str name: str + delivery_mode: str recurrence_kind: str interval_count: int timezone: str @@ -133,6 +140,8 @@ class CampaignScheduleResponse(BaseModel): last_fired_at: datetime | None = None last_campaign_id: str | None = None last_error: str | None = None + last_outcome: str | None = None + last_recovery_state: str | None = None created_at: datetime updated_at: datetime occurrences: list[CampaignScheduleOccurrenceResponse] = Field(default_factory=list) diff --git a/tests/test_campaign_scheduling.py b/tests/test_campaign_scheduling.py index d4c9421..cc6f21b 100644 --- a/tests/test_campaign_scheduling.py +++ b/tests/test_campaign_scheduling.py @@ -1,7 +1,9 @@ from __future__ import annotations +from contextlib import nullcontext from datetime import UTC, datetime -from unittest.mock import patch +from types import SimpleNamespace +from unittest.mock import Mock, patch from sqlalchemy import Column, String, Table, create_engine from sqlalchemy.orm import Session, sessionmaker @@ -15,6 +17,7 @@ from govoplan_campaign.backend.campaign.scheduling import ( ) from govoplan_campaign.backend.db.models import ( Campaign, + CampaignJob, CampaignSchedule, CampaignScheduleOccurrence, CampaignShare, @@ -71,6 +74,7 @@ class TestCampaignScheduling: groups, Campaign.__table__, CampaignVersion.__table__, + CampaignJob.__table__, CampaignShare.__table__, CampaignSchedule.__table__, CampaignScheduleOccurrence.__table__, @@ -238,3 +242,289 @@ class TestCampaignScheduling: assert generated.settings == {"retention": "sealed"} assert generated.mail_profile_policy == {"profile_id": "profile-1"} assert schedule.resource_revision == 2 + + def test_autonomous_occurrences_allocate_commands_once_and_complete_bound(self): + context = SimpleNamespace( + snapshot=SimpleNamespace( + mail_profile_id="profile-1", + smtp_transport_revision="transport-1", + smtp_server_id="smtp-1", + smtp_credential_id="credential-1", + ), + message_bytes=b"From: Sender \r\nTo: one@example.test\r\n\r\nHello", + envelope_from="sender@example.test", + envelope_recipients=["one@example.test"], + ) + job = SimpleNamespace( + id="job-1", + resolved_recipients={"from": {"email": "sender@example.test"}}, + ) + mail = Mock() + mail.durable_delivery_available = True + mail.delivery_command_summary.return_value = { + "id": "command-1", + "status": "accepted", + "accepted_count": 1, + "refused_count": 0, + "failure_code": None, + } + mail.submit_delivery_command.side_effect = [ + {"id": "command-1", "status": "pending", "duplicate": False}, + {"id": "command-2", "status": "pending", "duplicate": False}, + ] + with self.SessionLocal() as session: + schedule = session.get(CampaignSchedule, "schedule-1") + assert schedule is not None + schedule.delivery_mode = "autonomous" + schedule.approved_execution_snapshot_hash = "a" * 64 + session.commit() + + validation = { + "execution_snapshot_hash": "a" * 64, + "approval_request_id": "approval-1", + "approval_subject_digest": "b" * 64, + "job_count": 1, + "job_manifest_sha256": "c" * 64, + } + patches = ( + patch( + "govoplan_campaign.backend.campaign.scheduling.validate_autonomous_schedule_source", + return_value=validation, + ), + patch( + "govoplan_campaign.backend.campaign.scheduling._autonomous_source_jobs", + return_value=[job], + ), + patch( + "govoplan_campaign.backend.campaign.scheduling._send_job_delivery_context", + return_value=context, + ), + patch( + "govoplan_campaign.backend.campaign.scheduling._synchronous_smtp_batch_manager", + return_value=nullcontext(None), + ), + patch( + "govoplan_campaign.backend.campaign.scheduling.mail_integration", + return_value=mail, + ), + patch("govoplan_campaign.backend.campaign.scheduling.audit_event"), + ) + with patches[0], patches[1], patches[2], patches[3], patches[4], patches[5]: + first = dispatch_due_campaign_schedules( + session, + tenant_id="tenant-1", + now=datetime(2026, 8, 7, 8, tzinfo=UTC), + ) + session.commit() + second = dispatch_due_campaign_schedules( + session, + tenant_id="tenant-1", + now=datetime(2026, 8, 8, 8, tzinfo=UTC), + ) + session.commit() + + assert first["autonomous_prepared"] == 1 + assert second["autonomous_prepared"] == 1 + assert schedule.active is False + occurrences = ( + session.query(CampaignScheduleOccurrence) + .order_by(CampaignScheduleOccurrence.scheduled_for) + .all() + ) + assert [item.status for item in occurrences] == ["accepted", "prepared"] + assert [item.delivery_command_ids for item in occurrences] == [ + ["command-1"], + ["command-2"], + ] + assert len({item.idempotency_key for item in occurrences}) == 2 + assert [item.recovery_state for item in occurrences] == [ + "complete", + "pending", + ] + assert occurrences[0].evidence["source_campaign_id"] == "campaign-1" + assert occurrences[0].evidence["source_version_id"] == "version-1" + assert ( + occurrences[0].evidence["source_snapshot_hash"] + == schedule.source_snapshot_hash + ) + assert mail.submit_delivery_command.call_count == 2 + + def test_autonomous_unknown_outcome_pauses_without_resubmission(self): + mail = Mock() + mail.durable_delivery_available = True + mail.delivery_command_summary.return_value = { + "id": "command-1", + "status": "outcome_unknown", + "accepted_count": 0, + "refused_count": 0, + "failure_code": "smtp_outcome_unknown", + } + with self.SessionLocal() as session: + schedule = session.get(CampaignSchedule, "schedule-1") + assert schedule is not None + schedule.delivery_mode = "autonomous" + occurrence = CampaignScheduleOccurrence( + tenant_id="tenant-1", + schedule_id=schedule.id, + scheduled_for=datetime(2026, 8, 6, 8, tzinfo=UTC), + status="prepared", + idempotency_key="occurrence-1", + delivery_command_ids=["command-1"], + recovery_state="pending", + ) + session.add(occurrence) + session.commit() + with patch( + "govoplan_campaign.backend.campaign.scheduling.mail_integration", + return_value=mail, + ), patch( + "govoplan_campaign.backend.campaign.scheduling._notify_schedule_operator" + ) as notify: + result = dispatch_due_campaign_schedules( + session, + tenant_id="tenant-1", + now=datetime(2026, 8, 6, 9, tzinfo=UTC), + ) + session.commit() + + assert result["refreshed"]["uncertain"] == 1 + assert result["selected"] == 0 + assert occurrence.status == "uncertain" + assert occurrence.recovery_state == "operator_required" + assert schedule.active is False + assert schedule.last_outcome == "uncertain" + notify.assert_called_once() + mail.submit_delivery_command.assert_not_called() + + def test_autonomous_pending_occurrence_defers_the_next_delivery(self): + mail = Mock() + mail.durable_delivery_available = True + mail.delivery_command_summary.return_value = { + "id": "command-1", + "status": "pending", + "accepted_count": 0, + "refused_count": 0, + "failure_code": None, + } + with self.SessionLocal() as session: + schedule = session.get(CampaignSchedule, "schedule-1") + assert schedule is not None + schedule.delivery_mode = "autonomous" + session.add( + CampaignScheduleOccurrence( + tenant_id="tenant-1", + schedule_id=schedule.id, + scheduled_for=datetime(2026, 8, 6, 8, tzinfo=UTC), + status="prepared", + idempotency_key="occurrence-1", + delivery_command_ids=["command-1"], + recovery_state="pending", + ) + ) + session.commit() + + with patch( + "govoplan_campaign.backend.campaign.scheduling.mail_integration", + return_value=mail, + ): + result = dispatch_due_campaign_schedules( + session, + tenant_id="tenant-1", + now=datetime(2026, 8, 7, 8, tzinfo=UTC), + ) + + assert result["refreshed"]["checked"] == 1 + assert result["deferred"] == 1 + assert result["autonomous_prepared"] == 0 + assert schedule.active is True + assert schedule.occurrence_count == 0 + mail.submit_delivery_command.assert_not_called() + + def test_missing_mail_recovery_capability_pauses_an_open_occurrence(self): + mail = Mock() + mail.durable_delivery_available = False + with self.SessionLocal() as session: + schedule = session.get(CampaignSchedule, "schedule-1") + assert schedule is not None + schedule.delivery_mode = "autonomous" + occurrence = CampaignScheduleOccurrence( + tenant_id="tenant-1", + schedule_id=schedule.id, + scheduled_for=datetime(2026, 8, 6, 8, tzinfo=UTC), + status="prepared", + idempotency_key="occurrence-1", + delivery_command_ids=["command-1"], + recovery_state="pending", + ) + session.add(occurrence) + session.commit() + + with patch( + "govoplan_campaign.backend.campaign.scheduling.mail_integration", + return_value=mail, + ), patch( + "govoplan_campaign.backend.campaign.scheduling._notify_schedule_operator" + ) as notify: + result = dispatch_due_campaign_schedules( + session, + tenant_id="tenant-1", + now=datetime(2026, 8, 7, 8, tzinfo=UTC), + ) + + assert result["refreshed"]["uncertain"] == 1 + assert occurrence.status == "uncertain" + assert occurrence.evidence["recovery_reason"] == ( + "mail_delivery_outbox_unavailable" + ) + assert schedule.active is False + notify.assert_called_once() + + def test_autonomous_source_requires_an_explicit_approval(self): + with self.SessionLocal() as session, patch( + "govoplan_campaign.backend.campaign.scheduling.campaign_approval_gate", + return_value=None, + ): + campaign = session.get(Campaign, "campaign-1") + version = session.get(CampaignVersion, "version-1") + assert campaign is not None and version is not None + from govoplan_campaign.backend.campaign.scheduling import ( + validate_autonomous_schedule_source, + ) + + try: + validate_autonomous_schedule_source( + session, + campaign=campaign, + version=version, + ) + except RuntimeError as exc: + assert "explicit Approval request" in str(exc) + else: # pragma: no cover - defensive assertion + raise AssertionError("Autonomous source validation unexpectedly passed") + + def test_duplicate_occurrence_recovers_schedule_without_another_effect(self): + with self.SessionLocal() as session: + schedule = session.get(CampaignSchedule, "schedule-1") + assert schedule is not None + recorded = CampaignScheduleOccurrence( + tenant_id="tenant-1", + schedule_id=schedule.id, + scheduled_for=schedule.next_fire_at, + status="prepared", + idempotency_key="existing-key", + recovery_state="pending", + ) + session.add(recorded) + session.commit() + with patch("govoplan_campaign.backend.campaign.scheduling.audit_event"): + result = dispatch_due_campaign_schedules( + session, + tenant_id="tenant-1", + now=datetime(2026, 8, 7, 8, tzinfo=UTC), + ) + session.commit() + assert result["duplicates"] == 1 + assert result["failed"] == 0 + assert schedule.occurrence_count == 1 + assert schedule.next_fire_at == datetime(2026, 8, 8, 8, tzinfo=UTC) + assert session.query(CampaignScheduleOccurrence).count() == 1 diff --git a/tests/test_retention.py b/tests/test_retention.py index 737d77c..10c575c 100644 --- a/tests/test_retention.py +++ b/tests/test_retention.py @@ -138,3 +138,39 @@ def test_eml_retention_removes_only_terminal_artifact(tmp_path) -> None: assert job.eml_local_path is None assert job.eml_storage_key is None session.add.assert_called_once_with(job) + + +def test_eml_retention_preserves_unfinished_autonomous_schedule_source(tmp_path) -> None: + now = datetime.now(timezone.utc) + eml_path = tmp_path / "approved-source.eml" + eml_path.write_bytes(b"approved message") + job = SimpleNamespace( + campaign_id="campaign-1", + campaign_version_id="version-1", + updated_at=now - timedelta(days=10), + queue_status="draft", + send_status="smtp_accepted", + imap_status="appended", + eml_local_path=str(eml_path), + eml_storage_key=None, + ) + schedule_query = MagicMock() + schedule_query.filter.return_value.all.return_value = [("version-1",)] + job_query = MagicMock() + job_query.filter.return_value.order_by.return_value.all.return_value = [job] + session = MagicMock() + session.query.side_effect = [schedule_query, job_query] + policy = SimpleNamespace(generated_eml_retention_days=1) + + result = _apply_eml_retention( + session, + dry_run=False, + now=now, + policy_for_campaign_id=lambda _campaign_id: policy, + ) + + assert result["skipped_schedule_source"] == 1 + assert result["metadata_cleared"] == 0 + assert eml_path.exists() + assert job.eml_local_path == str(eml_path) + session.add.assert_not_called() diff --git a/webui/src/api/campaigns.ts b/webui/src/api/campaigns.ts index 8480388..df39826 100644 --- a/webui/src/api/campaigns.ts +++ b/webui/src/api/campaigns.ts @@ -138,10 +138,15 @@ export type CampaignScheduleOccurrence = { id: string; schedule_id: string; scheduled_for: string; - status: "prepared" | "failed" | string; + status: "prepared" | "accepted" | "uncertain" | "failed" | "skipped" | "superseded" | string; + idempotency_key?: string | null; generated_campaign_id?: string | null; generated_version_id?: string | null; error?: string | null; + delivery_command_ids: string[]; + recovery_state: string; + evidence: Record; + last_checked_at?: string | null; created_at: string; }; @@ -150,6 +155,7 @@ export type CampaignSchedule = { campaign_id: string; source_version_id: string; name: string; + delivery_mode: "manual" | "autonomous"; recurrence_kind: "once" | "daily" | "weekly" | "monthly"; interval_count: number; timezone: string; @@ -163,6 +169,8 @@ export type CampaignSchedule = { last_fired_at?: string | null; last_campaign_id?: string | null; last_error?: string | null; + last_outcome?: string | null; + last_recovery_state?: string | null; created_at: string; updated_at: string; occurrences: CampaignScheduleOccurrence[]; @@ -171,6 +179,7 @@ export type CampaignSchedule = { export type CampaignScheduleCreate = { source_version_id: string; name: string; + delivery_mode: CampaignSchedule["delivery_mode"]; recurrence_kind: CampaignSchedule["recurrence_kind"]; interval_count: number; timezone: string; diff --git a/webui/src/features/campaigns/CampaignOverviewPage.tsx b/webui/src/features/campaigns/CampaignOverviewPage.tsx index 3485aac..edfefce 100644 --- a/webui/src/features/campaigns/CampaignOverviewPage.tsx +++ b/webui/src/features/campaigns/CampaignOverviewPage.tsx @@ -75,6 +75,7 @@ function defaultScheduleDraft(): CampaignScheduleCreate { return { source_version_id: "", name: "", + delivery_mode: "manual", recurrence_kind: "once", interval_count: 1, timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC", @@ -113,6 +114,10 @@ export default function CampaignOverviewPage({ settings, auth, campaignId }: {se const canDelete = Boolean(campaign) && campaign?.status === "draft" && hasScope(auth, "campaigns:campaign:delete"); const canCopy = Boolean(data.currentVersion) && hasScope(auth, "campaigns:campaign:copy"); const canSchedule = Boolean(data.currentVersion) && hasScope(auth, "campaigns:campaign:schedule") && hasScope(auth, "campaigns:campaign:copy"); + const canAutonomousSchedule = canSchedule + && hasScope(auth, "campaigns:campaign:queue") + && hasScope(auth, "campaigns:campaign:send") + && hasScope(auth, "mail:profile:use"); function openSection(section: string, fragment = "") { const params = new URLSearchParams(); @@ -317,7 +322,9 @@ export default function CampaignOverviewPage({ settings, auth, campaignId }: {se }); setSchedules((current) => [created, ...current]); setScheduleDialogOpen(false); - setMessage("Campaign schedule created. Each occurrence prepares a fresh draft for review; it does not send automatically."); + setMessage(created.delivery_mode === "autonomous" + ? "Autonomous schedule created from the exact approved source execution." + : "Manual schedule created. Each occurrence prepares a fresh draft for review."); } catch (err) { setError(err instanceof Error ? err.message : String(err)); } finally { @@ -433,17 +440,19 @@ export default function CampaignOverviewPage({ settings, auth, campaignId }: {se