docs(campaign): distinguish worker process evidence

dev/mail-testbed/README.md

@@ -67,10 +67,23 @@ set +a
 
 The default run also uses controlled loopback protocol endpoints to prove that
 an SMTP connection loss before transmission is temporary, an explicit SMTP
-authentication rejection is permanent, and an IMAP authentication rejection
+authentication rejection is permanent, a final `451` response after DATA is
-after SMTP acceptance leaves the send accepted while the append fails. A
+temporary, one accepted and one refused RCPT command is retained as partial
+envelope acceptance, a connection loss after complete DATA is frozen as
+`outcome_unknown`, and an IMAP authentication rejection after SMTP acceptance
+leaves the send accepted while the append fails. A
 second ordinary send must be rejected before another provider effect.
 
+The worker drill queues one job, starts the registered
+`govoplan.campaigns.send_email` task body in a dedicated OS process, waits until
+the controlled endpoint has received complete DATA, terminates that process,
+and starts the same task body in a fresh process. It proves the durable
+`sending`/unfinished-attempt boundary is recovered as `outcome_unknown`
+without a second SMTP connection or DATA transaction. This is a real process
+and task-boundary interruption, but it does not start a Celery daemon, Redis
+broker, or broker redelivery; `celery_broker_redelivery` therefore remains
+`false` in the evidence.
+
 The bounded JSON contains no endpoint, account, address, credential, profile,
 campaign, version, or job identifiers. It records module versions, the fixture
 hash, normalized classifications/counts, the Mail-profile boundary, required
@@ -84,11 +97,11 @@ This runner is restricted to literal loopback IP addresses and the synchronous
 Campaign delivery mode. Hostnames such as `localhost` and every non-loopback
 address fail before profile creation, avoiding a DNS change between validation
 and connection. It is local target-like evidence, not approval of an
-institution's SMTP/IMAP service. A post-DATA connection loss
+institution's SMTP/IMAP service. The controlled post-DATA, temporary-response,
-with ambiguous SMTP outcome, an explicit temporary SMTP response, partial
+partial-refusal, and task-process interruption drills are local effect-level
-recipient refusal, and a real worker restart remain separate drills and are
+proof, not proof of a target provider's behavior or Redis/Celery broker
-explicitly `false` in the coverage projection. Use `--success-only` only when
+redelivery. Use `--success-only` only when testing the success journey without
-testing the success journey without the local failure endpoints.
+the local failure endpoints.
 
 Starting the maintained test bed requires a working Docker CLI, Compose plugin,
 daemon/socket access, and permission to pull `greenmail/standalone:2.1.9`. The

dev/mail-testbed/run_campaign_acceptance.py

@@ -20,6 +20,7 @@ import json
 import os
 import shutil
 import socketserver
+import subprocess
 import sys
 import tempfile
 import threading
@@ -37,7 +38,7 @@ from uuid import uuid4
 SCRIPT_ROOT = Path(__file__).resolve().parent
 REPOSITORY_ROOT = SCRIPT_ROOT.parents[1]
 DEFAULT_FIXTURE = REPOSITORY_ROOT / "examples" / "greenmail-delivery" / "campaign.json"
-EVIDENCE_SCHEMA = "govoplan.campaign.greenmail-acceptance.v1"
+EVIDENCE_SCHEMA = "govoplan.campaign.greenmail-acceptance.v2"
 EXPECTED_AUDIT_ACTIONS = frozenset(
     {
         "campaign.created",
@@ -47,6 +48,7 @@ EXPECTED_AUDIT_ACTIONS = frozenset(
         "campaign.sent_now",
         "campaign.send_now_rejected",
         "campaign.append_sent_enqueued",
+        "campaign.queued",
     }
 )
 FORBIDDEN_CAMPAIGN_KEYS = frozenset(
@@ -114,6 +116,35 @@ REPORT_IMAP_STATUSES = frozenset(
         "skipped",
     }
 )
+DURABLE_ATTEMPT_STATUSES = frozenset(
+    {
+        "failed_permanent",
+        "failed_temporary",
+        "outcome_unknown",
+        "smtp_accepted",
+        "smtp_accepted_with_refusals",
+        "smtp_in_progress",
+    }
+)
+SMTP_FAULT_MODES = frozenset(
+    {
+        "temporary_data_response",
+        "partial_recipient_refusal",
+        "post_data_disconnect",
+        "post_data_hold",
+    }
+)
+WORKER_TASK_CODE = """
+import sys
+from govoplan_core.celery_app import send_email
+
+result = send_email.run(sys.argv[1])
+if not isinstance(result, dict) or result.get("status") not in {
+    "outcome_unknown",
+    "smtp_accepted",
+}:
+    raise SystemExit(2)
+"""
 
 
 class AcceptanceError(RuntimeError):
@@ -195,6 +226,23 @@ class CampaignBoundary:
         }
 
 
+@dataclass(frozen=True, slots=True)
+class PreparedCampaignScenario:
+    campaign_id: str
+    version_id: str
+    subject: str
+    validation: dict[str, Any]
+    build: dict[str, Any]
+    boundary: CampaignBoundary
+
+    def public_evidence(self) -> dict[str, Any]:
+        return {
+            "validation": dict(self.validation),
+            "build": dict(self.build),
+            "campaign_mail_boundary": self.boundary.as_dict(),
+        }
+
+
 def _env(name: str, default: str) -> str:
     return os.environ.get(name, default).strip() or default
 
@@ -337,6 +385,7 @@ def materialize_campaign_fixture(
     settings: TestbedSettings,
     scenario: str,
     run_token: str,
+    additional_envelope_recipient: bool = False,
 ) -> tuple[dict[str, Any], str]:
     raw = json.loads(fixture_path.read_text(encoding="utf-8"))
     if not isinstance(raw, dict):
@@ -358,6 +407,18 @@ def materialize_campaign_fixture(
         raise AcceptanceError("Campaign fixture address shape is invalid")
     from_addresses[0]["email"] = settings.sender
     to_addresses[0]["email"] = settings.recipient
+    if additional_envelope_recipient:
+        try:
+            local_part, domain = settings.recipient.rsplit("@", 1)
+        except ValueError as exc:
+            raise AcceptanceError("Campaign acceptance recipient must be an email address") from exc
+        to_addresses.append(
+            {
+                "email": f"{local_part}+partial-{run_token}@{domain}",
+                "name": "Controlled partial-refusal recipient",
+                "type": "to",
+            }
+        )
     fields["acceptance_run"] = run_token
     template = raw.get("template")
     if not isinstance(template, dict):
@@ -550,7 +611,39 @@ def _allowlisted_count_map(
     return projected
 
 
-def execute_campaign_scenario(
+def _durable_state_evidence(payload: Mapping[str, Any]) -> dict[str, Any]:
+    job_count = payload.get("job_count")
+    unfinished_attempt_count = payload.get("unfinished_attempt_count")
+    if (
+        not isinstance(job_count, int)
+        or isinstance(job_count, bool)
+        or job_count < 0
+        or not isinstance(unfinished_attempt_count, int)
+        or isinstance(unfinished_attempt_count, bool)
+        or unfinished_attempt_count < 0
+    ):
+        raise AcceptanceError("Campaign durable delivery state count is invalid")
+    send_statuses = payload.get("send_status_counts")
+    attempt_statuses = payload.get("attempt_status_counts")
+    if not isinstance(send_statuses, dict) or not isinstance(attempt_statuses, dict):
+        raise AcceptanceError("Campaign durable delivery state is invalid")
+    return {
+        "job_count": job_count,
+        "send_status_counts": _allowlisted_count_map(
+            send_statuses,
+            allowed=REPORT_SEND_STATUSES,
+            label="Campaign durable send status",
+        ),
+        "attempt_status_counts": _allowlisted_count_map(
+            attempt_statuses,
+            allowed=DURABLE_ATTEMPT_STATUSES,
+            label="Campaign durable attempt status",
+        ),
+        "unfinished_attempt_count": unfinished_attempt_count,
+    }
+
+
+def prepare_campaign_scenario(
     client: ApiClient,
     headers: Mapping[str, str],
     *,
@@ -559,10 +652,8 @@ def execute_campaign_scenario(
     settings: TestbedSettings,
     scenario: str,
     snapshot_probe: Callable[[str], tuple[Mapping[str, Any], Mapping[str, Any]]],
-    audit_probe: Callable[[str, str], Mapping[str, int]],
+    additional_envelope_recipient: bool = False,
-    append_sent: bool,
+) -> PreparedCampaignScenario:
-    repeat_send: bool,
-) -> tuple[dict[str, Any], str]:
     run_token = uuid4().hex[:12]
     raw, subject = materialize_campaign_fixture(
         fixture_path,
@@ -570,6 +661,7 @@ def execute_campaign_scenario(
         settings=settings,
         scenario=scenario,
         run_token=run_token,
+        additional_envelope_recipient=additional_envelope_recipient,
     )
     created = _expect(
         client.post("/api/v1/campaigns", headers=dict(headers), json={"config": raw}),
@@ -608,6 +700,51 @@ def execute_campaign_scenario(
 
     stored_raw, execution_snapshot = snapshot_probe(version_id)
     boundary = assert_campaign_boundary(stored_raw, execution_snapshot, profile_id=profile_id)
+    return PreparedCampaignScenario(
+        campaign_id=campaign_id,
+        version_id=version_id,
+        subject=subject,
+        validation={
+            "ok": True,
+            "error_count": int(validation.get("error_count") or 0),
+            "warning_count": int(validation.get("warning_count") or 0),
+        },
+        build={
+            "built_count": int(build.get("built_count") or 0),
+            "build_failed_count": int(build.get("build_failed_count") or 0),
+            "queueable_count": int(build.get("queueable_count") or 0),
+        },
+        boundary=boundary,
+    )
+
+
+def execute_campaign_scenario(
+    client: ApiClient,
+    headers: Mapping[str, str],
+    *,
+    fixture_path: Path,
+    profile_id: str,
+    settings: TestbedSettings,
+    scenario: str,
+    snapshot_probe: Callable[[str], tuple[Mapping[str, Any], Mapping[str, Any]]],
+    audit_probe: Callable[[str, str], Mapping[str, int]],
+    append_sent: bool,
+    repeat_send: bool,
+    delivery_probe: Callable[[str, str], Mapping[str, Any]] | None = None,
+    additional_envelope_recipient: bool = False,
+) -> tuple[dict[str, Any], str]:
+    prepared = prepare_campaign_scenario(
+        client,
+        headers,
+        fixture_path=fixture_path,
+        profile_id=profile_id,
+        settings=settings,
+        scenario=scenario,
+        snapshot_probe=snapshot_probe,
+        additional_envelope_recipient=additional_envelope_recipient,
+    )
+    campaign_id = prepared.campaign_id
+    version_id = prepared.version_id
     send_payload = _expect(
         client.post(
             f"/api/v1/campaigns/{campaign_id}/send-now",
@@ -699,17 +836,7 @@ def execute_campaign_scenario(
         raise AcceptanceError(f"{scenario} Campaign audit evidence is incomplete")
 
     evidence = {
-        "validation": {
+        **prepared.public_evidence(),
-            "ok": True,
-            "error_count": int(validation.get("error_count") or 0),
-            "warning_count": int(validation.get("warning_count") or 0),
-        },
-        "build": {
-            "built_count": int(build.get("built_count") or 0),
-            "build_failed_count": int(build.get("build_failed_count") or 0),
-            "queueable_count": int(build.get("queueable_count") or 0),
-        },
-        "campaign_mail_boundary": boundary.as_dict(),
         "send": _send_evidence(send_payload),
         "report": _report_evidence(report),
         "audit_actions": audit_actions,
@@ -718,7 +845,9 @@ def execute_campaign_scenario(
         evidence["repeated_send"] = repeat_evidence
     if append_evidence is not None:
         evidence["append_sent"] = append_evidence
-    return evidence, subject
+    if delivery_probe is not None:
+        evidence["durable_state"] = _durable_state_evidence(delivery_probe(campaign_id, version_id))
+    return evidence, prepared.subject
 
 
 def _mailbox_state(
@@ -800,6 +929,111 @@ class _LoopbackServer(socketserver.TCPServer):
     allow_reuse_address = True
 
 
+class _SmtpFaultServer(_LoopbackServer):
+    def __init__(self, mode: str) -> None:
+        if mode not in SMTP_FAULT_MODES:
+            raise AcceptanceError("Unsupported SMTP fault mode")
+        self.mode = mode
+        self.data_received = threading.Event()
+        self.release_data_response = threading.Event()
+        self._counter_lock = threading.Lock()
+        self._counters = {
+            "connection_count": 0,
+            "accepted_rcpt_commands": 0,
+            "refused_rcpt_commands": 0,
+            "data_transactions": 0,
+        }
+        super().__init__(("127.0.0.1", 0), _FaultSmtpHandler)
+
+    def increment(self, key: str) -> int:
+        with self._counter_lock:
+            self._counters[key] += 1
+            return self._counters[key]
+
+    def evidence(self) -> dict[str, int]:
+        with self._counter_lock:
+            return dict(self._counters)
+
+
+class _FaultSmtpHandler(socketserver.StreamRequestHandler):
+    def handle(self) -> None:
+        server = self.server
+        if not isinstance(server, _SmtpFaultServer):
+            return
+        server.increment("connection_count")
+        self._write(b"220 local acceptance SMTP\r\n")
+        while line := self.rfile.readline(65_536):
+            command = line.split(None, 1)[0].upper()
+            if command in {b"EHLO", b"HELO"}:
+                self._write(b"250-localhost\r\n250 AUTH PLAIN\r\n")
+            elif command == b"AUTH":
+                self._write(b"235 2.7.0 Authentication successful\r\n")
+            elif command == b"MAIL":
+                self._write(b"250 2.1.0 Sender accepted\r\n")
+            elif command == b"RCPT":
+                rcpt_number = (
+                    server.evidence()["accepted_rcpt_commands"]
+                    + server.evidence()["refused_rcpt_commands"]
+                    + 1
+                )
+                if server.mode == "partial_recipient_refusal" and rcpt_number > 1:
+                    server.increment("refused_rcpt_commands")
+                    self._write(b"550 5.1.1 Controlled recipient refusal\r\n")
+                else:
+                    server.increment("accepted_rcpt_commands")
+                    self._write(b"250 2.1.5 Recipient accepted\r\n")
+            elif command == b"DATA":
+                self._write(b"354 End data with <CR><LF>.<CR><LF>\r\n")
+                if not self._read_data_transaction():
+                    return
+                server.increment("data_transactions")
+                server.data_received.set()
+                if server.mode == "temporary_data_response":
+                    self._write(b"451 4.3.0 Controlled temporary rejection\r\n")
+                elif server.mode == "post_data_disconnect":
+                    return
+                elif server.mode == "post_data_hold":
+                    server.release_data_response.wait()
+                    return
+                else:
+                    self._write(b"250 2.0.0 Message accepted\r\n")
+            elif command == b"RSET":
+                self._write(b"250 2.0.0 Reset\r\n")
+            elif command == b"NOOP":
+                self._write(b"250 2.0.0 OK\r\n")
+            elif command == b"QUIT":
+                self._write(b"221 2.0.0 Bye\r\n")
+                return
+            else:
+                self._write(b"500 5.5.1 Unsupported acceptance command\r\n")
+
+    def _read_data_transaction(self) -> bool:
+        while line := self.rfile.readline(65_536):
+            if line in {b".\n", b".\r\n"}:
+                return True
+        return False
+
+    def _write(self, value: bytes) -> None:
+        self.wfile.write(value)
+        self.wfile.flush()
+
+
+@dataclass(frozen=True, slots=True)
+class SmtpFaultEndpoint:
+    host: str
+    port: int
+    _server: _SmtpFaultServer
+
+    def wait_for_data(self, timeout_seconds: float) -> bool:
+        return self._server.data_received.wait(timeout_seconds)
+
+    def evidence(self) -> dict[str, int]:
+        return self._server.evidence()
+
+    def release_held_connection(self) -> None:
+        self._server.release_data_response.set()
+
+
 class _RejectingSmtpHandler(socketserver.StreamRequestHandler):
     def handle(self) -> None:
         self.wfile.write(b"220 local acceptance SMTP\r\n")
@@ -866,6 +1100,25 @@ def rejecting_imap_endpoint() -> Iterator[tuple[str, int]]:
         yield endpoint
 
 
+@contextmanager
+def smtp_fault_endpoint(mode: str) -> Iterator[SmtpFaultEndpoint]:
+    server = _SmtpFaultServer(mode)
+    thread = threading.Thread(
+        target=server.serve_forever,
+        name=f"govoplan-smtp-{mode.replace('_', '-')}",
+        daemon=True,
+    )
+    thread.start()
+    try:
+        host, port = server.server_address
+        yield SmtpFaultEndpoint(str(host), int(port), server)
+    finally:
+        server.release_data_response.set()
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=2)
+
+
 @contextmanager
 def _scripted_endpoint(
     handler: type[socketserver.BaseRequestHandler],
@@ -911,6 +1164,187 @@ def ensure_sent_folder(settings: TestbedSettings) -> None:
     raise AcceptanceError("GreenMail IMAP did not become ready for the Campaign acceptance run") from last_error
 
 
+def _start_campaign_worker_task(job_id: str) -> subprocess.Popen[bytes]:
+    """Start the exact Celery send task body in an isolated OS process.
+
+    The acceptance environment deliberately has no Redis broker. Calling the
+    registered task's ``run`` method exercises the same task/capability path a
+    Celery child process uses, without claiming broker redelivery coverage.
+    """
+
+    return subprocess.Popen(
+        [sys.executable, "-c", WORKER_TASK_CODE, job_id],
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        close_fds=True,
+    )
+
+
+def _terminate_worker_process(process: subprocess.Popen[bytes]) -> None:
+    if process.poll() is not None:
+        raise AcceptanceError("Campaign worker task exited before controlled interruption")
+    process.terminate()
+    try:
+        process.wait(timeout=5)
+    except subprocess.TimeoutExpired:
+        process.kill()
+        try:
+            process.wait(timeout=5)
+        except subprocess.TimeoutExpired as exc:
+            raise AcceptanceError("Campaign worker task could not be stopped") from exc
+
+
+def _wait_for_worker_process(process: subprocess.Popen[bytes], *, timeout_seconds: int) -> None:
+    try:
+        return_code = process.wait(timeout=timeout_seconds)
+    except subprocess.TimeoutExpired as exc:
+        process.kill()
+        process.wait(timeout=5)
+        raise AcceptanceError("Restarted Campaign worker task did not complete") from exc
+    if return_code != 0:
+        raise AcceptanceError("Restarted Campaign worker task failed")
+
+
+def execute_worker_interruption_scenario(
+    client: ApiClient,
+    headers: Mapping[str, str],
+    *,
+    fixture_path: Path,
+    profile_id: str,
+    settings: TestbedSettings,
+    endpoint: SmtpFaultEndpoint,
+    snapshot_probe: Callable[[str], tuple[Mapping[str, Any], Mapping[str, Any]]],
+    audit_probe: Callable[[str, str], Mapping[str, int]],
+    delivery_probe: Callable[[str, str], Mapping[str, Any]],
+    worker_job_probe: Callable[[str], str],
+) -> dict[str, Any]:
+    prepared = prepare_campaign_scenario(
+        client,
+        headers,
+        fixture_path=fixture_path,
+        profile_id=profile_id,
+        settings=settings,
+        scenario="worker_interruption",
+        snapshot_probe=snapshot_probe,
+    )
+    queue_payload = _expect(
+        client.post(
+            f"/api/v1/campaigns/{prepared.campaign_id}/queue",
+            headers=dict(headers),
+            json={
+                "version_id": prepared.version_id,
+                "include_warnings": True,
+                "enqueue_celery": False,
+                "dry_run": False,
+            },
+        ),
+        200,
+        "Worker-interruption Campaign queue",
+    )
+    expected_queue = {
+        "queued_count": 1,
+        "skipped_count": 0,
+        "blocked_count": 0,
+        "enqueued_count": 0,
+        "delivery_mode": "database_queue",
+        "worker_queue_available": False,
+        "dry_run": False,
+    }
+    queue_evidence = {key: queue_payload.get(key) for key in expected_queue}
+    if queue_evidence != expected_queue:
+        raise AcceptanceError("Worker-interruption Campaign was not durably queued once")
+
+    job_id = worker_job_probe(prepared.version_id)
+    first_worker = _start_campaign_worker_task(job_id)
+    try:
+        if not endpoint.wait_for_data(settings.provider_timeout_seconds):
+            raise AcceptanceError("Campaign worker did not reach the controlled SMTP DATA boundary")
+        _terminate_worker_process(first_worker)
+    finally:
+        if first_worker.poll() is None:
+            first_worker.kill()
+            first_worker.wait(timeout=5)
+        endpoint.release_held_connection()
+
+    interrupted_state = _durable_state_evidence(
+        delivery_probe(prepared.campaign_id, prepared.version_id)
+    )
+    if interrupted_state != {
+        "job_count": 1,
+        "send_status_counts": {"sending": 1},
+        "attempt_status_counts": {"smtp_in_progress": 1},
+        "unfinished_attempt_count": 1,
+    }:
+        raise AcceptanceError("Interrupted worker state was not durably retained at SMTP in-progress")
+
+    restarted_worker = _start_campaign_worker_task(job_id)
+    _wait_for_worker_process(
+        restarted_worker,
+        timeout_seconds=settings.provider_timeout_seconds,
+    )
+    restarted_state = _durable_state_evidence(
+        delivery_probe(prepared.campaign_id, prepared.version_id)
+    )
+    expected_restarted_state = {
+        "job_count": 1,
+        "send_status_counts": {"outcome_unknown": 1},
+        "attempt_status_counts": {"outcome_unknown": 1},
+        "unfinished_attempt_count": 0,
+    }
+    if restarted_state != expected_restarted_state:
+        raise AcceptanceError("Restarted worker did not freeze the unfinished SMTP attempt")
+
+    protocol_evidence = endpoint.evidence()
+    if protocol_evidence != {
+        "connection_count": 1,
+        "accepted_rcpt_commands": 1,
+        "refused_rcpt_commands": 0,
+        "data_transactions": 1,
+    }:
+        raise AcceptanceError("Restarted worker contacted SMTP or produced an unexpected transaction")
+
+    report = _expect(
+        client.get(
+            f"/api/v1/campaigns/{prepared.campaign_id}/report",
+            headers=dict(headers),
+            params={"version_id": prepared.version_id},
+        ),
+        200,
+        "Worker-interruption Campaign report",
+    )
+    report_evidence = _report_evidence(report)
+    if report_evidence["send_status_counts"] != {"outcome_unknown": 1}:
+        raise AcceptanceError("Worker-interruption Campaign report did not retain outcome unknown")
+
+    audit_actions = dict(
+        sorted(audit_probe(prepared.campaign_id, prepared.version_id).items())
+    )
+    if not {
+        "campaign.created",
+        "campaign.validated",
+        "campaign.messages_built",
+        "campaign.queued",
+    }.issubset(audit_actions):
+        raise AcceptanceError("Worker-interruption Campaign audit evidence is incomplete")
+
+    return {
+        **prepared.public_evidence(),
+        "queue": queue_evidence,
+        "interrupted_durable_state": interrupted_state,
+        "restarted_durable_state": restarted_state,
+        "protocol": protocol_evidence,
+        "report": report_evidence,
+        "audit_actions": audit_actions,
+        "process_boundary": {
+            "dedicated_task_process_terminated_after_data": True,
+            "fresh_task_process_completed": True,
+            "duplicate_smtp_transaction_prevented": True,
+            "celery_broker_redelivery_exercised": False,
+        },
+    }
+
+
 def run_acceptance(
     client: ApiClient,
     headers: Mapping[str, str],
@@ -919,6 +1353,8 @@ def run_acceptance(
     fixture_path: Path,
     snapshot_probe: Callable[[str], tuple[Mapping[str, Any], Mapping[str, Any]]],
     audit_probe: Callable[[str, str], Mapping[str, int]],
+    delivery_probe: Callable[[str, str], Mapping[str, Any]],
+    worker_job_probe: Callable[[str], str],
     include_failure_drills: bool,
     module_versions: Mapping[str, str],
 ) -> dict[str, Any]:
@@ -954,6 +1390,7 @@ def run_acceptance(
         audit_probe=audit_probe,
         append_sent=True,
         repeat_send=True,
+        delivery_probe=delivery_probe,
     )
     success["provider_verification"] = _wait_for_provider_evidence(
         client,
@@ -989,6 +1426,7 @@ def run_acceptance(
                 audit_probe=audit_probe,
                 append_sent=False,
                 repeat_send=False,
+                delivery_probe=delivery_probe,
             )
         if temporary["report"]["send_status_counts"] != {"failed_temporary": 1}:
             raise AcceptanceError("SMTP connection failure was not classified as temporary")
@@ -1014,6 +1452,7 @@ def run_acceptance(
                 audit_probe=audit_probe,
                 append_sent=False,
                 repeat_send=False,
+                delivery_probe=delivery_probe,
             )
         if permanent["report"]["send_status_counts"] != {"failed_permanent": 1}:
             raise AcceptanceError("SMTP authentication failure was not classified as permanent")
@@ -1039,11 +1478,150 @@ def run_acceptance(
                 audit_probe=audit_probe,
                 append_sent=True,
                 repeat_send=True,
+                delivery_probe=delivery_probe,
             )
         if imap_failure["report"]["imap_status_counts"] != {"failed": 1}:
             raise AcceptanceError("IMAP authentication failure was not retained as a failed append")
         drills["imap_authentication_failure"] = imap_failure
 
+        with smtp_fault_endpoint("temporary_data_response") as endpoint:
+            temporary_response_profile = create_mail_profile(
+                client,
+                headers,
+                settings,
+                name="Explicit temporary SMTP response drill",
+                smtp_host=endpoint.host,
+                smtp_port=endpoint.port,
+            )
+            temporary_response, _ = execute_campaign_scenario(
+                client,
+                headers,
+                fixture_path=fixture_path,
+                profile_id=temporary_response_profile,
+                settings=settings,
+                scenario="smtp_temporary_response",
+                snapshot_probe=snapshot_probe,
+                audit_probe=audit_probe,
+                append_sent=False,
+                repeat_send=False,
+                delivery_probe=delivery_probe,
+            )
+            temporary_response["protocol"] = endpoint.evidence()
+        if (
+            temporary_response["report"]["send_status_counts"] != {"failed_temporary": 1}
+            or temporary_response["durable_state"]["attempt_status_counts"]
+            != {"failed_temporary": 1}
+            or temporary_response["protocol"]
+            != {
+                "connection_count": 1,
+                "accepted_rcpt_commands": 1,
+                "refused_rcpt_commands": 0,
+                "data_transactions": 1,
+            }
+        ):
+            raise AcceptanceError("Explicit SMTP 451 response was not retained as temporary")
+        drills["smtp_temporary_response"] = temporary_response
+
+        with smtp_fault_endpoint("partial_recipient_refusal") as endpoint:
+            partial_profile = create_mail_profile(
+                client,
+                headers,
+                settings,
+                name="Partial SMTP recipient-refusal drill",
+                smtp_host=endpoint.host,
+                smtp_port=endpoint.port,
+            )
+            partial, _ = execute_campaign_scenario(
+                client,
+                headers,
+                fixture_path=fixture_path,
+                profile_id=partial_profile,
+                settings=settings,
+                scenario="partial_envelope_refusal",
+                snapshot_probe=snapshot_probe,
+                audit_probe=audit_probe,
+                append_sent=False,
+                repeat_send=False,
+                delivery_probe=delivery_probe,
+                additional_envelope_recipient=True,
+            )
+            partial["protocol"] = endpoint.evidence()
+        if (
+            partial["report"]["send_status_counts"] != {"smtp_accepted": 1}
+            or partial["durable_state"]["attempt_status_counts"]
+            != {"smtp_accepted_with_refusals": 1}
+            or partial["protocol"]
+            != {
+                "connection_count": 1,
+                "accepted_rcpt_commands": 1,
+                "refused_rcpt_commands": 1,
+                "data_transactions": 1,
+            }
+        ):
+            raise AcceptanceError("Partial SMTP envelope refusal was not durably distinguished")
+        drills["partial_envelope_refusal"] = partial
+
+        with smtp_fault_endpoint("post_data_disconnect") as endpoint:
+            ambiguous_profile = create_mail_profile(
+                client,
+                headers,
+                settings,
+                name="Post-DATA SMTP ambiguity drill",
+                smtp_host=endpoint.host,
+                smtp_port=endpoint.port,
+            )
+            ambiguous, _ = execute_campaign_scenario(
+                client,
+                headers,
+                fixture_path=fixture_path,
+                profile_id=ambiguous_profile,
+                settings=settings,
+                scenario="post_data_connection_loss",
+                snapshot_probe=snapshot_probe,
+                audit_probe=audit_probe,
+                append_sent=False,
+                repeat_send=False,
+                delivery_probe=delivery_probe,
+            )
+            ambiguous["protocol"] = endpoint.evidence()
+        if (
+            ambiguous["report"]["send_status_counts"] != {"outcome_unknown": 1}
+            or ambiguous["durable_state"]["attempt_status_counts"]
+            != {"outcome_unknown": 1}
+            or ambiguous["protocol"]
+            != {
+                "connection_count": 1,
+                "accepted_rcpt_commands": 1,
+                "refused_rcpt_commands": 0,
+                "data_transactions": 1,
+            }
+        ):
+            raise AcceptanceError("Post-DATA connection loss was not frozen as outcome unknown")
+        drills["post_data_connection_loss"] = ambiguous
+
+        with smtp_fault_endpoint("post_data_hold") as endpoint:
+            worker_profile = create_mail_profile(
+                client,
+                headers,
+                settings,
+                name="Campaign worker interruption drill",
+                smtp_host=endpoint.host,
+                smtp_port=endpoint.port,
+            )
+            worker_interruption = execute_worker_interruption_scenario(
+                client,
+                headers,
+                fixture_path=fixture_path,
+                profile_id=worker_profile,
+                settings=settings,
+                endpoint=endpoint,
+                snapshot_probe=snapshot_probe,
+                audit_probe=audit_probe,
+                delivery_probe=delivery_probe,
+                worker_job_probe=worker_job_probe,
+            )
+        drills["worker_interruption"] = worker_interruption
+
     evidence = {
         "schema_version": EVIDENCE_SCHEMA,
         "generated_at": datetime.now(timezone.utc).isoformat(),
@@ -1062,13 +1640,14 @@ def run_acceptance(
             "imap_append": True,
             "repeat_ordinary_send_no_duplicate": True,
             "smtp_connection_failure": include_failure_drills,
-            "smtp_temporary_response": False,
+            "smtp_temporary_response": include_failure_drills,
             "smtp_permanent_authentication_failure": include_failure_drills,
-            "partial_envelope_refusal": False,
+            "partial_envelope_refusal": include_failure_drills,
             "imap_authentication_failure": include_failure_drills,
-            "post_data_connection_loss_outcome_unknown": False,
+            "post_data_connection_loss_outcome_unknown": include_failure_drills,
             "source_artifact_provenance": False,
-            "worker_restart_interruption": False,
+            "worker_restart_interruption": include_failure_drills,
+            "celery_broker_redelivery": False,
         },
     }
     _assert_evidence_safe(evidence, settings=settings)
@@ -1235,6 +1814,48 @@ def _bootstrap_and_run(
                 ]
             return dict(Counter(actions))
 
+        def delivery_probe(campaign_id: str, version_id: str) -> Mapping[str, Any]:
+            from govoplan_campaign.backend.db.models import CampaignJob, SendAttempt
+
+            with database.SessionLocal() as session:
+                jobs = (
+                    session.query(CampaignJob)
+                    .filter(
+                        CampaignJob.campaign_id == campaign_id,
+                        CampaignJob.campaign_version_id == version_id,
+                    )
+                    .all()
+                )
+                job_ids = [job.id for job in jobs]
+                attempts = (
+                    session.query(SendAttempt)
+                    .filter(SendAttempt.job_id.in_(job_ids))
+                    .all()
+                    if job_ids
+                    else []
+                )
+                return {
+                    "job_count": len(jobs),
+                    "send_status_counts": dict(Counter(job.send_status for job in jobs)),
+                    "attempt_status_counts": dict(Counter(attempt.status for attempt in attempts)),
+                    "unfinished_attempt_count": sum(
+                        1 for attempt in attempts if attempt.finished_at is None
+                    ),
+                }
+
+        def worker_job_probe(version_id: str) -> str:
+            from govoplan_campaign.backend.db.models import CampaignJob
+
+            with database.SessionLocal() as session:
+                jobs = (
+                    session.query(CampaignJob)
+                    .filter(CampaignJob.campaign_version_id == version_id)
+                    .all()
+                )
+                if len(jobs) != 1:
+                    raise AcceptanceError("Worker-interruption Campaign did not contain one job")
+                return jobs[0].id
+
         with TestClient(app) as client:
             login = _expect(
                 client.post(
@@ -1265,6 +1886,8 @@ def _bootstrap_and_run(
                 fixture_path=fixture_path,
                 snapshot_probe=snapshot_probe,
                 audit_probe=audit_probe,
+                delivery_probe=delivery_probe,
+                worker_job_probe=worker_job_probe,
                 include_failure_drills=include_failure_drills,
                 module_versions=module_versions,
             )

docs/CAMPAIGN_DELIVERY_RUNBOOK.md

@@ -107,15 +107,20 @@ bed where possible:
 For the maintained loopback baseline, run
 `dev/mail-testbed/run_campaign_acceptance.py`. It proves the public Campaign
 path for SMTP acceptance, IMAP append, repeat-send blocking, an SMTP connection
-failure before transmission, an explicit SMTP authentication rejection, and an
+failure before transmission, an explicit SMTP authentication rejection, an
-IMAP authentication rejection after SMTP acceptance. Its evidence is an
+explicit temporary `451` response after DATA, partial RCPT refusal, a
-allowlisted classification/count projection; raw provider diagnostics and
+connection loss after complete DATA, and an IMAP authentication rejection
-transport/account identifiers are deliberately excluded.
+after SMTP acceptance. Its evidence is an allowlisted classification/count
+projection; raw provider diagnostics and transport/account identifiers are
+deliberately excluded.
 
-The runner does not simulate a connection loss after SMTP DATA and does not
+The runner also terminates a dedicated OS process executing the registered
-restart a real worker. Keep those two checklist items open until a controlled
+Campaign send task after complete DATA, then invokes the task in a fresh
-fault proxy/target provider and supervised worker environment can produce the
+process. The unfinished durable attempt must become `outcome_unknown` and the
-corresponding `outcome_unknown`, restart, retry, and reconciliation evidence.
+endpoint must observe no second connection or DATA transaction. This covers
+the worker task/process boundary. It does not cover Redis/Celery delivery,
+acknowledgement, broker redelivery, or daemon supervision; repeat the drill in
+that infrastructure before approving a production worker deployment.
 
 ## Reporting Checks
 

docs/EXAMPLE_CAMPAIGNS_AND_RELEASE_CHECKLIST.md

@@ -59,9 +59,12 @@ Before tagging a campaign release:
   projection. It must show one SMTP acceptance, one IMAP append, no duplicate
   effect from a repeated ordinary send, matching Campaign report/audit state,
   and no resolved transport material in Campaign JSON or its execution
-  snapshot.
+  snapshot. Its controlled endpoint evidence must also show explicit SMTP 451,
-- Do not interpret the local failure drills as post-DATA ambiguity or worker
+  partial RCPT refusal, post-DATA ambiguity, and task-process interruption
-  restart proof; the evidence coverage flags must keep those gaps explicit.
+  classifications without retaining addresses or provider diagnostics.
+- Treat the task-process restart proof separately from the still-open
+  Redis/Celery broker redelivery and daemon-supervision check; the coverage
+  projection must keep `celery_broker_redelivery` false.
 - Confirm reusable mail profile selection is revalidated after campaign owner
   transfer.
 - Confirm every inline SMTP/IMAP field is rejected on import/write, omitted

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@govoplan/campaign-webui",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "private": true,
   "type": "module",
   "main": "webui/src/index.ts",

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "govoplan-campaign"
-version = "0.1.10"
+version = "0.1.11"
 description = "GovOPlaN campaigns module with backend and WebUI integration."
 readme = "README.md"
 requires-python = ">=3.12"

src/govoplan_campaign/backend/capabilities.py

@@ -279,6 +279,12 @@ class CampaignDeliveryTaskService(CampaignDeliveryTaskProvider):
 
 
 def delivery_tasks_capability(context: object) -> CampaignDeliveryTaskService:
+    from govoplan_campaign.backend.runtime import configure_runtime
+
+    configure_runtime(
+        registry=getattr(context, "registry", None),
+        settings=getattr(context, "settings", None),
+    )
     return CampaignDeliveryTaskService()
 
 

src/govoplan_campaign/backend/manifest.py

@@ -156,7 +156,7 @@ def _campaigns_router(context: ModuleContext):
 manifest = ModuleManifest(
     id="campaigns",
     name="Campaigns",
-    version="0.1.10",
+    version="0.1.11",
     required_capabilities=(CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR),
     optional_dependencies=("files", "mail", "notifications", "addresses"),
     provides_interfaces=(

tests/test_mail_testbed_acceptance.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import importlib.util
 import json
+import smtplib
 import sys
 from pathlib import Path
 from typing import Any
@@ -317,6 +318,16 @@ def test_evidence_projection_rejects_unknown_status_keys() -> None:
             }
         )
 
+    with pytest.raises(runner.AcceptanceError, match="durable attempt status"):
+        runner._durable_state_evidence(
+            {
+                "job_count": 1,
+                "send_status_counts": {"sending": 1},
+                "attempt_status_counts": {"provider-secret": 1},
+                "unfinished_attempt_count": 1,
+            }
+        )
+
     with pytest.raises(runner.AcceptanceError, match="synchronous delivery mode"):
         runner._send_evidence(
             {
@@ -337,6 +348,85 @@ def test_evidence_projection_rejects_unknown_status_keys() -> None:
         )
 
 
+def _open_fault_smtp(endpoint):
+    client = smtplib.SMTP(endpoint.host, endpoint.port, timeout=5)
+    client.ehlo()
+    client.login("acceptance-user", "acceptance-password")
+    return client
+
+
+def test_explicit_temporary_smtp_response_occurs_after_complete_data() -> None:
+    with runner.smtp_fault_endpoint("temporary_data_response") as endpoint:
+        client = _open_fault_smtp(endpoint)
+        try:
+            with pytest.raises(smtplib.SMTPDataError) as captured:
+                client.sendmail(
+                    "sender@example.test",
+                    ["recipient@example.test"],
+                    b"Subject: temporary\r\n\r\nmessage",
+                )
+        finally:
+            client.close()
+        assert captured.value.smtp_code == 451
+        assert endpoint.evidence() == {
+            "connection_count": 1,
+            "accepted_rcpt_commands": 1,
+            "refused_rcpt_commands": 0,
+            "data_transactions": 1,
+        }
+
+
+def test_partial_recipient_refusal_retains_one_accepted_envelope() -> None:
+    with runner.smtp_fault_endpoint("partial_recipient_refusal") as endpoint:
+        client = _open_fault_smtp(endpoint)
+        try:
+            refused = client.sendmail(
+                "sender@example.test",
+                ["accepted@example.test", "refused@example.test"],
+                b"Subject: partial\r\n\r\nmessage",
+            )
+        finally:
+            client.quit()
+        assert set(refused) == {"refused@example.test"}
+        assert endpoint.evidence() == {
+            "connection_count": 1,
+            "accepted_rcpt_commands": 1,
+            "refused_rcpt_commands": 1,
+            "data_transactions": 1,
+        }
+
+
+def test_post_data_disconnect_is_a_real_ambiguous_protocol_boundary() -> None:
+    with runner.smtp_fault_endpoint("post_data_disconnect") as endpoint:
+        client = _open_fault_smtp(endpoint)
+        try:
+            with pytest.raises(smtplib.SMTPServerDisconnected):
+                client.sendmail(
+                    "sender@example.test",
+                    ["recipient@example.test"],
+                    b"Subject: ambiguous\r\n\r\nmessage",
+                )
+        finally:
+            client.close()
+        assert endpoint.wait_for_data(1)
+        assert endpoint.evidence()["data_transactions"] == 1
+
+
+def test_partial_refusal_fixture_adds_a_second_distinct_recipient() -> None:
+    raw, _subject = runner.materialize_campaign_fixture(
+        FIXTURE_PATH,
+        profile_id="profile-1",
+        settings=_settings(),
+        scenario="partial_envelope_refusal",
+        run_token="0123456789ab",
+        additional_envelope_recipient=True,
+    )
+
+    recipients = raw["entries"]["inline"][0]["to"]
+    assert len(recipients) == 2
+    assert recipients[0]["email"] != recipients[1]["email"]
+    assert recipients[1]["email"].endswith("@govoplan.test")
+
 def test_fixture_contains_no_transport_credentials() -> None:
     raw = json.loads(FIXTURE_PATH.read_text(encoding="utf-8"))
     runner._assert_no_forbidden_campaign_keys(raw)
@@ -350,7 +440,7 @@ def test_fixture_composition_versions_are_complete_and_exact() -> None:
         "core": "0.1.13",
         "access": "0.1.11",
         "audit": "0.1.8",
-        "campaigns": "0.1.10",
+        "campaigns": "0.1.11",
         "mail": "0.1.10",
         "files": "0.1.9",
     }
@@ -359,7 +449,7 @@ def test_fixture_composition_versions_are_complete_and_exact() -> None:
         "core": "0.1.13",
         "access": "0.1.11",
         "audit": "0.1.8",
-        "campaigns": "0.1.10",
+        "campaigns": "0.1.11",
         "mail": "0.1.10",
     }
 
@@ -371,7 +461,7 @@ def test_fixture_composition_fails_closed_when_a_required_version_is_missing() -
             {
                 "core": "0.1.13",
                 "access": "0.1.11",
-                "campaigns": "0.1.10",
+                "campaigns": "0.1.11",
                 "mail": "0.1.10",
             },
         )
@@ -382,7 +472,9 @@ def test_testbed_documentation_distinguishes_proven_and_open_failure_drills() ->
     runbook = (REPOSITORY_ROOT / "docs" / "CAMPAIGN_DELIVERY_RUNBOOK.md").read_text(encoding="utf-8")
 
     assert "second ordinary send must be rejected before another provider effect" in testbed
-    assert "post-DATA connection loss" in testbed
+    assert "connection loss after complete DATA is frozen" in testbed
-    assert "real worker restart" in testbed
+    assert "dedicated OS process" in testbed
-    assert "does not simulate a connection loss after SMTP DATA" in runbook
+    assert "Redis/Celery delivery" in runbook
+    assert "broker redelivery" in runbook
+    assert "celery_broker_redelivery" in testbed
     assert "raw provider diagnostics" in runbook

tests/test_sending_jobs.py

@@ -10,6 +10,7 @@ from govoplan_campaign.backend.db.models import (
     JobSendStatus,
     JobValidationStatus,
 )
+from govoplan_campaign.backend.capabilities import delivery_tasks_capability
 from govoplan_campaign.backend.sending.jobs import (
     SendJobResult,
     _queue_validation_statuses,
@@ -48,6 +49,17 @@ def _job(entry_id: str, **overrides):
 
 
 class CampaignQueueSelectionTests(unittest.TestCase):
+    def test_delivery_task_capability_configures_module_runtime_for_worker_processes(self):
+        registry = object()
+        settings = object()
+        context = SimpleNamespace(registry=registry, settings=settings)
+
+        with patch("govoplan_campaign.backend.runtime.configure_runtime") as configure:
+            capability = delivery_tasks_capability(context)
+
+        self.assertIsNotNone(capability)
+        configure.assert_called_once_with(registry=registry, settings=settings)
+
     def test_selects_queueable_jobs_without_reclassifying_retry_states(self):
         skipped_send = _job("1", send_status=JobSendStatus.FAILED_TEMPORARY.value)
         skipped_queue = _job("2", queue_status=JobQueueStatus.PAUSED.value)

webui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@govoplan/campaign-webui",
-  "version": "0.1.10",
+  "version": "0.1.11",
   "private": true,
   "type": "module",
   "main": "src/index.ts",