Release govoplan-campaign v0.1.28: stabilize saving, review and delivery recovery

dev/mail-testbed/run_celery_redelivery_acceptance.py

@@ -5,8 +5,9 @@ The default run starts an isolated Redis Compose service, two successive real
 Celery worker processes, and a controlled loopback SMTP endpoint. It kills the
 first worker after complete DATA but before a final SMTP response. The same
 unacknowledged broker task must be delivered to the replacement worker, which
-must freeze the unfinished durable attempt as ``outcome_unknown`` without a
-second SMTP connection or DATA transaction.
+must leave the unfinished durable attempt unchanged without a second SMTP
+connection or DATA transaction. Only then does explicit fenced recovery use
+verified process exit and an expired fixture lease to record outcome-unknown.
 """
 
 from __future__ import annotations
@@ -52,6 +53,7 @@ from run_campaign_acceptance import (  # noqa: E402
     create_mail_profile,
     prepare_campaign_scenario,
     required_composition_versions,
+    recover_stopped_fixture_claim,
     smtp_fault_endpoint,
 )
 
@@ -72,6 +74,13 @@ import os
 import sys
 
 from govoplan_core.celery_app import celery
+from govoplan_core import celery_app as worker_runtime
+
+# Test-only process identity evidence, confined to this disposable child.
+original_worker_metadata = worker_runtime._worker_metadata
+worker_runtime._worker_metadata = lambda: {
+    **original_worker_metadata(), "acceptance_worker_pid": os.getpid(),
+}
 
 visibility_timeout = int(os.environ["GOVOPLAN_CAMPAIGN_TEST_REDIS_VISIBILITY_TIMEOUT_SECONDS"])
 celery.conf.broker_transport_options = {
@@ -408,6 +417,7 @@ def execute_redelivery_scenario(
     snapshot_probe: Callable[[str], tuple[Mapping[str, Any], Mapping[str, Any]]],
     audit_probe: Callable[[str, str], Mapping[str, int]],
     delivery_probe: Callable[[str, str], Mapping[str, Any]],
+    recover_claim: Callable[[str, str, subprocess.Popen[bytes]], Mapping[str, Any]],
 ) -> dict[str, Any]:
     profile_id = create_mail_profile(
         client,
@@ -488,9 +498,23 @@ def execute_redelivery_scenario(
             task_id=redelivered_task_id,
             timeout_seconds=settings.provider_timeout_seconds,
         )
-        recovered_state = _durable_state_evidence(
+        redelivered_state = _durable_state_evidence(
             delivery_probe(prepared.campaign_id, prepared.version_id)
         )
+        if redelivered_state != interrupted_state:
+            raise AcceptanceError("Redelivered task changed an unfinished attempt without stopped-runtime proof")
+        expected_protocol = {
+            "connection_count": 1,
+            "accepted_rcpt_commands": 1,
+            "refused_rcpt_commands": 0,
+            "data_transactions": 1,
+        }
+        if endpoint.evidence() != expected_protocol:
+            raise AcceptanceError("Broker redelivery caused an unexpected SMTP transaction")
+        recovery_evidence = dict(recover_claim(
+            prepared.campaign_id, prepared.version_id, first_worker.process,
+        ))
+        recovered_state = _durable_state_evidence(delivery_probe(prepared.campaign_id, prepared.version_id))
         expected_recovered = {
             "job_count": 1,
             "send_status_counts": {"outcome_unknown": 1},
@@ -498,17 +522,11 @@ def execute_redelivery_scenario(
             "unfinished_attempt_count": 0,
         }
         if recovered_state != expected_recovered:
-            raise AcceptanceError("Redelivered task did not freeze the unfinished attempt")
+            raise AcceptanceError("Explicit fenced recovery did not freeze the unfinished attempt")
 
         protocol = endpoint.evidence()
-        expected_protocol = {
-            "connection_count": 1,
-            "accepted_rcpt_commands": 1,
-            "refused_rcpt_commands": 0,
-            "data_transactions": 1,
-        }
         if protocol != expected_protocol:
-            raise AcceptanceError("Broker redelivery caused an unexpected SMTP transaction")
+            raise AcceptanceError("Explicit claim recovery caused an unexpected SMTP transaction")
         broker_after = _wait_for_broker_drained(
             redis_url,
             timeout_seconds=settings.provider_timeout_seconds,
@@ -549,7 +567,9 @@ def execute_redelivery_scenario(
             **prepared.public_evidence(),
             "queue": queue_evidence,
             "interrupted_durable_state": interrupted_state,
+            "redelivered_durable_state": redelivered_state,
             "recovered_durable_state": recovered_state,
+            "claim_recovery": recovery_evidence,
             "protocol": protocol,
             "report": report,
             "audit_actions": audit_actions,
@@ -566,6 +586,7 @@ def execute_redelivery_scenario(
                 "first_worker_forced_exit": first_exit_code != 0,
                 "replacement_worker_started": True,
                 "replacement_worker_completed_redelivery": True,
+                "duplicate_task_left_sending_unchanged": True,
             },
         }
     finally:
@@ -743,6 +764,11 @@ def _bootstrap_and_run(
                     snapshot_probe=snapshot_probe,
                     audit_probe=audit_probe,
                     delivery_probe=delivery_probe,
+                    recover_claim=lambda campaign_id, version_id, process: recover_stopped_fixture_claim(
+                        client, {"Authorization": f"Bearer {access_token}"}, database=database,
+                        runtime_root=runtime_root, campaign_id=campaign_id, version_id=version_id,
+                        stopped_process=process,
+                    ),
                 )
 
         evidence = {
@@ -766,6 +792,8 @@ def _bootstrap_and_run(
                 "celery_worker_processes": True,
                 "forced_worker_loss_after_complete_data": True,
                 "same_task_broker_redelivery": True,
+                "redelivery_leaves_active_claim_unchanged": True,
+                "explicit_stopped_runtime_fenced_recovery": True,
                 "durable_outcome_unknown_recovery": True,
                 "duplicate_smtp_transaction_prevented": True,
                 "production_daemon_supervisor": False,