Release govoplan-campaign v0.1.28: stabilize saving, review and delivery recovery

dev/mail-testbed/run_campaign_acceptance.py

@@ -29,7 +29,7 @@ import tomllib
 from collections import Counter
 from contextlib import contextmanager
 from dataclasses import dataclass
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any, Callable, Iterator, Mapping, Protocol
 from uuid import uuid4
@@ -66,6 +66,7 @@ SEND_RESULT_STATUSES = frozenset(
     {
         "already_accepted",
         "already_claimed",
+        "already_sending",
         "cancelled",
         "dry_run",
         "failed",
@@ -135,11 +136,21 @@ SMTP_FAULT_MODES = frozenset(
     }
 )
 WORKER_TASK_CODE = """
+import os
 import sys
-from govoplan_core.celery_app import send_email
+from types import SimpleNamespace
+from govoplan_core.celery_app import send_email, _worker_runtime_identity
+from govoplan_core.core.runtime_coordination import register_runtime_node
+from govoplan_core.db.session import get_database
+
+identity = _worker_runtime_identity(SimpleNamespace(hostname=f"campaign-acceptance-{os.getpid()}"))
+with get_database().SessionLocal() as session:
+    register_runtime_node(session, identity, metadata={"acceptance_worker_pid": os.getpid()})
+    session.commit()
 
 result = send_email.run(sys.argv[1])
 if not isinstance(result, dict) or result.get("status") not in {
+    "already_sending",
     "outcome_unknown",
     "smtp_accepted",
 }:
@@ -1206,6 +1217,92 @@ def _wait_for_worker_process(process: subprocess.Popen[bytes], *, timeout_second
         raise AcceptanceError("Restarted Campaign worker task failed")
 
 
+def recover_stopped_fixture_claim(
+    client: ApiClient, headers: Mapping[str, str], *, database: Any,
+    runtime_root: Path, campaign_id: str, version_id: str,
+    stopped_process: subprocess.Popen[bytes],
+) -> dict[str, bool]:
+    """Model supervisor proof ONLY in this runner's disposable SQLite fixture.
+
+    No job, attempt or recovery-operation state is edited here. After proving
+    the exact fixture process exited, expire only its lease and mark its own
+    runtime stopped. The real fenced HTTP action performs domain recovery.
+    """
+    from sqlalchemy.engine import make_url
+    from sqlalchemy.exc import ArgumentError
+    from govoplan_core.core.runtime_coordination import DistributedLease, RuntimeNode, process_runtime_identity
+    from govoplan_campaign.backend.db.models import CampaignJob
+    from govoplan_campaign.backend.services.delivery_recovery import job_recovery_metadata
+
+    root = runtime_root.resolve()
+    expected_database = root / "acceptance.db"
+    allowed_parents = {Path(tempfile.gettempdir()).resolve(), Path("/tmp").resolve()}
+    if (
+        os.environ.get("APP_ENV") != "test"
+        or root.parent not in allowed_parents
+        or not root.name.startswith(("govoplan-campaign-greenmail-", "govoplan-campaign-celery-redelivery-"))
+        or not expected_database.is_file() or expected_database.is_symlink()
+        or database.engine.url.get_backend_name() != "sqlite"
+        or not database.engine.url.database
+        or Path(database.engine.url.database).resolve() != expected_database
+    ):
+        raise AcceptanceError("Claim proof is restricted to the runner's isolated temporary SQLite fixture")
+    try:
+        environment_url = make_url(os.environ.get("DATABASE_URL", ""))
+    except ArgumentError:
+        raise AcceptanceError("Claim proof requires the isolated fixture database environment") from None
+    if (
+        environment_url.get_backend_name() != "sqlite" or not environment_url.database
+        or Path(environment_url.database).resolve() != expected_database
+    ):
+        raise AcceptanceError("Claim proof database does not match the isolated fixture environment")
+    if stopped_process.poll() is None or stopped_process.returncode is None:
+        raise AcceptanceError("Claim proof requires the exact fixture worker to have exited")
+    identity = process_runtime_identity()
+    with database.SessionLocal() as session:
+        jobs = session.query(CampaignJob).filter(
+            CampaignJob.campaign_id == campaign_id,
+            CampaignJob.campaign_version_id == version_id,
+        ).all()
+        if len(jobs) != 1 or jobs[0].send_status != "sending" or not jobs[0].claim_token:
+            raise AcceptanceError("Claim proof requires exactly one unfinished fixture send")
+        job = jobs[0]
+        lease = session.query(DistributedLease).filter(
+            DistributedLease.installation_id == identity.installation_id,
+            DistributedLease.resource_key == f"campaign:delivery:{job.tenant_id}:{job.id}",
+        ).one_or_none()
+        node = session.query(RuntimeNode).filter(
+            RuntimeNode.installation_id == identity.installation_id,
+            RuntimeNode.node_id == lease.holder_node_id,
+        ).one_or_none() if lease else None
+        if (
+            lease is None or node is None or node.incarnation != lease.holder_incarnation
+            or (node.metadata_ or {}).get("acceptance_worker_pid") != stopped_process.pid
+        ):
+            raise AcceptanceError("Stopped fixture process does not own this exact runtime claim")
+        node.state = "stopped"
+        node.stopped_at = datetime.now(timezone.utc)
+        lease.expires_at = datetime.now(timezone.utc) - timedelta(seconds=1)
+        session.commit()
+        metadata = job_recovery_metadata(session, [job])[job.id]["smtp"]
+        if not metadata["eligible"]:
+            raise AcceptanceError("Proven stopped fixture claim is not eligible for fenced recovery")
+        job_id = job.id
+        revision = metadata["revision"]
+    payload = _expect(client.post(
+        f"/api/v1/campaigns/{campaign_id}/jobs/{job_id}/recover-claim",
+        headers=dict(headers),
+        json={
+            "channel": "smtp", "expected_revision": revision,
+            "note": "Verified isolated fixture worker exited after DATA; expired its test lease. Mailbox/provider reconciliation is still required.",
+        },
+    ), 200, "Explicit fenced fixture claim recovery")
+    result = payload.get("result") if isinstance(payload.get("result"), dict) else {}
+    if result.get("send_status") != "outcome_unknown" or result.get("reconciliation_required") is not True:
+        raise AcceptanceError("Fenced recovery did not preserve the unknown SMTP outcome")
+    return {"stopped_process_verified": True, "fixture_lease_expired": True, "explicit_fenced_recovery": True}
+
+
 def execute_worker_interruption_scenario(
     client: ApiClient,
     headers: Mapping[str, str],
@@ -1218,6 +1315,7 @@ def execute_worker_interruption_scenario(
     audit_probe: Callable[[str, str], Mapping[str, int]],
     delivery_probe: Callable[[str, str], Mapping[str, Any]],
     worker_job_probe: Callable[[str], str],
+    recover_claim: Callable[[str, str, subprocess.Popen[bytes]], Mapping[str, Any]],
 ) -> dict[str, Any]:
     prepared = prepare_campaign_scenario(
         client,
@@ -1286,23 +1384,30 @@ def execute_worker_interruption_scenario(
     restarted_state = _durable_state_evidence(
         delivery_probe(prepared.campaign_id, prepared.version_id)
     )
-    expected_restarted_state = {
+    if restarted_state != interrupted_state:
+        raise AcceptanceError("Duplicate task changed an unfinished SMTP attempt without stopped-runtime proof")
+
+    protocol_evidence = endpoint.evidence()
+    expected_protocol = {
+        "connection_count": 1,
+        "accepted_rcpt_commands": 1,
+        "refused_rcpt_commands": 0,
+        "data_transactions": 1,
+    }
+    if protocol_evidence != expected_protocol:
+        raise AcceptanceError("Restarted worker contacted SMTP or produced an unexpected transaction")
+    recovery_evidence = dict(recover_claim(prepared.campaign_id, prepared.version_id, first_worker))
+    recovered_state = _durable_state_evidence(delivery_probe(prepared.campaign_id, prepared.version_id))
+    expected_recovered_state = {
         "job_count": 1,
         "send_status_counts": {"outcome_unknown": 1},
         "attempt_status_counts": {"outcome_unknown": 1},
         "unfinished_attempt_count": 0,
     }
-    if restarted_state != expected_restarted_state:
-        raise AcceptanceError("Restarted worker did not freeze the unfinished SMTP attempt")
-
-    protocol_evidence = endpoint.evidence()
-    if protocol_evidence != {
-        "connection_count": 1,
-        "accepted_rcpt_commands": 1,
-        "refused_rcpt_commands": 0,
-        "data_transactions": 1,
-    }:
-        raise AcceptanceError("Restarted worker contacted SMTP or produced an unexpected transaction")
+    if recovered_state != expected_recovered_state:
+        raise AcceptanceError("Explicit fenced recovery did not freeze the unfinished SMTP attempt")
+    if endpoint.evidence() != expected_protocol:
+        raise AcceptanceError("Explicit claim recovery contacted SMTP")
 
     report = _expect(
         client.get(
@@ -1333,12 +1438,15 @@ def execute_worker_interruption_scenario(
         "queue": queue_evidence,
         "interrupted_durable_state": interrupted_state,
         "restarted_durable_state": restarted_state,
+        "recovered_durable_state": recovered_state,
+        "claim_recovery": recovery_evidence,
         "protocol": protocol_evidence,
         "report": report_evidence,
         "audit_actions": audit_actions,
         "process_boundary": {
             "dedicated_task_process_terminated_after_data": True,
             "fresh_task_process_completed": True,
+            "duplicate_task_left_sending_unchanged": True,
             "duplicate_smtp_transaction_prevented": True,
             "celery_broker_redelivery_exercised": False,
         },
@@ -1355,6 +1463,7 @@ def run_acceptance(
     audit_probe: Callable[[str, str], Mapping[str, int]],
     delivery_probe: Callable[[str, str], Mapping[str, Any]],
     worker_job_probe: Callable[[str], str],
+    recover_claim: Callable[[str, str, subprocess.Popen[bytes]], Mapping[str, Any]],
     include_failure_drills: bool,
     module_versions: Mapping[str, str],
 ) -> dict[str, Any]:
@@ -1619,6 +1728,7 @@ def run_acceptance(
                 audit_probe=audit_probe,
                 delivery_probe=delivery_probe,
                 worker_job_probe=worker_job_probe,
+                recover_claim=recover_claim,
             )
         drills["worker_interruption"] = worker_interruption
 
@@ -1647,6 +1757,8 @@ def run_acceptance(
             "post_data_connection_loss_outcome_unknown": include_failure_drills,
             "source_artifact_provenance": False,
             "worker_restart_interruption": include_failure_drills,
+            "duplicate_worker_leaves_active_claim_unchanged": include_failure_drills,
+            "explicit_stopped_runtime_fenced_recovery": include_failure_drills,
             "celery_broker_redelivery": False,
         },
     }
@@ -1888,6 +2000,11 @@ def _bootstrap_and_run(
                 audit_probe=audit_probe,
                 delivery_probe=delivery_probe,
                 worker_job_probe=worker_job_probe,
+                recover_claim=lambda campaign_id, version_id, process: recover_stopped_fixture_claim(
+                    client, {"Authorization": f"Bearer {access_token}"}, database=database,
+                    runtime_root=runtime_root, campaign_id=campaign_id, version_id=version_id,
+                    stopped_process=process,
+                ),
                 include_failure_drills=include_failure_drills,
                 module_versions=module_versions,
             )