Implement durable module recovery operations

tests/test_api_smoke.py

@@ -44,6 +44,11 @@ from govoplan_core.db.migrations import alembic_config
 from govoplan_core.db.session import configure_database, set_database
 from govoplan_core.core.change_sequence import decode_sequence_watermark, prune_sequence_entries
 from govoplan_core.core.pagination import encode_keyset_cursor, keyset_query_fingerprint
+from govoplan_core.core.recovery import (
+    RecoveryOperation,
+    RecoveryStatus,
+    verify_recovery_evidence_chain,
+)
 from govoplan_core.tenancy.scope import create_scope_tables, scope_registry
 from govoplan_access.backend.permissions.catalog import permission_catalog as access_permission_catalog
 
@@ -101,6 +106,14 @@ class ApiSmokeTests(unittest.TestCase):
         payload = response.json()
         return {"Authorization": f"Bearer {payload['access_token']}"}, payload
 
+    def _stored_campaign_eml(self, job: object) -> bytes:
+        from govoplan_files.backend.storage.backends import get_storage_backend
+
+        self.assertIsNone(getattr(job, "eml_local_path", None))
+        storage_key = getattr(job, "eml_storage_key", None)
+        self.assertTrue(storage_key)
+        return get_storage_backend().get_bytes(str(storage_key))
+
     def _create_test_mail_profile(
         self,
         headers: dict[str, str],
@@ -3304,6 +3317,29 @@ class ApiSmokeTests(unittest.TestCase):
             json={"write_eml": False},
         )
         self.assertEqual(built.status_code, 200, built.text)
+        replayed_build = self.client.post(
+            f"/api/v1/campaigns/versions/{version_id}/build",
+            headers=headers,
+            json={"write_eml": False},
+        )
+        self.assertEqual(replayed_build.status_code, 200, replayed_build.text)
+        with SessionLocal() as session:
+            operations = (
+                session.query(RecoveryOperation)
+                .filter(
+                    RecoveryOperation.module_id == "campaigns",
+                    RecoveryOperation.resource_id == version_id,
+                )
+                .all()
+            )
+            self.assertEqual(1, len(operations))
+            self.assertEqual(
+                RecoveryStatus.SUCCEEDED.value,
+                operations[0].status,
+            )
+            self.assertTrue(
+                verify_recovery_evidence_chain(session, operations[0].id)
+            )
         self.assertEqual(built.json()["built_count"], 1)
 
         mocked = self.client.post(
@@ -3742,7 +3778,8 @@ class ApiSmokeTests(unittest.TestCase):
 
         with SessionLocal() as session:
             job = session.query(CampaignJob).filter(CampaignJob.campaign_version_id == version_id).one()
-            self.assertTrue(job.eml_local_path)
+            self.assertIsNone(job.eml_local_path)
+            self.assertTrue(job.eml_storage_key)
             built_use = (
                 session.query(CampaignAttachmentUse)
                 .filter(
@@ -3979,7 +4016,9 @@ class ApiSmokeTests(unittest.TestCase):
             job = session.query(CampaignJob).filter(CampaignJob.campaign_id == campaign_id).one()
             self.assertEqual([item["email"] for item in job.resolved_recipients["from_all"]], ["local-from@example.org"])
             self.assertEqual([item["email"] for item in job.resolved_recipients["to"]], ["global-to@example.org", "local-to@example.org"])
-            message = BytesParser(policy=policy.default).parsebytes(Path(job.eml_local_path).read_bytes())
+            message = BytesParser(policy=policy.default).parsebytes(
+                self._stored_campaign_eml(job)
+            )
         self.assertIsNone(message["Sender"])
         self.assertEqual([address.addr_spec for address in message["From"].addresses], ["local-from@example.org"])
         self.assertEqual([address.addr_spec for address in message["To"].addresses], ["global-to@example.org", "local-to@example.org"])
@@ -4229,8 +4268,9 @@ class ApiSmokeTests(unittest.TestCase):
 
         with SessionLocal() as session:
             job = session.query(CampaignJob).filter(CampaignJob.campaign_id == campaign_id).one()
-            eml_path = Path(job.eml_local_path)
-            message = BytesParser(policy=policy.default).parsebytes(eml_path.read_bytes())
+            message = BytesParser(policy=policy.default).parsebytes(
+                self._stored_campaign_eml(job)
+            )
             uses = (
                 session.query(CampaignAttachmentUse)
                 .filter(
@@ -4337,8 +4377,7 @@ class ApiSmokeTests(unittest.TestCase):
 
         with SessionLocal() as session:
             job = session.query(CampaignJob).filter(CampaignJob.campaign_version_id == version_id).one()
-            self.assertIsNotNone(job.eml_local_path)
-            generated_eml = Path(job.eml_local_path).read_bytes()
+            generated_eml = self._stored_campaign_eml(job)
 
         sent = self.client.post(
             f"/api/v1/campaigns/{campaign_id}/send-now",
@@ -4368,13 +4407,19 @@ class ApiSmokeTests(unittest.TestCase):
         )
 
         from govoplan_campaign.backend.db.models import CampaignJob
+        from govoplan_files.backend.storage.backends import get_storage_backend
         from govoplan_mail.backend.dev.mock_mailbox import list_records
 
         with SessionLocal() as session:
             job = session.query(CampaignJob).filter(CampaignJob.campaign_version_id == version_id).one()
-            self.assertIsNotNone(job.eml_local_path)
-            eml_path = Path(job.eml_local_path)
-            eml_path.write_bytes(eml_path.read_bytes() + b"\r\nX-Tampered: true\r\n")
+            storage_key = job.eml_storage_key
+            generated_eml = self._stored_campaign_eml(job)
+        assert storage_key is not None
+        get_storage_backend().put_bytes(
+            storage_key,
+            generated_eml + b"\r\nX-Tampered: true\r\n",
+            content_type="message/rfc822",
+        )
 
         sent = self.client.post(
             f"/api/v1/campaigns/{campaign_id}/send-now",