Fence managed file object effects

2026-08-03 04:21:52 +02:00
parent b993d8e31a
commit 6baf2a421b
10 changed files with 1162 additions and 25 deletions
@@ -4,6 +4,7 @@ import hashlib
 import tempfile
 import unittest
 from pathlib import Path
+from unittest.mock import patch

 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
@@ -160,20 +161,23 @@ class IntegrityReconciliationTests(unittest.TestCase):
        )
        self.assertEqual("would_delete", preview_cleanup.action)
        self.assertTrue(self.backend.exists(orphan_key))
-        cleanup = cleanup_orphan_finding(
-            self.session,
-            orphan_finding,
-            user_id=USER_ID,
-            dry_run=False,
-            backend=self.backend,
-        )
-        repeated = cleanup_orphan_finding(
-            self.session,
-            orphan_finding,
-            user_id=USER_ID,
-            dry_run=False,
-            backend=self.backend,
-        )
+        with patch(
+            "govoplan_files.backend.storage.integrity.begin_orphan_cleanup_recovery"
+        ):
+            cleanup = cleanup_orphan_finding(
+                self.session,
+                orphan_finding,
+                user_id=USER_ID,
+                dry_run=False,
+                backend=self.backend,
+            )
+            repeated = cleanup_orphan_finding(
+                self.session,
+                orphan_finding,
+                user_id=USER_ID,
+                dry_run=False,
+                backend=self.backend,
+            )
        self.assertEqual("deleted", cleanup.action)
        self.assertFalse(self.backend.exists(orphan_key))
        self.assertEqual("already_deleted", repeated.action)
@@ -160,6 +160,8 @@ class FilesManifestDocumentationTests(unittest.TestCase):
        self.assertIn("quarantined", topic.body)
        self.assertIn("MASTER_KEY_B64", topic.metadata["recovery_unit"])
        self.assertIn("Encryption envelope and wrapped-key rows", topic.metadata["recovery_unit"])
+        self.assertIn("lease-fenced Core recovery", topic.body)
+        self.assertIn("Ops", topic.body)
        self.assertTrue(topic.metadata["verification"])
        self.assertIn(
            "/api/v1/files/integrity/scans",
@@ -0,0 +1,326 @@
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from govoplan_access.backend.db.models import Account, Group, User
+from govoplan_core.core.recovery import (
+    RecoveryCheckpoint,
+    RecoveryOperation,
+    RecoveryStatus,
+)
+from govoplan_core.core.runtime_coordination import (
+    DistributedLease,
+    RuntimeIdentity,
+    bind_process_runtime_identity,
+)
+from govoplan_core.db.base import Base
+from govoplan_core.db.session import configure_database, reset_database
+from govoplan_files.backend.db.models import (
+    FileBlob,
+    FileIntegrityFinding,
+    FileIntegrityScan,
+)
+from govoplan_files.backend.storage.backends import (
+    LocalFilesystemStorageBackend,
+)
+from govoplan_files.backend.storage.common import FileStorageError
+from govoplan_files.backend.storage.files import _get_or_create_blob
+from govoplan_files.backend.storage.integrity import cleanup_orphan_finding
+from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
+
+
+TENANT_ID = "tenant-1"
+USER_ID = "user-1"
+
+
+class StorageRecoveryTests(unittest.TestCase):
+    def setUp(self) -> None:
+        self.temporary_directory = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temporary_directory.cleanup)
+        root = Path(self.temporary_directory.name)
+        self.backend = LocalFilesystemStorageBackend(root / "objects")
+        database_path = root / "recovery.sqlite3"
+        self.engine = create_engine(f"sqlite:///{database_path}", future=True)
+        Base.metadata.create_all(
+            bind=self.engine,
+            tables=[
+                Account.__table__,
+                User.__table__,
+                Group.__table__,
+                DistributedLease.__table__,
+                RecoveryOperation.__table__,
+                RecoveryCheckpoint.__table__,
+                FileBlob.__table__,
+                FileIntegrityScan.__table__,
+                FileIntegrityFinding.__table__,
+            ],
+        )
+        configure_database(
+            f"sqlite:///{database_path}",
+            engine=self.engine,
+            dispose_previous=True,
+        )
+        bind_process_runtime_identity(
+            RuntimeIdentity(
+                installation_id="files-recovery-test",
+                node_id="node-1",
+                incarnation="incarnation-1",
+                role="api",
+                software_version="test",
+                composition_hash="a" * 64,
+            )
+        )
+        self.Session = sessionmaker(
+            bind=self.engine,
+            expire_on_commit=False,
+            future=True,
+        )
+        self.enterContext(
+            patch(
+                "govoplan_files.backend.storage.files._storage_backend_name",
+                return_value=self.backend.name,
+            )
+        )
+        self.enterContext(
+            patch(
+                "govoplan_files.backend.storage.files._storage_bucket_name",
+                return_value="",
+            )
+        )
+        self.session = self.Session()
+        self.addCleanup(self._cleanup_runtime)
+
+    def _cleanup_runtime(self) -> None:
+        self.session.close()
+        bind_process_runtime_identity(None)
+        reset_database()
+        self.engine.dispose()
+
+    def test_committed_blob_write_is_independently_verified(self) -> None:
+        observed_running_operation: list[bool] = []
+        put_bytes = self.backend.put_bytes
+
+        class ObservingBackend:
+            name = self.backend.name
+
+            def __getattr__(backend_self, name):
+                return getattr(self.backend, name)
+
+            def put_bytes(backend_self, key, data, *, content_type=None):
+                with self.Session() as evidence_session:
+                    operation = evidence_session.query(RecoveryOperation).one()
+                    observed_running_operation.append(
+                        operation.status == RecoveryStatus.RUNNING.value
+                        and len(operation.request_sha256) == 64
+                    )
+                put_bytes(key, data, content_type=content_type)
+
+        observing_backend = ObservingBackend()
+
+        with patch(
+            "govoplan_files.backend.storage.files.get_storage_backend",
+            return_value=observing_backend,
+        ):
+            blob = _get_or_create_blob(
+                self.session,
+                tenant_id=TENANT_ID,
+                data=b"durable",
+                filename="private-name.txt",
+                content_type="text/plain",
+                actor_id=USER_ID,
+            )
+            self.session.commit()
+
+        operation = self._only_operation()
+        self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
+        self.assertEqual([True], observed_running_operation)
+        self.assertTrue(self.backend.exists(blob.storage_key))
+        self.assertNotIn("private-name", blob.storage_key)
+        self.assertEqual(".blob", Path(blob.storage_key).suffix)
+
+    def test_rolled_back_blob_write_is_compensated(self) -> None:
+        with patch(
+            "govoplan_files.backend.storage.files.get_storage_backend",
+            return_value=self.backend,
+        ):
+            blob = _get_or_create_blob(
+                self.session,
+                tenant_id=TENANT_ID,
+                data=b"rollback",
+                filename="rollback.txt",
+                content_type="text/plain",
+                actor_id=USER_ID,
+            )
+            storage_key = blob.storage_key
+            self.assertTrue(self.backend.exists(storage_key))
+            self.session.rollback()
+
+        operation = self._only_operation()
+        self.assertEqual(RecoveryStatus.RECOVERED.value, operation.status)
+        self.assertFalse(self.backend.exists(storage_key))
+        with self.Session() as evidence_session:
+            self.assertIsNone(evidence_session.get(FileBlob, blob.id))
+
+    def test_post_write_tamper_is_quarantined_and_recovery_required(self) -> None:
+        with patch(
+            "govoplan_files.backend.storage.files.get_storage_backend",
+            return_value=self.backend,
+        ):
+            blob = _get_or_create_blob(
+                self.session,
+                tenant_id=TENANT_ID,
+                data=b"expected",
+                filename="evidence.bin",
+                content_type="application/octet-stream",
+                actor_id=USER_ID,
+            )
+            self.backend.put_bytes(blob.storage_key, b"tampered")
+            self.session.commit()
+
+        operation = self._only_operation()
+        self.assertEqual(
+            RecoveryStatus.RECOVERY_REQUIRED.value,
+            operation.status,
+        )
+        with self.Session() as evidence_session:
+            persisted = evidence_session.get(FileBlob, blob.id)
+            self.assertIsNotNone(persisted)
+            self.assertEqual("checksum_mismatch", persisted.integrity_status)
+            self.assertIsNotNone(persisted.quarantined_at)
+
+    def test_missing_optional_encryption_fails_before_object_effect(self) -> None:
+        with patch(
+            "govoplan_files.backend.storage.files.get_storage_backend",
+            return_value=self.backend,
+        ), patch(
+            "govoplan_files.backend.storage.content_protection.encryption_content_cipher",
+            return_value=None,
+        ), self.assertRaisesRegex(FileStorageError, "Encryption module"):
+            _get_or_create_blob(
+                self.session,
+                tenant_id=TENANT_ID,
+                data=b"protected",
+                filename="protected.bin",
+                content_type="application/octet-stream",
+                actor_id=USER_ID,
+                encryption_vault_id="vault-1",
+            )
+        self.session.rollback()
+
+        operation = self._only_operation()
+        self.assertEqual(RecoveryStatus.REJECTED.value, operation.status)
+        objects = self.backend.list_objects(
+            prefix=f"tenants/{TENANT_ID}/files/",
+            limit=10,
+        )
+        self.assertEqual((), objects.objects)
+
+    def test_orphan_cleanup_forward_completes_after_business_rollback(self) -> None:
+        key = f"tenants/{TENANT_ID}/files/orphan.bin"
+        self.backend.put_bytes(key, b"orphan")
+        scan = FileIntegrityScan(
+            id="scan-1",
+            tenant_id=TENANT_ID,
+            storage_backend=self.backend.name,
+            storage_prefix=f"tenants/{TENANT_ID}/files/",
+            status="completed",
+        )
+        finding = FileIntegrityFinding(
+            id="finding-1",
+            scan_id=scan.id,
+            tenant_id=TENANT_ID,
+            kind="orphan_object",
+            state="open",
+            storage_key=key,
+            observed_size_bytes=6,
+            observed_checksum_sha256=hashlib.sha256(b"orphan").hexdigest(),
+        )
+        self.session.add_all([scan, finding])
+        self.session.commit()
+
+        cleanup_orphan_finding(
+            self.session,
+            finding,
+            user_id=USER_ID,
+            dry_run=False,
+            backend=self.backend,
+        )
+        self.assertFalse(self.backend.exists(key))
+        self.session.rollback()
+
+        operation = self._only_operation()
+        self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
+        with self.Session() as evidence_session:
+            persisted = evidence_session.get(FileIntegrityFinding, finding.id)
+            self.assertIsNotNone(persisted)
+            self.assertEqual("deleted", persisted.state)
+
+    def test_missing_runtime_identity_blocks_before_object_write(self) -> None:
+        bind_process_runtime_identity(None)
+        with patch(
+            "govoplan_files.backend.storage.files.get_storage_backend",
+            return_value=self.backend,
+        ), self.assertRaisesRegex(FileStorageError, "recovery ledger"):
+            _get_or_create_blob(
+                self.session,
+                tenant_id=TENANT_ID,
+                data=b"blocked",
+                filename="blocked.bin",
+                content_type="application/octet-stream",
+                actor_id=USER_ID,
+            )
+        self.session.rollback()
+        objects = self.backend.list_objects(
+            prefix=f"tenants/{TENANT_ID}/files/",
+            limit=10,
+        )
+        self.assertEqual((), objects.objects)
+
+    def test_blob_fence_blocks_a_competing_runtime_before_effect(self) -> None:
+        checksum = hashlib.sha256(b"fenced").hexdigest()
+        begin_blob_write_recovery(
+            self.session,
+            backend=self.backend,
+            tenant_id=TENANT_ID,
+            blob_id="blob-fenced",
+            storage_key=f"tenants/{TENANT_ID}/files/fenced.blob",
+            semantic_checksum_sha256=checksum,
+            semantic_size_bytes=6,
+            protection_discriminator="plaintext",
+            created_new=True,
+        )
+        with self.Session() as competing_session, self.assertRaisesRegex(
+            FileStorageError,
+            "already owned",
+        ):
+            begin_blob_write_recovery(
+                competing_session,
+                backend=self.backend,
+                tenant_id=TENANT_ID,
+                blob_id="blob-fenced",
+                storage_key=f"tenants/{TENANT_ID}/files/fenced.blob",
+                semantic_checksum_sha256=checksum,
+                semantic_size_bytes=6,
+                protection_discriminator="plaintext",
+                created_new=True,
+            )
+        self.session.rollback()
+        self.assertEqual(RecoveryStatus.REJECTED.value, self._only_operation().status)
+
+    def _only_operation(self) -> RecoveryOperation:
+        with self.Session() as session:
+            operations = session.query(RecoveryOperation).all()
+            self.assertEqual(1, len(operations))
+            session.expunge(operations[0])
+            return operations[0]
+
+
+if __name__ == "__main__":
+    unittest.main()