from __future__ import annotations import hashlib from pathlib import Path import tempfile import unittest from unittest.mock import patch from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from govoplan_access.backend.db.models import Account, Group, User from govoplan_core.core.recovery import ( RecoveryCheckpoint, RecoveryOperation, RecoveryStatus, ) from govoplan_core.core.runtime_coordination import ( DistributedLease, RuntimeIdentity, bind_process_runtime_identity, ) from govoplan_core.db.base import Base from govoplan_core.db.session import configure_database, reset_database from govoplan_files.backend.db.models import ( FileBlob, FileIntegrityFinding, FileIntegrityScan, ) from govoplan_files.backend.storage.backends import ( LocalFilesystemStorageBackend, ) from govoplan_files.backend.storage.common import FileStorageError from govoplan_files.backend.storage.files import _get_or_create_blob from govoplan_files.backend.storage.integrity import cleanup_orphan_finding from govoplan_files.backend.storage.recovery import begin_blob_write_recovery TENANT_ID = "tenant-1" USER_ID = "user-1" class StorageRecoveryTests(unittest.TestCase): def setUp(self) -> None: self.temporary_directory = tempfile.TemporaryDirectory() self.addCleanup(self.temporary_directory.cleanup) root = Path(self.temporary_directory.name) self.backend = LocalFilesystemStorageBackend(root / "objects") database_path = root / "recovery.sqlite3" self.engine = create_engine(f"sqlite:///{database_path}", future=True) Base.metadata.create_all( bind=self.engine, tables=[ Account.__table__, User.__table__, Group.__table__, DistributedLease.__table__, RecoveryOperation.__table__, RecoveryCheckpoint.__table__, FileBlob.__table__, FileIntegrityScan.__table__, FileIntegrityFinding.__table__, ], ) configure_database( f"sqlite:///{database_path}", engine=self.engine, dispose_previous=True, ) bind_process_runtime_identity( RuntimeIdentity( installation_id="files-recovery-test", node_id="node-1", incarnation="incarnation-1", role="api", software_version="test", composition_hash="a" * 64, ) ) self.Session = sessionmaker( bind=self.engine, expire_on_commit=False, future=True, ) self.enterContext( patch( "govoplan_files.backend.storage.files._storage_backend_name", return_value=self.backend.name, ) ) self.enterContext( patch( "govoplan_files.backend.storage.files._storage_bucket_name", return_value="", ) ) self.session = self.Session() self.addCleanup(self._cleanup_runtime) def _cleanup_runtime(self) -> None: self.session.close() bind_process_runtime_identity(None) reset_database() self.engine.dispose() def test_committed_blob_write_is_independently_verified(self) -> None: observed_running_operation: list[bool] = [] put_bytes = self.backend.put_bytes class ObservingBackend: name = self.backend.name def __getattr__(backend_self, name): return getattr(self.backend, name) def put_bytes(backend_self, key, data, *, content_type=None): with self.Session() as evidence_session: operation = evidence_session.query(RecoveryOperation).one() observed_running_operation.append( operation.status == RecoveryStatus.RUNNING.value and len(operation.request_sha256) == 64 ) put_bytes(key, data, content_type=content_type) observing_backend = ObservingBackend() with patch( "govoplan_files.backend.storage.files.get_storage_backend", return_value=observing_backend, ): blob = _get_or_create_blob( self.session, tenant_id=TENANT_ID, data=b"durable", filename="private-name.txt", content_type="text/plain", actor_id=USER_ID, ) self.session.commit() operation = self._only_operation() self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status) self.assertEqual([True], observed_running_operation) self.assertTrue(self.backend.exists(blob.storage_key)) self.assertNotIn("private-name", blob.storage_key) self.assertEqual(".blob", Path(blob.storage_key).suffix) def test_rolled_back_blob_write_is_compensated(self) -> None: with patch( "govoplan_files.backend.storage.files.get_storage_backend", return_value=self.backend, ): blob = _get_or_create_blob( self.session, tenant_id=TENANT_ID, data=b"rollback", filename="rollback.txt", content_type="text/plain", actor_id=USER_ID, ) storage_key = blob.storage_key self.assertTrue(self.backend.exists(storage_key)) self.session.rollback() operation = self._only_operation() self.assertEqual(RecoveryStatus.RECOVERED.value, operation.status) self.assertFalse(self.backend.exists(storage_key)) with self.Session() as evidence_session: self.assertIsNone(evidence_session.get(FileBlob, blob.id)) def test_post_write_tamper_is_quarantined_and_recovery_required(self) -> None: with patch( "govoplan_files.backend.storage.files.get_storage_backend", return_value=self.backend, ): blob = _get_or_create_blob( self.session, tenant_id=TENANT_ID, data=b"expected", filename="evidence.bin", content_type="application/octet-stream", actor_id=USER_ID, ) self.backend.put_bytes(blob.storage_key, b"tampered") self.session.commit() operation = self._only_operation() self.assertEqual( RecoveryStatus.RECOVERY_REQUIRED.value, operation.status, ) with self.Session() as evidence_session: persisted = evidence_session.get(FileBlob, blob.id) self.assertIsNotNone(persisted) self.assertEqual("checksum_mismatch", persisted.integrity_status) self.assertIsNotNone(persisted.quarantined_at) def test_missing_optional_encryption_fails_before_object_effect(self) -> None: with patch( "govoplan_files.backend.storage.files.get_storage_backend", return_value=self.backend, ), patch( "govoplan_files.backend.storage.content_protection.encryption_content_cipher", return_value=None, ), self.assertRaisesRegex(FileStorageError, "Encryption module"): _get_or_create_blob( self.session, tenant_id=TENANT_ID, data=b"protected", filename="protected.bin", content_type="application/octet-stream", actor_id=USER_ID, encryption_vault_id="vault-1", ) self.session.rollback() operation = self._only_operation() self.assertEqual(RecoveryStatus.REJECTED.value, operation.status) objects = self.backend.list_objects( prefix=f"tenants/{TENANT_ID}/files/", limit=10, ) self.assertEqual((), objects.objects) def test_orphan_cleanup_forward_completes_after_business_rollback(self) -> None: key = f"tenants/{TENANT_ID}/files/orphan.bin" self.backend.put_bytes(key, b"orphan") scan = FileIntegrityScan( id="scan-1", tenant_id=TENANT_ID, storage_backend=self.backend.name, storage_prefix=f"tenants/{TENANT_ID}/files/", status="completed", ) finding = FileIntegrityFinding( id="finding-1", scan_id=scan.id, tenant_id=TENANT_ID, kind="orphan_object", state="open", storage_key=key, observed_size_bytes=6, observed_checksum_sha256=hashlib.sha256(b"orphan").hexdigest(), ) self.session.add_all([scan, finding]) self.session.commit() cleanup_orphan_finding( self.session, finding, user_id=USER_ID, dry_run=False, backend=self.backend, ) self.assertFalse(self.backend.exists(key)) self.session.rollback() operation = self._only_operation() self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status) with self.Session() as evidence_session: persisted = evidence_session.get(FileIntegrityFinding, finding.id) self.assertIsNotNone(persisted) self.assertEqual("deleted", persisted.state) def test_missing_runtime_identity_blocks_before_object_write(self) -> None: bind_process_runtime_identity(None) with patch( "govoplan_files.backend.storage.files.get_storage_backend", return_value=self.backend, ), self.assertRaisesRegex(FileStorageError, "recovery ledger"): _get_or_create_blob( self.session, tenant_id=TENANT_ID, data=b"blocked", filename="blocked.bin", content_type="application/octet-stream", actor_id=USER_ID, ) self.session.rollback() objects = self.backend.list_objects( prefix=f"tenants/{TENANT_ID}/files/", limit=10, ) self.assertEqual((), objects.objects) def test_blob_fence_blocks_a_competing_runtime_before_effect(self) -> None: checksum = hashlib.sha256(b"fenced").hexdigest() begin_blob_write_recovery( self.session, backend=self.backend, tenant_id=TENANT_ID, blob_id="blob-fenced", storage_key=f"tenants/{TENANT_ID}/files/fenced.blob", semantic_checksum_sha256=checksum, semantic_size_bytes=6, protection_discriminator="plaintext", created_new=True, ) with self.Session() as competing_session, self.assertRaisesRegex( FileStorageError, "already owned", ): begin_blob_write_recovery( competing_session, backend=self.backend, tenant_id=TENANT_ID, blob_id="blob-fenced", storage_key=f"tenants/{TENANT_ID}/files/fenced.blob", semantic_checksum_sha256=checksum, semantic_size_bytes=6, protection_discriminator="plaintext", created_new=True, ) self.session.rollback() self.assertEqual(RecoveryStatus.REJECTED.value, self._only_operation().status) def _only_operation(self) -> RecoveryOperation: with self.Session() as session: operations = session.query(RecoveryOperation).all() self.assertEqual(1, len(operations)) session.expunge(operations[0]) return operations[0] if __name__ == "__main__": unittest.main()