327 lines
12 KiB
Python
327 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from govoplan_access.backend.db.models import Account, Group, User
|
|
from govoplan_core.core.recovery import (
|
|
RecoveryCheckpoint,
|
|
RecoveryOperation,
|
|
RecoveryStatus,
|
|
)
|
|
from govoplan_core.core.runtime_coordination import (
|
|
DistributedLease,
|
|
RuntimeIdentity,
|
|
bind_process_runtime_identity,
|
|
)
|
|
from govoplan_core.db.base import Base
|
|
from govoplan_core.db.session import configure_database, reset_database
|
|
from govoplan_files.backend.db.models import (
|
|
FileBlob,
|
|
FileIntegrityFinding,
|
|
FileIntegrityScan,
|
|
)
|
|
from govoplan_files.backend.storage.backends import (
|
|
LocalFilesystemStorageBackend,
|
|
)
|
|
from govoplan_files.backend.storage.common import FileStorageError
|
|
from govoplan_files.backend.storage.files import _get_or_create_blob
|
|
from govoplan_files.backend.storage.integrity import cleanup_orphan_finding
|
|
from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
|
|
|
|
|
|
TENANT_ID = "tenant-1"
|
|
USER_ID = "user-1"
|
|
|
|
|
|
class StorageRecoveryTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.temporary_directory = tempfile.TemporaryDirectory()
|
|
self.addCleanup(self.temporary_directory.cleanup)
|
|
root = Path(self.temporary_directory.name)
|
|
self.backend = LocalFilesystemStorageBackend(root / "objects")
|
|
database_path = root / "recovery.sqlite3"
|
|
self.engine = create_engine(f"sqlite:///{database_path}", future=True)
|
|
Base.metadata.create_all(
|
|
bind=self.engine,
|
|
tables=[
|
|
Account.__table__,
|
|
User.__table__,
|
|
Group.__table__,
|
|
DistributedLease.__table__,
|
|
RecoveryOperation.__table__,
|
|
RecoveryCheckpoint.__table__,
|
|
FileBlob.__table__,
|
|
FileIntegrityScan.__table__,
|
|
FileIntegrityFinding.__table__,
|
|
],
|
|
)
|
|
configure_database(
|
|
f"sqlite:///{database_path}",
|
|
engine=self.engine,
|
|
dispose_previous=True,
|
|
)
|
|
bind_process_runtime_identity(
|
|
RuntimeIdentity(
|
|
installation_id="files-recovery-test",
|
|
node_id="node-1",
|
|
incarnation="incarnation-1",
|
|
role="api",
|
|
software_version="test",
|
|
composition_hash="a" * 64,
|
|
)
|
|
)
|
|
self.Session = sessionmaker(
|
|
bind=self.engine,
|
|
expire_on_commit=False,
|
|
future=True,
|
|
)
|
|
self.enterContext(
|
|
patch(
|
|
"govoplan_files.backend.storage.files._storage_backend_name",
|
|
return_value=self.backend.name,
|
|
)
|
|
)
|
|
self.enterContext(
|
|
patch(
|
|
"govoplan_files.backend.storage.files._storage_bucket_name",
|
|
return_value="",
|
|
)
|
|
)
|
|
self.session = self.Session()
|
|
self.addCleanup(self._cleanup_runtime)
|
|
|
|
def _cleanup_runtime(self) -> None:
|
|
self.session.close()
|
|
bind_process_runtime_identity(None)
|
|
reset_database()
|
|
self.engine.dispose()
|
|
|
|
def test_committed_blob_write_is_independently_verified(self) -> None:
|
|
observed_running_operation: list[bool] = []
|
|
put_bytes = self.backend.put_bytes
|
|
|
|
class ObservingBackend:
|
|
name = self.backend.name
|
|
|
|
def __getattr__(backend_self, name):
|
|
return getattr(self.backend, name)
|
|
|
|
def put_bytes(backend_self, key, data, *, content_type=None):
|
|
with self.Session() as evidence_session:
|
|
operation = evidence_session.query(RecoveryOperation).one()
|
|
observed_running_operation.append(
|
|
operation.status == RecoveryStatus.RUNNING.value
|
|
and len(operation.request_sha256) == 64
|
|
)
|
|
put_bytes(key, data, content_type=content_type)
|
|
|
|
observing_backend = ObservingBackend()
|
|
|
|
with patch(
|
|
"govoplan_files.backend.storage.files.get_storage_backend",
|
|
return_value=observing_backend,
|
|
):
|
|
blob = _get_or_create_blob(
|
|
self.session,
|
|
tenant_id=TENANT_ID,
|
|
data=b"durable",
|
|
filename="private-name.txt",
|
|
content_type="text/plain",
|
|
actor_id=USER_ID,
|
|
)
|
|
self.session.commit()
|
|
|
|
operation = self._only_operation()
|
|
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
|
|
self.assertEqual([True], observed_running_operation)
|
|
self.assertTrue(self.backend.exists(blob.storage_key))
|
|
self.assertNotIn("private-name", blob.storage_key)
|
|
self.assertEqual(".blob", Path(blob.storage_key).suffix)
|
|
|
|
def test_rolled_back_blob_write_is_compensated(self) -> None:
|
|
with patch(
|
|
"govoplan_files.backend.storage.files.get_storage_backend",
|
|
return_value=self.backend,
|
|
):
|
|
blob = _get_or_create_blob(
|
|
self.session,
|
|
tenant_id=TENANT_ID,
|
|
data=b"rollback",
|
|
filename="rollback.txt",
|
|
content_type="text/plain",
|
|
actor_id=USER_ID,
|
|
)
|
|
storage_key = blob.storage_key
|
|
self.assertTrue(self.backend.exists(storage_key))
|
|
self.session.rollback()
|
|
|
|
operation = self._only_operation()
|
|
self.assertEqual(RecoveryStatus.RECOVERED.value, operation.status)
|
|
self.assertFalse(self.backend.exists(storage_key))
|
|
with self.Session() as evidence_session:
|
|
self.assertIsNone(evidence_session.get(FileBlob, blob.id))
|
|
|
|
def test_post_write_tamper_is_quarantined_and_recovery_required(self) -> None:
|
|
with patch(
|
|
"govoplan_files.backend.storage.files.get_storage_backend",
|
|
return_value=self.backend,
|
|
):
|
|
blob = _get_or_create_blob(
|
|
self.session,
|
|
tenant_id=TENANT_ID,
|
|
data=b"expected",
|
|
filename="evidence.bin",
|
|
content_type="application/octet-stream",
|
|
actor_id=USER_ID,
|
|
)
|
|
self.backend.put_bytes(blob.storage_key, b"tampered")
|
|
self.session.commit()
|
|
|
|
operation = self._only_operation()
|
|
self.assertEqual(
|
|
RecoveryStatus.RECOVERY_REQUIRED.value,
|
|
operation.status,
|
|
)
|
|
with self.Session() as evidence_session:
|
|
persisted = evidence_session.get(FileBlob, blob.id)
|
|
self.assertIsNotNone(persisted)
|
|
self.assertEqual("checksum_mismatch", persisted.integrity_status)
|
|
self.assertIsNotNone(persisted.quarantined_at)
|
|
|
|
def test_missing_optional_encryption_fails_before_object_effect(self) -> None:
|
|
with patch(
|
|
"govoplan_files.backend.storage.files.get_storage_backend",
|
|
return_value=self.backend,
|
|
), patch(
|
|
"govoplan_files.backend.storage.content_protection.encryption_content_cipher",
|
|
return_value=None,
|
|
), self.assertRaisesRegex(FileStorageError, "Encryption module"):
|
|
_get_or_create_blob(
|
|
self.session,
|
|
tenant_id=TENANT_ID,
|
|
data=b"protected",
|
|
filename="protected.bin",
|
|
content_type="application/octet-stream",
|
|
actor_id=USER_ID,
|
|
encryption_vault_id="vault-1",
|
|
)
|
|
self.session.rollback()
|
|
|
|
operation = self._only_operation()
|
|
self.assertEqual(RecoveryStatus.REJECTED.value, operation.status)
|
|
objects = self.backend.list_objects(
|
|
prefix=f"tenants/{TENANT_ID}/files/",
|
|
limit=10,
|
|
)
|
|
self.assertEqual((), objects.objects)
|
|
|
|
def test_orphan_cleanup_forward_completes_after_business_rollback(self) -> None:
|
|
key = f"tenants/{TENANT_ID}/files/orphan.bin"
|
|
self.backend.put_bytes(key, b"orphan")
|
|
scan = FileIntegrityScan(
|
|
id="scan-1",
|
|
tenant_id=TENANT_ID,
|
|
storage_backend=self.backend.name,
|
|
storage_prefix=f"tenants/{TENANT_ID}/files/",
|
|
status="completed",
|
|
)
|
|
finding = FileIntegrityFinding(
|
|
id="finding-1",
|
|
scan_id=scan.id,
|
|
tenant_id=TENANT_ID,
|
|
kind="orphan_object",
|
|
state="open",
|
|
storage_key=key,
|
|
observed_size_bytes=6,
|
|
observed_checksum_sha256=hashlib.sha256(b"orphan").hexdigest(),
|
|
)
|
|
self.session.add_all([scan, finding])
|
|
self.session.commit()
|
|
|
|
cleanup_orphan_finding(
|
|
self.session,
|
|
finding,
|
|
user_id=USER_ID,
|
|
dry_run=False,
|
|
backend=self.backend,
|
|
)
|
|
self.assertFalse(self.backend.exists(key))
|
|
self.session.rollback()
|
|
|
|
operation = self._only_operation()
|
|
self.assertEqual(RecoveryStatus.SUCCEEDED.value, operation.status)
|
|
with self.Session() as evidence_session:
|
|
persisted = evidence_session.get(FileIntegrityFinding, finding.id)
|
|
self.assertIsNotNone(persisted)
|
|
self.assertEqual("deleted", persisted.state)
|
|
|
|
def test_missing_runtime_identity_blocks_before_object_write(self) -> None:
|
|
bind_process_runtime_identity(None)
|
|
with patch(
|
|
"govoplan_files.backend.storage.files.get_storage_backend",
|
|
return_value=self.backend,
|
|
), self.assertRaisesRegex(FileStorageError, "recovery ledger"):
|
|
_get_or_create_blob(
|
|
self.session,
|
|
tenant_id=TENANT_ID,
|
|
data=b"blocked",
|
|
filename="blocked.bin",
|
|
content_type="application/octet-stream",
|
|
actor_id=USER_ID,
|
|
)
|
|
self.session.rollback()
|
|
objects = self.backend.list_objects(
|
|
prefix=f"tenants/{TENANT_ID}/files/",
|
|
limit=10,
|
|
)
|
|
self.assertEqual((), objects.objects)
|
|
|
|
def test_blob_fence_blocks_a_competing_runtime_before_effect(self) -> None:
|
|
checksum = hashlib.sha256(b"fenced").hexdigest()
|
|
begin_blob_write_recovery(
|
|
self.session,
|
|
backend=self.backend,
|
|
tenant_id=TENANT_ID,
|
|
blob_id="blob-fenced",
|
|
storage_key=f"tenants/{TENANT_ID}/files/fenced.blob",
|
|
semantic_checksum_sha256=checksum,
|
|
semantic_size_bytes=6,
|
|
protection_discriminator="plaintext",
|
|
created_new=True,
|
|
)
|
|
with self.Session() as competing_session, self.assertRaisesRegex(
|
|
FileStorageError,
|
|
"already owned",
|
|
):
|
|
begin_blob_write_recovery(
|
|
competing_session,
|
|
backend=self.backend,
|
|
tenant_id=TENANT_ID,
|
|
blob_id="blob-fenced",
|
|
storage_key=f"tenants/{TENANT_ID}/files/fenced.blob",
|
|
semantic_checksum_sha256=checksum,
|
|
semantic_size_bytes=6,
|
|
protection_discriminator="plaintext",
|
|
created_new=True,
|
|
)
|
|
self.session.rollback()
|
|
self.assertEqual(RecoveryStatus.REJECTED.value, self._only_operation().status)
|
|
|
|
def _only_operation(self) -> RecoveryOperation:
|
|
with self.Session() as session:
|
|
operations = session.query(RecoveryOperation).all()
|
|
self.assertEqual(1, len(operations))
|
|
session.expunge(operations[0])
|
|
return operations[0]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|