GovOPlaN/govoplan-files
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: govern file lifecycle and connector writes

Browse Source
This commit is contained in:
zemion
2026-08-20 22:06:57 +02:00
parent 95aef18955
commit 6c3cf1c55e
27 changed files with 2721 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_storage_recovery.py
+268
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import hashlib
from datetime import UTC, datetime
from pathlib import Path
import tempfile
import unittest
@@ -15,6 +16,7 @@ from govoplan_core.core.recovery import (
RecoveryOperation,
RecoveryStatus,
)
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.core.runtime_coordination import (
DistributedLease,
RuntimeIdentity,
@@ -23,9 +25,13 @@ from govoplan_core.core.runtime_coordination import (
from govoplan_core.db.base import Base
from govoplan_core.db.session import configure_database, reset_database
from govoplan_files.backend.db.models import (
FileAsset,
FileBlob,
FileFormEvidenceGrant,
FileIntegrityFinding,
FileIntegrityScan,
FileShare,
FileVersion,
)
from govoplan_files.backend.storage.backends import (
LocalFilesystemStorageBackend,
@@ -34,12 +40,61 @@ from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.files import _get_or_create_blob
from govoplan_files.backend.storage.integrity import cleanup_orphan_finding
from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
from govoplan_files.backend.storage.connector_writes import write_connector_file
from govoplan_files.backend.storage.lifecycle import (
execute_asset_purge,
garbage_collect_unreferenced_blobs,
preview_asset_purge,
)
TENANT_ID = "tenant-1"
USER_ID = "user-1"
class _MissingS3Object(Exception):
response = {
"Error": {"Code": "NoSuchKey"},
"ResponseMetadata": {"HTTPStatusCode": 404},
}
class _WriteS3Client:
def __init__(
self, *, tamper_metadata: bool = False, fail_post_write_probe: bool = False
) -> None:
self.object: dict[str, object] | None = None
self.tamper_metadata = tamper_metadata
self.fail_post_write_probe = fail_post_write_probe
self.put_request: dict[str, object] | None = None
self.closed = False
self.write_completed = False
def head_object(self, **_kwargs: object) -> dict[str, object]:
if self.write_completed and self.fail_post_write_probe:
raise RuntimeError("provider unavailable")
if self.object is None:
raise _MissingS3Object()
return dict(self.object)
def put_object(self, **kwargs: object) -> dict[str, object]:
self.put_request = dict(kwargs)
metadata = dict(kwargs.get("Metadata") or {})
if self.tamper_metadata:
metadata["govoplan-sha256"] = "0" * 64
self.object = {
"ETag": '"etag-written"',
"VersionId": "version-written",
"Metadata": metadata,
}
self.write_completed = True
return {"ETag": '"etag-written"', "VersionId": "version-written"}
def close(self) -> None:
self.closed = True
class StorageRecoveryTests(unittest.TestCase):
def setUp(self) -> None:
self.temporary_directory = tempfile.TemporaryDirectory()
@@ -54,10 +109,15 @@ class StorageRecoveryTests(unittest.TestCase):
Account.__table__,
User.__table__,
Group.__table__,
ChangeSequenceEntry.__table__,
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
FileBlob.__table__,
FileAsset.__table__,
FileVersion.__table__,
FileFormEvidenceGrant.__table__,
FileShare.__table__,
FileIntegrityScan.__table__,
FileIntegrityFinding.__table__,
],
@@ -366,6 +426,214 @@ class StorageRecoveryTests(unittest.TestCase):
self.session.rollback()
self.assertEqual(RecoveryStatus.REJECTED.value, self._only_operation().status)
def test_purge_is_idempotent_and_gc_deletes_only_after_reference_check(self) -> None:
key = f"tenants/{TENANT_ID}/files/purge-me.blob"
self.backend.put_bytes(key, b"purge-me")
blob = FileBlob(
id="blob-purge",
tenant_id=TENANT_ID,
storage_backend=self.backend.name,
storage_key=key,
checksum_sha256=hashlib.sha256(b"purge-me").hexdigest(),
size_bytes=8,
ref_count=1,
)
asset = FileAsset(
id="asset-purge",
tenant_id=TENANT_ID,
owner_type="user",
owner_user_id=USER_ID,
current_version_id="version-purge",
display_path="purge-me.txt",
filename="purge-me.txt",
deleted_at=datetime.now(UTC),
)
version = FileVersion(
id="version-purge",
tenant_id=TENANT_ID,
file_asset_id=asset.id,
blob_id=blob.id,
version_number=1,
filename_at_upload=asset.filename,
display_path_at_upload=asset.display_path,
size_bytes=blob.size_bytes,
checksum_sha256=blob.checksum_sha256,
)
self.session.add_all([blob, asset, version])
self.session.commit()
preview = preview_asset_purge(
self.session, tenant_id=TENANT_ID, file_ids=[asset.id]
)
result = execute_asset_purge(
self.session,
tenant_id=TENANT_ID,
file_ids=[asset.id],
preview_sha256=preview.preview_sha256,
idempotency_key="purge-request-1",
approval_reference="approval-1",
)
self.assertEqual(1, result.purged_files)
self.assertEqual(1, result.released_blobs)
self.assertIsNone(self.session.get(FileAsset, asset.id))
retained_blob = self.session.get(FileBlob, blob.id)
self.assertIsNotNone(retained_blob)
self.assertEqual(0, retained_blob.ref_count)
self.assertTrue(self.backend.exists(key))
replay = execute_asset_purge(
self.session,
tenant_id=TENANT_ID,
file_ids=[asset.id],
preview_sha256=preview.preview_sha256,
idempotency_key="purge-request-1",
approval_reference="approval-1",
)
self.assertTrue(replay.replayed)
with patch(
"govoplan_files.backend.storage.lifecycle.get_storage_backend",
return_value=self.backend,
):
gc_result = garbage_collect_unreferenced_blobs(
self.session,
tenant_id=TENANT_ID,
limit=10,
approval_reference="approval-gc-1",
)
self.assertEqual(1, gc_result.deleted_blobs)
self.assertFalse(self.backend.exists(key))
self.assertIsNone(self.session.get(FileBlob, blob.id))
with self.Session() as evidence_session:
operations = evidence_session.query(RecoveryOperation).all()
self.assertEqual(2, len(operations))
self.assertEqual(
{RecoveryStatus.SUCCEEDED.value},
{operation.status for operation in operations},
)
def test_s3_write_is_conditional_verified_and_idempotent(self) -> None:
client = _WriteS3Client()
profile = ConnectorProfile(
id="s3-write",
label="S3 write",
provider="s3",
base_path="root",
capabilities=("browse", "write"),
metadata={"bucket": "files"},
)
with patch(
"govoplan_files.backend.storage.connector_writes._s3_client",
return_value=client,
):
result = write_connector_file(
profile,
tenant_id=TENANT_ID,
library_id=None,
remote_path="out/report.txt",
data=b"report",
content_type="text/plain",
expected_revision=None,
idempotency_key="connector-write-1",
)
replay = write_connector_file(
profile,
tenant_id=TENANT_ID,
library_id=None,
remote_path="out/report.txt",
data=b"report",
content_type="text/plain",
expected_revision=None,
idempotency_key="connector-write-1",
)
self.assertEqual(RecoveryStatus.SUCCEEDED.value, result.status)
self.assertTrue(replay.replayed)
self.assertEqual("*", client.put_request["IfNoneMatch"])
self.assertEqual("root/out/report.txt", client.put_request["Key"])
self.assertNotIn(b"report", repr(self._only_operation().metadata_).encode())
self.assertTrue(client.closed)
def test_s3_write_tamper_is_visible_and_blocks_another_writer(self) -> None:
client = _WriteS3Client(tamper_metadata=True)
profile = ConnectorProfile(
id="s3-write",
label="S3 write",
provider="s3",
capabilities=("write",),
metadata={"bucket": "files"},
)
with patch(
"govoplan_files.backend.storage.connector_writes._s3_client",
return_value=client,
):
result = write_connector_file(
profile,
tenant_id=TENANT_ID,
library_id=None,
remote_path="report.txt",
data=b"report",
content_type="text/plain",
expected_revision=None,
idempotency_key="connector-write-tampered",
)
with self.assertRaisesRegex(FileStorageError, "unresolved recovery"):
write_connector_file(
profile,
tenant_id=TENANT_ID,
library_id=None,
remote_path="report.txt",
data=b"replacement",
content_type="text/plain",
expected_revision="version-written",
idempotency_key="connector-write-after-tamper",
)
self.assertEqual(RecoveryStatus.RECOVERY_REQUIRED.value, result.status)
self.assertEqual(
RecoveryStatus.RECOVERY_REQUIRED.value, self._only_operation().status
)
def test_s3_write_probe_failure_is_outcome_unknown_not_confirmed_absent(self) -> None:
client = _WriteS3Client(fail_post_write_probe=True)
profile = ConnectorProfile(
id="s3-write",
label="S3 write",
provider="s3",
capabilities=("write",),
metadata={"bucket": "files"},
)
with patch(
"govoplan_files.backend.storage.connector_writes._s3_client",
return_value=client,
):
result = write_connector_file(
profile,
tenant_id=TENANT_ID,
library_id=None,
remote_path="report.txt",
data=b"report",
content_type="text/plain",
expected_revision=None,
idempotency_key="connector-write-probe-failed",
)
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, result.status)
operation = self._only_operation()
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, operation.status)
with self.Session() as session:
checkpoint = (
session.query(RecoveryCheckpoint)
.filter(RecoveryCheckpoint.operation_id == operation.id)
.order_by(RecoveryCheckpoint.sequence.desc())
.first()
)
self.assertIsNotNone(checkpoint)
self.assertFalse(checkpoint.evidence["observed"]["probe_verified"])
self.assertFalse(checkpoint.evidence["observed"]["present"])
def _only_operation(self) -> RecoveryOperation:
with self.Session() as session:
operations = session.query(RecoveryOperation).all()