GovOPlaN/govoplan-files
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: govern file lifecycle and connector writes

Browse Source
This commit is contained in:
zemion
2026-08-20 22:06:57 +02:00
parent 95aef18955
commit 6c3cf1c55e
27 changed files with 2721 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_connector_space_deletion.py
+34 -1
View File
@@ -5,7 +5,11 @@ from types import SimpleNamespace
from unittest.mock import Mock, patch
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_spaces import soft_delete_connector_space
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
from govoplan_files.backend.storage.connector_spaces import (
soft_delete_connector_space,
validate_connector_space_write_mode,
)
class ConnectorSpaceDeletionTests(unittest.TestCase):
@@ -63,6 +67,35 @@ class ConnectorSpaceDeletionTests(unittest.TestCase):
self.session.add.assert_not_called()
self.session.flush.assert_not_called()
def test_read_only_mode_does_not_require_remote_write_capability(self) -> None:
profile = ConnectorProfile(
id="webdav-read",
label="Read only",
provider="webdav",
capabilities=("browse", "import"),
)
validate_connector_space_write_mode(profile, read_only=True)
def test_two_way_mode_requires_explicit_supported_write_capability(self) -> None:
missing_capability = ConnectorProfile(
id="s3-read",
label="S3 read only",
provider="s3",
capabilities=("browse",),
)
unsupported_provider = ConnectorProfile(
id="webdav-write",
label="WebDAV write",
provider="webdav",
capabilities=("write",),
)
for profile in (missing_capability, unsupported_provider):
with self.subTest(profile=profile.id):
with self.assertRaisesRegex(FileStorageError, "requires an S3"):
validate_connector_space_write_mode(profile, read_only=False)
if __name__ == "__main__":
unittest.main()
tests/test_file_lifecycle.py
+164
View File
@@ -0,0 +1,164 @@
from __future__ import annotations
from datetime import timedelta
import unittest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from govoplan_access.backend.db.models import Account, Group, User
from govoplan_core.db.base import Base
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_files.backend.db.models import (
FileAsset,
FileBlob,
FileFolder,
FileFormEvidenceGrant,
FileShare,
FileVersion,
)
from govoplan_files.backend.storage.common import FileStorageError, utcnow
from govoplan_files.backend.storage.lifecycle import (
preview_asset_purge,
restore_asset,
set_asset_lifecycle,
)
class FileLifecycleTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite:///:memory:", future=True)
Base.metadata.create_all(
bind=self.engine,
tables=[
Account.__table__,
User.__table__,
Group.__table__,
ChangeSequenceEntry.__table__,
FileBlob.__table__,
FileAsset.__table__,
FileVersion.__table__,
FileFolder.__table__,
FileFormEvidenceGrant.__table__,
FileShare.__table__,
],
)
self.session = sessionmaker(bind=self.engine, future=True)()
self.blob = FileBlob(
id="blob-1",
tenant_id="tenant-1",
storage_backend="local",
storage_key="tenants/tenant-1/files/blob-1",
checksum_sha256="a" * 64,
size_bytes=1,
ref_count=1,
)
self.asset = FileAsset(
id="file-1",
tenant_id="tenant-1",
owner_type="user",
owner_user_id="user-1",
current_version_id="version-1",
display_path="records/report.txt",
filename="report.txt",
deleted_at=utcnow(),
)
self.version = FileVersion(
id="version-1",
tenant_id="tenant-1",
file_asset_id="file-1",
blob_id="blob-1",
version_number=1,
filename_at_upload="report.txt",
display_path_at_upload="records/report.txt",
size_bytes=1,
checksum_sha256="a" * 64,
)
self.session.add_all([self.blob, self.asset, self.version])
self.session.commit()
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_preview_hash_changes_with_retention_and_blocks_active_policy(self) -> None:
first = preview_asset_purge(
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
)
self.assertTrue(first.eligible)
set_asset_lifecycle(
self.session,
self.asset,
retained_until=utcnow() + timedelta(days=7),
legal_hold=True,
reason="Active proceeding",
expected_revision=1,
)
self.session.commit()
second = preview_asset_purge(
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
)
self.assertFalse(second.eligible)
self.assertNotEqual(first.preview_sha256, second.preview_sha256)
self.assertEqual(
{"legal_hold", "retention_active"}, set(second.items[0].blockers)
)
def test_restore_preserves_version_and_provenance_and_rejects_collision(self) -> None:
self.asset.metadata_ = {"source": {"provider": "s3", "revision": "v1"}}
self.session.commit()
self.assertTrue(restore_asset(self.session, self.asset))
self.assertEqual("version-1", self.asset.current_version_id)
self.assertEqual("v1", self.asset.metadata_["source"]["revision"])
self.session.commit()
self.asset.deleted_at = utcnow()
collision = FileAsset(
id="file-2",
tenant_id="tenant-1",
owner_type="user",
owner_user_id="user-1",
display_path=self.asset.display_path,
filename=self.asset.filename,
)
self.session.add(collision)
self.session.commit()
with self.assertRaisesRegex(FileStorageError, "path is already in use"):
restore_asset(self.session, self.asset)
def test_active_share_blocks_purge_until_revoked(self) -> None:
share = FileShare(
id="share-1",
tenant_id="tenant-1",
file_asset_id=self.asset.id,
target_type="tenant",
target_id="tenant-1",
permission="read",
)
self.session.add(share)
self.session.commit()
blocked = preview_asset_purge(
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
)
self.assertIn("active_share", blocked.items[0].blockers)
share.revoked_at = utcnow()
self.session.commit()
allowed = preview_asset_purge(
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
)
self.assertTrue(allowed.eligible)
def test_missing_campaign_table_is_an_optional_module_safe_path(self) -> None:
preview = preview_asset_purge(
self.session, tenant_id="tenant-1", file_ids=[self.asset.id]
)
self.assertNotIn("campaign_evidence", preview.items[0].blockers)
if __name__ == "__main__":
unittest.main()
tests/test_manifest_documentation.py
+16 -4
View File
@@ -11,6 +11,7 @@ STATIC_TOPIC_IDS = {
"files.workflow.find-and-download-files",
"files.workflow.share-managed-files",
"files.workflow.delete-managed-files",
"files.workflow.restore-retain-and-purge",
"files.governed-connectors-and-provenance",
"files.reference.integrity-recovery-and-fail-closed-transports",
"files.reference.shared-storage-profile",
@@ -119,11 +120,16 @@ class FilesManifestDocumentationTests(unittest.TestCase):
delete = self.topic("files.workflow.delete-managed-files")
self.assertIn("Soft-delete", delete.summary)
self.assertIn("no self-service restore or hard-purge", delete.body)
self.assertIn("Authorized restoration", delete.body)
self.assertTrue(
any("not a hard purge" in item for item in delete.metadata["limitations"])
any("soft deletion" in item.casefold() for item in delete.metadata["limitations"])
)
lifecycle = self.topic("files.workflow.restore-retain-and-purge")
self.assertIn("legal hold", lifecycle.body)
self.assertIn("preview hash", lifecycle.body)
self.assertIn("recovery ledger", lifecycle.body)
def test_admin_topic_covers_policy_redaction_and_atomic_credential_deletion(
self,
) -> None:
@@ -144,6 +150,9 @@ class FilesManifestDocumentationTests(unittest.TestCase):
self.assertIn("deny rules win", topic.body)
self.assertIn("redact secret values", topic.body)
self.assertIn("same transaction", topic.body)
self.assertIn("read-only by default", topic.body)
self.assertIn("S3", topic.body)
self.assertIn("Automatic remote deletion", topic.body)
self.assertTrue(
any(
"non-owned external references" in item
@@ -179,6 +188,8 @@ class FilesManifestDocumentationTests(unittest.TestCase):
)
self.assertIn("lease-fenced Core recovery", topic.body)
self.assertIn("Ops", topic.body)
self.assertIn("Hard purge", topic.body)
self.assertIn("S3 connector write-back", topic.body)
self.assertTrue(topic.metadata["verification"])
self.assertIn(
"/api/v1/files/integrity/scans",
@@ -224,8 +235,9 @@ class FilesManifestDocumentationTests(unittest.TestCase):
self.assertIn("process_owner", topic.audience)
self.assertIn("release_manager", topic.audience)
self.assertIn("versions align", topic.body)
self.assertIn("Share grant, change, expiry, and revocation", topic.body)
self.assertIn("no enforced retention or legal hold", topic.body)
self.assertIn("share lifecycle", topic.body)
self.assertIn("legal hold", topic.body)
self.assertIn("S3 write-back", topic.body)
for key in ("prerequisites", "steps", "outcome", "verification"):
self.assertTrue(topic.metadata[key])
self.assertTrue(
tests/test_migrations.py
+1 -1
View File
@@ -31,7 +31,7 @@ class FilesMigrationTests(unittest.TestCase):
)
with engine.connect() as connection:
self.assertIn(
"a2b3c4d5e6f8",
"a2b3c4d5e6f9",
set(MigrationContext.configure(connection).get_current_heads()),
)
finally:
tests/test_router_contract.py
+15 -1
View File
@@ -17,6 +17,7 @@ from govoplan_files.backend.routes.folders import router as folders_router
from govoplan_files.backend.routes.form_evidence import router as form_evidence_router
from govoplan_files.backend.routes.integrity import router as integrity_router
from govoplan_files.backend.routes.listing import router as listing_router
from govoplan_files.backend.routes.lifecycle import router as lifecycle_router
from govoplan_files.backend.routes.shares import router as shares_router
from govoplan_files.backend.routes.spaces import router as spaces_router
from govoplan_files.backend.routes.transfers import router as transfers_router
@@ -38,6 +39,7 @@ class FilesRouterContractTests(unittest.TestCase):
folders_router,
form_evidence_router,
integrity_router,
lifecycle_router,
listing_router,
uploads_router,
connector_settings_router,
@@ -55,7 +57,7 @@ class FilesRouterContractTests(unittest.TestCase):
actual = self._operation_keys(router)
self.assertEqual(expected, actual)
self.assertEqual(54, len(actual))
self.assertEqual(62, len(actual))
self.assertFalse(
[operation for operation, count in Counter(actual).items() if count > 1]
)
@@ -81,6 +83,7 @@ class FilesRouterContractTests(unittest.TestCase):
(("GET",), "/files/connectors/profiles/{profile_id}/browse"),
(("POST",), "/files/connectors/profiles/{profile_id}/import"),
(("POST",), "/files/connectors/profiles/{profile_id}/sync"),
(("POST",), "/files/connector-spaces/{space_id}/write-back"),
(("GET",), "/files/connectors/credentials"),
(("POST",), "/files/connectors/credentials"),
(("GET",), "/files/connector-spaces"),
@@ -119,6 +122,17 @@ class FilesRouterContractTests(unittest.TestCase):
routes,
)
def test_governed_lifecycle_routes_are_exposed(self) -> None:
routes = {
(tuple(sorted(route.methods or ())), route.path) for route in router.routes
}
self.assertIn((("PATCH",), "/files/{file_id}/lifecycle"), routes)
self.assertIn((("POST",), "/files/assets/{file_id}/restore"), routes)
self.assertIn((("POST",), "/files/purge/preview"), routes)
self.assertIn((("POST",), "/files/purge/execute"), routes)
self.assertIn((("POST",), "/files/purge/blobs"), routes)
def test_file_listing_exposes_structured_property_filters(self) -> None:
route = next(
route
tests/test_storage_recovery.py
+268
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import hashlib
from datetime import UTC, datetime
from pathlib import Path
import tempfile
import unittest
@@ -15,6 +16,7 @@ from govoplan_core.core.recovery import (
RecoveryOperation,
RecoveryStatus,
)
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.core.runtime_coordination import (
DistributedLease,
RuntimeIdentity,
@@ -23,9 +25,13 @@ from govoplan_core.core.runtime_coordination import (
from govoplan_core.db.base import Base
from govoplan_core.db.session import configure_database, reset_database
from govoplan_files.backend.db.models import (
FileAsset,
FileBlob,
FileFormEvidenceGrant,
FileIntegrityFinding,
FileIntegrityScan,
FileShare,
FileVersion,
)
from govoplan_files.backend.storage.backends import (
LocalFilesystemStorageBackend,
@@ -34,12 +40,61 @@ from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.files import _get_or_create_blob
from govoplan_files.backend.storage.integrity import cleanup_orphan_finding
from govoplan_files.backend.storage.recovery import begin_blob_write_recovery
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
from govoplan_files.backend.storage.connector_writes import write_connector_file
from govoplan_files.backend.storage.lifecycle import (
execute_asset_purge,
garbage_collect_unreferenced_blobs,
preview_asset_purge,
)
TENANT_ID = "tenant-1"
USER_ID = "user-1"
class _MissingS3Object(Exception):
response = {
"Error": {"Code": "NoSuchKey"},
"ResponseMetadata": {"HTTPStatusCode": 404},
}
class _WriteS3Client:
def __init__(
self, *, tamper_metadata: bool = False, fail_post_write_probe: bool = False
) -> None:
self.object: dict[str, object] | None = None
self.tamper_metadata = tamper_metadata
self.fail_post_write_probe = fail_post_write_probe
self.put_request: dict[str, object] | None = None
self.closed = False
self.write_completed = False
def head_object(self, **_kwargs: object) -> dict[str, object]:
if self.write_completed and self.fail_post_write_probe:
raise RuntimeError("provider unavailable")
if self.object is None:
raise _MissingS3Object()
return dict(self.object)
def put_object(self, **kwargs: object) -> dict[str, object]:
self.put_request = dict(kwargs)
metadata = dict(kwargs.get("Metadata") or {})
if self.tamper_metadata:
metadata["govoplan-sha256"] = "0" * 64
self.object = {
"ETag": '"etag-written"',
"VersionId": "version-written",
"Metadata": metadata,
}
self.write_completed = True
return {"ETag": '"etag-written"', "VersionId": "version-written"}
def close(self) -> None:
self.closed = True
class StorageRecoveryTests(unittest.TestCase):
def setUp(self) -> None:
self.temporary_directory = tempfile.TemporaryDirectory()
@@ -54,10 +109,15 @@ class StorageRecoveryTests(unittest.TestCase):
Account.__table__,
User.__table__,
Group.__table__,
ChangeSequenceEntry.__table__,
DistributedLease.__table__,
RecoveryOperation.__table__,
RecoveryCheckpoint.__table__,
FileBlob.__table__,
FileAsset.__table__,
FileVersion.__table__,
FileFormEvidenceGrant.__table__,
FileShare.__table__,
FileIntegrityScan.__table__,
FileIntegrityFinding.__table__,
],
@@ -366,6 +426,214 @@ class StorageRecoveryTests(unittest.TestCase):
self.session.rollback()
self.assertEqual(RecoveryStatus.REJECTED.value, self._only_operation().status)
def test_purge_is_idempotent_and_gc_deletes_only_after_reference_check(self) -> None:
key = f"tenants/{TENANT_ID}/files/purge-me.blob"
self.backend.put_bytes(key, b"purge-me")
blob = FileBlob(
id="blob-purge",
tenant_id=TENANT_ID,
storage_backend=self.backend.name,
storage_key=key,
checksum_sha256=hashlib.sha256(b"purge-me").hexdigest(),
size_bytes=8,
ref_count=1,
)
asset = FileAsset(
id="asset-purge",
tenant_id=TENANT_ID,
owner_type="user",
owner_user_id=USER_ID,
current_version_id="version-purge",
display_path="purge-me.txt",
filename="purge-me.txt",
deleted_at=datetime.now(UTC),
)
version = FileVersion(
id="version-purge",
tenant_id=TENANT_ID,
file_asset_id=asset.id,
blob_id=blob.id,
version_number=1,
filename_at_upload=asset.filename,
display_path_at_upload=asset.display_path,
size_bytes=blob.size_bytes,
checksum_sha256=blob.checksum_sha256,
)
self.session.add_all([blob, asset, version])
self.session.commit()
preview = preview_asset_purge(
self.session, tenant_id=TENANT_ID, file_ids=[asset.id]
)
result = execute_asset_purge(
self.session,
tenant_id=TENANT_ID,
file_ids=[asset.id],
preview_sha256=preview.preview_sha256,
idempotency_key="purge-request-1",
approval_reference="approval-1",
)
self.assertEqual(1, result.purged_files)
self.assertEqual(1, result.released_blobs)
self.assertIsNone(self.session.get(FileAsset, asset.id))
retained_blob = self.session.get(FileBlob, blob.id)
self.assertIsNotNone(retained_blob)
self.assertEqual(0, retained_blob.ref_count)
self.assertTrue(self.backend.exists(key))
replay = execute_asset_purge(
self.session,
tenant_id=TENANT_ID,
file_ids=[asset.id],
preview_sha256=preview.preview_sha256,
idempotency_key="purge-request-1",
approval_reference="approval-1",
)
self.assertTrue(replay.replayed)
with patch(
"govoplan_files.backend.storage.lifecycle.get_storage_backend",
return_value=self.backend,
):
gc_result = garbage_collect_unreferenced_blobs(
self.session,
tenant_id=TENANT_ID,
limit=10,
approval_reference="approval-gc-1",
)
self.assertEqual(1, gc_result.deleted_blobs)
self.assertFalse(self.backend.exists(key))
self.assertIsNone(self.session.get(FileBlob, blob.id))
with self.Session() as evidence_session:
operations = evidence_session.query(RecoveryOperation).all()
self.assertEqual(2, len(operations))
self.assertEqual(
{RecoveryStatus.SUCCEEDED.value},
{operation.status for operation in operations},
)
def test_s3_write_is_conditional_verified_and_idempotent(self) -> None:
client = _WriteS3Client()
profile = ConnectorProfile(
id="s3-write",
label="S3 write",
provider="s3",
base_path="root",
capabilities=("browse", "write"),
metadata={"bucket": "files"},
)
with patch(
"govoplan_files.backend.storage.connector_writes._s3_client",
return_value=client,
):
result = write_connector_file(
profile,
tenant_id=TENANT_ID,
library_id=None,
remote_path="out/report.txt",
data=b"report",
content_type="text/plain",
expected_revision=None,
idempotency_key="connector-write-1",
)
replay = write_connector_file(
profile,
tenant_id=TENANT_ID,
library_id=None,
remote_path="out/report.txt",
data=b"report",
content_type="text/plain",
expected_revision=None,
idempotency_key="connector-write-1",
)
self.assertEqual(RecoveryStatus.SUCCEEDED.value, result.status)
self.assertTrue(replay.replayed)
self.assertEqual("*", client.put_request["IfNoneMatch"])
self.assertEqual("root/out/report.txt", client.put_request["Key"])
self.assertNotIn(b"report", repr(self._only_operation().metadata_).encode())
self.assertTrue(client.closed)
def test_s3_write_tamper_is_visible_and_blocks_another_writer(self) -> None:
client = _WriteS3Client(tamper_metadata=True)
profile = ConnectorProfile(
id="s3-write",
label="S3 write",
provider="s3",
capabilities=("write",),
metadata={"bucket": "files"},
)
with patch(
"govoplan_files.backend.storage.connector_writes._s3_client",
return_value=client,
):
result = write_connector_file(
profile,
tenant_id=TENANT_ID,
library_id=None,
remote_path="report.txt",
data=b"report",
content_type="text/plain",
expected_revision=None,
idempotency_key="connector-write-tampered",
)
with self.assertRaisesRegex(FileStorageError, "unresolved recovery"):
write_connector_file(
profile,
tenant_id=TENANT_ID,
library_id=None,
remote_path="report.txt",
data=b"replacement",
content_type="text/plain",
expected_revision="version-written",
idempotency_key="connector-write-after-tamper",
)
self.assertEqual(RecoveryStatus.RECOVERY_REQUIRED.value, result.status)
self.assertEqual(
RecoveryStatus.RECOVERY_REQUIRED.value, self._only_operation().status
)
def test_s3_write_probe_failure_is_outcome_unknown_not_confirmed_absent(self) -> None:
client = _WriteS3Client(fail_post_write_probe=True)
profile = ConnectorProfile(
id="s3-write",
label="S3 write",
provider="s3",
capabilities=("write",),
metadata={"bucket": "files"},
)
with patch(
"govoplan_files.backend.storage.connector_writes._s3_client",
return_value=client,
):
result = write_connector_file(
profile,
tenant_id=TENANT_ID,
library_id=None,
remote_path="report.txt",
data=b"report",
content_type="text/plain",
expected_revision=None,
idempotency_key="connector-write-probe-failed",
)
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, result.status)
operation = self._only_operation()
self.assertEqual(RecoveryStatus.OUTCOME_UNKNOWN.value, operation.status)
with self.Session() as session:
checkpoint = (
session.query(RecoveryCheckpoint)
.filter(RecoveryCheckpoint.operation_id == operation.id)
.order_by(RecoveryCheckpoint.sequence.desc())
.first()
)
self.assertIsNotNone(checkpoint)
self.assertFalse(checkpoint.evidence["observed"]["probe_verified"])
self.assertFalse(checkpoint.evidence["observed"]["present"])
def _only_operation(self) -> RecoveryOperation:
with self.Session() as session:
operations = session.query(RecoveryOperation).all()