fix(files): verify acquired source identity and index import lookups

Release v0.1.27. Coordinated integrity review: GovOPlaN/govoplan-core#298.

tests/test_connector_provenance_binding.py

@@ -0,0 +1,244 @@
+import hashlib
+import io
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+
+from govoplan_files.backend.route_support import (
+    _download_connector_payload,
+    _http_error,
+)
+from govoplan_files.backend.schemas import FileConnectorImportRequest
+from govoplan_files.backend.storage.connector_imports import (
+    ConnectorDownloadedFile,
+    ConnectorRevisionConflict,
+    _read_seafile_file,
+    _read_smb_file,
+    _read_s3_file,
+)
+from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
+
+
+class ConnectorProvenanceTests(unittest.TestCase):
+    def acquire(self, download, *, expected=None, annotations=None):
+        profile = ConnectorProfile(id="profile", label="Synthetic", provider="s3")
+        request = FileConnectorImportRequest(
+            library_id="bucket",
+            path="source.txt",
+            source_revision=expected,
+            metadata=annotations or {},
+        )
+        with (
+            patch(
+                "govoplan_files.backend.route_support.connector_policy_decision",
+                return_value=SimpleNamespace(allowed=True),
+            ),
+            patch(
+                "govoplan_files.backend.route_support.read_connector_file",
+                return_value=download,
+            ),
+        ):
+            return _download_connector_payload(profile, request, operation="sync")
+
+    def test_stale_or_unavailable_expected_revision_cannot_label_new_bytes(self):
+        for revision in ("B", None):
+            with (
+                self.subTest(revision=revision),
+                self.assertRaises(ConnectorRevisionConflict) as caught,
+            ):
+                self.acquire(
+                    ConnectorDownloadedFile(
+                        filename="file", data=b"B", revision=revision
+                    ),
+                    expected="A",
+                )
+            self.assertEqual(409, _http_error(caught.exception).status_code)
+
+    def test_actual_digest_is_computed_and_claims_and_annotations_are_not_authority(
+        self,
+    ):
+        annotations = {
+            "profile_id": "forged",
+            "size": 999,
+            "checksum_sha256": "caller-claim",
+            "acquired_sha256": "forged",
+            "note": "keep this",
+        }
+        download = ConnectorDownloadedFile(
+            filename="source.txt",
+            data=b"actual",
+            revision="version-B",
+            metadata={"etag": "etag-B", "checksum_sha256": "provider-claim"},
+        )
+        _, actual, metadata = self.acquire(
+            download, expected="etag-B", annotations=annotations
+        )
+        self.assertEqual("version-B", metadata["source_revision"])
+        evidence = metadata["source_provenance"]["metadata"]
+        self.assertEqual(
+            hashlib.sha256(actual.data).hexdigest(), evidence["acquired_sha256"]
+        )
+        self.assertEqual(len(actual.data), evidence["size"])
+        self.assertEqual("profile", evidence["profile_id"])
+        self.assertNotIn("checksum_sha256", evidence)
+        self.assertEqual(
+            {"checksum_sha256": "provider-claim"}, evidence["provider_checksum_claims"]
+        )
+        self.assertEqual(annotations, evidence["import_annotations"])
+        self.assertEqual(
+            metadata,
+            self.acquire(download, expected="version-B", annotations=annotations)[2],
+        )
+
+    def test_caller_fields_stay_annotations_when_provider_omits_evidence(self):
+        annotations = {
+            "sha256": "caller-hash",
+            "etag": "caller-etag",
+            "mtime": "caller-modified-time",
+            "checksum_sha256": "caller-checksum",
+            "provider_checksum_claims": {"checksum_sha256": "caller-claim"},
+            "connector_space_id": "browse-space",
+            "browse_etag": "browse-revision",
+            "folder_sync": True,
+            "note": {"values": ["001", "  keep spaces  ", None]},
+        }
+        download = ConnectorDownloadedFile(filename="source.txt", data=b"actual")
+        _, _, metadata = self.acquire(download, annotations=annotations)
+        evidence = metadata["source_provenance"]["metadata"]
+        self.assertEqual(annotations, evidence["import_annotations"])
+        self.assertEqual({}, evidence["provider_checksum_claims"])
+        self.assertEqual(
+            hashlib.sha256(download.data).hexdigest(), evidence["acquired_sha256"]
+        )
+        self.assertEqual(
+            {
+                "profile_id", "library_id", "library_path", "size",
+                "acquired_sha256", "provider_checksum_claims", "import_annotations",
+            },
+            set(evidence),
+        )
+
+    def test_seafile_rechecks_revision_after_download(self):
+        profile = ConnectorProfile(
+            id="profile",
+            label="Synthetic",
+            provider="seafile",
+            endpoint_url="https://example.invalid",
+        )
+        before = {"id": "A", "name": "source.txt", "size": 4}
+        for after in ({**before, "id": "B"}, before):
+            with (
+                patch(
+                    "govoplan_files.backend.storage.connector_imports._seafile_headers",
+                    return_value={},
+                ),
+                patch(
+                    "govoplan_files.backend.storage.connector_imports._request_json",
+                    side_effect=[before, "https://example.invalid/download", after],
+                ),
+                patch(
+                    "govoplan_files.backend.storage.connector_imports.request_connector_bytes",
+                    return_value=SimpleNamespace(
+                        status_code=200, content=b"DATA", headers={}
+                    ),
+                ),
+            ):
+                if after["id"] == "B":
+                    with self.assertRaises(ConnectorRevisionConflict):
+                        _read_seafile_file(
+                            profile, library_id="repo", path="source.txt", max_bytes=10
+                        )
+                else:
+                    result = _read_seafile_file(
+                        profile, library_id="repo", path="source.txt", max_bytes=10
+                    )
+                    self.assertEqual(("A", b"DATA"), (result.revision, result.data))
+
+    def test_smb_observes_stable_metadata_while_write_and_delete_sharing_are_denied(
+        self,
+    ):
+        profile = ConnectorProfile(id="profile", label="Synthetic", provider="smb")
+        before = SimpleNamespace(st_size=4, st_mtime_ns=10, st_mtime=1, st_ino=1)
+        for after in (
+            SimpleNamespace(st_size=4, st_mtime_ns=11, st_mtime=1, st_ino=1),
+            before,
+        ):
+            with (
+                patch(
+                    "govoplan_files.backend.storage.connector_imports._smb_location",
+                    return_value=SimpleNamespace(
+                        share="share", server="example.invalid", port=445
+                    ),
+                ),
+                patch(
+                    "govoplan_files.backend.storage.connector_imports._smb_unc_path",
+                    return_value="synthetic",
+                ),
+                patch(
+                    "govoplan_files.backend.storage.connector_imports._smb_client_kwargs",
+                    return_value={},
+                ),
+                patch(
+                    "govoplan_files.backend.storage.connector_imports._smbclient_module"
+                ) as factory,
+            ):
+                sdk = factory.return_value
+                sdk.stat.side_effect = [before, after]
+                sdk.open_file.return_value = io.BytesIO(b"DATA")
+                if after is before:
+                    self.assertEqual(
+                        b"DATA",
+                        _read_smb_file(profile, path="source.txt", max_bytes=10).data,
+                    )
+                else:
+                    with self.assertRaises(ConnectorRevisionConflict):
+                        _read_smb_file(profile, path="source.txt", max_bytes=10)
+                self.assertEqual("r", sdk.open_file.call_args.kwargs["share_access"])
+
+    def test_s3_conditional_or_versioned_read_and_response_binding(self):
+        profile = ConnectorProfile(id="profile", label="Synthetic", provider="s3")
+        for version in (None, "version-A", "null"):
+            for returned_etag in ("etag-A", "etag-B"):
+                with (
+                    self.subTest(version=version, etag=returned_etag),
+                    patch(
+                        "govoplan_files.backend.storage.connector_imports._s3_import_client"
+                    ) as factory,
+                ):
+                    client = factory.return_value
+                    client.head_object.return_value = {
+                        "ContentLength": 4,
+                        "ETag": "etag-A",
+                        "VersionId": version,
+                    }
+                    body = io.BytesIO(b"DATA")
+                    client.get_object.return_value = {
+                        "Body": body,
+                        "ETag": returned_etag,
+                        "VersionId": version,
+                    }
+                    if returned_etag == "etag-A":
+                        result = _read_s3_file(
+                            profile,
+                            library_id="bucket",
+                            path="source.txt",
+                            max_bytes=10,
+                        )
+                        self.assertEqual(version if version and version != "null" else "etag-A", result.revision)
+                    else:
+                        with self.assertRaises(ConnectorRevisionConflict):
+                            _read_s3_file(
+                                profile,
+                                library_id="bucket",
+                                path="source.txt",
+                                max_bytes=10,
+                            )
+                    request = client.get_object.call_args.kwargs
+                    immutable = version and version != "null"
+                    self.assertEqual(version if immutable else "etag-A", request["VersionId" if immutable else "IfMatch"])
+                    self.assertTrue(body.closed)
+                    client.close.assert_called_once()
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_manifest_documentation.py

@@ -4,6 +4,7 @@ import unittest
 
 
 STATIC_TOPIC_IDS = {
+    "files.source-identity-provenance",
     "files.archive-worker-limits",
     "files.configuration-package.managed-storage",
     "files.quick-access-and-product-area",
@@ -46,6 +47,15 @@ class FilesManifestDocumentationTests(unittest.TestCase):
     def topic(self, topic_id: str):
         return self.topics[topic_id]
 
+    def test_source_provenance_and_legacy_ambiguity_are_bilingual(self):
+        topic = self.topic("files.source-identity-provenance")
+        self.assertEqual({"admin", "user"}, set(topic.documentation_types))
+        for body in (topic.body, topic.translations["de"]["body"]):
+            for field in ("source_revision", "acquired_sha256", "provider_checksum_claims", "import_annotations"):
+                self.assertIn(field, body)
+        self.assertIn("non-unique", topic.body)
+        self.assertIn("nicht eindeutig", topic.translations["de"]["body"])
+
     def test_archive_resource_boundary_is_static_and_bilingual(self):
         topic = self.topic("files.archive-worker-limits")
         self.assertEqual("available", topic.layer)

tests/test_migrations.py

@@ -31,7 +31,7 @@ class FilesMigrationTests(unittest.TestCase):
                 )
                 with engine.connect() as connection:
                     self.assertIn(
-                        "a2b3c4d5e6f9",
+                        "a2b3c4d5e701",
                         set(MigrationContext.configure(connection).get_current_heads()),
                     )
             finally:

tests/test_source_identity_index.py

@@ -0,0 +1,199 @@
+from datetime import datetime, timezone
+import importlib
+import unittest
+
+from alembic.migration import MigrationContext
+from alembic.operations import Operations
+from sqlalchemy import (
+    JSON,
+    Column,
+    MetaData,
+    String,
+    Table,
+    create_engine,
+    inspect,
+    select,
+    text,
+)
+from sqlalchemy.orm import Session
+
+from govoplan_access.backend.db import models as _access_models  # register FK metadata
+from govoplan_core.core.change_sequence import ChangeSequenceEntry
+from govoplan_files.backend.db.models import FileAsset
+from govoplan_files.backend.storage.common import FileSourceConflict
+from govoplan_files.backend.storage.files import find_asset_by_source
+from govoplan_files.backend.storage.provenance import source_identity_hash
+
+
+def provenance(external="remote-1"):
+    return {"connector_id": " connector ", "provider": "s3", "external_id": external}
+
+
+class SourceIdentityIndexTests(unittest.TestCase):
+    def setUp(self):
+        self.engine = create_engine("sqlite:///:memory:")
+        for table in (
+            _access_models.Account.__table__,
+            _access_models.User.__table__,
+            _access_models.Group.__table__,
+            FileAsset.__table__,
+            ChangeSequenceEntry.__table__,
+        ):
+            table.create(self.engine)
+        self.session = Session(self.engine)
+
+    def tearDown(self):
+        self.session.close()
+        self.engine.dispose()
+
+    def asset(self, name, *, tenant="tenant", owner="owner", source=None, group=False):
+        row = FileAsset(
+            id=name,
+            tenant_id=tenant,
+            owner_type="group" if group else "user",
+            owner_user_id=None if group else owner,
+            owner_group_id=owner if group else None,
+            display_path=f"{name}.txt",
+            filename=f"{name}.txt",
+            metadata_={"source_provenance": source or provenance()},
+        )
+        self.session.add(row)
+        self.session.flush()
+        return row
+
+    def find(self, source=None, *, tenant="tenant", owner="owner", group=False):
+        return find_asset_by_source(
+            self.session,
+            tenant_id=tenant,
+            owner_type="group" if group else "user",
+            owner_id=owner,
+            source_provenance=source or provenance(),
+        )
+
+    def test_index_lookup_preserves_tenant_owner_provider_and_external_identity(self):
+        row = self.asset("wanted")
+        self.asset("other-tenant", tenant="another")
+        self.asset("other-owner", owner="another")
+        self.asset("group", group=True)
+        self.asset("other-provider", source={**provenance(), "provider": "webdav"})
+        for index in range(100):
+            self.asset(f"unrelated-{index}", source=provenance(f"remote-{index + 2}"))
+        self.assertIs(row, self.find())
+        self.assertIs(row, self.find())  # retry
+        self.assertEqual("group", self.find(group=True).id)
+        statement = (
+            select(FileAsset.id)
+            .where(
+                FileAsset.tenant_id == "tenant",
+                FileAsset.owner_type == "user",
+                FileAsset.owner_user_id == "owner",
+                FileAsset.source_identity_hash == source_identity_hash(provenance()),
+                FileAsset.deleted_at.is_(None),
+            )
+            .limit(2)
+        )
+        sql = str(
+            statement.compile(self.engine, compile_kwargs={"literal_binds": True})
+        )
+        plan = self.session.execute(text("EXPLAIN QUERY PLAN " + sql)).all()
+        self.assertIn("ix_file_assets_user_source", str(plan))
+        self.assertIn("LIMIT 2", sql)
+
+    def test_metadata_copy_delete_restore_and_duplicates_never_choose_a_winner(self):
+        row = self.asset("original")
+        copy = self.asset("copy", source=dict(row.metadata_["source_provenance"]))
+        self.assertEqual(row.source_identity_hash, copy.source_identity_hash)
+        with self.assertRaisesRegex(FileSourceConflict, "Multiple active"):
+            self.find()
+        copy.deleted_at = datetime.now(timezone.utc)
+        self.session.flush()
+        self.assertIs(row, self.find())
+        copy.deleted_at = None
+        self.session.flush()
+        with self.assertRaises(FileSourceConflict):
+            self.find()
+        copy.metadata_ = {"source_provenance": provenance("changed")}
+        self.session.flush()
+        self.assertIs(row, self.find())
+        self.assertIs(copy, self.find(provenance("changed")))
+        copy.metadata_ = {}
+        self.session.flush()
+        self.assertIsNone(copy.source_identity_hash)
+        self.assertIsNone(self.find(provenance("changed")))
+
+    def test_backfill_retains_all_legacy_records_and_metadata_in_bounded_batches(self):
+        migration = importlib.import_module(
+            "govoplan_files.backend.migrations.versions.a2b3c4d5e701_file_source_identity_index"
+        )
+        engine = create_engine("sqlite:///:memory:")
+        metadata = MetaData()
+        old = Table(
+            "file_assets",
+            metadata,
+            Column("id", String, primary_key=True),
+            Column("tenant_id", String),
+            Column("owner_type", String),
+            Column("owner_user_id", String),
+            Column("owner_group_id", String),
+            Column("metadata", JSON),
+        )
+        metadata.create_all(engine)
+        rows = [
+            {
+                "id": f"{index:04d}",
+                "tenant_id": "tenant",
+                "owner_type": "user",
+                "owner_user_id": "owner",
+                "metadata": {
+                    "source_provenance": provenance(
+                        "duplicate" if index < 2 else str(index)
+                    )
+                },
+            }
+            for index in range(503)
+        ]
+        rows.append(
+            {
+                "id": "no-source",
+                "tenant_id": "tenant",
+                "owner_type": "user",
+                "owner_user_id": "owner",
+                "metadata": {"unrelated": [1, 2]},
+            }
+        )
+        try:
+            with engine.begin() as connection:
+                connection.execute(old.insert(), rows)
+                with Operations.context(MigrationContext.configure(connection)):
+                    migration.upgrade()
+                new = Table("file_assets", MetaData(), autoload_with=connection)
+                actual = (
+                    connection.execute(select(new).order_by(new.c.id)).mappings().all()
+                )
+                self.assertEqual(len(rows), len(actual))
+                self.assertEqual(
+                    [row["metadata"] for row in rows],
+                    [row["metadata"] for row in actual],
+                )
+                self.assertEqual(
+                    actual[0]["source_identity_hash"], actual[1]["source_identity_hash"]
+                )
+                for row in actual:
+                    self.assertEqual(
+                        source_identity_hash(row["metadata"].get("source_provenance")),
+                        row["source_identity_hash"],
+                    )
+                indexes = inspect(connection).get_indexes("file_assets")
+                self.assertEqual(2, len(indexes))
+                self.assertFalse(any(index["unique"] for index in indexes))
+                with Operations.context(MigrationContext.configure(connection)):
+                    migration.downgrade()
+                self.assertEqual(
+                    len(rows), len(connection.execute(select(old.c.id)).all())
+                )
+        finally:
+            engine.dispose()
+
+
+if __name__ == "__main__":
+    unittest.main()