fix(files): verify acquired source identity and index import lookups
Module Package Release / publish-packages (push) Successful in 14s
Module Package Release / publish-packages (push) Successful in 14s
Release v0.1.27. Coordinated integrity review: GovOPlaN/govoplan-core#298.
This commit is contained in:
Executable
+244
@@ -0,0 +1,244 @@
|
||||
import hashlib
|
||||
import io
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from govoplan_files.backend.route_support import (
|
||||
_download_connector_payload,
|
||||
_http_error,
|
||||
)
|
||||
from govoplan_files.backend.schemas import FileConnectorImportRequest
|
||||
from govoplan_files.backend.storage.connector_imports import (
|
||||
ConnectorDownloadedFile,
|
||||
ConnectorRevisionConflict,
|
||||
_read_seafile_file,
|
||||
_read_smb_file,
|
||||
_read_s3_file,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||
|
||||
|
||||
class ConnectorProvenanceTests(unittest.TestCase):
|
||||
def acquire(self, download, *, expected=None, annotations=None):
|
||||
profile = ConnectorProfile(id="profile", label="Synthetic", provider="s3")
|
||||
request = FileConnectorImportRequest(
|
||||
library_id="bucket",
|
||||
path="source.txt",
|
||||
source_revision=expected,
|
||||
metadata=annotations or {},
|
||||
)
|
||||
with (
|
||||
patch(
|
||||
"govoplan_files.backend.route_support.connector_policy_decision",
|
||||
return_value=SimpleNamespace(allowed=True),
|
||||
),
|
||||
patch(
|
||||
"govoplan_files.backend.route_support.read_connector_file",
|
||||
return_value=download,
|
||||
),
|
||||
):
|
||||
return _download_connector_payload(profile, request, operation="sync")
|
||||
|
||||
def test_stale_or_unavailable_expected_revision_cannot_label_new_bytes(self):
|
||||
for revision in ("B", None):
|
||||
with (
|
||||
self.subTest(revision=revision),
|
||||
self.assertRaises(ConnectorRevisionConflict) as caught,
|
||||
):
|
||||
self.acquire(
|
||||
ConnectorDownloadedFile(
|
||||
filename="file", data=b"B", revision=revision
|
||||
),
|
||||
expected="A",
|
||||
)
|
||||
self.assertEqual(409, _http_error(caught.exception).status_code)
|
||||
|
||||
def test_actual_digest_is_computed_and_claims_and_annotations_are_not_authority(
|
||||
self,
|
||||
):
|
||||
annotations = {
|
||||
"profile_id": "forged",
|
||||
"size": 999,
|
||||
"checksum_sha256": "caller-claim",
|
||||
"acquired_sha256": "forged",
|
||||
"note": "keep this",
|
||||
}
|
||||
download = ConnectorDownloadedFile(
|
||||
filename="source.txt",
|
||||
data=b"actual",
|
||||
revision="version-B",
|
||||
metadata={"etag": "etag-B", "checksum_sha256": "provider-claim"},
|
||||
)
|
||||
_, actual, metadata = self.acquire(
|
||||
download, expected="etag-B", annotations=annotations
|
||||
)
|
||||
self.assertEqual("version-B", metadata["source_revision"])
|
||||
evidence = metadata["source_provenance"]["metadata"]
|
||||
self.assertEqual(
|
||||
hashlib.sha256(actual.data).hexdigest(), evidence["acquired_sha256"]
|
||||
)
|
||||
self.assertEqual(len(actual.data), evidence["size"])
|
||||
self.assertEqual("profile", evidence["profile_id"])
|
||||
self.assertNotIn("checksum_sha256", evidence)
|
||||
self.assertEqual(
|
||||
{"checksum_sha256": "provider-claim"}, evidence["provider_checksum_claims"]
|
||||
)
|
||||
self.assertEqual(annotations, evidence["import_annotations"])
|
||||
self.assertEqual(
|
||||
metadata,
|
||||
self.acquire(download, expected="version-B", annotations=annotations)[2],
|
||||
)
|
||||
|
||||
def test_caller_fields_stay_annotations_when_provider_omits_evidence(self):
|
||||
annotations = {
|
||||
"sha256": "caller-hash",
|
||||
"etag": "caller-etag",
|
||||
"mtime": "caller-modified-time",
|
||||
"checksum_sha256": "caller-checksum",
|
||||
"provider_checksum_claims": {"checksum_sha256": "caller-claim"},
|
||||
"connector_space_id": "browse-space",
|
||||
"browse_etag": "browse-revision",
|
||||
"folder_sync": True,
|
||||
"note": {"values": ["001", " keep spaces ", None]},
|
||||
}
|
||||
download = ConnectorDownloadedFile(filename="source.txt", data=b"actual")
|
||||
_, _, metadata = self.acquire(download, annotations=annotations)
|
||||
evidence = metadata["source_provenance"]["metadata"]
|
||||
self.assertEqual(annotations, evidence["import_annotations"])
|
||||
self.assertEqual({}, evidence["provider_checksum_claims"])
|
||||
self.assertEqual(
|
||||
hashlib.sha256(download.data).hexdigest(), evidence["acquired_sha256"]
|
||||
)
|
||||
self.assertEqual(
|
||||
{
|
||||
"profile_id", "library_id", "library_path", "size",
|
||||
"acquired_sha256", "provider_checksum_claims", "import_annotations",
|
||||
},
|
||||
set(evidence),
|
||||
)
|
||||
|
||||
def test_seafile_rechecks_revision_after_download(self):
|
||||
profile = ConnectorProfile(
|
||||
id="profile",
|
||||
label="Synthetic",
|
||||
provider="seafile",
|
||||
endpoint_url="https://example.invalid",
|
||||
)
|
||||
before = {"id": "A", "name": "source.txt", "size": 4}
|
||||
for after in ({**before, "id": "B"}, before):
|
||||
with (
|
||||
patch(
|
||||
"govoplan_files.backend.storage.connector_imports._seafile_headers",
|
||||
return_value={},
|
||||
),
|
||||
patch(
|
||||
"govoplan_files.backend.storage.connector_imports._request_json",
|
||||
side_effect=[before, "https://example.invalid/download", after],
|
||||
),
|
||||
patch(
|
||||
"govoplan_files.backend.storage.connector_imports.request_connector_bytes",
|
||||
return_value=SimpleNamespace(
|
||||
status_code=200, content=b"DATA", headers={}
|
||||
),
|
||||
),
|
||||
):
|
||||
if after["id"] == "B":
|
||||
with self.assertRaises(ConnectorRevisionConflict):
|
||||
_read_seafile_file(
|
||||
profile, library_id="repo", path="source.txt", max_bytes=10
|
||||
)
|
||||
else:
|
||||
result = _read_seafile_file(
|
||||
profile, library_id="repo", path="source.txt", max_bytes=10
|
||||
)
|
||||
self.assertEqual(("A", b"DATA"), (result.revision, result.data))
|
||||
|
||||
def test_smb_observes_stable_metadata_while_write_and_delete_sharing_are_denied(
|
||||
self,
|
||||
):
|
||||
profile = ConnectorProfile(id="profile", label="Synthetic", provider="smb")
|
||||
before = SimpleNamespace(st_size=4, st_mtime_ns=10, st_mtime=1, st_ino=1)
|
||||
for after in (
|
||||
SimpleNamespace(st_size=4, st_mtime_ns=11, st_mtime=1, st_ino=1),
|
||||
before,
|
||||
):
|
||||
with (
|
||||
patch(
|
||||
"govoplan_files.backend.storage.connector_imports._smb_location",
|
||||
return_value=SimpleNamespace(
|
||||
share="share", server="example.invalid", port=445
|
||||
),
|
||||
),
|
||||
patch(
|
||||
"govoplan_files.backend.storage.connector_imports._smb_unc_path",
|
||||
return_value="synthetic",
|
||||
),
|
||||
patch(
|
||||
"govoplan_files.backend.storage.connector_imports._smb_client_kwargs",
|
||||
return_value={},
|
||||
),
|
||||
patch(
|
||||
"govoplan_files.backend.storage.connector_imports._smbclient_module"
|
||||
) as factory,
|
||||
):
|
||||
sdk = factory.return_value
|
||||
sdk.stat.side_effect = [before, after]
|
||||
sdk.open_file.return_value = io.BytesIO(b"DATA")
|
||||
if after is before:
|
||||
self.assertEqual(
|
||||
b"DATA",
|
||||
_read_smb_file(profile, path="source.txt", max_bytes=10).data,
|
||||
)
|
||||
else:
|
||||
with self.assertRaises(ConnectorRevisionConflict):
|
||||
_read_smb_file(profile, path="source.txt", max_bytes=10)
|
||||
self.assertEqual("r", sdk.open_file.call_args.kwargs["share_access"])
|
||||
|
||||
def test_s3_conditional_or_versioned_read_and_response_binding(self):
|
||||
profile = ConnectorProfile(id="profile", label="Synthetic", provider="s3")
|
||||
for version in (None, "version-A", "null"):
|
||||
for returned_etag in ("etag-A", "etag-B"):
|
||||
with (
|
||||
self.subTest(version=version, etag=returned_etag),
|
||||
patch(
|
||||
"govoplan_files.backend.storage.connector_imports._s3_import_client"
|
||||
) as factory,
|
||||
):
|
||||
client = factory.return_value
|
||||
client.head_object.return_value = {
|
||||
"ContentLength": 4,
|
||||
"ETag": "etag-A",
|
||||
"VersionId": version,
|
||||
}
|
||||
body = io.BytesIO(b"DATA")
|
||||
client.get_object.return_value = {
|
||||
"Body": body,
|
||||
"ETag": returned_etag,
|
||||
"VersionId": version,
|
||||
}
|
||||
if returned_etag == "etag-A":
|
||||
result = _read_s3_file(
|
||||
profile,
|
||||
library_id="bucket",
|
||||
path="source.txt",
|
||||
max_bytes=10,
|
||||
)
|
||||
self.assertEqual(version if version and version != "null" else "etag-A", result.revision)
|
||||
else:
|
||||
with self.assertRaises(ConnectorRevisionConflict):
|
||||
_read_s3_file(
|
||||
profile,
|
||||
library_id="bucket",
|
||||
path="source.txt",
|
||||
max_bytes=10,
|
||||
)
|
||||
request = client.get_object.call_args.kwargs
|
||||
immutable = version and version != "null"
|
||||
self.assertEqual(version if immutable else "etag-A", request["VersionId" if immutable else "IfMatch"])
|
||||
self.assertTrue(body.closed)
|
||||
client.close.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user