zemion
61625fb00f
Module Package Release / publish-packages (push) Successful in 14s
Release v0.1.27. Coordinated integrity review: GovOPlaN/govoplan-core#298.
245 lines
10 KiB
Python
Executable File
245 lines
10 KiB
Python
Executable File
import hashlib
|
|
import io
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from govoplan_files.backend.route_support import (
|
|
_download_connector_payload,
|
|
_http_error,
|
|
)
|
|
from govoplan_files.backend.schemas import FileConnectorImportRequest
|
|
from govoplan_files.backend.storage.connector_imports import (
|
|
ConnectorDownloadedFile,
|
|
ConnectorRevisionConflict,
|
|
_read_seafile_file,
|
|
_read_smb_file,
|
|
_read_s3_file,
|
|
)
|
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
|
|
|
|
|
class ConnectorProvenanceTests(unittest.TestCase):
|
|
def acquire(self, download, *, expected=None, annotations=None):
|
|
profile = ConnectorProfile(id="profile", label="Synthetic", provider="s3")
|
|
request = FileConnectorImportRequest(
|
|
library_id="bucket",
|
|
path="source.txt",
|
|
source_revision=expected,
|
|
metadata=annotations or {},
|
|
)
|
|
with (
|
|
patch(
|
|
"govoplan_files.backend.route_support.connector_policy_decision",
|
|
return_value=SimpleNamespace(allowed=True),
|
|
),
|
|
patch(
|
|
"govoplan_files.backend.route_support.read_connector_file",
|
|
return_value=download,
|
|
),
|
|
):
|
|
return _download_connector_payload(profile, request, operation="sync")
|
|
|
|
def test_stale_or_unavailable_expected_revision_cannot_label_new_bytes(self):
|
|
for revision in ("B", None):
|
|
with (
|
|
self.subTest(revision=revision),
|
|
self.assertRaises(ConnectorRevisionConflict) as caught,
|
|
):
|
|
self.acquire(
|
|
ConnectorDownloadedFile(
|
|
filename="file", data=b"B", revision=revision
|
|
),
|
|
expected="A",
|
|
)
|
|
self.assertEqual(409, _http_error(caught.exception).status_code)
|
|
|
|
def test_actual_digest_is_computed_and_claims_and_annotations_are_not_authority(
|
|
self,
|
|
):
|
|
annotations = {
|
|
"profile_id": "forged",
|
|
"size": 999,
|
|
"checksum_sha256": "caller-claim",
|
|
"acquired_sha256": "forged",
|
|
"note": "keep this",
|
|
}
|
|
download = ConnectorDownloadedFile(
|
|
filename="source.txt",
|
|
data=b"actual",
|
|
revision="version-B",
|
|
metadata={"etag": "etag-B", "checksum_sha256": "provider-claim"},
|
|
)
|
|
_, actual, metadata = self.acquire(
|
|
download, expected="etag-B", annotations=annotations
|
|
)
|
|
self.assertEqual("version-B", metadata["source_revision"])
|
|
evidence = metadata["source_provenance"]["metadata"]
|
|
self.assertEqual(
|
|
hashlib.sha256(actual.data).hexdigest(), evidence["acquired_sha256"]
|
|
)
|
|
self.assertEqual(len(actual.data), evidence["size"])
|
|
self.assertEqual("profile", evidence["profile_id"])
|
|
self.assertNotIn("checksum_sha256", evidence)
|
|
self.assertEqual(
|
|
{"checksum_sha256": "provider-claim"}, evidence["provider_checksum_claims"]
|
|
)
|
|
self.assertEqual(annotations, evidence["import_annotations"])
|
|
self.assertEqual(
|
|
metadata,
|
|
self.acquire(download, expected="version-B", annotations=annotations)[2],
|
|
)
|
|
|
|
def test_caller_fields_stay_annotations_when_provider_omits_evidence(self):
|
|
annotations = {
|
|
"sha256": "caller-hash",
|
|
"etag": "caller-etag",
|
|
"mtime": "caller-modified-time",
|
|
"checksum_sha256": "caller-checksum",
|
|
"provider_checksum_claims": {"checksum_sha256": "caller-claim"},
|
|
"connector_space_id": "browse-space",
|
|
"browse_etag": "browse-revision",
|
|
"folder_sync": True,
|
|
"note": {"values": ["001", " keep spaces ", None]},
|
|
}
|
|
download = ConnectorDownloadedFile(filename="source.txt", data=b"actual")
|
|
_, _, metadata = self.acquire(download, annotations=annotations)
|
|
evidence = metadata["source_provenance"]["metadata"]
|
|
self.assertEqual(annotations, evidence["import_annotations"])
|
|
self.assertEqual({}, evidence["provider_checksum_claims"])
|
|
self.assertEqual(
|
|
hashlib.sha256(download.data).hexdigest(), evidence["acquired_sha256"]
|
|
)
|
|
self.assertEqual(
|
|
{
|
|
"profile_id", "library_id", "library_path", "size",
|
|
"acquired_sha256", "provider_checksum_claims", "import_annotations",
|
|
},
|
|
set(evidence),
|
|
)
|
|
|
|
def test_seafile_rechecks_revision_after_download(self):
|
|
profile = ConnectorProfile(
|
|
id="profile",
|
|
label="Synthetic",
|
|
provider="seafile",
|
|
endpoint_url="https://example.invalid",
|
|
)
|
|
before = {"id": "A", "name": "source.txt", "size": 4}
|
|
for after in ({**before, "id": "B"}, before):
|
|
with (
|
|
patch(
|
|
"govoplan_files.backend.storage.connector_imports._seafile_headers",
|
|
return_value={},
|
|
),
|
|
patch(
|
|
"govoplan_files.backend.storage.connector_imports._request_json",
|
|
side_effect=[before, "https://example.invalid/download", after],
|
|
),
|
|
patch(
|
|
"govoplan_files.backend.storage.connector_imports.request_connector_bytes",
|
|
return_value=SimpleNamespace(
|
|
status_code=200, content=b"DATA", headers={}
|
|
),
|
|
),
|
|
):
|
|
if after["id"] == "B":
|
|
with self.assertRaises(ConnectorRevisionConflict):
|
|
_read_seafile_file(
|
|
profile, library_id="repo", path="source.txt", max_bytes=10
|
|
)
|
|
else:
|
|
result = _read_seafile_file(
|
|
profile, library_id="repo", path="source.txt", max_bytes=10
|
|
)
|
|
self.assertEqual(("A", b"DATA"), (result.revision, result.data))
|
|
|
|
def test_smb_observes_stable_metadata_while_write_and_delete_sharing_are_denied(
|
|
self,
|
|
):
|
|
profile = ConnectorProfile(id="profile", label="Synthetic", provider="smb")
|
|
before = SimpleNamespace(st_size=4, st_mtime_ns=10, st_mtime=1, st_ino=1)
|
|
for after in (
|
|
SimpleNamespace(st_size=4, st_mtime_ns=11, st_mtime=1, st_ino=1),
|
|
before,
|
|
):
|
|
with (
|
|
patch(
|
|
"govoplan_files.backend.storage.connector_imports._smb_location",
|
|
return_value=SimpleNamespace(
|
|
share="share", server="example.invalid", port=445
|
|
),
|
|
),
|
|
patch(
|
|
"govoplan_files.backend.storage.connector_imports._smb_unc_path",
|
|
return_value="synthetic",
|
|
),
|
|
patch(
|
|
"govoplan_files.backend.storage.connector_imports._smb_client_kwargs",
|
|
return_value={},
|
|
),
|
|
patch(
|
|
"govoplan_files.backend.storage.connector_imports._smbclient_module"
|
|
) as factory,
|
|
):
|
|
sdk = factory.return_value
|
|
sdk.stat.side_effect = [before, after]
|
|
sdk.open_file.return_value = io.BytesIO(b"DATA")
|
|
if after is before:
|
|
self.assertEqual(
|
|
b"DATA",
|
|
_read_smb_file(profile, path="source.txt", max_bytes=10).data,
|
|
)
|
|
else:
|
|
with self.assertRaises(ConnectorRevisionConflict):
|
|
_read_smb_file(profile, path="source.txt", max_bytes=10)
|
|
self.assertEqual("r", sdk.open_file.call_args.kwargs["share_access"])
|
|
|
|
def test_s3_conditional_or_versioned_read_and_response_binding(self):
|
|
profile = ConnectorProfile(id="profile", label="Synthetic", provider="s3")
|
|
for version in (None, "version-A", "null"):
|
|
for returned_etag in ("etag-A", "etag-B"):
|
|
with (
|
|
self.subTest(version=version, etag=returned_etag),
|
|
patch(
|
|
"govoplan_files.backend.storage.connector_imports._s3_import_client"
|
|
) as factory,
|
|
):
|
|
client = factory.return_value
|
|
client.head_object.return_value = {
|
|
"ContentLength": 4,
|
|
"ETag": "etag-A",
|
|
"VersionId": version,
|
|
}
|
|
body = io.BytesIO(b"DATA")
|
|
client.get_object.return_value = {
|
|
"Body": body,
|
|
"ETag": returned_etag,
|
|
"VersionId": version,
|
|
}
|
|
if returned_etag == "etag-A":
|
|
result = _read_s3_file(
|
|
profile,
|
|
library_id="bucket",
|
|
path="source.txt",
|
|
max_bytes=10,
|
|
)
|
|
self.assertEqual(version if version and version != "null" else "etag-A", result.revision)
|
|
else:
|
|
with self.assertRaises(ConnectorRevisionConflict):
|
|
_read_s3_file(
|
|
profile,
|
|
library_id="bucket",
|
|
path="source.txt",
|
|
max_bytes=10,
|
|
)
|
|
request = client.get_object.call_args.kwargs
|
|
immutable = version and version != "null"
|
|
self.assertEqual(version if immutable else "etag-A", request["VersionId" if immutable else "IfMatch"])
|
|
self.assertTrue(body.closed)
|
|
client.close.assert_called_once()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|