325 lines
9.1 KiB
Python
325 lines
9.1 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import timedelta
|
|
import hashlib
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from urllib.error import URLError
|
|
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_connectors.backend.db.models import (
|
|
ConnectorSanctionsAcquisitionRun,
|
|
ConnectorSanctionsSnapshot,
|
|
)
|
|
from govoplan_connectors.backend.sanctions_sources import (
|
|
SANCTIONS_READ_SCOPE,
|
|
SANCTIONS_REFRESH_SCOPE,
|
|
SOURCE_DEFINITIONS,
|
|
SYNTHETIC_PROVIDER_ID,
|
|
SYNTHETIC_UN_XML,
|
|
SanctionsSourceError,
|
|
SqlSanctionsSnapshotProvider,
|
|
TransportResponse,
|
|
UNSC_PROVIDER_ID,
|
|
UrllibSanctionsTransport,
|
|
)
|
|
from govoplan_core.auth import ApiPrincipal
|
|
from govoplan_core.core.access import PrincipalRef
|
|
from govoplan_core.db.base import Base, utcnow
|
|
|
|
|
|
def principal(
|
|
tenant_id: str = "tenant-1",
|
|
*,
|
|
scopes: tuple[str, ...] = (
|
|
SANCTIONS_READ_SCOPE,
|
|
SANCTIONS_REFRESH_SCOPE,
|
|
),
|
|
) -> ApiPrincipal:
|
|
return ApiPrincipal(
|
|
principal=PrincipalRef(
|
|
account_id="account-1",
|
|
membership_id="membership-1",
|
|
tenant_id=tenant_id,
|
|
scopes=frozenset(scopes),
|
|
),
|
|
account=object(),
|
|
user=object(),
|
|
)
|
|
|
|
|
|
class _Transport:
|
|
def __init__(self, responses):
|
|
self.responses = list(responses)
|
|
self.headers = []
|
|
|
|
def fetch(self, definition, *, headers):
|
|
del definition
|
|
self.headers.append(dict(headers))
|
|
response = self.responses.pop(0)
|
|
if isinstance(response, Exception):
|
|
raise response
|
|
return response
|
|
|
|
|
|
def response(
|
|
content: bytes = SYNTHETIC_UN_XML,
|
|
*,
|
|
status: int = 200,
|
|
content_type: str = "application/xml",
|
|
etag: str = '"fixture-v1"',
|
|
) -> TransportResponse:
|
|
return TransportResponse(
|
|
status=status,
|
|
final_url="https://scsanctions.un.org/consolidated.xml",
|
|
headers={
|
|
"content-type": content_type,
|
|
"etag": etag,
|
|
},
|
|
content=content,
|
|
attempts=1,
|
|
)
|
|
|
|
|
|
class SanctionsSourcesTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.engine = create_engine("sqlite:///:memory:")
|
|
Base.metadata.create_all(
|
|
self.engine,
|
|
tables=(
|
|
ConnectorSanctionsAcquisitionRun.__table__,
|
|
ConnectorSanctionsSnapshot.__table__,
|
|
),
|
|
)
|
|
self.session = Session(self.engine)
|
|
|
|
def tearDown(self) -> None:
|
|
self.session.close()
|
|
self.engine.dispose()
|
|
|
|
def test_fixture_refreshes_are_immutable_and_evidence_is_readable(
|
|
self,
|
|
) -> None:
|
|
provider = SqlSanctionsSnapshotProvider()
|
|
|
|
first = provider.refresh_source(
|
|
self.session,
|
|
principal(),
|
|
provider_id=SYNTHETIC_PROVIDER_ID,
|
|
)
|
|
second = provider.refresh_source(
|
|
self.session,
|
|
principal(),
|
|
provider_id=SYNTHETIC_PROVIDER_ID,
|
|
)
|
|
self.session.commit()
|
|
|
|
self.assertEqual("succeeded", first.status)
|
|
self.assertEqual("succeeded", second.status)
|
|
self.assertNotEqual(first.snapshot.ref, second.snapshot.ref)
|
|
self.assertEqual(
|
|
hashlib.sha256(SYNTHETIC_UN_XML).hexdigest(),
|
|
first.snapshot.sha256,
|
|
)
|
|
self.assertEqual(
|
|
SYNTHETIC_UN_XML,
|
|
provider.read_snapshot(
|
|
self.session,
|
|
principal(),
|
|
snapshot_ref=first.snapshot.ref,
|
|
).content,
|
|
)
|
|
runs = provider.list_runs(
|
|
self.session,
|
|
principal(),
|
|
)
|
|
self.assertTrue(
|
|
all(
|
|
run.request_evidence["subject_data_transmitted"]
|
|
is False
|
|
for run in runs
|
|
)
|
|
)
|
|
|
|
def test_conditional_fetch_reuses_prior_immutable_snapshot(self) -> None:
|
|
transport = _Transport(
|
|
(
|
|
response(),
|
|
response(b"", status=304),
|
|
)
|
|
)
|
|
provider = SqlSanctionsSnapshotProvider(transport)
|
|
first = provider.refresh_source(
|
|
self.session,
|
|
principal(),
|
|
provider_id=UNSC_PROVIDER_ID,
|
|
)
|
|
second = provider.refresh_source(
|
|
self.session,
|
|
principal(),
|
|
provider_id=UNSC_PROVIDER_ID,
|
|
)
|
|
|
|
self.assertEqual("not_modified", second.status)
|
|
self.assertEqual(first.snapshot.ref, second.snapshot.ref)
|
|
self.assertEqual(
|
|
{'If-None-Match': '"fixture-v1"'},
|
|
transport.headers[1],
|
|
)
|
|
self.assertEqual(
|
|
1,
|
|
self.session.query(ConnectorSanctionsSnapshot).count(),
|
|
)
|
|
|
|
def test_malformed_and_changed_sources_have_explicit_health(
|
|
self,
|
|
) -> None:
|
|
cases = (
|
|
(
|
|
b"<CONSOLIDATED_LIST>",
|
|
"application/xml",
|
|
"malformed",
|
|
),
|
|
(
|
|
b"<DIFFERENT><INDIVIDUALS/><ENTITIES/></DIFFERENT>",
|
|
"application/xml",
|
|
"unexpected_change",
|
|
),
|
|
(
|
|
SYNTHETIC_UN_XML,
|
|
"text/html",
|
|
"unexpected_change",
|
|
),
|
|
)
|
|
for payload, content_type, expected in cases:
|
|
with self.subTest(expected=expected, content_type=content_type):
|
|
provider = SqlSanctionsSnapshotProvider(
|
|
_Transport(
|
|
(
|
|
response(
|
|
payload,
|
|
content_type=content_type,
|
|
),
|
|
)
|
|
)
|
|
)
|
|
result = provider.refresh_source(
|
|
self.session,
|
|
principal(),
|
|
provider_id=UNSC_PROVIDER_ID,
|
|
)
|
|
self.assertEqual(expected, result.status)
|
|
self.assertIsNotNone(result.error)
|
|
|
|
def test_unavailable_source_becomes_stale_when_evidence_is_old(
|
|
self,
|
|
) -> None:
|
|
provider = SqlSanctionsSnapshotProvider(
|
|
_Transport(
|
|
(
|
|
response(),
|
|
SanctionsSourceError("offline"),
|
|
)
|
|
)
|
|
)
|
|
first = provider.refresh_source(
|
|
self.session,
|
|
principal(),
|
|
provider_id=UNSC_PROVIDER_ID,
|
|
)
|
|
record = self.session.get(
|
|
ConnectorSanctionsSnapshot,
|
|
first.snapshot.ref.removeprefix("sanctions-snapshot:"),
|
|
)
|
|
record.acquired_at = utcnow() - timedelta(days=3)
|
|
|
|
result = provider.refresh_source(
|
|
self.session,
|
|
principal(),
|
|
provider_id=UNSC_PROVIDER_ID,
|
|
)
|
|
|
|
self.assertEqual("stale", result.status)
|
|
self.assertEqual(first.snapshot.ref, result.snapshot.ref)
|
|
|
|
def test_snapshot_access_is_tenant_and_scope_isolated(self) -> None:
|
|
provider = SqlSanctionsSnapshotProvider()
|
|
created = provider.refresh_source(
|
|
self.session,
|
|
principal(),
|
|
provider_id=SYNTHETIC_PROVIDER_ID,
|
|
)
|
|
|
|
self.assertIsNone(
|
|
provider.get_snapshot(
|
|
self.session,
|
|
principal("tenant-2"),
|
|
snapshot_ref=created.snapshot.ref,
|
|
)
|
|
)
|
|
with self.assertRaisesRegex(Exception, "Missing scope"):
|
|
provider.list_snapshots(
|
|
self.session,
|
|
principal(scopes=()),
|
|
)
|
|
|
|
def test_transport_retries_transient_network_failures(self) -> None:
|
|
class _Headers(dict):
|
|
pass
|
|
|
|
class _Response:
|
|
status = 200
|
|
headers = _Headers(
|
|
{
|
|
"Content-Type": "application/xml",
|
|
"Content-Length": str(len(SYNTHETIC_UN_XML)),
|
|
}
|
|
)
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
return False
|
|
|
|
def geturl(self):
|
|
return (
|
|
"https://scsanctions.un.org/"
|
|
"resources/xml/en/consolidated.xml"
|
|
)
|
|
|
|
def read(self, size):
|
|
del size
|
|
if hasattr(self, "_read"):
|
|
return b""
|
|
self._read = True
|
|
return SYNTHETIC_UN_XML
|
|
|
|
opener = unittest.mock.Mock()
|
|
opener.open.side_effect = (
|
|
URLError("temporary"),
|
|
URLError("temporary"),
|
|
_Response(),
|
|
)
|
|
sleeps = []
|
|
transport = UrllibSanctionsTransport(
|
|
sleeper=sleeps.append
|
|
)
|
|
with patch(
|
|
"govoplan_connectors.backend.sanctions_sources.build_opener",
|
|
return_value=opener,
|
|
):
|
|
fetched = transport.fetch(
|
|
SOURCE_DEFINITIONS[UNSC_PROVIDER_ID],
|
|
headers={},
|
|
)
|
|
|
|
self.assertEqual(3, fetched.attempts)
|
|
self.assertEqual([1.0, 2.0], sleeps)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|