feat: acquire immutable sanctions snapshots
This commit is contained in:
@@ -6,6 +6,9 @@ from govoplan_core.core.tabular_sources import (
|
||||
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER,
|
||||
CAPABILITY_CONNECTORS_TABULAR_SOURCES,
|
||||
)
|
||||
from govoplan_core.core.sanctions import (
|
||||
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
|
||||
)
|
||||
from govoplan_connectors.backend.manifest import manifest
|
||||
|
||||
|
||||
@@ -25,6 +28,10 @@ class ConnectorsManifestTests(unittest.TestCase):
|
||||
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER,
|
||||
manifest.capability_factories,
|
||||
)
|
||||
self.assertIn(
|
||||
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
|
||||
manifest.capability_factories,
|
||||
)
|
||||
self.assertIsNotNone(manifest.migration_spec)
|
||||
|
||||
|
||||
|
||||
@@ -24,13 +24,21 @@ class ConnectorsMigrationTests(unittest.TestCase):
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
self.assertIn(
|
||||
"e6b7c8d9f0a1",
|
||||
"f7c8d9e0a1b2",
|
||||
set(MigrationContext.configure(connection).get_current_heads()),
|
||||
)
|
||||
self.assertIn(
|
||||
"connector_tabular_sources",
|
||||
inspect(connection).get_table_names(),
|
||||
)
|
||||
self.assertIn(
|
||||
"connector_sanctions_snapshots",
|
||||
inspect(connection).get_table_names(),
|
||||
)
|
||||
self.assertIn(
|
||||
"connector_sanctions_acquisition_runs",
|
||||
inspect(connection).get_table_names(),
|
||||
)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
import hashlib
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from urllib.error import URLError
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_connectors.backend.db.models import (
|
||||
ConnectorSanctionsAcquisitionRun,
|
||||
ConnectorSanctionsSnapshot,
|
||||
)
|
||||
from govoplan_connectors.backend.sanctions_sources import (
|
||||
SANCTIONS_READ_SCOPE,
|
||||
SANCTIONS_REFRESH_SCOPE,
|
||||
SOURCE_DEFINITIONS,
|
||||
SYNTHETIC_PROVIDER_ID,
|
||||
SYNTHETIC_UN_XML,
|
||||
SanctionsSourceError,
|
||||
SqlSanctionsSnapshotProvider,
|
||||
TransportResponse,
|
||||
UNSC_PROVIDER_ID,
|
||||
UrllibSanctionsTransport,
|
||||
)
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.core.access import PrincipalRef
|
||||
from govoplan_core.db.base import Base, utcnow
|
||||
|
||||
|
||||
def principal(
|
||||
tenant_id: str = "tenant-1",
|
||||
*,
|
||||
scopes: tuple[str, ...] = (
|
||||
SANCTIONS_READ_SCOPE,
|
||||
SANCTIONS_REFRESH_SCOPE,
|
||||
),
|
||||
) -> ApiPrincipal:
|
||||
return ApiPrincipal(
|
||||
principal=PrincipalRef(
|
||||
account_id="account-1",
|
||||
membership_id="membership-1",
|
||||
tenant_id=tenant_id,
|
||||
scopes=frozenset(scopes),
|
||||
),
|
||||
account=object(),
|
||||
user=object(),
|
||||
)
|
||||
|
||||
|
||||
class _Transport:
|
||||
def __init__(self, responses):
|
||||
self.responses = list(responses)
|
||||
self.headers = []
|
||||
|
||||
def fetch(self, definition, *, headers):
|
||||
del definition
|
||||
self.headers.append(dict(headers))
|
||||
response = self.responses.pop(0)
|
||||
if isinstance(response, Exception):
|
||||
raise response
|
||||
return response
|
||||
|
||||
|
||||
def response(
|
||||
content: bytes = SYNTHETIC_UN_XML,
|
||||
*,
|
||||
status: int = 200,
|
||||
content_type: str = "application/xml",
|
||||
etag: str = '"fixture-v1"',
|
||||
) -> TransportResponse:
|
||||
return TransportResponse(
|
||||
status=status,
|
||||
final_url="https://scsanctions.un.org/consolidated.xml",
|
||||
headers={
|
||||
"content-type": content_type,
|
||||
"etag": etag,
|
||||
},
|
||||
content=content,
|
||||
attempts=1,
|
||||
)
|
||||
|
||||
|
||||
class SanctionsSourcesTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite:///:memory:")
|
||||
Base.metadata.create_all(
|
||||
self.engine,
|
||||
tables=(
|
||||
ConnectorSanctionsAcquisitionRun.__table__,
|
||||
ConnectorSanctionsSnapshot.__table__,
|
||||
),
|
||||
)
|
||||
self.session = Session(self.engine)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def test_fixture_refreshes_are_immutable_and_evidence_is_readable(
|
||||
self,
|
||||
) -> None:
|
||||
provider = SqlSanctionsSnapshotProvider()
|
||||
|
||||
first = provider.refresh_source(
|
||||
self.session,
|
||||
principal(),
|
||||
provider_id=SYNTHETIC_PROVIDER_ID,
|
||||
)
|
||||
second = provider.refresh_source(
|
||||
self.session,
|
||||
principal(),
|
||||
provider_id=SYNTHETIC_PROVIDER_ID,
|
||||
)
|
||||
self.session.commit()
|
||||
|
||||
self.assertEqual("succeeded", first.status)
|
||||
self.assertEqual("succeeded", second.status)
|
||||
self.assertNotEqual(first.snapshot.ref, second.snapshot.ref)
|
||||
self.assertEqual(
|
||||
hashlib.sha256(SYNTHETIC_UN_XML).hexdigest(),
|
||||
first.snapshot.sha256,
|
||||
)
|
||||
self.assertEqual(
|
||||
SYNTHETIC_UN_XML,
|
||||
provider.read_snapshot(
|
||||
self.session,
|
||||
principal(),
|
||||
snapshot_ref=first.snapshot.ref,
|
||||
).content,
|
||||
)
|
||||
runs = provider.list_runs(
|
||||
self.session,
|
||||
principal(),
|
||||
)
|
||||
self.assertTrue(
|
||||
all(
|
||||
run.request_evidence["subject_data_transmitted"]
|
||||
is False
|
||||
for run in runs
|
||||
)
|
||||
)
|
||||
|
||||
def test_conditional_fetch_reuses_prior_immutable_snapshot(self) -> None:
|
||||
transport = _Transport(
|
||||
(
|
||||
response(),
|
||||
response(b"", status=304),
|
||||
)
|
||||
)
|
||||
provider = SqlSanctionsSnapshotProvider(transport)
|
||||
first = provider.refresh_source(
|
||||
self.session,
|
||||
principal(),
|
||||
provider_id=UNSC_PROVIDER_ID,
|
||||
)
|
||||
second = provider.refresh_source(
|
||||
self.session,
|
||||
principal(),
|
||||
provider_id=UNSC_PROVIDER_ID,
|
||||
)
|
||||
|
||||
self.assertEqual("not_modified", second.status)
|
||||
self.assertEqual(first.snapshot.ref, second.snapshot.ref)
|
||||
self.assertEqual(
|
||||
{'If-None-Match': '"fixture-v1"'},
|
||||
transport.headers[1],
|
||||
)
|
||||
self.assertEqual(
|
||||
1,
|
||||
self.session.query(ConnectorSanctionsSnapshot).count(),
|
||||
)
|
||||
|
||||
def test_malformed_and_changed_sources_have_explicit_health(
|
||||
self,
|
||||
) -> None:
|
||||
cases = (
|
||||
(
|
||||
b"<CONSOLIDATED_LIST>",
|
||||
"application/xml",
|
||||
"malformed",
|
||||
),
|
||||
(
|
||||
b"<DIFFERENT><INDIVIDUALS/><ENTITIES/></DIFFERENT>",
|
||||
"application/xml",
|
||||
"unexpected_change",
|
||||
),
|
||||
(
|
||||
SYNTHETIC_UN_XML,
|
||||
"text/html",
|
||||
"unexpected_change",
|
||||
),
|
||||
)
|
||||
for payload, content_type, expected in cases:
|
||||
with self.subTest(expected=expected, content_type=content_type):
|
||||
provider = SqlSanctionsSnapshotProvider(
|
||||
_Transport(
|
||||
(
|
||||
response(
|
||||
payload,
|
||||
content_type=content_type,
|
||||
),
|
||||
)
|
||||
)
|
||||
)
|
||||
result = provider.refresh_source(
|
||||
self.session,
|
||||
principal(),
|
||||
provider_id=UNSC_PROVIDER_ID,
|
||||
)
|
||||
self.assertEqual(expected, result.status)
|
||||
self.assertIsNotNone(result.error)
|
||||
|
||||
def test_unavailable_source_becomes_stale_when_evidence_is_old(
|
||||
self,
|
||||
) -> None:
|
||||
provider = SqlSanctionsSnapshotProvider(
|
||||
_Transport(
|
||||
(
|
||||
response(),
|
||||
SanctionsSourceError("offline"),
|
||||
)
|
||||
)
|
||||
)
|
||||
first = provider.refresh_source(
|
||||
self.session,
|
||||
principal(),
|
||||
provider_id=UNSC_PROVIDER_ID,
|
||||
)
|
||||
record = self.session.get(
|
||||
ConnectorSanctionsSnapshot,
|
||||
first.snapshot.ref.removeprefix("sanctions-snapshot:"),
|
||||
)
|
||||
record.acquired_at = utcnow() - timedelta(days=3)
|
||||
|
||||
result = provider.refresh_source(
|
||||
self.session,
|
||||
principal(),
|
||||
provider_id=UNSC_PROVIDER_ID,
|
||||
)
|
||||
|
||||
self.assertEqual("stale", result.status)
|
||||
self.assertEqual(first.snapshot.ref, result.snapshot.ref)
|
||||
|
||||
def test_snapshot_access_is_tenant_and_scope_isolated(self) -> None:
|
||||
provider = SqlSanctionsSnapshotProvider()
|
||||
created = provider.refresh_source(
|
||||
self.session,
|
||||
principal(),
|
||||
provider_id=SYNTHETIC_PROVIDER_ID,
|
||||
)
|
||||
|
||||
self.assertIsNone(
|
||||
provider.get_snapshot(
|
||||
self.session,
|
||||
principal("tenant-2"),
|
||||
snapshot_ref=created.snapshot.ref,
|
||||
)
|
||||
)
|
||||
with self.assertRaisesRegex(Exception, "Missing scope"):
|
||||
provider.list_snapshots(
|
||||
self.session,
|
||||
principal(scopes=()),
|
||||
)
|
||||
|
||||
def test_transport_retries_transient_network_failures(self) -> None:
|
||||
class _Headers(dict):
|
||||
pass
|
||||
|
||||
class _Response:
|
||||
status = 200
|
||||
headers = _Headers(
|
||||
{
|
||||
"Content-Type": "application/xml",
|
||||
"Content-Length": str(len(SYNTHETIC_UN_XML)),
|
||||
}
|
||||
)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *args):
|
||||
return False
|
||||
|
||||
def geturl(self):
|
||||
return (
|
||||
"https://scsanctions.un.org/"
|
||||
"resources/xml/en/consolidated.xml"
|
||||
)
|
||||
|
||||
def read(self, size):
|
||||
del size
|
||||
if hasattr(self, "_read"):
|
||||
return b""
|
||||
self._read = True
|
||||
return SYNTHETIC_UN_XML
|
||||
|
||||
opener = unittest.mock.Mock()
|
||||
opener.open.side_effect = (
|
||||
URLError("temporary"),
|
||||
URLError("temporary"),
|
||||
_Response(),
|
||||
)
|
||||
sleeps = []
|
||||
transport = UrllibSanctionsTransport(
|
||||
sleeper=sleeps.append
|
||||
)
|
||||
with patch(
|
||||
"govoplan_connectors.backend.sanctions_sources.build_opener",
|
||||
return_value=opener,
|
||||
):
|
||||
fetched = transport.fetch(
|
||||
SOURCE_DEFINITIONS[UNSC_PROVIDER_ID],
|
||||
headers={},
|
||||
)
|
||||
|
||||
self.assertEqual(3, fetched.attempts)
|
||||
self.assertEqual([1.0, 2.0], sleeps)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user