zemion
889cafaf20
Module Package Release / publish-packages (push) Successful in 17s
Release v0.1.27. Coordinated integrity review: GovOPlaN/govoplan-core#298.
321 lines
13 KiB
Python
321 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from dataclasses import replace
|
|
|
|
from fastapi import HTTPException
|
|
from sqlalchemy import create_engine, inspect
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from govoplan_core.auth import ApiPrincipal
|
|
from govoplan_core.core.access import PrincipalRef
|
|
from govoplan_core.core.tabular_sources import (
|
|
TabularCsvSource,
|
|
TabularReadRequest,
|
|
TabularSnapshotInput,
|
|
TabularSourceAccessError,
|
|
TabularSourceUnavailableError,
|
|
TabularSourceValidationError,
|
|
TabularSourceNotFoundError,
|
|
)
|
|
from govoplan_core.db.base import Base
|
|
from govoplan_connectors.backend.db.models import ConnectorTabularSource
|
|
from govoplan_connectors.backend.router import api_create_tabular_snapshot, api_original_tabular_csv
|
|
from govoplan_connectors.backend.schemas import SnapshotCreateRequest
|
|
from govoplan_connectors.backend.tabular_sources import (
|
|
READ_SCOPE,
|
|
WRITE_SCOPE,
|
|
SqlTabularSourceProvider,
|
|
parse_csv_snapshot,
|
|
)
|
|
|
|
|
|
def principal(
|
|
tenant_id: str = "tenant-1",
|
|
*,
|
|
scopes: tuple[str, ...] = (READ_SCOPE, WRITE_SCOPE),
|
|
) -> ApiPrincipal:
|
|
return ApiPrincipal(
|
|
principal=PrincipalRef(
|
|
account_id="account-1",
|
|
membership_id="membership-1",
|
|
tenant_id=tenant_id,
|
|
scopes=frozenset(scopes),
|
|
),
|
|
account=object(),
|
|
user=object(),
|
|
)
|
|
|
|
|
|
class ConnectorsTabularSourceTests(unittest.TestCase):
|
|
def setUp(self) -> None:
|
|
self.engine = create_engine("sqlite:///:memory:")
|
|
Base.metadata.create_all(self.engine, tables=[ConnectorTabularSource.__table__])
|
|
self.Session = sessionmaker(bind=self.engine)
|
|
self.session = self.Session()
|
|
self.provider = SqlTabularSourceProvider()
|
|
|
|
def tearDown(self) -> None:
|
|
self.session.close()
|
|
Base.metadata.drop_all(self.engine, tables=[ConnectorTabularSource.__table__])
|
|
self.engine.dispose()
|
|
|
|
def test_snapshot_round_trip_preserves_schema_fingerprint_and_bounds(self) -> None:
|
|
created = self.provider.create_snapshot(
|
|
self.session,
|
|
principal(),
|
|
snapshot=TabularSnapshotInput(
|
|
name="Monthly cases",
|
|
source_name="monthly_cases_2026_07",
|
|
rows=(
|
|
{"case_id": "A-1", "amount": 12, "active": True},
|
|
{"case_id": "A-2", "amount": None, "active": False},
|
|
),
|
|
),
|
|
)
|
|
self.session.commit()
|
|
|
|
listed = self.provider.list_sources(self.session, principal())
|
|
preview = self.provider.read_source(
|
|
self.session,
|
|
principal(),
|
|
request=TabularReadRequest(
|
|
source_ref=created.ref,
|
|
limit=1,
|
|
expected_fingerprint=created.fingerprint,
|
|
),
|
|
)
|
|
|
|
self.assertEqual((created.ref,), tuple(source.ref for source in listed))
|
|
self.assertEqual(
|
|
["case_id", "amount", "active"],
|
|
[column.name for column in created.schema],
|
|
)
|
|
self.assertEqual(2, preview.total_rows)
|
|
self.assertEqual(1, len(preview.rows))
|
|
self.assertTrue(preview.truncated)
|
|
self.assertEqual(created.fingerprint, preview.source.fingerprint)
|
|
self.assertEqual("cached", preview.source.source_mode)
|
|
self.assertTrue(preview.source.pushdown.projections)
|
|
self.assertTrue(preview.source.pushdown.pagination)
|
|
self.assertEqual("healthy", preview.source.health.status)
|
|
self.assertGreater(preview.returned_bytes, 2)
|
|
self.assertEqual(1, preview.effective_row_limit)
|
|
self.assertEqual("preview.row_limit_reached", preview.diagnostics[0].code)
|
|
|
|
def test_preview_enforces_byte_time_and_provider_ceiling_budgets(self) -> None:
|
|
created = self.provider.create_snapshot(
|
|
self.session,
|
|
principal(),
|
|
snapshot=TabularSnapshotInput(
|
|
name="Bounded",
|
|
source_name="bounded",
|
|
rows=(
|
|
{"id": 1, "value": "first"},
|
|
{"id": 2, "value": "second"},
|
|
),
|
|
),
|
|
)
|
|
self.session.commit()
|
|
|
|
bounded = self.provider.read_source(
|
|
self.session,
|
|
principal(),
|
|
request=TabularReadRequest(
|
|
source_ref=created.ref,
|
|
limit=500,
|
|
max_bytes=35,
|
|
timeout_ms=2_000,
|
|
),
|
|
)
|
|
self.assertEqual(1, len(bounded.rows))
|
|
self.assertTrue(bounded.truncated)
|
|
self.assertEqual(
|
|
"preview.byte_limit_reached",
|
|
bounded.diagnostics[-1].code,
|
|
)
|
|
with self.assertRaisesRegex(
|
|
TabularSourceValidationError,
|
|
"single source row exceeds",
|
|
):
|
|
self.provider.read_source(
|
|
self.session,
|
|
principal(),
|
|
request=TabularReadRequest(
|
|
source_ref=created.ref,
|
|
max_bytes=2,
|
|
),
|
|
)
|
|
|
|
tightened = self.provider.read_source(
|
|
self.session,
|
|
principal(),
|
|
request=TabularReadRequest(
|
|
source_ref=created.ref,
|
|
limit=5_000,
|
|
max_bytes=5_000_000,
|
|
timeout_ms=10_000,
|
|
),
|
|
)
|
|
self.assertEqual(500, tightened.effective_row_limit)
|
|
self.assertEqual(1_000_000, tightened.effective_byte_limit)
|
|
self.assertEqual(2_000, tightened.effective_timeout_ms)
|
|
self.assertEqual(
|
|
{
|
|
"preview.row_limit_tightened",
|
|
"preview.byte_limit_tightened",
|
|
"preview.timeout_tightened",
|
|
},
|
|
{item.code for item in tightened.diagnostics},
|
|
)
|
|
|
|
times = iter((0.0, 0.01))
|
|
timeout_provider = SqlTabularSourceProvider(clock=lambda: next(times))
|
|
with self.assertRaisesRegex(
|
|
TabularSourceUnavailableError,
|
|
"time budget",
|
|
):
|
|
timeout_provider.read_source(
|
|
self.session,
|
|
principal(),
|
|
request=TabularReadRequest(
|
|
source_ref=created.ref,
|
|
timeout_ms=1,
|
|
),
|
|
)
|
|
|
|
def test_tenant_and_scope_isolation_are_enforced(self) -> None:
|
|
created = self.provider.create_snapshot(
|
|
self.session,
|
|
principal(),
|
|
snapshot=TabularSnapshotInput(
|
|
name="Private",
|
|
source_name="private_source",
|
|
rows=({"id": 1},),
|
|
),
|
|
)
|
|
self.session.commit()
|
|
|
|
self.assertEqual((), self.provider.list_sources(self.session, principal("tenant-2")))
|
|
self.assertIsNone(
|
|
self.provider.get_source(
|
|
self.session,
|
|
principal("tenant-2"),
|
|
source_ref=created.ref,
|
|
)
|
|
)
|
|
with self.assertRaises(TabularSourceAccessError):
|
|
self.provider.list_sources(
|
|
self.session,
|
|
principal(scopes=()),
|
|
)
|
|
|
|
def test_duplicate_source_name_and_stale_fingerprint_are_rejected(self) -> None:
|
|
snapshot = TabularSnapshotInput(
|
|
name="Cases",
|
|
source_name="cases",
|
|
rows=({"id": 1},),
|
|
)
|
|
created = self.provider.create_snapshot(self.session, principal(), snapshot=snapshot)
|
|
self.session.commit()
|
|
|
|
with self.assertRaises(TabularSourceValidationError):
|
|
self.provider.create_snapshot(self.session, principal(), snapshot=snapshot)
|
|
with self.assertRaises(TabularSourceValidationError):
|
|
self.provider.read_source(
|
|
self.session,
|
|
principal(),
|
|
request=TabularReadRequest(
|
|
source_ref=created.ref,
|
|
expected_fingerprint="stale",
|
|
),
|
|
)
|
|
|
|
def test_csv_parser_infers_scalar_values_and_rejects_duplicate_headers(self) -> None:
|
|
rows = parse_csv_snapshot(
|
|
"\ufeffid;amount;active;note\n0012;12.5;true;\n2;7;false;ok\n",
|
|
delimiter=";",
|
|
)
|
|
|
|
self.assertEqual(
|
|
(
|
|
{"id": "0012", "amount": 12.5, "active": True, "note": None},
|
|
{"id": 2, "amount": 7, "active": False, "note": "ok"},
|
|
),
|
|
rows,
|
|
)
|
|
with self.assertRaises(TabularSourceValidationError):
|
|
parse_csv_snapshot("id,id\n1,2\n", delimiter=",")
|
|
with self.assertRaises(TabularSourceValidationError):
|
|
parse_csv_snapshot("id,name\n1,Ada,extra\n", delimiter=",")
|
|
|
|
def test_malformed_csv_api_request_is_reported_as_validation_error(self) -> None:
|
|
payload = SnapshotCreateRequest(
|
|
name="Malformed",
|
|
source_name="malformed",
|
|
format="csv",
|
|
csv_text="id,name\n1,Ada,extra\n",
|
|
)
|
|
|
|
with self.assertRaises(HTTPException) as raised:
|
|
api_create_tabular_snapshot(
|
|
payload,
|
|
session=self.session,
|
|
principal=principal(),
|
|
)
|
|
|
|
self.assertEqual(422, raised.exception.status_code)
|
|
|
|
def test_original_csv_round_trip_is_private_bounded_and_not_catalogue_content(self) -> None:
|
|
text = '\ufeffid,value\r\n9007199254740993," keep me "\r\ntrue,0.123456789012345678901234567890\r\n" ",""\r\n'
|
|
payload = SnapshotCreateRequest(name="CSV", source_name="csv", format="csv", csv_text=text)
|
|
with patch("govoplan_connectors.backend.router.audit_event"):
|
|
created = api_create_tabular_snapshot(payload, session=self.session, principal=principal())
|
|
self.session.expunge_all()
|
|
listed = self.provider.list_sources(self.session, principal())
|
|
self.assertNotIn("text", listed[0].metadata["csv_source"])
|
|
self.assertEqual("legacy_typed", listed[0].metadata["csv_source"]["value_mode"])
|
|
record = self.session.get(ConnectorTabularSource, created.ref.split(":", 1)[1])
|
|
self.assertIn("csv_source_", inspect(record).unloaded)
|
|
with patch("govoplan_connectors.backend.router.audit_event") as audit:
|
|
response = api_original_tabular_csv(created.ref, session=self.session, principal=principal())
|
|
self.assertEqual("connectors.original_csv.exported", audit.call_args.kwargs["action"])
|
|
self.assertEqual({"sha256"}, set(audit.call_args.kwargs["details"]))
|
|
with patch("govoplan_connectors.backend.router.provider.original_csv") as read, self.assertRaises(HTTPException) as denied:
|
|
api_original_tabular_csv(created.ref, session=self.session, principal=principal(scopes=()))
|
|
self.assertEqual(403, denied.exception.status_code)
|
|
read.assert_not_called()
|
|
self.assertEqual(text.encode("utf-8"), response.body)
|
|
self.assertEqual("no-store", response.headers["cache-control"])
|
|
self.assertIn("attachment", response.headers["content-disposition"])
|
|
with self.assertRaises(TabularSourceNotFoundError):
|
|
self.provider.original_csv(self.session, principal("tenant-2"), source_ref=created.ref)
|
|
with self.assertRaises(TabularSourceAccessError):
|
|
self.provider.original_csv(self.session, principal(scopes=()), source_ref=created.ref)
|
|
record.csv_source_ = {**record.csv_source_, "text": "tampered"}
|
|
self.session.flush()
|
|
with self.assertRaises(TabularSourceUnavailableError):
|
|
self.provider.original_csv(self.session, principal(), source_ref=created.ref)
|
|
|
|
def test_text_snapshot_matches_exact_projection_and_rejects_inconsistent_evidence(self) -> None:
|
|
source = TabularCsvSource(text='value\n" "\n0.123456789012345678901234567890\n', value_mode="text")
|
|
snapshot = TabularSnapshotInput(name="Text", source_name="text", rows=parse_csv_snapshot(source.text, delimiter=",", value_mode="text"), csv_source=source)
|
|
with self.assertRaises(TabularSourceValidationError):
|
|
self.provider.create_snapshot(self.session, principal(), snapshot=replace(snapshot, rows=({"value": "changed"},)))
|
|
created = self.provider.create_snapshot(self.session, principal(), snapshot=snapshot)
|
|
self.assertEqual(2, created.row_count)
|
|
self.assertEqual(source.text, self.provider.original_csv(self.session, principal(), source_ref=created.ref))
|
|
legacy = self.provider.create_snapshot(self.session, principal(), snapshot=TabularSnapshotInput(name="Old", source_name="old", rows=({"id": 1},)))
|
|
with self.assertRaises(TabularSourceNotFoundError):
|
|
self.provider.original_csv(self.session, principal(), source_ref=legacy.ref)
|
|
|
|
def test_original_csv_projection_binding_rejects_equal_but_different_types(self) -> None:
|
|
for text, value in (("value\ntrue\n", 1), ("value\n1\n", True), ("value\n1\n", 1.0)):
|
|
with self.subTest(text=text, value=value), self.assertRaises(TabularSourceValidationError):
|
|
self.provider.create_snapshot(self.session, principal(), snapshot=TabularSnapshotInput(name="Invalid", source_name="invalid", rows=({"value": value},), csv_source=TabularCsvSource(text=text)))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|