from __future__ import annotations import unittest from unittest.mock import patch from dataclasses import replace from fastapi import HTTPException from sqlalchemy import create_engine, inspect from sqlalchemy.orm import sessionmaker from govoplan_core.auth import ApiPrincipal from govoplan_core.core.access import PrincipalRef from govoplan_core.core.tabular_sources import ( TabularCsvSource, TabularReadRequest, TabularSnapshotInput, TabularSourceAccessError, TabularSourceUnavailableError, TabularSourceValidationError, TabularSourceNotFoundError, ) from govoplan_core.db.base import Base from govoplan_connectors.backend.db.models import ConnectorTabularSource from govoplan_connectors.backend.router import api_create_tabular_snapshot, api_original_tabular_csv from govoplan_connectors.backend.schemas import SnapshotCreateRequest from govoplan_connectors.backend.tabular_sources import ( READ_SCOPE, WRITE_SCOPE, SqlTabularSourceProvider, parse_csv_snapshot, ) def principal( tenant_id: str = "tenant-1", *, scopes: tuple[str, ...] = (READ_SCOPE, WRITE_SCOPE), ) -> ApiPrincipal: return ApiPrincipal( principal=PrincipalRef( account_id="account-1", membership_id="membership-1", tenant_id=tenant_id, scopes=frozenset(scopes), ), account=object(), user=object(), ) class ConnectorsTabularSourceTests(unittest.TestCase): def setUp(self) -> None: self.engine = create_engine("sqlite:///:memory:") Base.metadata.create_all(self.engine, tables=[ConnectorTabularSource.__table__]) self.Session = sessionmaker(bind=self.engine) self.session = self.Session() self.provider = SqlTabularSourceProvider() def tearDown(self) -> None: self.session.close() Base.metadata.drop_all(self.engine, tables=[ConnectorTabularSource.__table__]) self.engine.dispose() def test_snapshot_round_trip_preserves_schema_fingerprint_and_bounds(self) -> None: created = self.provider.create_snapshot( self.session, principal(), snapshot=TabularSnapshotInput( name="Monthly cases", source_name="monthly_cases_2026_07", rows=( {"case_id": "A-1", "amount": 12, "active": True}, {"case_id": "A-2", "amount": None, "active": False}, ), ), ) self.session.commit() listed = self.provider.list_sources(self.session, principal()) preview = self.provider.read_source( self.session, principal(), request=TabularReadRequest( source_ref=created.ref, limit=1, expected_fingerprint=created.fingerprint, ), ) self.assertEqual((created.ref,), tuple(source.ref for source in listed)) self.assertEqual( ["case_id", "amount", "active"], [column.name for column in created.schema], ) self.assertEqual(2, preview.total_rows) self.assertEqual(1, len(preview.rows)) self.assertTrue(preview.truncated) self.assertEqual(created.fingerprint, preview.source.fingerprint) self.assertEqual("cached", preview.source.source_mode) self.assertTrue(preview.source.pushdown.projections) self.assertTrue(preview.source.pushdown.pagination) self.assertEqual("healthy", preview.source.health.status) self.assertGreater(preview.returned_bytes, 2) self.assertEqual(1, preview.effective_row_limit) self.assertEqual("preview.row_limit_reached", preview.diagnostics[0].code) def test_preview_enforces_byte_time_and_provider_ceiling_budgets(self) -> None: created = self.provider.create_snapshot( self.session, principal(), snapshot=TabularSnapshotInput( name="Bounded", source_name="bounded", rows=( {"id": 1, "value": "first"}, {"id": 2, "value": "second"}, ), ), ) self.session.commit() bounded = self.provider.read_source( self.session, principal(), request=TabularReadRequest( source_ref=created.ref, limit=500, max_bytes=35, timeout_ms=2_000, ), ) self.assertEqual(1, len(bounded.rows)) self.assertTrue(bounded.truncated) self.assertEqual( "preview.byte_limit_reached", bounded.diagnostics[-1].code, ) with self.assertRaisesRegex( TabularSourceValidationError, "single source row exceeds", ): self.provider.read_source( self.session, principal(), request=TabularReadRequest( source_ref=created.ref, max_bytes=2, ), ) tightened = self.provider.read_source( self.session, principal(), request=TabularReadRequest( source_ref=created.ref, limit=5_000, max_bytes=5_000_000, timeout_ms=10_000, ), ) self.assertEqual(500, tightened.effective_row_limit) self.assertEqual(1_000_000, tightened.effective_byte_limit) self.assertEqual(2_000, tightened.effective_timeout_ms) self.assertEqual( { "preview.row_limit_tightened", "preview.byte_limit_tightened", "preview.timeout_tightened", }, {item.code for item in tightened.diagnostics}, ) times = iter((0.0, 0.01)) timeout_provider = SqlTabularSourceProvider(clock=lambda: next(times)) with self.assertRaisesRegex( TabularSourceUnavailableError, "time budget", ): timeout_provider.read_source( self.session, principal(), request=TabularReadRequest( source_ref=created.ref, timeout_ms=1, ), ) def test_tenant_and_scope_isolation_are_enforced(self) -> None: created = self.provider.create_snapshot( self.session, principal(), snapshot=TabularSnapshotInput( name="Private", source_name="private_source", rows=({"id": 1},), ), ) self.session.commit() self.assertEqual((), self.provider.list_sources(self.session, principal("tenant-2"))) self.assertIsNone( self.provider.get_source( self.session, principal("tenant-2"), source_ref=created.ref, ) ) with self.assertRaises(TabularSourceAccessError): self.provider.list_sources( self.session, principal(scopes=()), ) def test_duplicate_source_name_and_stale_fingerprint_are_rejected(self) -> None: snapshot = TabularSnapshotInput( name="Cases", source_name="cases", rows=({"id": 1},), ) created = self.provider.create_snapshot(self.session, principal(), snapshot=snapshot) self.session.commit() with self.assertRaises(TabularSourceValidationError): self.provider.create_snapshot(self.session, principal(), snapshot=snapshot) with self.assertRaises(TabularSourceValidationError): self.provider.read_source( self.session, principal(), request=TabularReadRequest( source_ref=created.ref, expected_fingerprint="stale", ), ) def test_csv_parser_infers_scalar_values_and_rejects_duplicate_headers(self) -> None: rows = parse_csv_snapshot( "\ufeffid;amount;active;note\n0012;12.5;true;\n2;7;false;ok\n", delimiter=";", ) self.assertEqual( ( {"id": "0012", "amount": 12.5, "active": True, "note": None}, {"id": 2, "amount": 7, "active": False, "note": "ok"}, ), rows, ) with self.assertRaises(TabularSourceValidationError): parse_csv_snapshot("id,id\n1,2\n", delimiter=",") with self.assertRaises(TabularSourceValidationError): parse_csv_snapshot("id,name\n1,Ada,extra\n", delimiter=",") def test_malformed_csv_api_request_is_reported_as_validation_error(self) -> None: payload = SnapshotCreateRequest( name="Malformed", source_name="malformed", format="csv", csv_text="id,name\n1,Ada,extra\n", ) with self.assertRaises(HTTPException) as raised: api_create_tabular_snapshot( payload, session=self.session, principal=principal(), ) self.assertEqual(422, raised.exception.status_code) def test_original_csv_round_trip_is_private_bounded_and_not_catalogue_content(self) -> None: text = '\ufeffid,value\r\n9007199254740993," keep me "\r\ntrue,0.123456789012345678901234567890\r\n" ",""\r\n' payload = SnapshotCreateRequest(name="CSV", source_name="csv", format="csv", csv_text=text) with patch("govoplan_connectors.backend.router.audit_event"): created = api_create_tabular_snapshot(payload, session=self.session, principal=principal()) self.session.expunge_all() listed = self.provider.list_sources(self.session, principal()) self.assertNotIn("text", listed[0].metadata["csv_source"]) self.assertEqual("legacy_typed", listed[0].metadata["csv_source"]["value_mode"]) record = self.session.get(ConnectorTabularSource, created.ref.split(":", 1)[1]) self.assertIn("csv_source_", inspect(record).unloaded) with patch("govoplan_connectors.backend.router.audit_event") as audit: response = api_original_tabular_csv(created.ref, session=self.session, principal=principal()) self.assertEqual("connectors.original_csv.exported", audit.call_args.kwargs["action"]) self.assertEqual({"sha256"}, set(audit.call_args.kwargs["details"])) with patch("govoplan_connectors.backend.router.provider.original_csv") as read, self.assertRaises(HTTPException) as denied: api_original_tabular_csv(created.ref, session=self.session, principal=principal(scopes=())) self.assertEqual(403, denied.exception.status_code) read.assert_not_called() self.assertEqual(text.encode("utf-8"), response.body) self.assertEqual("no-store", response.headers["cache-control"]) self.assertIn("attachment", response.headers["content-disposition"]) with self.assertRaises(TabularSourceNotFoundError): self.provider.original_csv(self.session, principal("tenant-2"), source_ref=created.ref) with self.assertRaises(TabularSourceAccessError): self.provider.original_csv(self.session, principal(scopes=()), source_ref=created.ref) record.csv_source_ = {**record.csv_source_, "text": "tampered"} self.session.flush() with self.assertRaises(TabularSourceUnavailableError): self.provider.original_csv(self.session, principal(), source_ref=created.ref) def test_text_snapshot_matches_exact_projection_and_rejects_inconsistent_evidence(self) -> None: source = TabularCsvSource(text='value\n" "\n0.123456789012345678901234567890\n', value_mode="text") snapshot = TabularSnapshotInput(name="Text", source_name="text", rows=parse_csv_snapshot(source.text, delimiter=",", value_mode="text"), csv_source=source) with self.assertRaises(TabularSourceValidationError): self.provider.create_snapshot(self.session, principal(), snapshot=replace(snapshot, rows=({"value": "changed"},))) created = self.provider.create_snapshot(self.session, principal(), snapshot=snapshot) self.assertEqual(2, created.row_count) self.assertEqual(source.text, self.provider.original_csv(self.session, principal(), source_ref=created.ref)) legacy = self.provider.create_snapshot(self.session, principal(), snapshot=TabularSnapshotInput(name="Old", source_name="old", rows=({"id": 1},))) with self.assertRaises(TabularSourceNotFoundError): self.provider.original_csv(self.session, principal(), source_ref=legacy.ref) def test_original_csv_projection_binding_rejects_equal_but_different_types(self) -> None: for text, value in (("value\ntrue\n", 1), ("value\n1\n", True), ("value\n1\n", 1.0)): with self.subTest(text=text, value=value), self.assertRaises(TabularSourceValidationError): self.provider.create_snapshot(self.session, principal(), snapshot=TabularSnapshotInput(name="Invalid", source_name="invalid", rows=({"value": value},), csv_source=TabularCsvSource(text=text))) if __name__ == "__main__": unittest.main()