fix(connectors): retain exact CSV source evidence

Release v0.1.27. Coordinated integrity review: GovOPlaN/govoplan-core#298.

tests/test_migrations.py

@@ -2,16 +2,45 @@ from __future__ import annotations
 
 import tempfile
 import unittest
+from datetime import UTC, datetime
 from pathlib import Path
 
+from alembic import command
 from alembic.runtime.migration import MigrationContext
-from sqlalchemy import create_engine, inspect
+from sqlalchemy import MetaData, Table, create_engine, inspect, select
 
 from govoplan_connectors.backend.manifest import get_manifest
-from govoplan_core.db.migrations import migrate_database
+from govoplan_core.db.migrations import alembic_config, migrate_database
 
 
 class ConnectorsMigrationTests(unittest.TestCase):
+    def test_csv_evidence_upgrade_preserves_legacy_snapshot_without_inventing_source(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="govoplan-connectors-csv-migration-") as directory:
+            url = f"sqlite:///{Path(directory) / 'connectors.db'}"
+            config = alembic_config(database_url=url, enabled_modules=("connectors",), manifest_factories=(get_manifest,))
+            command.upgrade(config, "c0f1a2b3c4d5")
+            engine = create_engine(url)
+            try:
+                table = Table("connector_tabular_sources", MetaData(), autoload_with=engine)
+                now = datetime.now(UTC)
+                with engine.begin() as connection:
+                    connection.execute(table.insert().values(
+                        id="legacy-csv", tenant_id="tenant-1", provider="snapshot",
+                        source_name="legacy", name="Legacy", status="active", schema_version=1,
+                        schema=[{"name": "id", "data_type": "integer", "nullable": False}],
+                        rows=[{"id": 1}], fingerprint="a" * 64, row_count=1, byte_count=10,
+                        metadata={"original_label": "CSV"}, created_at=now, updated_at=now,
+                    ))
+                    before = dict(connection.execute(select(table)).mappings().one())
+                command.upgrade(config, "d2a4c6e8f0b1")
+                upgraded = Table("connector_tabular_sources", MetaData(), autoload_with=engine)
+                with engine.connect() as connection:
+                    after = dict(connection.execute(select(upgraded)).mappings().one())
+                self.assertIsNone(after.pop("csv_source"))
+                self.assertEqual(before, after)
+            finally:
+                engine.dispose()
+
     def test_baseline_creates_connector_tables_and_head(self) -> None:
         with tempfile.TemporaryDirectory(prefix="govoplan-connectors-migration-") as directory:
             url = f"sqlite:///{Path(directory) / 'connectors.db'}"
@@ -24,7 +53,7 @@ class ConnectorsMigrationTests(unittest.TestCase):
             try:
                 with engine.connect() as connection:
                     self.assertIn(
-                        "c0f1a2b3c4d5",
+                        "d2a4c6e8f0b1",
                         set(MigrationContext.configure(connection).get_current_heads()),
                     )
                     self.assertTrue(

tests/test_search_principal.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import unittest
+from types import SimpleNamespace
+
+from govoplan_connectors.backend.search_principal import principal_acl_tokens
+
+
+class SearchPrincipalTests(unittest.TestCase):
+    def test_legacy_first_seen_projection_and_cap_are_unchanged(self) -> None:
+        principal = SimpleNamespace(
+            account_id=" actor ",
+            membership_id="m",
+            identity_id="i",
+            group_ids=("g", "", "g", "2"),
+            role_ids=("r", "r"),
+            function_assignment_ids=("f",),
+            scopes=tuple(f"scope-{i}" for i in range(700)),
+        )
+        legacy = []
+        for prefix, attribute in (
+            ("account", "account_id"),
+            ("membership", "membership_id"),
+            ("identity", "identity_id"),
+        ):
+            value = getattr(principal, attribute, None)
+            if value:
+                legacy.append(f"{prefix}:{value}")
+        for prefix, attribute in (
+            ("group", "group_ids"),
+            ("role", "role_ids"),
+            ("function", "function_assignment_ids"),
+            ("scope", "scopes"),
+        ):
+            legacy.extend(
+                f"{prefix}:{value}"
+                for value in getattr(principal, attribute, ())
+                if value
+            )
+        expected = tuple(dict.fromkeys(legacy))[:500]
+        self.assertEqual(expected, principal_acl_tokens(principal))
+        self.assertEqual(500, len(expected))
+        self.assertEqual((), principal_acl_tokens(object()))
+
+    def test_projection_stops_consuming_at_authorization_cap(self) -> None:
+        def bounded_scopes():
+            yield from (str(i) for i in range(500))
+            raise AssertionError("ACL projection scanned beyond its effective cap")
+
+        self.assertEqual(
+            500, len(principal_acl_tokens(SimpleNamespace(scopes=bounded_scopes())))
+        )

tests/test_tabular_sources.py

@@ -1,23 +1,27 @@
 from __future__ import annotations
 
 import unittest
+from unittest.mock import patch
+from dataclasses import replace
 
 from fastapi import HTTPException
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, inspect
 from sqlalchemy.orm import sessionmaker
 
 from govoplan_core.auth import ApiPrincipal
 from govoplan_core.core.access import PrincipalRef
 from govoplan_core.core.tabular_sources import (
+    TabularCsvSource,
     TabularReadRequest,
     TabularSnapshotInput,
     TabularSourceAccessError,
     TabularSourceUnavailableError,
     TabularSourceValidationError,
+    TabularSourceNotFoundError,
 )
 from govoplan_core.db.base import Base
 from govoplan_connectors.backend.db.models import ConnectorTabularSource
-from govoplan_connectors.backend.router import api_create_tabular_snapshot
+from govoplan_connectors.backend.router import api_create_tabular_snapshot, api_original_tabular_csv
 from govoplan_connectors.backend.schemas import SnapshotCreateRequest
 from govoplan_connectors.backend.tabular_sources import (
     READ_SCOPE,
@@ -263,6 +267,54 @@ class ConnectorsTabularSourceTests(unittest.TestCase):
 
         self.assertEqual(422, raised.exception.status_code)
 
+    def test_original_csv_round_trip_is_private_bounded_and_not_catalogue_content(self) -> None:
+        text = '\ufeffid,value\r\n9007199254740993,"  keep me  "\r\ntrue,0.123456789012345678901234567890\r\n"   ",""\r\n'
+        payload = SnapshotCreateRequest(name="CSV", source_name="csv", format="csv", csv_text=text)
+        with patch("govoplan_connectors.backend.router.audit_event"):
+            created = api_create_tabular_snapshot(payload, session=self.session, principal=principal())
+        self.session.expunge_all()
+        listed = self.provider.list_sources(self.session, principal())
+        self.assertNotIn("text", listed[0].metadata["csv_source"])
+        self.assertEqual("legacy_typed", listed[0].metadata["csv_source"]["value_mode"])
+        record = self.session.get(ConnectorTabularSource, created.ref.split(":", 1)[1])
+        self.assertIn("csv_source_", inspect(record).unloaded)
+        with patch("govoplan_connectors.backend.router.audit_event") as audit:
+            response = api_original_tabular_csv(created.ref, session=self.session, principal=principal())
+        self.assertEqual("connectors.original_csv.exported", audit.call_args.kwargs["action"])
+        self.assertEqual({"sha256"}, set(audit.call_args.kwargs["details"]))
+        with patch("govoplan_connectors.backend.router.provider.original_csv") as read, self.assertRaises(HTTPException) as denied:
+            api_original_tabular_csv(created.ref, session=self.session, principal=principal(scopes=()))
+        self.assertEqual(403, denied.exception.status_code)
+        read.assert_not_called()
+        self.assertEqual(text.encode("utf-8"), response.body)
+        self.assertEqual("no-store", response.headers["cache-control"])
+        self.assertIn("attachment", response.headers["content-disposition"])
+        with self.assertRaises(TabularSourceNotFoundError):
+            self.provider.original_csv(self.session, principal("tenant-2"), source_ref=created.ref)
+        with self.assertRaises(TabularSourceAccessError):
+            self.provider.original_csv(self.session, principal(scopes=()), source_ref=created.ref)
+        record.csv_source_ = {**record.csv_source_, "text": "tampered"}
+        self.session.flush()
+        with self.assertRaises(TabularSourceUnavailableError):
+            self.provider.original_csv(self.session, principal(), source_ref=created.ref)
+
+    def test_text_snapshot_matches_exact_projection_and_rejects_inconsistent_evidence(self) -> None:
+        source = TabularCsvSource(text='value\n"   "\n0.123456789012345678901234567890\n', value_mode="text")
+        snapshot = TabularSnapshotInput(name="Text", source_name="text", rows=parse_csv_snapshot(source.text, delimiter=",", value_mode="text"), csv_source=source)
+        with self.assertRaises(TabularSourceValidationError):
+            self.provider.create_snapshot(self.session, principal(), snapshot=replace(snapshot, rows=({"value": "changed"},)))
+        created = self.provider.create_snapshot(self.session, principal(), snapshot=snapshot)
+        self.assertEqual(2, created.row_count)
+        self.assertEqual(source.text, self.provider.original_csv(self.session, principal(), source_ref=created.ref))
+        legacy = self.provider.create_snapshot(self.session, principal(), snapshot=TabularSnapshotInput(name="Old", source_name="old", rows=({"id": 1},)))
+        with self.assertRaises(TabularSourceNotFoundError):
+            self.provider.original_csv(self.session, principal(), source_ref=legacy.ref)
+
+    def test_original_csv_projection_binding_rejects_equal_but_different_types(self) -> None:
+        for text, value in (("value\ntrue\n", 1), ("value\n1\n", True), ("value\n1\n", 1.0)):
+            with self.subTest(text=text, value=value), self.assertRaises(TabularSourceValidationError):
+                self.provider.create_snapshot(self.session, principal(), snapshot=TabularSnapshotInput(name="Invalid", source_name="invalid", rows=({"value": value},), csv_source=TabularCsvSource(text=text)))
+
 
 if __name__ == "__main__":
     unittest.main()