fix(datasources): preserve governed CSV originals and fresh detail state

Release v0.1.26. Coordinated integrity review: GovOPlaN/govoplan-core#298.

tests/test_lifecycle.py

@@ -1,6 +1,10 @@
 from __future__ import annotations
 
 import unittest
+from dataclasses import replace
+from unittest.mock import patch
+
+from fastapi import HTTPException
 
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
@@ -24,6 +28,9 @@ from govoplan_core.core.datasources import (
     DatasourceValidationError,
 )
 from govoplan_core.core.tabular_sources import (
+    TabularCsvSource,
+    parse_tabular_csv,
+    csv_source_payload,
     TabularPreviewDiagnostic,
     TabularPushdown,
     TabularSourceHealth,
@@ -40,6 +47,7 @@ from govoplan_datasources.backend.db.models import (
     DatasourceStageRecord,
 )
 from govoplan_datasources.backend.service import (
+    ADMIN_SCOPE,
     CATALOGUE_READ_SCOPE,
     SOURCE_WRITE_SCOPE,
     STAGE_WRITE_SCOPE,
@@ -263,6 +271,85 @@ class DatasourceLifecycleTests(unittest.TestCase):
         )
         self.engine.dispose()
 
+    def test_original_csv_preserves_source_but_respects_current_and_historical_visibility(self) -> None:
+        text = 'value\r\n"  keep me  "\r\n0.123456789012345678901234567890\r\n"   "\r\n'
+        source = TabularCsvSource(text=text, value_mode="text")
+        request = DatasourceStageInput(name="Text", source_name="csv_text", kind="upload", mode="static", shape="tabular", rows=parse_tabular_csv(text, value_mode="text"), csv_source=source)
+        with self.assertRaises(DatasourceValidationError):
+            self.provider.create_stage(self.session, principal(), stage=replace(request, rows=({"value": "changed"},)))
+        stage = self.provider.create_stage(self.session, principal(), stage=request)
+        descriptor, materialization = self.provider.promote_stage(self.session, principal(), stage_ref=stage.ref)
+        self.session.commit()
+        self.session.expunge_all()
+        admin = principal(scopes=(ADMIN_SCOPE,))
+        self.assertEqual(text, self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref))
+        from govoplan_datasources.backend.router import api_original_csv
+        with patch("govoplan_datasources.backend.router._provider", return_value=self.provider), patch("govoplan_datasources.backend.router._audit") as audit:
+            response = api_original_csv(descriptor.ref.split(":", 1)[1], materialization.ref.split(":", 1)[1], session=self.session, principal=admin)
+        self.assertEqual(text.encode("utf-8"), response.body)
+        self.assertEqual("no-store", response.headers["cache-control"])
+        self.assertEqual("datasources.original_csv.exported", audit.call_args.kwargs["action"])
+        self.assertNotIn(text, repr(audit.call_args.kwargs["details"]))
+        with patch("govoplan_datasources.backend.router._provider") as read, self.assertRaises(HTTPException) as denied:
+            api_original_csv("anything", "anything", session=self.session, principal=principal())
+        self.assertEqual(403, denied.exception.status_code)
+        read.assert_not_called()
+        frozen = self.provider.freeze_datasource(self.session, admin, datasource_ref=descriptor.ref)
+        self.assertEqual(text, self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=frozen.ref))
+        self.assertNotIn("text", descriptor.metadata["csv_source"])
+        self.assertNotIn("text", materialization.metadata["csv_source"])
+        with self.assertRaises(DatasourceAccessError):
+            self.provider.original_csv(self.session, principal(), datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
+        from govoplan_core.core.datasources import DatasourceNotFoundError
+        with self.assertRaises(DatasourceNotFoundError):
+            self.provider.original_csv(self.session, principal("other-tenant", scopes=(ADMIN_SCOPE,)), datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
+        record = self.session.get(DatasourceRecord, descriptor.ref.split(":", 1)[1])
+        record.visibility_policy = {"fields": {"value": {"classification": "restricted", "action": "omit", "allow": {"account_ids": ["other"]}}}}
+        self.session.flush()
+        with self.assertRaises(DatasourceAccessError):
+            self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
+        record.visibility_policy = {}
+        stored = self.session.get(DatasourceMaterializationRecord, materialization.ref.split(":", 1)[1])
+        stored.governance_snapshot_ = {**stored.governance_snapshot_, "visibility_policy": {"row_filters": [{"field": "value", "claim": "account_id"}]}}
+        self.session.flush()
+        with self.assertRaises(DatasourceAccessError):
+            self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
+        stored.governance_snapshot_ = {**stored.governance_snapshot_, "visibility_policy": {}}
+        stored.csv_source_ = {**stored.csv_source_, "text": "changed"}
+        self.session.flush()
+        with self.assertRaises(DatasourceUnavailableError):
+            self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
+
+    def test_csv_stage_projection_binding_rejects_equal_but_different_types(self) -> None:
+        for text, value in (("value\ntrue\n", 1), ("value\n1\n", True), ("value\n1\n", 1.0)):
+            with self.subTest(text=text, value=value), self.assertRaises(DatasourceValidationError):
+                self.provider.create_stage(self.session, principal(), stage=DatasourceStageInput(name="Invalid", source_name="invalid", kind="upload", mode="static", shape="tabular", rows=({"value": value},), csv_source=TabularCsvSource(text=text)))
+
+    def test_csv_invalid_unicode_is_validation_in_both_modes(self) -> None:
+        from govoplan_datasources.backend.tabular import parse_csv_rows
+        for mode in ("text", "legacy_typed"):
+            with self.subTest(mode=mode), self.assertRaisesRegex(DatasourceValidationError, "valid Unicode"):
+                parse_csv_rows("value\nprivate-\ud800\n", value_mode=mode)
+
+    def test_freezing_csv_preserves_historical_restrictions_and_rejects_resealed_drift(self) -> None:
+        source = TabularCsvSource(text='value\nprivate\n', value_mode="text")
+        governance = DatasourceGovernance(visibility_policy={"fields": {"value": {"classification": "restricted", "action": "omit", "allow": {"account_ids": ["other"]}}}})
+        stage = self.provider.create_stage(self.session, principal(), stage=DatasourceStageInput(name="Private CSV", source_name="private_csv", kind="upload", mode="static", shape="tabular", rows=parse_tabular_csv(source.text, value_mode="text"), csv_source=source, governance=governance))
+        descriptor, original = self.provider.promote_stage(self.session, principal(), stage_ref=stage.ref)
+        record = self.session.get(DatasourceRecord, descriptor.ref.split(":", 1)[1])
+        record.visibility_policy = {}
+        self.session.flush()
+        admin = principal(scopes=(ADMIN_SCOPE,))
+        frozen = self.provider.freeze_datasource(self.session, admin, datasource_ref=descriptor.ref)
+        for ref in (original.ref, frozen.ref):
+            with self.subTest(ref=ref), self.assertRaises(DatasourceAccessError):
+                self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=ref)
+        stored = self.session.get(DatasourceMaterializationRecord, original.ref.split(":", 1)[1])
+        stored.csv_source_ = csv_source_payload(TabularCsvSource(text="value\nchanged\n", value_mode="text"))
+        self.session.flush()
+        with self.assertRaises(DatasourceUnavailableError):
+            self.provider.freeze_datasource(self.session, admin, datasource_ref=descriptor.ref)
+
     def test_static_stage_promote_update_and_frozen_read(self) -> None:
         first_stage = self.provider.create_stage(
             self.session,

tests/test_lifecycle_governance.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from govoplan_core.core.tabular_sources import TabularCsvSource
+
 import unittest
 from datetime import timedelta
 
@@ -345,6 +347,7 @@ class DatasourceLifecycleGovernanceTests(unittest.TestCase):
                 mode="static",
                 shape="tabular",
                 rows=({"id": 1},),
+                csv_source=TabularCsvSource(text="id\n1\n"),
                 governance=governance,
             ),
         )
@@ -389,6 +392,7 @@ class DatasourceLifecycleGovernanceTests(unittest.TestCase):
             )
         )
         payload_id = first_row.payload_id
+        self.assertIsNotNone(first_row.csv_source_)
         disposed, evidence_hashes = self.provider.apply_retention(
             self.session,
             principal("admin", scopes=("datasources:source:admin",)),
@@ -400,6 +404,7 @@ class DatasourceLifecycleGovernanceTests(unittest.TestCase):
         self.assertEqual(1, len(evidence_hashes))
         self.assertIsNone(self.session.get(DatasourcePayloadRecord, payload_id))
         self.assertIsNotNone(first_row.disposed_at)
+        self.assertIsNone(first_row.csv_source_)
         self.assertEqual("disposed", first_row.state)
         with self.assertRaises(DatasourceUnavailableError):
             self.provider.read_datasource(
@@ -446,6 +451,7 @@ class DatasourceLifecycleGovernanceTests(unittest.TestCase):
                 mode="static",
                 shape="tabular",
                 rows=({"id": 1},),
+                csv_source=TabularCsvSource(text="id\n1\n"),
                 governance=DatasourceGovernance(
                     retention_policy={
                         "version": "stage-retention-v1",

tests/test_migrations.py

@@ -14,6 +14,47 @@ from govoplan_datasources.backend.manifest import get_manifest
 
 
 class DatasourceMigrationTests(unittest.TestCase):
+    def test_csv_evidence_upgrade_preserves_legacy_rows_and_fingerprints(self) -> None:
+        with tempfile.TemporaryDirectory(prefix="govoplan-datasources-csv-migration-") as directory:
+            url = f"sqlite:///{Path(directory) / 'datasources.db'}"
+            config = self._config(url)
+            command.upgrade(config, "d1a7c3e9f5b2")
+            engine = create_engine(url)
+            try:
+                metadata = MetaData()
+                tables = {name: Table(name, metadata, autoload_with=engine) for name in (
+                    "datasource_catalogue", "datasource_stages", "datasource_materializations",
+                )}
+                now = datetime.now(UTC)
+                common = dict(tenant_id="tenant-1", schema=[], fingerprint="a" * 64,
+                              metadata={"retained": True}, provenance={}, created_at=now, updated_at=now)
+                with engine.begin() as connection:
+                    connection.execute(tables["datasource_catalogue"].insert().values(
+                        **common, id="legacy-source", source_name="legacy", name="Legacy", kind="upload",
+                        mode="static", shape="tabular", status="active", schema_version=1,
+                    ))
+                    connection.execute(tables["datasource_stages"].insert().values(
+                        **common, id="legacy-stage", source_name="staged", name="Staged", kind="upload",
+                        mode="static", shape="tabular", state="ready", rows=[{"id": 1}],
+                        row_count=1, byte_count=10, validation={}, governance={}, approval={},
+                    ))
+                    connection.execute(tables["datasource_materializations"].insert().values(
+                        **common, id="legacy-materialization", datasource_id="legacy-source", revision=1,
+                        state="published", schema_version=1, rows=[{"id": 1}], row_count=1,
+                        byte_count=10, governance_snapshot={},
+                    ))
+                    before = {name: dict(connection.execute(select(table)).mappings().one()) for name, table in tables.items()}
+                command.upgrade(config, "e2b8d4a0f6c3")
+                with engine.connect() as connection:
+                    for name in tables:
+                        upgraded = Table(name, MetaData(), autoload_with=connection)
+                        after = dict(connection.execute(select(upgraded)).mappings().one())
+                        if name != "datasource_catalogue":
+                            self.assertIsNone(after.pop("csv_source"))
+                        self.assertEqual(before[name], after)
+            finally:
+                engine.dispose()
+
     @staticmethod
     def _config(url: str):
         return alembic_config(
@@ -34,7 +75,7 @@ class DatasourceMigrationTests(unittest.TestCase):
             try:
                 with engine.connect() as connection:
                     self.assertIn(
-                        "d1a7c3e9f5b2",
+                        "e2b8d4a0f6c3",
                         set(MigrationContext.configure(connection).get_current_heads()),
                     )
                     catalogue_columns = {