fix(datasources): preserve governed CSV originals and fresh detail state

Release v0.1.26. Coordinated integrity review: GovOPlaN/govoplan-core#298.

tests/test_lifecycle.py

@@ -1,6 +1,10 @@
 from __future__ import annotations
 
 import unittest
+from dataclasses import replace
+from unittest.mock import patch
+
+from fastapi import HTTPException
 
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
@@ -24,6 +28,9 @@ from govoplan_core.core.datasources import (
     DatasourceValidationError,
 )
 from govoplan_core.core.tabular_sources import (
+    TabularCsvSource,
+    parse_tabular_csv,
+    csv_source_payload,
     TabularPreviewDiagnostic,
     TabularPushdown,
     TabularSourceHealth,
@@ -40,6 +47,7 @@ from govoplan_datasources.backend.db.models import (
     DatasourceStageRecord,
 )
 from govoplan_datasources.backend.service import (
+    ADMIN_SCOPE,
     CATALOGUE_READ_SCOPE,
     SOURCE_WRITE_SCOPE,
     STAGE_WRITE_SCOPE,
@@ -263,6 +271,85 @@ class DatasourceLifecycleTests(unittest.TestCase):
         )
         self.engine.dispose()
 
+    def test_original_csv_preserves_source_but_respects_current_and_historical_visibility(self) -> None:
+        text = 'value\r\n"  keep me  "\r\n0.123456789012345678901234567890\r\n"   "\r\n'
+        source = TabularCsvSource(text=text, value_mode="text")
+        request = DatasourceStageInput(name="Text", source_name="csv_text", kind="upload", mode="static", shape="tabular", rows=parse_tabular_csv(text, value_mode="text"), csv_source=source)
+        with self.assertRaises(DatasourceValidationError):
+            self.provider.create_stage(self.session, principal(), stage=replace(request, rows=({"value": "changed"},)))
+        stage = self.provider.create_stage(self.session, principal(), stage=request)
+        descriptor, materialization = self.provider.promote_stage(self.session, principal(), stage_ref=stage.ref)
+        self.session.commit()
+        self.session.expunge_all()
+        admin = principal(scopes=(ADMIN_SCOPE,))
+        self.assertEqual(text, self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref))
+        from govoplan_datasources.backend.router import api_original_csv
+        with patch("govoplan_datasources.backend.router._provider", return_value=self.provider), patch("govoplan_datasources.backend.router._audit") as audit:
+            response = api_original_csv(descriptor.ref.split(":", 1)[1], materialization.ref.split(":", 1)[1], session=self.session, principal=admin)
+        self.assertEqual(text.encode("utf-8"), response.body)
+        self.assertEqual("no-store", response.headers["cache-control"])
+        self.assertEqual("datasources.original_csv.exported", audit.call_args.kwargs["action"])
+        self.assertNotIn(text, repr(audit.call_args.kwargs["details"]))
+        with patch("govoplan_datasources.backend.router._provider") as read, self.assertRaises(HTTPException) as denied:
+            api_original_csv("anything", "anything", session=self.session, principal=principal())
+        self.assertEqual(403, denied.exception.status_code)
+        read.assert_not_called()
+        frozen = self.provider.freeze_datasource(self.session, admin, datasource_ref=descriptor.ref)
+        self.assertEqual(text, self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=frozen.ref))
+        self.assertNotIn("text", descriptor.metadata["csv_source"])
+        self.assertNotIn("text", materialization.metadata["csv_source"])
+        with self.assertRaises(DatasourceAccessError):
+            self.provider.original_csv(self.session, principal(), datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
+        from govoplan_core.core.datasources import DatasourceNotFoundError
+        with self.assertRaises(DatasourceNotFoundError):
+            self.provider.original_csv(self.session, principal("other-tenant", scopes=(ADMIN_SCOPE,)), datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
+        record = self.session.get(DatasourceRecord, descriptor.ref.split(":", 1)[1])
+        record.visibility_policy = {"fields": {"value": {"classification": "restricted", "action": "omit", "allow": {"account_ids": ["other"]}}}}
+        self.session.flush()
+        with self.assertRaises(DatasourceAccessError):
+            self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
+        record.visibility_policy = {}
+        stored = self.session.get(DatasourceMaterializationRecord, materialization.ref.split(":", 1)[1])
+        stored.governance_snapshot_ = {**stored.governance_snapshot_, "visibility_policy": {"row_filters": [{"field": "value", "claim": "account_id"}]}}
+        self.session.flush()
+        with self.assertRaises(DatasourceAccessError):
+            self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
+        stored.governance_snapshot_ = {**stored.governance_snapshot_, "visibility_policy": {}}
+        stored.csv_source_ = {**stored.csv_source_, "text": "changed"}
+        self.session.flush()
+        with self.assertRaises(DatasourceUnavailableError):
+            self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=materialization.ref)
+
+    def test_csv_stage_projection_binding_rejects_equal_but_different_types(self) -> None:
+        for text, value in (("value\ntrue\n", 1), ("value\n1\n", True), ("value\n1\n", 1.0)):
+            with self.subTest(text=text, value=value), self.assertRaises(DatasourceValidationError):
+                self.provider.create_stage(self.session, principal(), stage=DatasourceStageInput(name="Invalid", source_name="invalid", kind="upload", mode="static", shape="tabular", rows=({"value": value},), csv_source=TabularCsvSource(text=text)))
+
+    def test_csv_invalid_unicode_is_validation_in_both_modes(self) -> None:
+        from govoplan_datasources.backend.tabular import parse_csv_rows
+        for mode in ("text", "legacy_typed"):
+            with self.subTest(mode=mode), self.assertRaisesRegex(DatasourceValidationError, "valid Unicode"):
+                parse_csv_rows("value\nprivate-\ud800\n", value_mode=mode)
+
+    def test_freezing_csv_preserves_historical_restrictions_and_rejects_resealed_drift(self) -> None:
+        source = TabularCsvSource(text='value\nprivate\n', value_mode="text")
+        governance = DatasourceGovernance(visibility_policy={"fields": {"value": {"classification": "restricted", "action": "omit", "allow": {"account_ids": ["other"]}}}})
+        stage = self.provider.create_stage(self.session, principal(), stage=DatasourceStageInput(name="Private CSV", source_name="private_csv", kind="upload", mode="static", shape="tabular", rows=parse_tabular_csv(source.text, value_mode="text"), csv_source=source, governance=governance))
+        descriptor, original = self.provider.promote_stage(self.session, principal(), stage_ref=stage.ref)
+        record = self.session.get(DatasourceRecord, descriptor.ref.split(":", 1)[1])
+        record.visibility_policy = {}
+        self.session.flush()
+        admin = principal(scopes=(ADMIN_SCOPE,))
+        frozen = self.provider.freeze_datasource(self.session, admin, datasource_ref=descriptor.ref)
+        for ref in (original.ref, frozen.ref):
+            with self.subTest(ref=ref), self.assertRaises(DatasourceAccessError):
+                self.provider.original_csv(self.session, admin, datasource_ref=descriptor.ref, materialization_ref=ref)
+        stored = self.session.get(DatasourceMaterializationRecord, original.ref.split(":", 1)[1])
+        stored.csv_source_ = csv_source_payload(TabularCsvSource(text="value\nchanged\n", value_mode="text"))
+        self.session.flush()
+        with self.assertRaises(DatasourceUnavailableError):
+            self.provider.freeze_datasource(self.session, admin, datasource_ref=descriptor.ref)
+
     def test_static_stage_promote_update_and_frozen_read(self) -> None:
         first_stage = self.provider.create_stage(
             self.session,