Add datasource quality and schema gates

2026-08-04 11:23:14 +02:00
parent a6d5c9839d
commit 075d2fc087
16 changed files with 1305 additions and 10 deletions
@@ -265,6 +265,96 @@ class DatasourceLifecycleTests(unittest.TestCase):
        self.assertEqual(first_record.payload_id, frozen_record.payload_id)
        self.assertEqual([], first_record.rows)

+    def test_stage_quality_and_schema_gates_block_only_error_diagnostics(self) -> None:
+        governance = DatasourceGovernance(
+            authority_mode="native_authoritative",
+            publication_state="internal",
+            quality_policy={
+                "version": "monthly-cases-v1",
+                "rules": [
+                    {"id": "unique-id", "type": "unique", "fields": ["id"]},
+                ],
+                "schema_policy": {"field_added_required": "warning"},
+            },
+        )
+        first_stage = self.provider.create_stage(
+            self.session,
+            principal(),
+            stage=DatasourceStageInput(
+                name="Monthly cases",
+                source_name="monthly_quality_cases",
+                kind="upload",
+                mode="static",
+                shape="tabular",
+                rows=({"id": 1},),
+                governance=governance,
+            ),
+        )
+        datasource, _first = self.provider.promote_stage(
+            self.session,
+            principal(),
+            stage_ref=first_stage.ref,
+        )
+
+        blocked_stage = self.provider.create_stage(
+            self.session,
+            principal(),
+            stage=DatasourceStageInput(
+                name="Monthly cases",
+                source_name="monthly_quality_cases",
+                kind="upload",
+                mode="static",
+                shape="tabular",
+                target_datasource_ref=datasource.ref,
+                rows=({"id": 2, "name": "Ada"}, {"id": 2, "name": "Lin"}),
+            ),
+        )
+        self.assertEqual("invalid", blocked_stage.state)
+        self.assertFalse(blocked_stage.validation["valid"])
+        self.assertEqual(
+            "quality.unique",
+            blocked_stage.validation["errors"][0]["code"],
+        )
+        self.assertEqual(
+            "warning",
+            blocked_stage.validation["schema_change"]["classification"],
+        )
+        with self.assertRaisesRegex(
+            DatasourceValidationError,
+            "blocking quality or schema diagnostics",
+        ):
+            self.provider.promote_stage(
+                self.session,
+                principal(),
+                stage_ref=blocked_stage.ref,
+            )
+
+        ready_stage = self.provider.create_stage(
+            self.session,
+            principal(),
+            stage=DatasourceStageInput(
+                name="Monthly cases",
+                source_name="monthly_quality_cases",
+                kind="upload",
+                mode="static",
+                shape="tabular",
+                target_datasource_ref=datasource.ref,
+                rows=({"id": 2, "name": "Ada"}, {"id": 3, "name": "Lin"}),
+            ),
+        )
+        self.assertEqual("ready", ready_stage.state)
+        self.assertTrue(ready_stage.validation["valid"])
+        self.assertEqual("schema.field_added_required", ready_stage.validation["warnings"][0]["code"])
+        _updated, materialization = self.provider.promote_stage(
+            self.session,
+            principal(),
+            stage_ref=ready_stage.ref,
+        )
+        self.assertEqual(
+            ready_stage.validation["policy_hash"],
+            materialization.provenance["stage_validation"]["policy_hash"],
+        )
+
    def test_governance_is_queryable_and_snapshotted_per_materialization(self) -> None:
        stage = self.provider.create_stage(
            self.session,