GovOPlaN/govoplan-datasources
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

Add datasource quality and schema gates

Browse Source
This commit is contained in:
zemion
2026-08-04 11:23:14 +02:00
parent a6d5c9839d
commit 075d2fc087
16 changed files with 1305 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_interface_documentation_contract.py
+4
View File
@@ -26,10 +26,14 @@ class DatasourcesInterfaceDocumentationContractTests(unittest.TestCase):
topics = {topic.id: topic for topic in manifest.documentation}
lifecycle = topics["datasources.lifecycle"]
governance = topics["datasources.governance"]
quality = topics["datasources.quality-gates"]
reference = topics["datasources.reference.fields-and-consequences"]
self.assertIn("datasources.staging", lifecycle.metadata["help_contexts"])
self.assertIn("datasources.field.authority-mode", governance.metadata["help_contexts"])
self.assertIn("datasources.staging.validation", quality.metadata["help_contexts"])
self.assertIn("policy version and hash", quality.body)
self.assertTrue(quality.metadata["limitations"])
self.assertIn("datasources.action.promote", reference.metadata["help_contexts"])
self.assertIn("freeze", reference.metadata["consequence_classes"])
self.assertIn("retire", reference.metadata["consequence_classes"])
tests/test_lifecycle.py
+90
View File
@@ -265,6 +265,96 @@ class DatasourceLifecycleTests(unittest.TestCase):
self.assertEqual(first_record.payload_id, frozen_record.payload_id)
self.assertEqual([], first_record.rows)
def test_stage_quality_and_schema_gates_block_only_error_diagnostics(self) -> None:
governance = DatasourceGovernance(
authority_mode="native_authoritative",
publication_state="internal",
quality_policy={
"version": "monthly-cases-v1",
"rules": [
{"id": "unique-id", "type": "unique", "fields": ["id"]},
],
"schema_policy": {"field_added_required": "warning"},
},
)
first_stage = self.provider.create_stage(
self.session,
principal(),
stage=DatasourceStageInput(
name="Monthly cases",
source_name="monthly_quality_cases",
kind="upload",
mode="static",
shape="tabular",
rows=({"id": 1},),
governance=governance,
),
)
datasource, _first = self.provider.promote_stage(
self.session,
principal(),
stage_ref=first_stage.ref,
)
blocked_stage = self.provider.create_stage(
self.session,
principal(),
stage=DatasourceStageInput(
name="Monthly cases",
source_name="monthly_quality_cases",
kind="upload",
mode="static",
shape="tabular",
target_datasource_ref=datasource.ref,
rows=({"id": 2, "name": "Ada"}, {"id": 2, "name": "Lin"}),
),
)
self.assertEqual("invalid", blocked_stage.state)
self.assertFalse(blocked_stage.validation["valid"])
self.assertEqual(
"quality.unique",
blocked_stage.validation["errors"][0]["code"],
)
self.assertEqual(
"warning",
blocked_stage.validation["schema_change"]["classification"],
)
with self.assertRaisesRegex(
DatasourceValidationError,
"blocking quality or schema diagnostics",
):
self.provider.promote_stage(
self.session,
principal(),
stage_ref=blocked_stage.ref,
)
ready_stage = self.provider.create_stage(
self.session,
principal(),
stage=DatasourceStageInput(
name="Monthly cases",
source_name="monthly_quality_cases",
kind="upload",
mode="static",
shape="tabular",
target_datasource_ref=datasource.ref,
rows=({"id": 2, "name": "Ada"}, {"id": 3, "name": "Lin"}),
),
)
self.assertEqual("ready", ready_stage.state)
self.assertTrue(ready_stage.validation["valid"])
self.assertEqual("schema.field_added_required", ready_stage.validation["warnings"][0]["code"])
_updated, materialization = self.provider.promote_stage(
self.session,
principal(),
stage_ref=ready_stage.ref,
)
self.assertEqual(
ready_stage.validation["policy_hash"],
materialization.provenance["stage_validation"]["policy_hash"],
)
def test_governance_is_queryable_and_snapshotted_per_materialization(self) -> None:
stage = self.provider.create_stage(
self.session,
tests/test_quality.py
+142
View File
@@ -0,0 +1,142 @@
from __future__ import annotations
import unittest
from govoplan_core.core.datasources import DatasourceField
from govoplan_datasources.backend.quality import MAX_DIAGNOSTIC_ROWS, validate_stage
from govoplan_datasources.backend.schemas import DatasourceStageValidationResponse
class DatasourceQualityTests(unittest.TestCase):
def test_quality_rules_report_counts_without_exposing_values(self) -> None:
rows = (
{"id": 1, "name": "Ada", "amount": 10, "status": "new"},
{"id": 1, "name": None, "amount": -2, "status": "unknown"},
)
schema = (
DatasourceField("id", "integer", False),
DatasourceField("name", "string", True),
DatasourceField("amount", "integer", False),
DatasourceField("status", "string", False),
)
result = validate_stage(
rows=rows,
schema=schema,
quality_policy={
"version": "case-import-v2",
"rules": [
{"id": "rows", "type": "row_count", "minimum": 3},
{"id": "columns", "type": "required_fields", "fields": ["id", "missing"]},
{"id": "id-shape", "type": "field", "field": "id", "data_type": "string"},
{"id": "names", "type": "not_null", "fields": ["name"]},
{"id": "ids", "type": "unique", "fields": ["id"]},
{"id": "amount", "type": "range", "field": "amount", "minimum": 0},
{
"id": "status",
"type": "referential",
"field": "status",
"allowed_values": ["new", "closed"],
},
],
},
)
self.assertFalse(result["valid"])
self.assertEqual("case-import-v2", result["policy_version"])
errors = result["errors"]
self.assertEqual(
{
"quality.row_count",
"quality.required_fields",
"quality.field_type",
"quality.not_null",
"quality.unique",
"quality.range",
"quality.referential",
},
{item["code"] for item in errors},
)
unique = next(item for item in errors if item["code"] == "quality.unique")
self.assertEqual(2, unique["affected_rows"])
self.assertEqual([1, 2], unique["row_numbers"])
self.assertNotIn("unknown", str(errors))
contract = DatasourceStageValidationResponse.model_validate(result)
self.assertEqual("case-import-v2", contract.policy_version)
self.assertEqual(7, contract.quality.rules_failed)
def test_warning_rules_do_not_block_promotion_readiness(self) -> None:
result = validate_stage(
rows=({"id": 1},),
schema=(DatasourceField("id", "integer", False),),
quality_policy={
"rules": [
{
"id": "large-batch",
"type": "row_count",
"minimum": 2,
"severity": "warning",
}
]
},
)
self.assertTrue(result["valid"])
self.assertEqual([], result["errors"])
self.assertEqual("quality.row_count", result["warnings"][0]["code"])
def test_schema_changes_are_classified_and_can_be_governed(self) -> None:
baseline = (
DatasourceField("id", "integer", False),
DatasourceField("label", "string", False),
)
candidate = (
DatasourceField("id", "number", False),
DatasourceField("note", "string", True),
)
blocked = validate_stage(
rows=(),
schema=candidate,
baseline_schema=baseline,
quality_policy={},
)
self.assertFalse(blocked["valid"])
self.assertEqual("breaking", blocked["schema_change"]["classification"])
self.assertEqual(
{"schema.type_widened", "schema.field_removed", "schema.field_added_nullable"},
{item["code"] for item in blocked["schema_change"]["changes"]},
)
governed = validate_stage(
rows=(),
schema=candidate,
baseline_schema=baseline,
quality_policy={"schema_policy": {"field_removed": "warning"}},
)
self.assertTrue(governed["valid"])
self.assertEqual("warning", governed["schema_change"]["classification"])
def test_malformed_policy_fails_closed_and_hash_is_stable(self) -> None:
policy = {"rules": [{"id": "bad", "type": "unique", "fields": []}]}
first = validate_stage(rows=(), schema=(), quality_policy=policy)
second = validate_stage(rows=(), schema=(), quality_policy=policy)
self.assertFalse(first["valid"])
self.assertEqual("quality.policy_invalid", first["errors"][0]["code"])
self.assertEqual(first["policy_hash"], second["policy_hash"])
def test_diagnostic_row_numbers_are_bounded(self) -> None:
rows = tuple({"id": None} for _ in range(MAX_DIAGNOSTIC_ROWS + 10))
result = validate_stage(
rows=rows,
schema=(DatasourceField("id", "unknown", True),),
quality_policy={"rules": [{"id": "id", "type": "not_null", "field": "id"}]},
)
error = result["errors"][0]
self.assertEqual(MAX_DIAGNOSTIC_ROWS + 10, error["affected_rows"])
self.assertEqual(MAX_DIAGNOSTIC_ROWS, len(error["row_numbers"]))
self.assertTrue(error["row_numbers_truncated"])
if __name__ == "__main__":
unittest.main()