Enforce quality gates on output publication

This commit is contained in:
2026-08-04 11:33:11 +02:00
parent 075d2fc087
commit 2d93ad07d2
4 changed files with 221 additions and 23 deletions
+5 -4
View File
@@ -406,15 +406,16 @@ manifest = ModuleManifest(
),
DocumentationTopic(
id="datasources.quality-gates",
title="Validate a Datasource stage",
summary="Apply deterministic quality rules and schema-change policy before staged data can become consumable.",
title="Validate staged and produced Datasource revisions",
summary="Apply deterministic quality rules and schema-change policy before staged or produced data can become consumable.",
body=(
"Tabular stages evaluate configured row count, required-field, field-shape, nullability, uniqueness, numeric-range, and bounded "
"referential-set rules. Errors keep the stage inspectable but block promotion; warnings stay visible and permit an explicit "
"promotion. Updates compare the detected schema with the current target and classify each change as compatible, warning, or "
"breaking. Diagnostics expose counts and bounded row numbers, never field values. The policy version and hash, diagnostics, and "
"schema diff are copied into immutable materialization provenance when promotion succeeds. Approval and retention execution are "
"not inferred from arbitrary JSON flags and remain separate governed lifecycle work."
"schema diff are copied into immutable materialization provenance when promotion succeeds. Producer publication applies the same "
"gate before any catalogue effect, retains validation evidence on the immutable output revision, and emits a transactional terminal event. Approval and retention execution "
"are not inferred from arbitrary JSON flags and remain separate governed lifecycle work."
),
layer="available",
documentation_types=("admin", "user"),
+96 -14
View File
@@ -10,6 +10,7 @@ from typing import Any, cast
from sqlalchemy import exists, func, or_, select
from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_event
from govoplan_core.auth import ApiPrincipal, has_scope
from govoplan_core.core.datasources import (
DatasourceAccessError,
@@ -108,12 +109,37 @@ class SqlDatasourceProvider:
return existing
actor_id = _actor_id(api_principal)
target, governance, baseline_schema = _publication_validation_context(
db,
tenant_id=api_principal.tenant_id,
request=request,
)
validation = validate_stage(
rows=prepared.rows,
schema=prepared.schema,
quality_policy=governance.quality_policy,
baseline_schema=baseline_schema,
)
if validation["valid"] is not True:
errors = validation.get("errors", [])
error_codes = ", ".join(
str(item.get("code") or "validation.error")
for item in errors
if isinstance(item, Mapping)
)
suffix = f" ({error_codes})" if error_codes else ""
raise DatasourceValidationError(
"Published output failed governed quality or schema validation"
f"{suffix}."
)
datasource = _publication_target(
db,
tenant_id=api_principal.tenant_id,
actor_id=actor_id,
request=request,
prepared=prepared,
target=target,
governance=governance,
)
materialization = _append_materialization(
db,
@@ -126,7 +152,10 @@ class SqlDatasourceProvider:
frozen=request.freeze,
frozen_label=request.frozen_label,
source_timestamp=request.source_timestamp,
provenance=_publication_provenance(request, prepared),
provenance={
**_publication_provenance(request, prepared),
"publication_validation": validation,
},
metadata=dict(request.metadata),
set_current=request.set_current,
)
@@ -139,6 +168,28 @@ class SqlDatasourceProvider:
request=request,
prepared=prepared,
)
audit_event(
db,
tenant_id=api_principal.tenant_id,
user_id=getattr(api_principal.user, "id", None)
or api_principal.account_id,
api_key_id=api_principal.api_key_id,
action="datasource.publication.published",
object_type="datasource_publication",
object_id=publication.id,
details={
"producer_module": prepared.producer_module,
"producer_run_ref": prepared.producer_run_ref,
"datasource_ref": _datasource_ref(datasource.id),
"materialization_ref": _materialization_ref(materialization.id),
"fingerprint": prepared.fingerprint,
"row_count": len(prepared.rows),
"policy_hash": validation["policy_hash"],
"schema_classification": validation["schema_change"][
"classification"
],
},
)
return DatasourcePublicationResult(
ref=_publication_ref(publication.id),
status=publication.status,
@@ -1607,19 +1658,13 @@ def _publication_target(
actor_id: str | None,
request: DatasourcePublicationRequest,
prepared: _PreparedPublication,
target: DatasourceRecord | None,
governance: DatasourceGovernance,
) -> DatasourceRecord:
if request.target_datasource_ref:
datasource = _required_datasource(
session,
tenant_id=tenant_id,
datasource_ref=request.target_datasource_ref,
)
if datasource.mode == "live" or datasource.shape != "tabular":
raise DatasourceValidationError(
"Produced rows require a static or cached tabular datasource."
)
if target is not None:
datasource = target
if request.governance is not None:
_apply_datasource_governance(datasource, request.governance)
_apply_datasource_governance(datasource, governance)
datasource.updated_by = actor_id
return datasource
name = str(request.name or "").strip()
@@ -1665,14 +1710,47 @@ def _publication_target(
)
_apply_datasource_governance(
datasource,
request.governance
or _default_governance(mode="static", provider_ref=None),
governance,
)
session.add(datasource)
session.flush()
return datasource
def _publication_validation_context(
session: Session,
*,
tenant_id: str,
request: DatasourcePublicationRequest,
) -> tuple[
DatasourceRecord | None,
DatasourceGovernance,
tuple[DatasourceField, ...] | None,
]:
if not request.target_datasource_ref:
return (
None,
request.governance
or _default_governance(mode="static", provider_ref=None),
None,
)
target = _required_datasource(
session,
tenant_id=tenant_id,
datasource_ref=request.target_datasource_ref,
for_update=True,
)
if target.mode == "live" or target.shape != "tabular":
raise DatasourceValidationError(
"Produced rows require a static or cached tabular datasource."
)
return (
target,
request.governance or _datasource_governance(target),
_fields(target.schema_) if target.current_materialization_id else None,
)
def _publication_provenance(
request: DatasourcePublicationRequest,
prepared: _PreparedPublication,
@@ -1710,6 +1788,10 @@ def _create_publication_record(
"row_count": len(prepared.rows),
"set_current": request.set_current,
"frozen": request.freeze,
"validation": materialization.provenance_.get(
"publication_validation",
{},
),
},
created_by=actor_id,
)