Enforce quality gates on output publication

src/govoplan_datasources/backend/service.py

@@ -10,6 +10,7 @@ from typing import Any, cast
 from sqlalchemy import exists, func, or_, select
 from sqlalchemy.orm import Session
 
+from govoplan_core.audit.logging import audit_event
 from govoplan_core.auth import ApiPrincipal, has_scope
 from govoplan_core.core.datasources import (
     DatasourceAccessError,
@@ -108,12 +109,37 @@ class SqlDatasourceProvider:
             return existing
 
         actor_id = _actor_id(api_principal)
+        target, governance, baseline_schema = _publication_validation_context(
+            db,
+            tenant_id=api_principal.tenant_id,
+            request=request,
+        )
+        validation = validate_stage(
+            rows=prepared.rows,
+            schema=prepared.schema,
+            quality_policy=governance.quality_policy,
+            baseline_schema=baseline_schema,
+        )
+        if validation["valid"] is not True:
+            errors = validation.get("errors", [])
+            error_codes = ", ".join(
+                str(item.get("code") or "validation.error")
+                for item in errors
+                if isinstance(item, Mapping)
+            )
+            suffix = f" ({error_codes})" if error_codes else ""
+            raise DatasourceValidationError(
+                "Published output failed governed quality or schema validation"
+                f"{suffix}."
+            )
         datasource = _publication_target(
             db,
             tenant_id=api_principal.tenant_id,
             actor_id=actor_id,
             request=request,
             prepared=prepared,
+            target=target,
+            governance=governance,
         )
         materialization = _append_materialization(
             db,
@@ -126,7 +152,10 @@ class SqlDatasourceProvider:
             frozen=request.freeze,
             frozen_label=request.frozen_label,
             source_timestamp=request.source_timestamp,
-            provenance=_publication_provenance(request, prepared),
+            provenance={
+                **_publication_provenance(request, prepared),
+                "publication_validation": validation,
+            },
             metadata=dict(request.metadata),
             set_current=request.set_current,
         )
@@ -139,6 +168,28 @@ class SqlDatasourceProvider:
             request=request,
             prepared=prepared,
         )
+        audit_event(
+            db,
+            tenant_id=api_principal.tenant_id,
+            user_id=getattr(api_principal.user, "id", None)
+            or api_principal.account_id,
+            api_key_id=api_principal.api_key_id,
+            action="datasource.publication.published",
+            object_type="datasource_publication",
+            object_id=publication.id,
+            details={
+                "producer_module": prepared.producer_module,
+                "producer_run_ref": prepared.producer_run_ref,
+                "datasource_ref": _datasource_ref(datasource.id),
+                "materialization_ref": _materialization_ref(materialization.id),
+                "fingerprint": prepared.fingerprint,
+                "row_count": len(prepared.rows),
+                "policy_hash": validation["policy_hash"],
+                "schema_classification": validation["schema_change"][
+                    "classification"
+                ],
+            },
+        )
         return DatasourcePublicationResult(
             ref=_publication_ref(publication.id),
             status=publication.status,
@@ -1607,19 +1658,13 @@ def _publication_target(
     actor_id: str | None,
     request: DatasourcePublicationRequest,
     prepared: _PreparedPublication,
+    target: DatasourceRecord | None,
+    governance: DatasourceGovernance,
 ) -> DatasourceRecord:
-    if request.target_datasource_ref:
-        datasource = _required_datasource(
-            session,
-            tenant_id=tenant_id,
-            datasource_ref=request.target_datasource_ref,
-        )
-        if datasource.mode == "live" or datasource.shape != "tabular":
-            raise DatasourceValidationError(
-                "Produced rows require a static or cached tabular datasource."
-            )
+    if target is not None:
+        datasource = target
         if request.governance is not None:
-            _apply_datasource_governance(datasource, request.governance)
+            _apply_datasource_governance(datasource, governance)
             datasource.updated_by = actor_id
         return datasource
     name = str(request.name or "").strip()
@@ -1665,14 +1710,47 @@ def _publication_target(
     )
     _apply_datasource_governance(
         datasource,
-        request.governance
-        or _default_governance(mode="static", provider_ref=None),
+        governance,
     )
     session.add(datasource)
     session.flush()
     return datasource
 
 
+def _publication_validation_context(
+    session: Session,
+    *,
+    tenant_id: str,
+    request: DatasourcePublicationRequest,
+) -> tuple[
+    DatasourceRecord | None,
+    DatasourceGovernance,
+    tuple[DatasourceField, ...] | None,
+]:
+    if not request.target_datasource_ref:
+        return (
+            None,
+            request.governance
+            or _default_governance(mode="static", provider_ref=None),
+            None,
+        )
+    target = _required_datasource(
+        session,
+        tenant_id=tenant_id,
+        datasource_ref=request.target_datasource_ref,
+        for_update=True,
+    )
+    if target.mode == "live" or target.shape != "tabular":
+        raise DatasourceValidationError(
+            "Produced rows require a static or cached tabular datasource."
+        )
+    return (
+        target,
+        request.governance or _datasource_governance(target),
+        _fields(target.schema_) if target.current_materialization_id else None,
+    )
+
+
 def _publication_provenance(
     request: DatasourcePublicationRequest,
     prepared: _PreparedPublication,
@@ -1710,6 +1788,10 @@ def _create_publication_record(
             "row_count": len(prepared.rows),
             "set_current": request.set_current,
             "frozen": request.freeze,
+            "validation": materialization.provenance_.get(
+                "publication_validation",
+                {},
+            ),
         },
         created_by=actor_id,
     )