feat(datasources): govern approvals and retention

src/govoplan_datasources/backend/service.py

@@ -5,6 +5,7 @@ import json
 import re
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, replace
+from datetime import datetime
 from typing import Any, cast
 
 from sqlalchemy import exists, func, or_, select, text
@@ -46,6 +47,18 @@ from govoplan_datasources.backend.db.models import (
     DatasourceRecord,
     DatasourceStageRecord,
 )
+from govoplan_datasources.backend.governance import (
+    RetentionPlan,
+    apply_retention_plan,
+    build_retention_plan,
+    decide_stage,
+    ensure_stage_approval_current,
+    initialize_stage_approval,
+    list_lifecycle_evidence,
+    normalize_approval_policy,
+    normalize_retention_policy,
+    record_lifecycle_evidence,
+)
 from govoplan_datasources.backend.payloads import (
     DatasourcePayloadBackend,
     PayloadBackendRegistry,
@@ -77,6 +90,7 @@ from govoplan_datasources.backend.visibility import (
 CATALOGUE_READ_SCOPE = "datasources:catalogue:read"
 SOURCE_WRITE_SCOPE = "datasources:source:write"
 STAGE_WRITE_SCOPE = "datasources:stage:write"
+STAGE_APPROVE_SCOPE = "datasources:stage:approve"
 ADMIN_SCOPE = "datasources:source:admin"
 
 
@@ -480,6 +494,10 @@ class SqlDatasourceProvider:
                 else "This datasource has no materialized state."
             )
             raise DatasourceUnavailableError(message)
+        if materialization.disposed_at is not None:
+            raise DatasourceUnavailableError(
+                "This materialization payload was disposed under its retention policy."
+            )
         source_schema = _fields(materialization.schema_)
         snapshot_governance = DatasourceGovernance.from_mapping(
             materialization.governance_snapshot_
@@ -721,6 +739,8 @@ class SqlDatasourceProvider:
                 governance.visibility_policy,
                 schema=schema,
             ),
+            approval_policy=normalize_approval_policy(governance.approval_policy),
+            retention_policy=normalize_retention_policy(governance.retention_policy),
         )
         fingerprint = fingerprint_rows(rows, schema)
         validation = validate_stage(
@@ -738,7 +758,7 @@ class SqlDatasourceProvider:
             kind=stage.kind,
             mode=stage.mode,
             shape=stage.shape,
-            state="ready" if validation["valid"] else "invalid",
+            state="invalid",
             provider=_clean_optional(stage.provider),
             provider_ref=_clean_optional(stage.provider_ref),
             schema_=[field_payload(field) for field in schema],
@@ -754,8 +774,91 @@ class SqlDatasourceProvider:
         )
         db.add(item)
         db.flush()
+        approval = initialize_stage_approval(item, created_at=item.created_at)
+        record_lifecycle_evidence(
+            db,
+            tenant_id=api_principal.tenant_id,
+            subject_ref=_stage_ref(item.id),
+            event_type="stage.validated",
+            actor_ref=_actor_id(api_principal),
+            subject_digest=str(approval["subject_digest"]),
+            policy_version=str(validation.get("policy_version") or "1"),
+            policy_hash=str(validation.get("policy_hash") or ""),
+            details={
+                "validation_valid": validation.get("valid"),
+                "schema_classification": (
+                    validation.get("schema_change", {}).get("classification")
+                    if isinstance(validation.get("schema_change"), Mapping)
+                    else None
+                ),
+                "approval_required": approval.get("state") != "not_required",
+                "approval_policy_hash": approval.get("policy_hash"),
+            },
+            occurred_at=item.created_at,
+        )
+        db.flush()
         return _stage_dto(item)
 
+    def decide_stage(
+        self,
+        session: object,
+        principal: object,
+        *,
+        stage_ref: str,
+        decision: str,
+        reason: str,
+        expected_policy_hash: str,
+        expected_subject_digest: str,
+    ) -> tuple[DatasourceStage, str | None, bool]:
+        db, api_principal = _context(session, principal, STAGE_APPROVE_SCOPE)
+        actor_ref = _actor_id(api_principal)
+        if actor_ref is None:
+            raise DatasourceValidationError(
+                "An attributable account is required for datasource approval."
+            )
+        stage = _required_stage(
+            db,
+            tenant_id=api_principal.tenant_id,
+            stage_ref=stage_ref,
+            for_update=True,
+        )
+        approval, replayed = decide_stage(
+            stage,
+            actor_ref=actor_ref,
+            actor_scopes=tuple(api_principal.scopes),
+            decision=decision,
+            reason=reason,
+            expected_policy_hash=expected_policy_hash,
+            expected_subject_digest=expected_subject_digest,
+        )
+        evidence_hash = None
+        if not replayed:
+            policy = approval.get("policy")
+            policy_mapping = policy if isinstance(policy, Mapping) else {}
+            evidence = record_lifecycle_evidence(
+                db,
+                tenant_id=api_principal.tenant_id,
+                subject_ref=_stage_ref(stage.id),
+                event_type=(
+                    "stage.approved" if decision == "approve" else "stage.rejected"
+                ),
+                actor_ref=actor_ref,
+                subject_digest=str(approval["subject_digest"]),
+                policy_version=str(policy_mapping.get("version") or "1"),
+                policy_hash=str(approval["policy_hash"]),
+                details={
+                    "decision": decision,
+                    "reason": reason.strip(),
+                    "approval_count": approval.get("approval_count"),
+                    "required_approvals": approval.get("required_approvals"),
+                    "resulting_state": approval.get("state"),
+                    "authority_scopes": sorted(api_principal.scopes),
+                },
+            )
+            evidence_hash = evidence.event_hash
+        db.flush()
+        return _stage_dto(stage), evidence_hash, replayed
+
     def promote_stage(
         self,
         session: object,
@@ -787,6 +890,15 @@ class SqlDatasourceProvider:
             or datasource.deleted_at is not None
         ):
             datasource = None
+        current_governance = (
+            _datasource_governance(datasource)
+            if datasource is not None
+            else DatasourceGovernance.from_mapping(stage.governance_)
+        )
+        ensure_stage_approval_current(
+            stage,
+            current_policy=current_governance.approval_policy,
+        )
         if datasource is None:
             _ensure_source_name_available(
                 db,
@@ -845,6 +957,7 @@ class SqlDatasourceProvider:
                 **dict(stage.provenance_),
                 "stage_ref": _stage_ref(stage.id),
                 "stage_validation": dict(stage.validation_),
+                "stage_approval": dict(stage.approval_),
             },
             metadata=dict(stage.metadata_),
             set_current=True,
@@ -852,9 +965,81 @@ class SqlDatasourceProvider:
         stage.state = "promoted"
         stage.promoted_at = utcnow()
         stage.promoted_materialization_id = materialization.id
+        approval_policy = stage.approval_.get("policy")
+        policy_mapping = (
+            approval_policy if isinstance(approval_policy, Mapping) else {}
+        )
+        promotion_evidence = record_lifecycle_evidence(
+            db,
+            tenant_id=api_principal.tenant_id,
+            subject_ref=_stage_ref(stage.id),
+            event_type="stage.promoted",
+            actor_ref=_actor_id(api_principal),
+            subject_digest=str(stage.approval_.get("subject_digest") or ""),
+            policy_version=str(policy_mapping.get("version") or "1"),
+            policy_hash=str(stage.approval_.get("policy_hash") or ""),
+            details={
+                "datasource_ref": _datasource_ref(datasource.id),
+                "materialization_ref": _materialization_ref(materialization.id),
+                "revision": materialization.revision,
+                "quality_policy_hash": stage.validation_.get("policy_hash"),
+                "approval_count": stage.approval_.get("approval_count"),
+            },
+        )
+        materialization.provenance_ = {
+            **dict(materialization.provenance_),
+            "promotion_evidence_hash": promotion_evidence.event_hash,
+        }
         db.flush()
         return _datasource_dto(datasource), _materialization_dto(materialization)
 
+    def prepare_refresh(
+        self,
+        session: object,
+        principal: object,
+        *,
+        datasource_ref: str,
+    ) -> DatasourceStage:
+        db, api_principal = _context(session, principal, SOURCE_WRITE_SCOPE)
+        item = _required_datasource(
+            db,
+            tenant_id=api_principal.tenant_id,
+            datasource_ref=datasource_ref,
+        )
+        if item.mode != "cached" or not item.provider_ref:
+            raise DatasourceValidationError(
+                "Only cached connector-backed datasources can prepare a refresh."
+            )
+        rows, origin = self._read_origin_all(
+            db,
+            api_principal,
+            origin_ref=item.provider_ref,
+        )
+        return self.create_stage(
+            db,
+            api_principal,
+            stage=DatasourceStageInput(
+                name=item.name,
+                source_name=item.source_name,
+                description=item.description,
+                kind=cast(Any, item.kind),
+                mode="cached",
+                shape=cast(Any, item.shape),
+                rows=tuple(rows),
+                target_datasource_ref=_datasource_ref(item.id),
+                provider=item.provider,
+                provider_ref=item.provider_ref,
+                provenance={
+                    "created_via": "datasources.refresh",
+                    "origin_ref": origin.ref,
+                    "origin_fingerprint": origin.fingerprint,
+                    "origin_schema_version": origin.schema_version,
+                },
+                metadata=dict(item.metadata_),
+                governance=_datasource_governance(item),
+            ),
+        )
+
     def register_origin(
         self,
         session: object,
@@ -973,6 +1158,13 @@ class SqlDatasourceProvider:
             raise DatasourceValidationError(
                 "Only cached connector-backed datasources can be refreshed."
             )
+        approval_policy = normalize_approval_policy(
+            _datasource_governance(item).approval_policy
+        )
+        if approval_policy["required"]:
+            raise DatasourceValidationError(
+                "This datasource requires an approved refresh stage before it can become current."
+            )
         rows, origin = self._read_origin_all(
             db,
             api_principal,
@@ -988,6 +1180,68 @@ class SqlDatasourceProvider:
         )
         return _datasource_dto(item), _materialization_dto(materialization)
 
+    def preview_retention(
+        self,
+        session: object,
+        principal: object,
+        *,
+        as_of: datetime,
+    ) -> RetentionPlan:
+        db, api_principal = _context(session, principal, ADMIN_SCOPE)
+        return build_retention_plan(
+            db,
+            tenant_id=api_principal.tenant_id,
+            as_of=as_of,
+        )
+
+    def apply_retention(
+        self,
+        session: object,
+        principal: object,
+        *,
+        as_of: datetime,
+        plan_hash: str,
+        target_refs: Sequence[str],
+    ) -> tuple[tuple[str, ...], tuple[str, ...]]:
+        db, api_principal = _context(session, principal, ADMIN_SCOPE)
+        actor_ref = _actor_id(api_principal)
+        if actor_ref is None:
+            raise DatasourceValidationError(
+                "An attributable account is required for retention execution."
+            )
+        plan = build_retention_plan(
+            db,
+            tenant_id=api_principal.tenant_id,
+            as_of=as_of,
+        )
+        if plan.plan_hash != plan_hash:
+            raise DatasourceValidationError(
+                "The retention plan changed; preview it again before applying."
+            )
+        return apply_retention_plan(
+            db,
+            tenant_id=api_principal.tenant_id,
+            actor_ref=actor_ref,
+            plan=plan,
+            target_refs=target_refs,
+        )
+
+    def list_lifecycle_evidence(
+        self,
+        session: object,
+        principal: object,
+        *,
+        subject_ref: str | None = None,
+        limit: int = 200,
+    ):
+        db, api_principal = _context(session, principal, CATALOGUE_READ_SCOPE)
+        return list_lifecycle_evidence(
+            db,
+            tenant_id=api_principal.tenant_id,
+            subject_ref=subject_ref,
+            limit=limit,
+        )
+
     def freeze_datasource(
         self,
         session: object,
@@ -1741,16 +1995,18 @@ def _required_stage(
     *,
     tenant_id: str,
     stage_ref: str,
+    for_update: bool = False,
 ) -> DatasourceStageRecord:
     stage_id = _strip_ref(stage_ref, "stage:")
     if stage_id is None:
         raise DatasourceNotFoundError("Datasource stage not found.")
-    item = session.scalar(
-        select(DatasourceStageRecord).where(
+    statement = select(DatasourceStageRecord).where(
             DatasourceStageRecord.id == stage_id,
             DatasourceStageRecord.tenant_id == tenant_id,
         )
-    )
+    if for_update:
+        statement = statement.with_for_update()
+    item = session.scalar(statement)
     if item is None:
         raise DatasourceNotFoundError("Datasource stage not found.")
     return item
@@ -1817,6 +2073,8 @@ def _datasource_governance(item: DatasourceRecord) -> DatasourceGovernance:
             "transfer_agreement_ref": item.transfer_agreement_ref,
             "freshness_policy": item.freshness_policy,
             "quality_policy": item.quality_policy,
+            "approval_policy": item.approval_policy,
+            "retention_policy": item.retention_policy,
             "known_limits": item.known_limits,
             "correction_procedure_ref": item.correction_procedure_ref,
             "affected_refs": item.affected_refs,
@@ -1855,6 +2113,8 @@ def _apply_datasource_governance(
     item.transfer_agreement_ref = governance.transfer_agreement_ref
     item.freshness_policy = dict(governance.freshness_policy)
     item.quality_policy = dict(governance.quality_policy)
+    item.approval_policy = normalize_approval_policy(governance.approval_policy)
+    item.retention_policy = normalize_retention_policy(governance.retention_policy)
     item.known_limits = list(governance.known_limits)
     item.correction_procedure_ref = governance.correction_procedure_ref
     item.affected_refs = list(governance.affected_refs)
@@ -1922,6 +2182,8 @@ def _materialization_dto(
         frozen_label=item.frozen_label,
         source_timestamp=item.source_timestamp,
         created_at=item.created_at,
+        disposed_at=item.disposed_at,
+        disposition=dict(item.disposition_),
         provenance=dict(item.provenance_),
         metadata=dict(item.metadata_),
         governance=DatasourceGovernance.from_mapping(item.governance_snapshot_),
@@ -1947,6 +2209,7 @@ def _stage_dto(item: DatasourceStageRecord) -> DatasourceStage:
         row_count=item.row_count,
         byte_count=item.byte_count,
         validation=dict(item.validation_),
+        approval=dict(item.approval_),
         created_at=item.created_at,
         promoted_at=item.promoted_at,
         promoted_materialization_ref=(
@@ -2689,6 +2952,7 @@ __all__ = [
     "ADMIN_SCOPE",
     "CATALOGUE_READ_SCOPE",
     "SOURCE_WRITE_SCOPE",
+    "STAGE_APPROVE_SCOPE",
     "STAGE_WRITE_SCOPE",
     "SqlDatasourceProvider",
 ]