GovOPlaN/govoplan-datasources
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(datasources): govern approvals and retention
Module Package Release / publish-packages (push) Successful in 11s
Details

Browse Source
This commit is contained in:
zemion
2026-08-22 19:37:44 +02:00
parent b54d1919e4
commit 7a7654cc0f
24 changed files with 2753 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_datasources/backend/router.py
+218
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping
from datetime import UTC, datetime
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy.orm import Session
@@ -30,6 +31,8 @@ from govoplan_datasources.backend.schemas import (
DatasourceFreezeRequest,
DatasourceGovernancePayload,
DatasourceGovernanceUpdateRequest,
DatasourceLifecycleEvidenceListResponse,
DatasourceLifecycleEvidenceResponse,
DatasourceListResponse,
DatasourceMaterializationListResponse,
DatasourceMaterializationResponse,
@@ -41,8 +44,13 @@ from govoplan_datasources.backend.schemas import (
DatasourcePreviewDiagnosticResponse,
DatasourcePreviewResponse,
DatasourceResponse,
DatasourceRetentionApplyRequest,
DatasourceRetentionApplyResponse,
DatasourceRetentionCandidateResponse,
DatasourceRetentionPlanResponse,
DatasourceRetireResponse,
DatasourceStageCreateRequest,
DatasourceStageDecisionRequest,
DatasourceStageListResponse,
DatasourceStagePromoteRequest,
DatasourceStagePromoteResponse,
@@ -52,6 +60,7 @@ from govoplan_datasources.backend.service import (
ADMIN_SCOPE,
CATALOGUE_READ_SCOPE,
SOURCE_WRITE_SCOPE,
STAGE_APPROVE_SCOPE,
STAGE_WRITE_SCOPE,
SqlDatasourceProvider,
)
@@ -236,6 +245,56 @@ def api_create_stage(
"schema_classification": _safe_mapping(
stage.validation.get("schema_change")
).get("classification"),
"approval_state": stage.approval.get("state"),
"approval_policy_hash": stage.approval.get("policy_hash"),
},
)
session.commit()
return _stage_response(stage)
@router.post(
"/stages/{stage_id}/decision",
response_model=DatasourceStageResponse,
)
def api_decide_stage(
stage_id: str,
payload: DatasourceStageDecisionRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DatasourceStageResponse:
_require_any_scope(principal, STAGE_APPROVE_SCOPE, ADMIN_SCOPE)
try:
stage, evidence_hash, replayed = _provider().decide_stage(
session,
principal,
stage_ref=f"stage:{stage_id}",
decision=payload.decision,
reason=payload.reason,
expected_policy_hash=payload.expected_policy_hash,
expected_subject_digest=payload.expected_subject_digest,
)
except DatasourceError as exc:
raise _http_error(exc) from exc
action = (
"datasources.stage.approved"
if payload.decision == "approve"
else "datasources.stage.rejected"
)
_audit(
session,
principal,
action=action,
object_type="datasource_stage",
object_id=stage.ref,
details={
"decision": payload.decision,
"resulting_state": stage.state,
"approval_count": stage.approval.get("approval_count"),
"required_approvals": stage.approval.get("required_approvals"),
"policy_hash": stage.approval.get("policy_hash"),
"evidence_hash": evidence_hash,
"replayed": replayed,
},
)
session.commit()
@@ -277,6 +336,12 @@ def api_promote_stage(
"quality_policy_hash": _safe_mapping(
materialization.provenance.get("stage_validation")
).get("policy_hash"),
"approval_policy_hash": _safe_mapping(
materialization.provenance.get("stage_approval")
).get("policy_hash"),
"promotion_evidence_hash": materialization.provenance.get(
"promotion_evidence_hash"
),
},
)
session.commit()
@@ -286,6 +351,100 @@ def api_promote_stage(
)
@router.get(
"/lifecycle-evidence",
response_model=DatasourceLifecycleEvidenceListResponse,
)
def api_list_lifecycle_evidence(
subject_ref: str | None = Query(default=None, max_length=160),
limit: int = Query(default=200, ge=1, le=500),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DatasourceLifecycleEvidenceListResponse:
_require_any_scope(principal, CATALOGUE_READ_SCOPE, ADMIN_SCOPE)
try:
rows = _provider().list_lifecycle_evidence(
session,
principal,
subject_ref=subject_ref,
limit=limit,
)
except DatasourceError as exc:
raise _http_error(exc) from exc
return DatasourceLifecycleEvidenceListResponse(
evidence=[_evidence_response(item) for item in rows]
)
@router.get(
"/retention/plan",
response_model=DatasourceRetentionPlanResponse,
)
def api_preview_retention(
as_of: datetime | None = Query(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DatasourceRetentionPlanResponse:
_require_any_scope(principal, ADMIN_SCOPE)
effective_at = as_of or datetime.now(UTC)
try:
plan = _provider().preview_retention(
session,
principal,
as_of=effective_at,
)
except DatasourceError as exc:
raise _http_error(exc) from exc
return DatasourceRetentionPlanResponse(
as_of=plan.as_of.isoformat(),
plan_hash=plan.plan_hash,
candidates=[
DatasourceRetentionCandidateResponse(**item.to_dict())
for item in plan.candidates
],
)
@router.post(
"/retention/apply",
response_model=DatasourceRetentionApplyResponse,
)
def api_apply_retention(
payload: DatasourceRetentionApplyRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DatasourceRetentionApplyResponse:
_require_any_scope(principal, ADMIN_SCOPE)
try:
disposed, evidence_hashes = _provider().apply_retention(
session,
principal,
as_of=payload.as_of,
plan_hash=payload.plan_hash,
target_refs=payload.target_refs,
)
except DatasourceError as exc:
raise _http_error(exc) from exc
_audit(
session,
principal,
action="datasources.retention.applied",
object_type="datasource_retention_plan",
object_id=payload.plan_hash,
details={
"as_of": payload.as_of.isoformat(),
"disposed_refs": list(disposed),
"evidence_hashes": list(evidence_hashes),
},
)
session.commit()
return DatasourceRetentionApplyResponse(
plan_hash=payload.plan_hash,
disposed_refs=list(disposed),
evidence_hashes=list(evidence_hashes),
)
@router.get("", response_model=DatasourceListResponse)
def api_list_datasources(
query: str = Query(default="", max_length=200),
@@ -507,6 +666,46 @@ def api_refresh_datasource(
)
@router.post(
"/{datasource_id}/refresh/stage",
response_model=DatasourceStageResponse,
status_code=status.HTTP_201_CREATED,
)
def api_prepare_refresh(
datasource_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DatasourceStageResponse:
_require_any_scope(
principal,
SOURCE_WRITE_SCOPE,
STAGE_WRITE_SCOPE,
ADMIN_SCOPE,
)
try:
stage = _provider().prepare_refresh(
session,
principal,
datasource_ref=f"datasource:{datasource_id}",
)
except DatasourceError as exc:
raise _http_error(exc) from exc
_audit(
session,
principal,
action="datasources.refresh.staged",
object_type="datasource_stage",
object_id=stage.ref,
details={
"datasource_ref": stage.target_datasource_ref,
"approval_state": stage.approval.get("state"),
"quality_policy_hash": stage.validation.get("policy_hash"),
},
)
session.commit()
return _stage_response(stage)
@router.post(
"/{datasource_id}/freeze",
response_model=DatasourceMaterializationResponse,
@@ -635,6 +834,8 @@ def _materialization_response(
item.source_timestamp.isoformat() if item.source_timestamp else None
),
created_at=item.created_at.isoformat() if item.created_at else None,
disposed_at=item.disposed_at.isoformat() if item.disposed_at else None,
disposition=dict(item.disposition),
provenance=dict(item.provenance),
metadata=dict(item.metadata),
governance=item.governance.to_dict(),
@@ -664,6 +865,7 @@ def _stage_response(item: DatasourceStage) -> DatasourceStageResponse:
row_count=item.row_count,
byte_count=item.byte_count,
validation=dict(item.validation),
approval=dict(item.approval),
created_at=item.created_at.isoformat() if item.created_at else None,
promoted_at=item.promoted_at.isoformat() if item.promoted_at else None,
promoted_materialization_ref=item.promoted_materialization_ref,
@@ -719,6 +921,22 @@ def _origin_response(item: DatasourceOrigin) -> DatasourceOriginResponse:
)
def _evidence_response(item) -> DatasourceLifecycleEvidenceResponse:
return DatasourceLifecycleEvidenceResponse(
ref=f"lifecycle-evidence:{item.id}",
subject_ref=item.subject_ref,
event_type=item.event_type,
occurred_at=item.occurred_at.isoformat(),
actor_ref=item.actor_ref,
policy_version=item.policy_version,
policy_hash=item.policy_hash,
subject_digest=item.subject_digest,
previous_event_hash=item.previous_event_hash,
event_hash=item.event_hash,
details=dict(item.details_),
)
def _governance(
payload: DatasourceGovernancePayload | None,
) -> DatasourceGovernance | None: