from __future__ import annotations from collections.abc import Sequence from dataclasses import dataclass from datetime import datetime, timezone from typing import Any from sqlalchemy import or_ from sqlalchemy.orm import Session from govoplan_core.core.dsar import ( DsarErasureActionRef, DsarExecutionResultRef, DsarRecordRef, DsarSubjectRef, dsar_capability_name, ) from govoplan_datasources.backend.db.models import ( DatasourceGovernanceReferenceRecord, DatasourceMaterializationRecord, DatasourcePayloadRecord, DatasourcePublicationRecord, DatasourceRecord, DatasourceStageRecord, ) DATASOURCES_DSAR_CAPABILITY = dsar_capability_name("datasources") _MAX_RECORDS = 5_000 _CONFLICT = object() _DIRECT_ALIASES = { "datasource_id": ("datasources.datasource", "datasources.catalogue"), "governance_reference_id": ("datasources.governance_reference",), "materialization_id": ("datasources.materialization",), "payload_id": ("datasources.payload",), "stage_id": ("datasources.stage",), "publication_id": ("datasources.publication",), } _RESOURCE_MODELS = { "datasource": DatasourceRecord, "datasource_governance_reference": DatasourceGovernanceReferenceRecord, "datasource_materialization": DatasourceMaterializationRecord, "datasource_payload": DatasourcePayloadRecord, "datasource_stage": DatasourceStageRecord, "datasource_publication": DatasourcePublicationRecord, } @dataclass(frozen=True, slots=True) class _Selectors: account_id: str | None identity_id: str | None membership_id: str | None direct: dict[str, str] @property def actor_ids(self) -> tuple[str, ...]: return tuple( value for value in (self.account_id, self.identity_id, self.membership_id) if value ) @dataclass(frozen=True, slots=True) class _Match: resource_type: str row: Any category: str class DatasourcesDsarProvider: provider_id = "datasources" module_id = "datasources" def search_subject( self, session: object, *, tenant_id: str, subject: DsarSubjectRef, ) -> Sequence[DsarRecordRef]: db = _session(session) selectors = _selectors(subject) if selectors is None or not (selectors.actor_ids or selectors.direct): return () direct = _direct_matches(db, tenant_id=tenant_id, selectors=selectors) if direct is None: return () if direct: if selectors.actor_ids and not any( _correlates(match, selectors.actor_ids) for match in direct ): return () matches = direct else: matches = _canonical_matches( db, tenant_id=tenant_id, actor_ids=selectors.actor_ids, ) records: list[DsarRecordRef] = [] seen: set[tuple[str, str]] = set() for match in matches: key = (match.resource_type, str(match.row.id)) if key in seen: continue if len(records) >= _MAX_RECORDS: raise ValueError( "Datasources DSAR result limit exceeded; narrow the selectors." ) seen.add(key) records.append(_record(match)) return tuple(records) def plan_erasure( self, session: object, *, tenant_id: str, subject: DsarSubjectRef, records: Sequence[DsarRecordRef], ) -> Sequence[DsarErasureActionRef]: del tenant_id _session(session) if _selectors(subject) is None: raise ValueError("Datasources DSAR subject selectors conflict.") actions: list[DsarErasureActionRef] = [] for record in records: _validate_record(record) if record.category in { "unpromoted_datasource_stage", "unreferenced_datasource_payload", }: kind = "delete" executable = True rationale = ( "Remove transient Datasources content that has not become " "immutable or referenced lifecycle evidence." ) elif record.category == "datasource_operator_attribution": kind = "retain" executable = False rationale = record.retention_reason or ( "Institutional data operations remain attributable." ) else: kind = "manual_review" executable = False rationale = ( "A data steward must correct the authoritative source and " "review immutable revisions, holds, consumers, and evidence." ) actions.append( DsarErasureActionRef( action_id=( f"datasources:{kind}:{record.resource_type}:" f"{record.resource_id}" ), provider_id=self.provider_id, module_id=self.module_id, kind=kind, resource_type=record.resource_type, resource_id=record.resource_id, title=( f"Delete {record.title}" if executable else f"Review {record.title}" ), rationale=rationale, executable=executable, irreversible=executable, metadata={"record_category": record.category}, ) ) return tuple(actions) def execute_erasure( self, session: object, *, tenant_id: str, subject: DsarSubjectRef, actions: Sequence[DsarErasureActionRef], request_id: str, ) -> Sequence[DsarExecutionResultRef]: db = _session(session) selectors = _selectors(subject) if selectors is None: raise ValueError("Datasources DSAR subject selectors conflict.") results: list[DsarExecutionResultRef] = [] for action in actions: _validate_action(action) if not action.executable: results.append( DsarExecutionResultRef( action_id=action.action_id, status="blocked", summary=( "Use governed source correction and Datasources " "retention review before changing published state." ), evidence={"request_id": request_id}, ) ) continue if action.kind != "delete" or action.resource_type not in { "datasource_stage", "datasource_payload", }: raise ValueError("Datasources DSAR executable action is unsupported.") model = _RESOURCE_MODELS[action.resource_type] row = ( db.query(model) .filter(model.tenant_id == tenant_id, model.id == action.resource_id) .with_for_update() .one_or_none() ) if row is None: status = "unchanged" summary = "Transient Datasources row was already absent." else: match = _Match(action.resource_type, row, "execution") if not ( _directly_targets(selectors, match) or _correlates(match, selectors.actor_ids) ): raise ValueError( "Datasources DSAR action is not corroborated by the subject." ) _assert_deletable(db, resource_type=action.resource_type, row=row) db.delete(row) db.flush() status = "executed" summary = "Transient, unreferenced Datasources row removed." results.append( DsarExecutionResultRef( action_id=action.action_id, status=status, summary=summary, evidence={"request_id": request_id}, ) ) return tuple(results) def _direct_matches( session: Session, *, tenant_id: str, selectors: _Selectors, ) -> list[_Match] | None: matches: list[_Match] = [] for selector, value in selectors.direct.items(): if selector == "datasource_id": datasource = _one( session, DatasourceRecord, tenant_id=tenant_id, field="id", value=value.removeprefix("datasource:"), ) if datasource is None: return None current = _datasource_package( session, tenant_id=tenant_id, datasource=datasource, ) else: model, resource_type = { "governance_reference_id": ( DatasourceGovernanceReferenceRecord, "datasource_governance_reference", ), "materialization_id": ( DatasourceMaterializationRecord, "datasource_materialization", ), "payload_id": (DatasourcePayloadRecord, "datasource_payload"), "stage_id": (DatasourceStageRecord, "datasource_stage"), "publication_id": ( DatasourcePublicationRecord, "datasource_publication", ), }[selector] row = _one( session, model, tenant_id=tenant_id, field="id", value=_strip_prefix(value), ) if row is None: return None current = [ _Match( resource_type, row, _direct_category(session, resource_type, row) ) ] matches.extend(current) if len(matches) > _MAX_RECORDS: raise ValueError( "Datasources DSAR result limit exceeded; narrow the selectors." ) roots = { root for match in matches for root in _root_datasource_ids(session, match) if root } if len(roots) > 1: return None return matches def _datasource_package( session: Session, *, tenant_id: str, datasource: DatasourceRecord, ) -> list[_Match]: matches = [_Match("datasource", datasource, "datasource_configuration")] specs = ( ( DatasourceGovernanceReferenceRecord, "datasource_governance_reference", ), (DatasourceMaterializationRecord, "datasource_materialization"), (DatasourceStageRecord, "datasource_stage"), (DatasourcePublicationRecord, "datasource_publication"), ) materializations: list[DatasourceMaterializationRecord] = [] for model, resource_type in specs: rows = ( session.query(model) .filter( model.tenant_id == tenant_id, ( model.target_datasource_id == datasource.id if model is DatasourceStageRecord else model.datasource_id == datasource.id ), ) .order_by(model.id) .limit(_MAX_RECORDS + 1) .all() ) if model is DatasourceMaterializationRecord: materializations = rows matches.extend( _Match( resource_type, row, ( "datasource_related_stage" if resource_type == "datasource_stage" else _direct_category(session, resource_type, row) ), ) for row in rows ) payload_ids = {row.payload_id for row in materializations if row.payload_id} if payload_ids: payloads = ( session.query(DatasourcePayloadRecord) .filter( DatasourcePayloadRecord.tenant_id == tenant_id, DatasourcePayloadRecord.id.in_(payload_ids), ) .order_by(DatasourcePayloadRecord.id) .limit(_MAX_RECORDS + 1) .all() ) matches.extend( _Match( "datasource_payload", row, _direct_category(session, "datasource_payload", row), ) for row in payloads ) return matches def _canonical_matches( session: Session, *, tenant_id: str, actor_ids: tuple[str, ...], ) -> list[_Match]: if not actor_ids: return [] specs = ( ( DatasourceRecord, or_( DatasourceRecord.created_by.in_(actor_ids), DatasourceRecord.updated_by.in_(actor_ids), ), "datasource", ), ( DatasourceMaterializationRecord, DatasourceMaterializationRecord.created_by.in_(actor_ids), "datasource_materialization", ), ( DatasourcePayloadRecord, DatasourcePayloadRecord.created_by.in_(actor_ids), "datasource_payload", ), ( DatasourceStageRecord, DatasourceStageRecord.created_by.in_(actor_ids), "datasource_stage", ), ( DatasourcePublicationRecord, DatasourcePublicationRecord.created_by.in_(actor_ids), "datasource_publication", ), ) matches: list[_Match] = [] for model, condition, resource_type in specs: rows = ( session.query(model) .filter(model.tenant_id == tenant_id, condition) .order_by(model.id) .limit(_MAX_RECORDS + 1) .all() ) matches.extend( _Match(resource_type, row, "datasource_operator_attribution") for row in rows ) if len(matches) > _MAX_RECORDS: raise ValueError( "Datasources DSAR result limit exceeded; narrow the selectors." ) return matches def _one( session: Session, model: Any, *, tenant_id: str, field: str, value: str, ) -> Any | None: return ( session.query(model) .filter(model.tenant_id == tenant_id, getattr(model, field) == value) .one_or_none() ) def _direct_category(session: Session, resource_type: str, row: Any) -> str: if resource_type == "datasource_stage" and row.promoted_at is None: return "unpromoted_datasource_stage" if resource_type == "datasource_payload": referenced = ( session.query(DatasourceMaterializationRecord.id) .filter( DatasourceMaterializationRecord.tenant_id == row.tenant_id, DatasourceMaterializationRecord.payload_id == row.id, ) .limit(1) .count() ) if not referenced: return "unreferenced_datasource_payload" return { "datasource": "datasource_configuration", "datasource_governance_reference": "datasource_governance_configuration", "datasource_materialization": "immutable_datasource_materialization", "datasource_payload": "referenced_datasource_payload", "datasource_stage": "promoted_datasource_stage", "datasource_publication": "immutable_datasource_publication", }[resource_type] def _root_datasource_ids(session: Session, match: _Match) -> set[str]: row = match.row if match.resource_type == "datasource": return {row.id} if match.resource_type == "datasource_stage": return {row.target_datasource_id} if row.target_datasource_id else set() if match.resource_type == "datasource_payload": return { value for (value,) in session.query(DatasourceMaterializationRecord.datasource_id) .filter( DatasourceMaterializationRecord.tenant_id == row.tenant_id, DatasourceMaterializationRecord.payload_id == row.id, ) .all() } return {row.datasource_id} def _correlates(match: _Match, actor_ids: tuple[str, ...]) -> bool: row = match.row return any( str(getattr(row, field, "") or "") in actor_ids for field in ("created_by", "updated_by") ) def _directly_targets(selectors: _Selectors, match: _Match) -> bool: row = match.row selector, field = { "datasource": ("datasource_id", "id"), "datasource_governance_reference": ( "governance_reference_id", "id", ), "datasource_materialization": ("materialization_id", "id"), "datasource_payload": ("payload_id", "id"), "datasource_stage": ("stage_id", "id"), "datasource_publication": ("publication_id", "id"), }[match.resource_type] value = _strip_prefix(selectors.direct.get(selector, "")) if value == str(getattr(row, field)): return True datasource_selector = _strip_prefix(selectors.direct.get("datasource_id", "")) return bool( datasource_selector and datasource_selector in _root_datasource_ids_for_row(match) ) def _root_datasource_ids_for_row(match: _Match) -> set[str]: row = match.row if match.resource_type == "datasource": return {row.id} if match.resource_type == "datasource_stage": return {row.target_datasource_id} if row.target_datasource_id else set() if match.resource_type == "datasource_payload": return set() return {row.datasource_id} def _assert_deletable(session: Session, *, resource_type: str, row: Any) -> None: if resource_type == "datasource_stage": if row.promoted_at is not None or row.promoted_materialization_id is not None: raise ValueError("Promoted Datasource stages require manual review.") return referenced = ( session.query(DatasourceMaterializationRecord.id) .filter( DatasourceMaterializationRecord.tenant_id == row.tenant_id, DatasourceMaterializationRecord.payload_id == row.id, ) .limit(1) .count() ) if referenced: raise ValueError("Referenced Datasource payloads require manual review.") def _record(match: _Match) -> DsarRecordRef: row = match.row immutable = match.category == "datasource_operator_attribution" return DsarRecordRef( provider_id="datasources", module_id="datasources", resource_type=match.resource_type, resource_id=str(row.id), category=match.category, title=_title(match.resource_type), data={ key: value for key, value in _record_data(match.resource_type, row).items() if value is not None }, observed_at=_observed_at(row), immutable_evidence=immutable, retention_reason=( "Institutional datasource creation, publication, and revision " "activity remains attributable for governance and audit." if immutable else None ), source_path="/datasources", ) def _record_data(resource_type: str, row: Any) -> dict[str, object]: if resource_type == "datasource": return { "kind": row.kind, "mode": row.mode, "shape": row.shape, "status": row.status, "schema_version": row.schema_version, "row_count": row.row_count, "byte_count": row.byte_count, "authority_mode": row.authority_mode, "classification": row.classification, "publication_state": row.publication_state, "deleted_at": _iso(row.deleted_at), "created_at": _iso(row.created_at), "updated_at": _iso(row.updated_at), } if resource_type == "datasource_governance_reference": return {"relation": row.relation} if resource_type == "datasource_materialization": return { "revision": row.revision, "state": row.state, "schema_version": row.schema_version, "row_count": row.row_count, "byte_count": row.byte_count, "frozen_at": _iso(row.frozen_at), "source_timestamp": _iso(row.source_timestamp), "created_at": _iso(row.created_at), } if resource_type == "datasource_payload": return { "backend": row.backend, "state": row.state, "media_type": row.media_type, "row_count": row.row_count, "byte_count": row.byte_count, "created_at": _iso(row.created_at), } if resource_type == "datasource_stage": return { "kind": row.kind, "mode": row.mode, "shape": row.shape, "state": row.state, "row_count": row.row_count, "byte_count": row.byte_count, "promoted": row.promoted_at is not None, "promoted_at": _iso(row.promoted_at), "created_at": _iso(row.created_at), } return { "producer_module": row.producer_module, "status": row.status, "created_at": _iso(row.created_at), } def _selectors(subject: DsarSubjectRef) -> _Selectors | None: references = subject.external_references account_id = _coalesce( subject.account_id, references.get("datasources.account"), references.get("access.account"), ) identity_id = _coalesce( subject.identity_id, references.get("datasources.identity"), references.get("identity.id"), ) membership_id = _coalesce( subject.membership_id, references.get("datasources.membership"), references.get("tenancy.membership"), ) if any(value is _CONFLICT for value in (account_id, identity_id, membership_id)): return None direct: dict[str, str] = {} for selector, aliases in _DIRECT_ALIASES.items(): value = _coalesce(*(references.get(alias) for alias in aliases)) if value is _CONFLICT: return None if value: direct[selector] = str(value) return _Selectors( account_id=_optional(account_id), identity_id=_optional(identity_id), membership_id=_optional(membership_id), direct=direct, ) def _coalesce(*values: str | None) -> str | None | object: normalized = {str(value).strip() for value in values if str(value or "").strip()} if len(normalized) > 1: return _CONFLICT return next(iter(normalized), None) def _optional(value: object) -> str | None: return value if isinstance(value, str) and value else None def _strip_prefix(value: str) -> str: return value.partition(":")[2] if ":" in value else value def _title(resource_type: str) -> str: return resource_type.replace("_", " ").title() def _observed_at(row: Any) -> datetime | None: for field in ("promoted_at", "source_timestamp", "updated_at", "created_at"): value = getattr(row, field, None) if isinstance(value, datetime): return _aware(value) return None def _iso(value: datetime | None) -> str | None: aware = _aware(value) return aware.isoformat() if aware else None def _aware(value: datetime | None) -> datetime | None: if value is None or value.tzinfo is not None: return value return value.replace(tzinfo=timezone.utc) def _session(value: object) -> Session: if not isinstance(value, Session): raise TypeError("Datasources DSAR requires a SQLAlchemy Session.") return value def _validate_record(record: DsarRecordRef) -> None: if record.provider_id != "datasources" or record.module_id != "datasources": raise ValueError("Datasources DSAR cannot plan a foreign provider record.") if record.resource_type not in _RESOURCE_MODELS or not record.resource_id: raise ValueError("Datasources DSAR record identity is invalid.") def _validate_action(action: DsarErasureActionRef) -> None: if action.provider_id != "datasources" or action.module_id != "datasources": raise ValueError("Datasources DSAR cannot execute a foreign provider action.") if action.resource_type not in _RESOURCE_MODELS or not action.action_id.startswith( "datasources:" ): raise ValueError("Datasources DSAR action identity is invalid.") __all__ = ["DATASOURCES_DSAR_CAPABILITY", "DatasourcesDsarProvider"]