feat(datasources): add governed DSAR coverage
This commit is contained in:
@@ -41,3 +41,18 @@ without changing consumers. Larger durable artifact-backed publications remain
|
||||
a later storage-provider slice.
|
||||
|
||||
See [docs/CONCEPT.md](docs/CONCEPT.md) for ownership and lifecycle details.
|
||||
|
||||
## Data-subject requests
|
||||
|
||||
Datasources publishes `privacy.dsar.datasources` for exact catalogue,
|
||||
governance-reference, materialization, payload, stage, and publication
|
||||
references and for minimized operator attribution. It never exports connector
|
||||
references, locators, credentials, arbitrary rows, schemas, validation
|
||||
samples, metadata, provenance bodies, checkpoints, replay material, or hashes.
|
||||
The module does not guess subject identity by scanning schema-dependent tabular
|
||||
payloads; the authoritative source module locates and corrects those facts.
|
||||
|
||||
Unpromoted stages and unreferenced payloads can be deleted idempotently.
|
||||
Published or referenced state, immutable materializations, governance evidence,
|
||||
holds, and operator attribution require data-steward review. Dataflow and
|
||||
Reporting derivatives must be refreshed after the source correction.
|
||||
|
||||
@@ -0,0 +1,743 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.dsar import (
|
||||
DsarErasureActionRef,
|
||||
DsarExecutionResultRef,
|
||||
DsarRecordRef,
|
||||
DsarSubjectRef,
|
||||
dsar_capability_name,
|
||||
)
|
||||
from govoplan_datasources.backend.db.models import (
|
||||
DatasourceGovernanceReferenceRecord,
|
||||
DatasourceMaterializationRecord,
|
||||
DatasourcePayloadRecord,
|
||||
DatasourcePublicationRecord,
|
||||
DatasourceRecord,
|
||||
DatasourceStageRecord,
|
||||
)
|
||||
|
||||
|
||||
DATASOURCES_DSAR_CAPABILITY = dsar_capability_name("datasources")
|
||||
_MAX_RECORDS = 5_000
|
||||
_CONFLICT = object()
|
||||
_DIRECT_ALIASES = {
|
||||
"datasource_id": ("datasources.datasource", "datasources.catalogue"),
|
||||
"governance_reference_id": ("datasources.governance_reference",),
|
||||
"materialization_id": ("datasources.materialization",),
|
||||
"payload_id": ("datasources.payload",),
|
||||
"stage_id": ("datasources.stage",),
|
||||
"publication_id": ("datasources.publication",),
|
||||
}
|
||||
_RESOURCE_MODELS = {
|
||||
"datasource": DatasourceRecord,
|
||||
"datasource_governance_reference": DatasourceGovernanceReferenceRecord,
|
||||
"datasource_materialization": DatasourceMaterializationRecord,
|
||||
"datasource_payload": DatasourcePayloadRecord,
|
||||
"datasource_stage": DatasourceStageRecord,
|
||||
"datasource_publication": DatasourcePublicationRecord,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Selectors:
|
||||
account_id: str | None
|
||||
identity_id: str | None
|
||||
membership_id: str | None
|
||||
direct: dict[str, str]
|
||||
|
||||
@property
|
||||
def actor_ids(self) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
value
|
||||
for value in (self.account_id, self.identity_id, self.membership_id)
|
||||
if value
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Match:
|
||||
resource_type: str
|
||||
row: Any
|
||||
category: str
|
||||
|
||||
|
||||
class DatasourcesDsarProvider:
|
||||
provider_id = "datasources"
|
||||
module_id = "datasources"
|
||||
|
||||
def search_subject(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
subject: DsarSubjectRef,
|
||||
) -> Sequence[DsarRecordRef]:
|
||||
db = _session(session)
|
||||
selectors = _selectors(subject)
|
||||
if selectors is None or not (selectors.actor_ids or selectors.direct):
|
||||
return ()
|
||||
direct = _direct_matches(db, tenant_id=tenant_id, selectors=selectors)
|
||||
if direct is None:
|
||||
return ()
|
||||
if direct:
|
||||
if selectors.actor_ids and not any(
|
||||
_correlates(match, selectors.actor_ids) for match in direct
|
||||
):
|
||||
return ()
|
||||
matches = direct
|
||||
else:
|
||||
matches = _canonical_matches(
|
||||
db,
|
||||
tenant_id=tenant_id,
|
||||
actor_ids=selectors.actor_ids,
|
||||
)
|
||||
|
||||
records: list[DsarRecordRef] = []
|
||||
seen: set[tuple[str, str]] = set()
|
||||
for match in matches:
|
||||
key = (match.resource_type, str(match.row.id))
|
||||
if key in seen:
|
||||
continue
|
||||
if len(records) >= _MAX_RECORDS:
|
||||
raise ValueError(
|
||||
"Datasources DSAR result limit exceeded; narrow the selectors."
|
||||
)
|
||||
seen.add(key)
|
||||
records.append(_record(match))
|
||||
return tuple(records)
|
||||
|
||||
def plan_erasure(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
subject: DsarSubjectRef,
|
||||
records: Sequence[DsarRecordRef],
|
||||
) -> Sequence[DsarErasureActionRef]:
|
||||
del tenant_id
|
||||
_session(session)
|
||||
if _selectors(subject) is None:
|
||||
raise ValueError("Datasources DSAR subject selectors conflict.")
|
||||
actions: list[DsarErasureActionRef] = []
|
||||
for record in records:
|
||||
_validate_record(record)
|
||||
if record.category in {
|
||||
"unpromoted_datasource_stage",
|
||||
"unreferenced_datasource_payload",
|
||||
}:
|
||||
kind = "delete"
|
||||
executable = True
|
||||
rationale = (
|
||||
"Remove transient Datasources content that has not become "
|
||||
"immutable or referenced lifecycle evidence."
|
||||
)
|
||||
elif record.category == "datasource_operator_attribution":
|
||||
kind = "retain"
|
||||
executable = False
|
||||
rationale = record.retention_reason or (
|
||||
"Institutional data operations remain attributable."
|
||||
)
|
||||
else:
|
||||
kind = "manual_review"
|
||||
executable = False
|
||||
rationale = (
|
||||
"A data steward must correct the authoritative source and "
|
||||
"review immutable revisions, holds, consumers, and evidence."
|
||||
)
|
||||
actions.append(
|
||||
DsarErasureActionRef(
|
||||
action_id=(
|
||||
f"datasources:{kind}:{record.resource_type}:"
|
||||
f"{record.resource_id}"
|
||||
),
|
||||
provider_id=self.provider_id,
|
||||
module_id=self.module_id,
|
||||
kind=kind,
|
||||
resource_type=record.resource_type,
|
||||
resource_id=record.resource_id,
|
||||
title=(
|
||||
f"Delete {record.title}"
|
||||
if executable
|
||||
else f"Review {record.title}"
|
||||
),
|
||||
rationale=rationale,
|
||||
executable=executable,
|
||||
irreversible=executable,
|
||||
metadata={"record_category": record.category},
|
||||
)
|
||||
)
|
||||
return tuple(actions)
|
||||
|
||||
def execute_erasure(
|
||||
self,
|
||||
session: object,
|
||||
*,
|
||||
tenant_id: str,
|
||||
subject: DsarSubjectRef,
|
||||
actions: Sequence[DsarErasureActionRef],
|
||||
request_id: str,
|
||||
) -> Sequence[DsarExecutionResultRef]:
|
||||
db = _session(session)
|
||||
selectors = _selectors(subject)
|
||||
if selectors is None:
|
||||
raise ValueError("Datasources DSAR subject selectors conflict.")
|
||||
results: list[DsarExecutionResultRef] = []
|
||||
for action in actions:
|
||||
_validate_action(action)
|
||||
if not action.executable:
|
||||
results.append(
|
||||
DsarExecutionResultRef(
|
||||
action_id=action.action_id,
|
||||
status="blocked",
|
||||
summary=(
|
||||
"Use governed source correction and Datasources "
|
||||
"retention review before changing published state."
|
||||
),
|
||||
evidence={"request_id": request_id},
|
||||
)
|
||||
)
|
||||
continue
|
||||
if action.kind != "delete" or action.resource_type not in {
|
||||
"datasource_stage",
|
||||
"datasource_payload",
|
||||
}:
|
||||
raise ValueError("Datasources DSAR executable action is unsupported.")
|
||||
model = _RESOURCE_MODELS[action.resource_type]
|
||||
row = (
|
||||
db.query(model)
|
||||
.filter(model.tenant_id == tenant_id, model.id == action.resource_id)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
if row is None:
|
||||
status = "unchanged"
|
||||
summary = "Transient Datasources row was already absent."
|
||||
else:
|
||||
match = _Match(action.resource_type, row, "execution")
|
||||
if not (
|
||||
_directly_targets(selectors, match)
|
||||
or _correlates(match, selectors.actor_ids)
|
||||
):
|
||||
raise ValueError(
|
||||
"Datasources DSAR action is not corroborated by the subject."
|
||||
)
|
||||
_assert_deletable(db, resource_type=action.resource_type, row=row)
|
||||
db.delete(row)
|
||||
db.flush()
|
||||
status = "executed"
|
||||
summary = "Transient, unreferenced Datasources row removed."
|
||||
results.append(
|
||||
DsarExecutionResultRef(
|
||||
action_id=action.action_id,
|
||||
status=status,
|
||||
summary=summary,
|
||||
evidence={"request_id": request_id},
|
||||
)
|
||||
)
|
||||
return tuple(results)
|
||||
|
||||
|
||||
def _direct_matches(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
selectors: _Selectors,
|
||||
) -> list[_Match] | None:
|
||||
matches: list[_Match] = []
|
||||
for selector, value in selectors.direct.items():
|
||||
if selector == "datasource_id":
|
||||
datasource = _one(
|
||||
session,
|
||||
DatasourceRecord,
|
||||
tenant_id=tenant_id,
|
||||
field="id",
|
||||
value=value.removeprefix("datasource:"),
|
||||
)
|
||||
if datasource is None:
|
||||
return None
|
||||
current = _datasource_package(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
datasource=datasource,
|
||||
)
|
||||
else:
|
||||
model, resource_type = {
|
||||
"governance_reference_id": (
|
||||
DatasourceGovernanceReferenceRecord,
|
||||
"datasource_governance_reference",
|
||||
),
|
||||
"materialization_id": (
|
||||
DatasourceMaterializationRecord,
|
||||
"datasource_materialization",
|
||||
),
|
||||
"payload_id": (DatasourcePayloadRecord, "datasource_payload"),
|
||||
"stage_id": (DatasourceStageRecord, "datasource_stage"),
|
||||
"publication_id": (
|
||||
DatasourcePublicationRecord,
|
||||
"datasource_publication",
|
||||
),
|
||||
}[selector]
|
||||
row = _one(
|
||||
session,
|
||||
model,
|
||||
tenant_id=tenant_id,
|
||||
field="id",
|
||||
value=_strip_prefix(value),
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
current = [
|
||||
_Match(
|
||||
resource_type, row, _direct_category(session, resource_type, row)
|
||||
)
|
||||
]
|
||||
matches.extend(current)
|
||||
if len(matches) > _MAX_RECORDS:
|
||||
raise ValueError(
|
||||
"Datasources DSAR result limit exceeded; narrow the selectors."
|
||||
)
|
||||
roots = {
|
||||
root
|
||||
for match in matches
|
||||
for root in _root_datasource_ids(session, match)
|
||||
if root
|
||||
}
|
||||
if len(roots) > 1:
|
||||
return None
|
||||
return matches
|
||||
|
||||
|
||||
def _datasource_package(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
datasource: DatasourceRecord,
|
||||
) -> list[_Match]:
|
||||
matches = [_Match("datasource", datasource, "datasource_configuration")]
|
||||
specs = (
|
||||
(
|
||||
DatasourceGovernanceReferenceRecord,
|
||||
"datasource_governance_reference",
|
||||
),
|
||||
(DatasourceMaterializationRecord, "datasource_materialization"),
|
||||
(DatasourceStageRecord, "datasource_stage"),
|
||||
(DatasourcePublicationRecord, "datasource_publication"),
|
||||
)
|
||||
materializations: list[DatasourceMaterializationRecord] = []
|
||||
for model, resource_type in specs:
|
||||
rows = (
|
||||
session.query(model)
|
||||
.filter(
|
||||
model.tenant_id == tenant_id,
|
||||
(
|
||||
model.target_datasource_id == datasource.id
|
||||
if model is DatasourceStageRecord
|
||||
else model.datasource_id == datasource.id
|
||||
),
|
||||
)
|
||||
.order_by(model.id)
|
||||
.limit(_MAX_RECORDS + 1)
|
||||
.all()
|
||||
)
|
||||
if model is DatasourceMaterializationRecord:
|
||||
materializations = rows
|
||||
matches.extend(
|
||||
_Match(
|
||||
resource_type,
|
||||
row,
|
||||
(
|
||||
"datasource_related_stage"
|
||||
if resource_type == "datasource_stage"
|
||||
else _direct_category(session, resource_type, row)
|
||||
),
|
||||
)
|
||||
for row in rows
|
||||
)
|
||||
payload_ids = {row.payload_id for row in materializations if row.payload_id}
|
||||
if payload_ids:
|
||||
payloads = (
|
||||
session.query(DatasourcePayloadRecord)
|
||||
.filter(
|
||||
DatasourcePayloadRecord.tenant_id == tenant_id,
|
||||
DatasourcePayloadRecord.id.in_(payload_ids),
|
||||
)
|
||||
.order_by(DatasourcePayloadRecord.id)
|
||||
.limit(_MAX_RECORDS + 1)
|
||||
.all()
|
||||
)
|
||||
matches.extend(
|
||||
_Match(
|
||||
"datasource_payload",
|
||||
row,
|
||||
_direct_category(session, "datasource_payload", row),
|
||||
)
|
||||
for row in payloads
|
||||
)
|
||||
return matches
|
||||
|
||||
|
||||
def _canonical_matches(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
actor_ids: tuple[str, ...],
|
||||
) -> list[_Match]:
|
||||
if not actor_ids:
|
||||
return []
|
||||
specs = (
|
||||
(
|
||||
DatasourceRecord,
|
||||
or_(
|
||||
DatasourceRecord.created_by.in_(actor_ids),
|
||||
DatasourceRecord.updated_by.in_(actor_ids),
|
||||
),
|
||||
"datasource",
|
||||
),
|
||||
(
|
||||
DatasourceMaterializationRecord,
|
||||
DatasourceMaterializationRecord.created_by.in_(actor_ids),
|
||||
"datasource_materialization",
|
||||
),
|
||||
(
|
||||
DatasourcePayloadRecord,
|
||||
DatasourcePayloadRecord.created_by.in_(actor_ids),
|
||||
"datasource_payload",
|
||||
),
|
||||
(
|
||||
DatasourceStageRecord,
|
||||
DatasourceStageRecord.created_by.in_(actor_ids),
|
||||
"datasource_stage",
|
||||
),
|
||||
(
|
||||
DatasourcePublicationRecord,
|
||||
DatasourcePublicationRecord.created_by.in_(actor_ids),
|
||||
"datasource_publication",
|
||||
),
|
||||
)
|
||||
matches: list[_Match] = []
|
||||
for model, condition, resource_type in specs:
|
||||
rows = (
|
||||
session.query(model)
|
||||
.filter(model.tenant_id == tenant_id, condition)
|
||||
.order_by(model.id)
|
||||
.limit(_MAX_RECORDS + 1)
|
||||
.all()
|
||||
)
|
||||
matches.extend(
|
||||
_Match(resource_type, row, "datasource_operator_attribution")
|
||||
for row in rows
|
||||
)
|
||||
if len(matches) > _MAX_RECORDS:
|
||||
raise ValueError(
|
||||
"Datasources DSAR result limit exceeded; narrow the selectors."
|
||||
)
|
||||
return matches
|
||||
|
||||
|
||||
def _one(
|
||||
session: Session,
|
||||
model: Any,
|
||||
*,
|
||||
tenant_id: str,
|
||||
field: str,
|
||||
value: str,
|
||||
) -> Any | None:
|
||||
return (
|
||||
session.query(model)
|
||||
.filter(model.tenant_id == tenant_id, getattr(model, field) == value)
|
||||
.one_or_none()
|
||||
)
|
||||
|
||||
|
||||
def _direct_category(session: Session, resource_type: str, row: Any) -> str:
|
||||
if resource_type == "datasource_stage" and row.promoted_at is None:
|
||||
return "unpromoted_datasource_stage"
|
||||
if resource_type == "datasource_payload":
|
||||
referenced = (
|
||||
session.query(DatasourceMaterializationRecord.id)
|
||||
.filter(
|
||||
DatasourceMaterializationRecord.tenant_id == row.tenant_id,
|
||||
DatasourceMaterializationRecord.payload_id == row.id,
|
||||
)
|
||||
.limit(1)
|
||||
.count()
|
||||
)
|
||||
if not referenced:
|
||||
return "unreferenced_datasource_payload"
|
||||
return {
|
||||
"datasource": "datasource_configuration",
|
||||
"datasource_governance_reference": "datasource_governance_configuration",
|
||||
"datasource_materialization": "immutable_datasource_materialization",
|
||||
"datasource_payload": "referenced_datasource_payload",
|
||||
"datasource_stage": "promoted_datasource_stage",
|
||||
"datasource_publication": "immutable_datasource_publication",
|
||||
}[resource_type]
|
||||
|
||||
|
||||
def _root_datasource_ids(session: Session, match: _Match) -> set[str]:
|
||||
row = match.row
|
||||
if match.resource_type == "datasource":
|
||||
return {row.id}
|
||||
if match.resource_type == "datasource_stage":
|
||||
return {row.target_datasource_id} if row.target_datasource_id else set()
|
||||
if match.resource_type == "datasource_payload":
|
||||
return {
|
||||
value
|
||||
for (value,) in session.query(DatasourceMaterializationRecord.datasource_id)
|
||||
.filter(
|
||||
DatasourceMaterializationRecord.tenant_id == row.tenant_id,
|
||||
DatasourceMaterializationRecord.payload_id == row.id,
|
||||
)
|
||||
.all()
|
||||
}
|
||||
return {row.datasource_id}
|
||||
|
||||
|
||||
def _correlates(match: _Match, actor_ids: tuple[str, ...]) -> bool:
|
||||
row = match.row
|
||||
return any(
|
||||
str(getattr(row, field, "") or "") in actor_ids
|
||||
for field in ("created_by", "updated_by")
|
||||
)
|
||||
|
||||
|
||||
def _directly_targets(selectors: _Selectors, match: _Match) -> bool:
|
||||
row = match.row
|
||||
selector, field = {
|
||||
"datasource": ("datasource_id", "id"),
|
||||
"datasource_governance_reference": (
|
||||
"governance_reference_id",
|
||||
"id",
|
||||
),
|
||||
"datasource_materialization": ("materialization_id", "id"),
|
||||
"datasource_payload": ("payload_id", "id"),
|
||||
"datasource_stage": ("stage_id", "id"),
|
||||
"datasource_publication": ("publication_id", "id"),
|
||||
}[match.resource_type]
|
||||
value = _strip_prefix(selectors.direct.get(selector, ""))
|
||||
if value == str(getattr(row, field)):
|
||||
return True
|
||||
datasource_selector = _strip_prefix(selectors.direct.get("datasource_id", ""))
|
||||
return bool(
|
||||
datasource_selector
|
||||
and datasource_selector in _root_datasource_ids_for_row(match)
|
||||
)
|
||||
|
||||
|
||||
def _root_datasource_ids_for_row(match: _Match) -> set[str]:
|
||||
row = match.row
|
||||
if match.resource_type == "datasource":
|
||||
return {row.id}
|
||||
if match.resource_type == "datasource_stage":
|
||||
return {row.target_datasource_id} if row.target_datasource_id else set()
|
||||
if match.resource_type == "datasource_payload":
|
||||
return set()
|
||||
return {row.datasource_id}
|
||||
|
||||
|
||||
def _assert_deletable(session: Session, *, resource_type: str, row: Any) -> None:
|
||||
if resource_type == "datasource_stage":
|
||||
if row.promoted_at is not None or row.promoted_materialization_id is not None:
|
||||
raise ValueError("Promoted Datasource stages require manual review.")
|
||||
return
|
||||
referenced = (
|
||||
session.query(DatasourceMaterializationRecord.id)
|
||||
.filter(
|
||||
DatasourceMaterializationRecord.tenant_id == row.tenant_id,
|
||||
DatasourceMaterializationRecord.payload_id == row.id,
|
||||
)
|
||||
.limit(1)
|
||||
.count()
|
||||
)
|
||||
if referenced:
|
||||
raise ValueError("Referenced Datasource payloads require manual review.")
|
||||
|
||||
|
||||
def _record(match: _Match) -> DsarRecordRef:
|
||||
row = match.row
|
||||
immutable = match.category == "datasource_operator_attribution"
|
||||
return DsarRecordRef(
|
||||
provider_id="datasources",
|
||||
module_id="datasources",
|
||||
resource_type=match.resource_type,
|
||||
resource_id=str(row.id),
|
||||
category=match.category,
|
||||
title=_title(match.resource_type),
|
||||
data={
|
||||
key: value
|
||||
for key, value in _record_data(match.resource_type, row).items()
|
||||
if value is not None
|
||||
},
|
||||
observed_at=_observed_at(row),
|
||||
immutable_evidence=immutable,
|
||||
retention_reason=(
|
||||
"Institutional datasource creation, publication, and revision "
|
||||
"activity remains attributable for governance and audit."
|
||||
if immutable
|
||||
else None
|
||||
),
|
||||
source_path="/datasources",
|
||||
)
|
||||
|
||||
|
||||
def _record_data(resource_type: str, row: Any) -> dict[str, object]:
|
||||
if resource_type == "datasource":
|
||||
return {
|
||||
"kind": row.kind,
|
||||
"mode": row.mode,
|
||||
"shape": row.shape,
|
||||
"status": row.status,
|
||||
"schema_version": row.schema_version,
|
||||
"row_count": row.row_count,
|
||||
"byte_count": row.byte_count,
|
||||
"authority_mode": row.authority_mode,
|
||||
"classification": row.classification,
|
||||
"publication_state": row.publication_state,
|
||||
"deleted_at": _iso(row.deleted_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
if resource_type == "datasource_governance_reference":
|
||||
return {"relation": row.relation}
|
||||
if resource_type == "datasource_materialization":
|
||||
return {
|
||||
"revision": row.revision,
|
||||
"state": row.state,
|
||||
"schema_version": row.schema_version,
|
||||
"row_count": row.row_count,
|
||||
"byte_count": row.byte_count,
|
||||
"frozen_at": _iso(row.frozen_at),
|
||||
"source_timestamp": _iso(row.source_timestamp),
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
if resource_type == "datasource_payload":
|
||||
return {
|
||||
"backend": row.backend,
|
||||
"state": row.state,
|
||||
"media_type": row.media_type,
|
||||
"row_count": row.row_count,
|
||||
"byte_count": row.byte_count,
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
if resource_type == "datasource_stage":
|
||||
return {
|
||||
"kind": row.kind,
|
||||
"mode": row.mode,
|
||||
"shape": row.shape,
|
||||
"state": row.state,
|
||||
"row_count": row.row_count,
|
||||
"byte_count": row.byte_count,
|
||||
"promoted": row.promoted_at is not None,
|
||||
"promoted_at": _iso(row.promoted_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
return {
|
||||
"producer_module": row.producer_module,
|
||||
"status": row.status,
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
def _selectors(subject: DsarSubjectRef) -> _Selectors | None:
|
||||
references = subject.external_references
|
||||
account_id = _coalesce(
|
||||
subject.account_id,
|
||||
references.get("datasources.account"),
|
||||
references.get("access.account"),
|
||||
)
|
||||
identity_id = _coalesce(
|
||||
subject.identity_id,
|
||||
references.get("datasources.identity"),
|
||||
references.get("identity.id"),
|
||||
)
|
||||
membership_id = _coalesce(
|
||||
subject.membership_id,
|
||||
references.get("datasources.membership"),
|
||||
references.get("tenancy.membership"),
|
||||
)
|
||||
if any(value is _CONFLICT for value in (account_id, identity_id, membership_id)):
|
||||
return None
|
||||
direct: dict[str, str] = {}
|
||||
for selector, aliases in _DIRECT_ALIASES.items():
|
||||
value = _coalesce(*(references.get(alias) for alias in aliases))
|
||||
if value is _CONFLICT:
|
||||
return None
|
||||
if value:
|
||||
direct[selector] = str(value)
|
||||
return _Selectors(
|
||||
account_id=_optional(account_id),
|
||||
identity_id=_optional(identity_id),
|
||||
membership_id=_optional(membership_id),
|
||||
direct=direct,
|
||||
)
|
||||
|
||||
|
||||
def _coalesce(*values: str | None) -> str | None | object:
|
||||
normalized = {str(value).strip() for value in values if str(value or "").strip()}
|
||||
if len(normalized) > 1:
|
||||
return _CONFLICT
|
||||
return next(iter(normalized), None)
|
||||
|
||||
|
||||
def _optional(value: object) -> str | None:
|
||||
return value if isinstance(value, str) and value else None
|
||||
|
||||
|
||||
def _strip_prefix(value: str) -> str:
|
||||
return value.partition(":")[2] if ":" in value else value
|
||||
|
||||
|
||||
def _title(resource_type: str) -> str:
|
||||
return resource_type.replace("_", " ").title()
|
||||
|
||||
|
||||
def _observed_at(row: Any) -> datetime | None:
|
||||
for field in ("promoted_at", "source_timestamp", "updated_at", "created_at"):
|
||||
value = getattr(row, field, None)
|
||||
if isinstance(value, datetime):
|
||||
return _aware(value)
|
||||
return None
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
aware = _aware(value)
|
||||
return aware.isoformat() if aware else None
|
||||
|
||||
|
||||
def _aware(value: datetime | None) -> datetime | None:
|
||||
if value is None or value.tzinfo is not None:
|
||||
return value
|
||||
return value.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _session(value: object) -> Session:
|
||||
if not isinstance(value, Session):
|
||||
raise TypeError("Datasources DSAR requires a SQLAlchemy Session.")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_record(record: DsarRecordRef) -> None:
|
||||
if record.provider_id != "datasources" or record.module_id != "datasources":
|
||||
raise ValueError("Datasources DSAR cannot plan a foreign provider record.")
|
||||
if record.resource_type not in _RESOURCE_MODELS or not record.resource_id:
|
||||
raise ValueError("Datasources DSAR record identity is invalid.")
|
||||
|
||||
|
||||
def _validate_action(action: DsarErasureActionRef) -> None:
|
||||
if action.provider_id != "datasources" or action.module_id != "datasources":
|
||||
raise ValueError("Datasources DSAR cannot execute a foreign provider action.")
|
||||
if action.resource_type not in _RESOURCE_MODELS or not action.action_id.startswith(
|
||||
"datasources:"
|
||||
):
|
||||
raise ValueError("Datasources DSAR action identity is invalid.")
|
||||
|
||||
|
||||
__all__ = ["DATASOURCES_DSAR_CAPABILITY", "DatasourcesDsarProvider"]
|
||||
@@ -17,6 +17,7 @@ from govoplan_core.core.module_guards import (
|
||||
persistent_table_uninstall_guard,
|
||||
)
|
||||
from govoplan_core.core.modules import (
|
||||
CapabilityDocumentation,
|
||||
DocumentationLink,
|
||||
DocumentationTopic,
|
||||
FrontendModule,
|
||||
@@ -40,6 +41,10 @@ from govoplan_core.core.search import SearchSourceProviderRegistration
|
||||
from govoplan_core.core.views import ViewSurface
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_datasources.backend.db import models as datasource_models
|
||||
from govoplan_datasources.backend.dsar_provider import (
|
||||
DATASOURCES_DSAR_CAPABILITY,
|
||||
DatasourcesDsarProvider,
|
||||
)
|
||||
from govoplan_datasources.backend.search_source import (
|
||||
create_datasources_search_source,
|
||||
)
|
||||
@@ -180,6 +185,11 @@ def _provider(context: ModuleContext) -> SqlDatasourceProvider:
|
||||
return SqlDatasourceProvider(registry=context.registry)
|
||||
|
||||
|
||||
def _dsar_provider(context: ModuleContext) -> DatasourcesDsarProvider:
|
||||
del context
|
||||
return DatasourcesDsarProvider()
|
||||
|
||||
|
||||
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
|
||||
return {
|
||||
"datasources": (
|
||||
@@ -252,6 +262,7 @@ manifest = ModuleManifest(
|
||||
name="datasources.publication",
|
||||
version=DATASOURCE_INTERFACE_VERSION,
|
||||
),
|
||||
ModuleInterfaceProvider(name=DATASOURCES_DSAR_CAPABILITY, version="0.1.0"),
|
||||
),
|
||||
requires_interfaces=(
|
||||
ModuleInterfaceRequirement(
|
||||
@@ -305,17 +316,56 @@ manifest = ModuleManifest(
|
||||
label="i18n:govoplan-core.product_area.data_assurance",
|
||||
icon="database-zap",
|
||||
description="i18n:govoplan-core.product_area.data_assurance_description",
|
||||
surface_ids=("datasources.nav.datasources", "datasources.route.datasources"),
|
||||
surface_ids=(
|
||||
"datasources.nav.datasources",
|
||||
"datasources.route.datasources",
|
||||
),
|
||||
order=60,
|
||||
),
|
||||
),
|
||||
view_surfaces=(
|
||||
ViewSurface(id="datasources.page", module_id=MODULE_ID, kind="route", label="Datasources", order=70),
|
||||
ViewSurface(id="datasources.catalogue", module_id=MODULE_ID, kind="section", label="Datasource catalogue", order=10),
|
||||
ViewSurface(id="datasources.staging", module_id=MODULE_ID, kind="section", label="Datasource staging", order=20),
|
||||
ViewSurface(id="datasources.origins", module_id=MODULE_ID, kind="section", label="Datasource origins", order=30),
|
||||
ViewSurface(id="datasources.governance", module_id=MODULE_ID, kind="action", label="Datasource governance", order=40),
|
||||
ViewSurface(id="datasources.preview", module_id=MODULE_ID, kind="section", label="Datasource preview and materializations", order=50),
|
||||
ViewSurface(
|
||||
id="datasources.page",
|
||||
module_id=MODULE_ID,
|
||||
kind="route",
|
||||
label="Datasources",
|
||||
order=70,
|
||||
),
|
||||
ViewSurface(
|
||||
id="datasources.catalogue",
|
||||
module_id=MODULE_ID,
|
||||
kind="section",
|
||||
label="Datasource catalogue",
|
||||
order=10,
|
||||
),
|
||||
ViewSurface(
|
||||
id="datasources.staging",
|
||||
module_id=MODULE_ID,
|
||||
kind="section",
|
||||
label="Datasource staging",
|
||||
order=20,
|
||||
),
|
||||
ViewSurface(
|
||||
id="datasources.origins",
|
||||
module_id=MODULE_ID,
|
||||
kind="section",
|
||||
label="Datasource origins",
|
||||
order=30,
|
||||
),
|
||||
ViewSurface(
|
||||
id="datasources.governance",
|
||||
module_id=MODULE_ID,
|
||||
kind="action",
|
||||
label="Datasource governance",
|
||||
order=40,
|
||||
),
|
||||
ViewSurface(
|
||||
id="datasources.preview",
|
||||
module_id=MODULE_ID,
|
||||
kind="section",
|
||||
label="Datasource preview and materializations",
|
||||
order=50,
|
||||
),
|
||||
),
|
||||
),
|
||||
route_factory=_router,
|
||||
@@ -323,6 +373,16 @@ manifest = ModuleManifest(
|
||||
CAPABILITY_DATASOURCE_CATALOGUE: _provider,
|
||||
CAPABILITY_DATASOURCE_LIFECYCLE: _provider,
|
||||
CAPABILITY_DATASOURCE_PUBLICATION: _provider,
|
||||
DATASOURCES_DSAR_CAPABILITY: _dsar_provider,
|
||||
},
|
||||
capability_documentation={
|
||||
DATASOURCES_DSAR_CAPABILITY: CapabilityDocumentation(
|
||||
label="Datasources data-subject request provider",
|
||||
summary="Finds governed datasource copies without exposing credentials or row payloads.",
|
||||
contract_version="0.1.0",
|
||||
documentation_types=("admin", "user"),
|
||||
audience=("privacy_officer", "data_steward", "user"),
|
||||
),
|
||||
},
|
||||
tenant_summary_providers=(_tenant_summary,),
|
||||
search_sources=(
|
||||
@@ -363,6 +423,29 @@ manifest = ModuleManifest(
|
||||
),
|
||||
architecture=ARCHITECTURE,
|
||||
documentation=(
|
||||
DocumentationTopic(
|
||||
id="datasources.data-subject-requests",
|
||||
title="Datasources data-subject requests",
|
||||
summary="Identify governed datasource copies while preserving credential, payload, and immutable-evidence boundaries.",
|
||||
body=(
|
||||
"Datasources matches exact tenant-scoped catalogue, governance-reference, materialization, payload, stage, and publication identifiers plus minimized account, identity, or membership operator attribution. DSAR results never copy connector/provider references, locators, credentials, arbitrary rows, schemas, validation samples, metadata, provenance bodies, checkpoints, idempotency material, or hashes. Arbitrary tabular payloads are not scanned for identifiers because that would be incomplete, schema-dependent, and liable to disclose unrelated people; the authoritative source module must locate and correct subject facts. "
|
||||
"An unpromoted stage or unreferenced payload can be deleted idempotently. Published catalogue state, promoted stages, referenced payloads, immutable materializations, governance references, publications, holds, and operator attribution require data-steward review and source correction. Downstream Dataflow and Reporting outputs must be refreshed after correction."
|
||||
),
|
||||
layer="configured",
|
||||
documentation_types=("admin", "user"),
|
||||
audience=("user", "operator", "module_admin", "data_steward", "auditor"),
|
||||
related_modules=("core", "connectors", "dataflow", "reporting"),
|
||||
order=69,
|
||||
metadata={
|
||||
"seed": True,
|
||||
"help_contexts": [
|
||||
"datasources.data-subject-requests",
|
||||
"datasources.catalogue",
|
||||
"datasources.staging",
|
||||
"datasources.preview",
|
||||
],
|
||||
},
|
||||
),
|
||||
DocumentationTopic(
|
||||
id="datasources.lifecycle",
|
||||
title="Datasource lifecycle",
|
||||
@@ -425,7 +508,14 @@ manifest = ModuleManifest(
|
||||
layer="available",
|
||||
documentation_types=("admin", "user"),
|
||||
audience=("operator", "module_admin", "data_steward", "product_owner"),
|
||||
related_modules=("policy", "organizations", "idm", "dataflow", "reporting", "risk_compliance"),
|
||||
related_modules=(
|
||||
"policy",
|
||||
"organizations",
|
||||
"idm",
|
||||
"dataflow",
|
||||
"reporting",
|
||||
"risk_compliance",
|
||||
),
|
||||
order=71,
|
||||
metadata={
|
||||
"seed": True,
|
||||
@@ -463,7 +553,13 @@ manifest = ModuleManifest(
|
||||
kind="repository",
|
||||
),
|
||||
),
|
||||
related_modules=("policy", "approvals", "audit", "dataflow", "workflow_engine"),
|
||||
related_modules=(
|
||||
"policy",
|
||||
"approvals",
|
||||
"audit",
|
||||
"dataflow",
|
||||
"workflow_engine",
|
||||
),
|
||||
order=72,
|
||||
metadata={
|
||||
"seed": True,
|
||||
@@ -495,7 +591,13 @@ manifest = ModuleManifest(
|
||||
layer="available",
|
||||
documentation_types=("admin", "user"),
|
||||
audience=("operator", "module_admin", "power_user", "product_owner"),
|
||||
related_modules=("connectors", "dataflow", "workflow_engine", "reporting", "audit"),
|
||||
related_modules=(
|
||||
"connectors",
|
||||
"dataflow",
|
||||
"workflow_engine",
|
||||
"reporting",
|
||||
"audit",
|
||||
),
|
||||
order=73,
|
||||
metadata={
|
||||
"seed": True,
|
||||
|
||||
@@ -0,0 +1,581 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.dsar import (
|
||||
DsarErasureActionRef,
|
||||
DsarProvider,
|
||||
DsarRecordRef,
|
||||
DsarSubjectRef,
|
||||
)
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_core.privacy.dsar_workflow import (
|
||||
create_data_subject_request,
|
||||
search_data_subject_request,
|
||||
)
|
||||
from govoplan_datasources.backend.db.models import (
|
||||
DatasourceGovernanceReferenceRecord,
|
||||
DatasourceMaterializationRecord,
|
||||
DatasourcePayloadRecord,
|
||||
DatasourcePayloadRowRecord,
|
||||
DatasourcePublicationRecord,
|
||||
DatasourceRecord,
|
||||
DatasourceStageRecord,
|
||||
)
|
||||
from govoplan_datasources.backend.dsar_provider import (
|
||||
DATASOURCES_DSAR_CAPABILITY,
|
||||
DatasourcesDsarProvider,
|
||||
)
|
||||
from govoplan_datasources.backend.manifest import manifest
|
||||
|
||||
|
||||
NOW = datetime(2026, 8, 21, 19, 0, tzinfo=UTC)
|
||||
SECRET = "private-datasource-detail-do-not-export"
|
||||
|
||||
|
||||
class _Registry:
|
||||
def __init__(
|
||||
self,
|
||||
provider: DatasourcesDsarProvider,
|
||||
*,
|
||||
active: bool = True,
|
||||
) -> None:
|
||||
self.provider = provider
|
||||
self.active = active
|
||||
|
||||
def capability_names(self):
|
||||
return (DATASOURCES_DSAR_CAPABILITY,)
|
||||
|
||||
def capability_owner(self, name):
|
||||
self._assert_capability(name)
|
||||
return "datasources"
|
||||
|
||||
def tenant_entitlement_resolver(self):
|
||||
active = self.active
|
||||
|
||||
class _Resolver:
|
||||
@staticmethod
|
||||
def resolve(session, tenant_id):
|
||||
del session, tenant_id
|
||||
return type(
|
||||
"State",
|
||||
(),
|
||||
{"effective_modules": ("datasources",) if active else ()},
|
||||
)()
|
||||
|
||||
return _Resolver()
|
||||
|
||||
def require_tenant_capability(self, name, session, **kwargs):
|
||||
del session, kwargs
|
||||
self._assert_capability(name)
|
||||
return self.provider
|
||||
|
||||
def manifests(self):
|
||||
return (type("Manifest", (), {"id": "datasources"})(),)
|
||||
|
||||
@staticmethod
|
||||
def _assert_capability(name: str) -> None:
|
||||
if name != DATASOURCES_DSAR_CAPABILITY:
|
||||
raise KeyError(name)
|
||||
|
||||
|
||||
class DatasourcesDsarProviderTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||
Base.metadata.create_all(self.engine)
|
||||
self.session = Session(self.engine)
|
||||
self.provider = DatasourcesDsarProvider()
|
||||
self.assertIsInstance(self.provider, DsarProvider)
|
||||
self._seed()
|
||||
self.session.commit()
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.session.close()
|
||||
self.engine.dispose()
|
||||
|
||||
def _seed(self) -> None:
|
||||
datasource = self._datasource("datasource-1", "tenant-1")
|
||||
other = self._datasource("datasource-other", "tenant-2")
|
||||
self.session.add_all((datasource, other))
|
||||
self.session.flush()
|
||||
referenced_payload = self._payload(
|
||||
"payload-referenced",
|
||||
"tenant-1",
|
||||
created_by="account-1",
|
||||
)
|
||||
orphan_payload = self._payload(
|
||||
"payload-orphan",
|
||||
"tenant-1",
|
||||
created_by="account-1",
|
||||
)
|
||||
self.session.add_all((referenced_payload, orphan_payload))
|
||||
self.session.flush()
|
||||
self.session.add(
|
||||
DatasourcePayloadRowRecord(
|
||||
payload_id=referenced_payload.id,
|
||||
row_index=0,
|
||||
row_={"secret": SECRET},
|
||||
checksum="a" * 64,
|
||||
)
|
||||
)
|
||||
materialization = DatasourceMaterializationRecord(
|
||||
id="materialization-1",
|
||||
tenant_id="tenant-1",
|
||||
datasource_id=datasource.id,
|
||||
revision=1,
|
||||
state="published",
|
||||
schema_version=1,
|
||||
schema_=[{"secret": SECRET}],
|
||||
payload_id=referenced_payload.id,
|
||||
payload_checksum="b" * 64,
|
||||
rows=[{"secret": SECRET}],
|
||||
fingerprint="c" * 64,
|
||||
row_count=1,
|
||||
byte_count=100,
|
||||
source_timestamp=NOW,
|
||||
provenance_={"secret": SECRET},
|
||||
metadata_={"secret": SECRET},
|
||||
governance_snapshot_={"secret": SECRET},
|
||||
created_by="account-1",
|
||||
)
|
||||
self.session.add(materialization)
|
||||
self.session.flush()
|
||||
datasource.current_materialization_id = materialization.id
|
||||
self.session.add_all(
|
||||
(
|
||||
DatasourceGovernanceReferenceRecord(
|
||||
id="governance-1",
|
||||
tenant_id="tenant-1",
|
||||
datasource_id=datasource.id,
|
||||
relation="authoritative_source",
|
||||
reference=SECRET,
|
||||
),
|
||||
self._stage(
|
||||
"stage-1",
|
||||
target_datasource_id=datasource.id,
|
||||
promoted=False,
|
||||
),
|
||||
self._stage(
|
||||
"stage-promoted",
|
||||
target_datasource_id=datasource.id,
|
||||
promoted=True,
|
||||
),
|
||||
DatasourcePublicationRecord(
|
||||
id="publication-1",
|
||||
tenant_id="tenant-1",
|
||||
producer_module="dataflow",
|
||||
producer_run_ref=SECRET,
|
||||
idempotency_key=SECRET,
|
||||
request_hash="d" * 64,
|
||||
datasource_id=datasource.id,
|
||||
materialization_id=materialization.id,
|
||||
status="published",
|
||||
details_={"secret": SECRET},
|
||||
created_by="account-1",
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _datasource(row_id: str, tenant_id: str) -> DatasourceRecord:
|
||||
return DatasourceRecord(
|
||||
id=row_id,
|
||||
tenant_id=tenant_id,
|
||||
source_name=f"source-{row_id}",
|
||||
name=SECRET,
|
||||
description=SECRET,
|
||||
kind="table",
|
||||
mode="cached",
|
||||
shape="tabular",
|
||||
status="active",
|
||||
provider="connector.provider",
|
||||
provider_ref=SECRET,
|
||||
schema_version=1,
|
||||
schema_=[{"secret": SECRET}],
|
||||
fingerprint="e" * 64,
|
||||
row_count=1,
|
||||
byte_count=100,
|
||||
provenance_={"secret": SECRET},
|
||||
metadata_={"secret": SECRET},
|
||||
owner_ref=SECRET,
|
||||
steward_ref=SECRET,
|
||||
authoritative_source_ref=SECRET,
|
||||
authority_mode="external_mirror",
|
||||
legal_basis_refs=[SECRET],
|
||||
purposes=[SECRET],
|
||||
semantic_definition=SECRET,
|
||||
official_keys=[SECRET],
|
||||
classification="personal",
|
||||
privacy_profile_ref=SECRET,
|
||||
retention_policy_ref=SECRET,
|
||||
hold_refs=[SECRET],
|
||||
publication_state="published",
|
||||
transfer_agreement_ref=SECRET,
|
||||
freshness_policy={"secret": SECRET},
|
||||
quality_policy={"secret": SECRET},
|
||||
known_limits=[SECRET],
|
||||
correction_procedure_ref=SECRET,
|
||||
affected_refs=[SECRET],
|
||||
dependency_refs=[SECRET],
|
||||
created_by="account-1",
|
||||
updated_by="account-1",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _payload(
|
||||
row_id: str,
|
||||
tenant_id: str,
|
||||
*,
|
||||
created_by: str,
|
||||
) -> DatasourcePayloadRecord:
|
||||
return DatasourcePayloadRecord(
|
||||
id=row_id,
|
||||
tenant_id=tenant_id,
|
||||
backend="database_rows",
|
||||
state="published",
|
||||
locator=SECRET,
|
||||
media_type="application/x-ndjson",
|
||||
checksum="f" * 64,
|
||||
row_count=1,
|
||||
byte_count=100,
|
||||
checkpoint_={"secret": SECRET},
|
||||
metadata_={"secret": SECRET},
|
||||
created_by=created_by,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _stage(
|
||||
row_id: str,
|
||||
*,
|
||||
target_datasource_id: str,
|
||||
promoted: bool,
|
||||
) -> DatasourceStageRecord:
|
||||
return DatasourceStageRecord(
|
||||
id=row_id,
|
||||
tenant_id="tenant-1",
|
||||
target_datasource_id=target_datasource_id,
|
||||
name=SECRET,
|
||||
source_name=SECRET,
|
||||
description=SECRET,
|
||||
kind="table",
|
||||
mode="static",
|
||||
shape="tabular",
|
||||
state="promoted" if promoted else "ready",
|
||||
provider="upload",
|
||||
provider_ref=SECRET,
|
||||
schema_=[{"secret": SECRET}],
|
||||
rows=[{"secret": SECRET}],
|
||||
fingerprint="0" * 64,
|
||||
row_count=1,
|
||||
byte_count=100,
|
||||
validation_={"secret": SECRET},
|
||||
provenance_={"secret": SECRET},
|
||||
metadata_={"secret": SECRET},
|
||||
governance_={"secret": SECRET},
|
||||
promoted_at=NOW if promoted else None,
|
||||
promoted_materialization_id="materialization-1" if promoted else None,
|
||||
created_by="account-1",
|
||||
)
|
||||
|
||||
def test_canonical_selector_exports_only_minimized_attribution(self) -> None:
|
||||
records = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=DsarSubjectRef(account_id="account-1"),
|
||||
)
|
||||
|
||||
self.assertEqual(7, len(records))
|
||||
self.assertEqual(
|
||||
{"datasource_operator_attribution"},
|
||||
{record.category for record in records},
|
||||
)
|
||||
exported = json.dumps([record.to_dict() for record in records])
|
||||
self.assertNotIn(SECRET, exported)
|
||||
self.assertNotIn("account-1", exported)
|
||||
self.assertNotIn("datasource-other", exported)
|
||||
|
||||
def test_exact_datasource_returns_minimized_lifecycle_package(self) -> None:
|
||||
records = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=DsarSubjectRef(
|
||||
account_id="account-1",
|
||||
external_references={
|
||||
"datasources.datasource": "datasource:datasource-1"
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
self.assertEqual(7, len(records))
|
||||
self.assertEqual(
|
||||
{
|
||||
"datasource",
|
||||
"datasource_governance_reference",
|
||||
"datasource_materialization",
|
||||
"datasource_payload",
|
||||
"datasource_stage",
|
||||
"datasource_publication",
|
||||
},
|
||||
{record.resource_type for record in records},
|
||||
)
|
||||
exported = json.dumps([record.to_dict() for record in records])
|
||||
self.assertNotIn(SECRET, exported)
|
||||
self.assertNotIn("payload-orphan", exported)
|
||||
actions = self.provider.plan_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=DsarSubjectRef(
|
||||
external_references={"datasources.datasource": "datasource-1"}
|
||||
),
|
||||
records=records,
|
||||
)
|
||||
self.assertEqual({"manual_review"}, {action.kind for action in actions})
|
||||
|
||||
def test_exact_references_conflicts_and_tenants_fail_closed(self) -> None:
|
||||
references = {
|
||||
"datasources.governance_reference": "governance-1",
|
||||
"datasources.materialization": "materialization-1",
|
||||
"datasources.payload": "payload-referenced",
|
||||
"datasources.stage": "stage-1",
|
||||
"datasources.publication": "publication-1",
|
||||
}
|
||||
for key, value in references.items():
|
||||
with self.subTest(key=key):
|
||||
records = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=DsarSubjectRef(external_references={key: value}),
|
||||
)
|
||||
self.assertEqual(1, len(records))
|
||||
|
||||
mismatch = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=DsarSubjectRef(
|
||||
account_id="account-2",
|
||||
external_references={"datasources.stage": "stage-1"},
|
||||
),
|
||||
)
|
||||
wrong_tenant = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-2",
|
||||
subject=DsarSubjectRef(
|
||||
external_references={"datasources.stage": "stage-1"}
|
||||
),
|
||||
)
|
||||
conflict = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=DsarSubjectRef(
|
||||
external_references={
|
||||
"datasources.datasource": "datasource-1",
|
||||
"datasources.catalogue": "different",
|
||||
}
|
||||
),
|
||||
)
|
||||
self.assertEqual((), mismatch)
|
||||
self.assertEqual((), wrong_tenant)
|
||||
self.assertEqual((), conflict)
|
||||
|
||||
def test_transient_deletion_is_safe_and_idempotent(self) -> None:
|
||||
subject = DsarSubjectRef(
|
||||
external_references={
|
||||
"datasources.stage": "stage-1",
|
||||
"datasources.payload": "payload-orphan",
|
||||
}
|
||||
)
|
||||
records = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=subject,
|
||||
)
|
||||
actions = self.provider.plan_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=subject,
|
||||
records=records,
|
||||
)
|
||||
self.assertEqual({"delete"}, {action.kind for action in actions})
|
||||
first = self.provider.execute_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=subject,
|
||||
actions=actions,
|
||||
request_id="dsar-1",
|
||||
)
|
||||
second = self.provider.execute_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=subject,
|
||||
actions=actions,
|
||||
request_id="dsar-1-retry",
|
||||
)
|
||||
self.assertTrue(all(result.status == "executed" for result in first))
|
||||
self.assertTrue(all(result.status == "unchanged" for result in second))
|
||||
self.assertIsNone(self.session.get(DatasourceStageRecord, "stage-1"))
|
||||
self.assertIsNone(self.session.get(DatasourcePayloadRecord, "payload-orphan"))
|
||||
self.assertIsNotNone(
|
||||
self.session.get(DatasourcePayloadRecord, "payload-referenced")
|
||||
)
|
||||
|
||||
def test_published_and_attribution_state_requires_review_or_retention(self) -> None:
|
||||
direct_subject = DsarSubjectRef(
|
||||
external_references={
|
||||
"datasources.materialization": "materialization-1",
|
||||
"datasources.payload": "payload-referenced",
|
||||
"datasources.stage": "stage-promoted",
|
||||
}
|
||||
)
|
||||
direct = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=direct_subject,
|
||||
)
|
||||
direct_actions = self.provider.plan_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=direct_subject,
|
||||
records=direct,
|
||||
)
|
||||
self.assertEqual(
|
||||
{"manual_review"},
|
||||
{action.kind for action in direct_actions},
|
||||
)
|
||||
|
||||
canonical_subject = DsarSubjectRef(account_id="account-1")
|
||||
canonical = self.provider.search_subject(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=canonical_subject,
|
||||
)
|
||||
canonical_actions = self.provider.plan_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=canonical_subject,
|
||||
records=canonical,
|
||||
)
|
||||
self.assertEqual({"retain"}, {action.kind for action in canonical_actions})
|
||||
results = self.provider.execute_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=canonical_subject,
|
||||
actions=canonical_actions,
|
||||
request_id="dsar-2",
|
||||
)
|
||||
self.assertTrue(all(result.status == "blocked" for result in results))
|
||||
|
||||
def test_foreign_records_and_actions_are_rejected(self) -> None:
|
||||
subject = DsarSubjectRef(account_id="account-1")
|
||||
with self.assertRaisesRegex(ValueError, "foreign provider record"):
|
||||
self.provider.plan_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=subject,
|
||||
records=(
|
||||
DsarRecordRef(
|
||||
provider_id="cases",
|
||||
module_id="cases",
|
||||
resource_type="case",
|
||||
resource_id="case-1",
|
||||
category="case",
|
||||
title="Case",
|
||||
),
|
||||
),
|
||||
)
|
||||
with self.assertRaisesRegex(ValueError, "foreign provider action"):
|
||||
self.provider.execute_erasure(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
subject=subject,
|
||||
actions=(
|
||||
DsarErasureActionRef(
|
||||
action_id="cases:delete:case:case-1",
|
||||
provider_id="cases",
|
||||
module_id="cases",
|
||||
kind="delete",
|
||||
resource_type="case",
|
||||
resource_id="case-1",
|
||||
title="Delete case",
|
||||
rationale="Foreign",
|
||||
executable=True,
|
||||
),
|
||||
),
|
||||
request_id="dsar-3",
|
||||
)
|
||||
|
||||
def test_core_workflow_reports_active_and_inactive_provider(self) -> None:
|
||||
row = create_data_subject_request(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
reference="DSAR-DATASOURCES-1",
|
||||
request_kind="access_and_erasure",
|
||||
subject=DsarSubjectRef(account_id="account-1"),
|
||||
purpose="Respond to a verified request.",
|
||||
legal_basis="Article 15 and 17 GDPR",
|
||||
due_at=None,
|
||||
requested_by_account_id="privacy-officer",
|
||||
)
|
||||
self.session.commit()
|
||||
search_data_subject_request(
|
||||
self.session,
|
||||
registry=_Registry(self.provider),
|
||||
row=row,
|
||||
expected_revision=1,
|
||||
)
|
||||
self.assertEqual(
|
||||
[DATASOURCES_DSAR_CAPABILITY],
|
||||
row.coverage["provider_capabilities"],
|
||||
)
|
||||
self.assertEqual(7, row.search_result["record_count"])
|
||||
|
||||
inactive = create_data_subject_request(
|
||||
self.session,
|
||||
tenant_id="tenant-1",
|
||||
reference="DSAR-DATASOURCES-2",
|
||||
request_kind="access",
|
||||
subject=DsarSubjectRef(account_id="account-1"),
|
||||
purpose="Respond to a verified request.",
|
||||
legal_basis="Article 15 GDPR",
|
||||
due_at=None,
|
||||
requested_by_account_id="privacy-officer",
|
||||
)
|
||||
self.session.commit()
|
||||
search_data_subject_request(
|
||||
self.session,
|
||||
registry=_Registry(self.provider, active=False),
|
||||
row=inactive,
|
||||
expected_revision=1,
|
||||
)
|
||||
self.assertEqual([], inactive.coverage["provider_capabilities"])
|
||||
self.assertEqual(
|
||||
[DATASOURCES_DSAR_CAPABILITY],
|
||||
inactive.coverage["inactive_provider_capabilities"],
|
||||
)
|
||||
self.assertEqual(0, inactive.search_result["record_count"])
|
||||
|
||||
def test_manifest_registers_and_documents_capability(self) -> None:
|
||||
self.assertIn(DATASOURCES_DSAR_CAPABILITY, manifest.capability_factories)
|
||||
self.assertIn(
|
||||
DATASOURCES_DSAR_CAPABILITY,
|
||||
manifest.capability_documentation,
|
||||
)
|
||||
self.assertIn(
|
||||
DATASOURCES_DSAR_CAPABILITY,
|
||||
{item.name for item in manifest.provides_interfaces},
|
||||
)
|
||||
self.assertTrue(
|
||||
any(
|
||||
topic.id == "datasources.data-subject-requests"
|
||||
and {"admin", "user"}.issubset(topic.documentation_types)
|
||||
for topic in manifest.documentation
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user