GovOPlaN/govoplan-datasources
2
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement governed datasource catalogue metadata

Browse Source
This commit is contained in:
zemion
2026-08-01 17:48:28 +02:00
parent 8da8035693
commit 98d93e707f
14 changed files with 1382 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_datasources/backend/service.py
+192 -4
View File
@@ -7,7 +7,7 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, replace
from typing import Any, cast
from sqlalchemy import func, or_, select
from sqlalchemy import exists, func, or_, select
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, has_scope
@@ -16,6 +16,7 @@ from govoplan_core.core.datasources import (
DatasourceDescriptor,
DatasourceError,
DatasourceField,
DatasourceGovernance,
DatasourceMaterialization,
DatasourceMode,
DatasourceNotFoundError,
@@ -33,6 +34,7 @@ from govoplan_core.core.datasources import (
)
from govoplan_core.db.base import utcnow
from govoplan_datasources.backend.db.models import (
DatasourceGovernanceReferenceRecord,
DatasourceMaterializationRecord,
DatasourcePayloadRecord,
DatasourcePublicationRecord,
@@ -151,6 +153,13 @@ class SqlDatasourceProvider:
*,
query: str = "",
limit: int = 100,
authority_mode: str | None = None,
classification: str | None = None,
publication_state: str | None = None,
owner_ref: str | None = None,
responsible_organization_ref: str | None = None,
affected_ref: str | None = None,
dependency_ref: str | None = None,
) -> Sequence[DatasourceDescriptor]:
db, api_principal = _context(session, principal, CATALOGUE_READ_SCOPE)
statement = (
@@ -172,6 +181,35 @@ class SqlDatasourceProvider:
DatasourceRecord.description.ilike(pattern, escape="\\"),
)
)
for column, value in (
(DatasourceRecord.authority_mode, authority_mode),
(DatasourceRecord.classification, classification),
(DatasourceRecord.publication_state, publication_state),
(DatasourceRecord.owner_ref, owner_ref),
(
DatasourceRecord.responsible_organization_ref,
responsible_organization_ref,
),
):
cleaned = str(value or "").strip()
if cleaned:
statement = statement.where(column == cleaned)
for relation, value in (
("affected", affected_ref),
("depends_on", dependency_ref),
):
cleaned = str(value or "").strip()
if cleaned:
statement = statement.where(
exists().where(
DatasourceGovernanceReferenceRecord.datasource_id
== DatasourceRecord.id,
DatasourceGovernanceReferenceRecord.tenant_id
== api_principal.tenant_id,
DatasourceGovernanceReferenceRecord.relation == relation,
DatasourceGovernanceReferenceRecord.reference == cleaned,
)
)
return tuple(_datasource_dto(item) for item in db.scalars(statement))
def get_datasource(
@@ -378,6 +416,14 @@ class SqlDatasourceProvider:
raise DatasourceValidationError(
"A stage can only update a datasource with the same mode and shape."
)
governance = (
stage.governance
or (_datasource_governance(target) if target is not None else None)
or _default_governance(
mode=stage.mode,
provider_ref=stage.provider_ref,
)
)
rows = normalize_rows(stage.rows)
schema = infer_schema(rows)
fingerprint = fingerprint_rows(rows, schema)
@@ -401,6 +447,7 @@ class SqlDatasourceProvider:
validation_={"valid": True, "errors": [], "warnings": []},
provenance_=dict(stage.provenance),
metadata_=dict(stage.metadata),
governance_=governance.to_dict(),
created_by=_actor_id(api_principal),
)
db.add(item)
@@ -461,6 +508,10 @@ class SqlDatasourceProvider:
created_by=_actor_id(api_principal),
updated_by=_actor_id(api_principal),
)
_apply_datasource_governance(
datasource,
DatasourceGovernance.from_mapping(stage.governance_),
)
db.add(datasource)
db.flush()
stage.target_datasource_id = datasource.id
@@ -468,6 +519,11 @@ class SqlDatasourceProvider:
raise DatasourceValidationError(
"A stage can only update a datasource with the same mode and shape."
)
else:
_apply_datasource_governance(
datasource,
DatasourceGovernance.from_mapping(stage.governance_),
)
materialization = _append_materialization(
db,
@@ -502,6 +558,7 @@ class SqlDatasourceProvider:
source_name: str,
mode: DatasourceMode,
description: str | None = None,
governance: DatasourceGovernance | None = None,
) -> DatasourceDescriptor:
db, api_principal = _context(session, principal, SOURCE_WRITE_SCOPE)
if mode not in {"live", "cached"}:
@@ -549,6 +606,11 @@ class SqlDatasourceProvider:
created_by=_actor_id(api_principal),
updated_by=_actor_id(api_principal),
)
_apply_datasource_governance(
item,
governance
or _default_governance(mode=mode, provider_ref=origin.ref),
)
db.add(item)
db.flush()
if mode == "cached":
@@ -567,6 +629,26 @@ class SqlDatasourceProvider:
)
return _datasource_dto(item)
def update_datasource_governance(
self,
session: object,
principal: object,
*,
datasource_ref: str,
governance: DatasourceGovernance,
) -> DatasourceDescriptor:
db, api_principal = _context(session, principal, SOURCE_WRITE_SCOPE)
item = _required_datasource(
db,
tenant_id=api_principal.tenant_id,
datasource_ref=datasource_ref,
for_update=True,
)
_apply_datasource_governance(item, governance)
item.updated_by = _actor_id(api_principal)
db.flush()
return _datasource_dto(item)
def refresh_datasource(
self,
session: object,
@@ -903,6 +985,7 @@ def _append_materialization(
source_timestamp=source_timestamp,
provenance_=dict(provenance or {}),
metadata_=dict(metadata or {}),
governance_snapshot_=_datasource_governance(datasource).to_dict(),
created_by=actor_id,
)
session.add(materialization)
@@ -1034,17 +1117,19 @@ def _datasource_record(
*,
tenant_id: str,
datasource_ref: str,
for_update: bool = False,
) -> DatasourceRecord | None:
datasource_id = _strip_ref(datasource_ref, "datasource:")
if datasource_id is None:
return None
return session.scalar(
select(DatasourceRecord).where(
statement = select(DatasourceRecord).where(
DatasourceRecord.id == datasource_id,
DatasourceRecord.tenant_id == tenant_id,
DatasourceRecord.deleted_at.is_(None),
)
)
if for_update:
statement = statement.with_for_update()
return session.scalar(statement)
def _required_datasource(
@@ -1052,11 +1137,13 @@ def _required_datasource(
*,
tenant_id: str,
datasource_ref: str,
for_update: bool = False,
) -> DatasourceRecord:
item = _datasource_record(
session,
tenant_id=tenant_id,
datasource_ref=datasource_ref,
for_update=for_update,
)
if item is None:
raise DatasourceNotFoundError("Datasource not found.")
@@ -1120,6 +1207,96 @@ def _ensure_source_name_available(
)
def _default_governance(
*,
mode: str,
provider_ref: str | None,
) -> DatasourceGovernance:
if mode == "live":
authority_mode = "external_authoritative"
elif mode == "cached" and provider_ref:
authority_mode = "external_mirror"
else:
authority_mode = "native_authoritative"
return DatasourceGovernance(
authoritative_source_ref=provider_ref,
authority_mode=cast(Any, authority_mode),
purposes=("governed_data_processing",),
publication_state="internal",
)
def _datasource_governance(item: DatasourceRecord) -> DatasourceGovernance:
return DatasourceGovernance.from_mapping(
{
"owner_ref": item.owner_ref,
"steward_ref": item.steward_ref,
"responsible_organization_ref": item.responsible_organization_ref,
"responsible_function_ref": item.responsible_function_ref,
"authoritative_source_ref": item.authoritative_source_ref,
"authority_mode": item.authority_mode,
"legal_basis_refs": item.legal_basis_refs,
"purposes": item.purposes,
"semantic_definition": item.semantic_definition,
"schema_owner_ref": item.schema_owner_ref,
"official_keys": item.official_keys,
"classification": item.classification,
"privacy_profile_ref": item.privacy_profile_ref,
"retention_policy_ref": item.retention_policy_ref,
"hold_refs": item.hold_refs,
"publication_state": item.publication_state,
"transfer_agreement_ref": item.transfer_agreement_ref,
"freshness_policy": item.freshness_policy,
"quality_policy": item.quality_policy,
"known_limits": item.known_limits,
"correction_procedure_ref": item.correction_procedure_ref,
"affected_refs": item.affected_refs,
"dependency_refs": item.dependency_refs,
}
)
def _apply_datasource_governance(
item: DatasourceRecord,
governance: DatasourceGovernance,
) -> None:
item.owner_ref = governance.owner_ref
item.steward_ref = governance.steward_ref
item.responsible_organization_ref = governance.responsible_organization_ref
item.responsible_function_ref = governance.responsible_function_ref
item.authoritative_source_ref = governance.authoritative_source_ref
item.authority_mode = governance.authority_mode
item.legal_basis_refs = list(governance.legal_basis_refs)
item.purposes = list(governance.purposes)
item.semantic_definition = governance.semantic_definition
item.schema_owner_ref = governance.schema_owner_ref
item.official_keys = list(governance.official_keys)
item.classification = governance.classification
item.privacy_profile_ref = governance.privacy_profile_ref
item.retention_policy_ref = governance.retention_policy_ref
item.hold_refs = list(governance.hold_refs)
item.publication_state = governance.publication_state
item.transfer_agreement_ref = governance.transfer_agreement_ref
item.freshness_policy = dict(governance.freshness_policy)
item.quality_policy = dict(governance.quality_policy)
item.known_limits = list(governance.known_limits)
item.correction_procedure_ref = governance.correction_procedure_ref
item.affected_refs = list(governance.affected_refs)
item.dependency_refs = list(governance.dependency_refs)
item.governance_references = [
DatasourceGovernanceReferenceRecord(
tenant_id=item.tenant_id,
relation=relation,
reference=reference,
)
for relation, references in (
("affected", governance.affected_refs),
("depends_on", governance.dependency_refs),
)
for reference in references
]
def _datasource_dto(item: DatasourceRecord) -> DatasourceDescriptor:
capabilities = ["read", "preview", "freeze"]
if item.mode == "cached" and item.provider_ref:
@@ -1149,6 +1326,7 @@ def _datasource_dto(item: DatasourceRecord) -> DatasourceDescriptor:
capabilities=tuple(capabilities),
provenance=dict(item.provenance_),
metadata=dict(item.metadata_),
governance=_datasource_governance(item),
)
@@ -1170,6 +1348,7 @@ def _materialization_dto(
created_at=item.created_at,
provenance=dict(item.provenance_),
metadata=dict(item.metadata_),
governance=DatasourceGovernance.from_mapping(item.governance_snapshot_),
)
@@ -1201,6 +1380,7 @@ def _stage_dto(item: DatasourceStageRecord) -> DatasourceStage:
),
provenance=dict(item.provenance_),
metadata=dict(item.metadata_),
governance=DatasourceGovernance.from_mapping(item.governance_),
)
@@ -1426,6 +1606,9 @@ def _publication_target(
raise DatasourceValidationError(
"Produced rows require a static or cached tabular datasource."
)
if request.governance is not None:
_apply_datasource_governance(datasource, request.governance)
datasource.updated_by = actor_id
return datasource
name = str(request.name or "").strip()
source_name = str(request.source_name or "").strip()
@@ -1468,6 +1651,11 @@ def _publication_target(
created_by=actor_id,
updated_by=actor_id,
)
_apply_datasource_governance(
datasource,
request.governance
or _default_governance(mode="static", provider_ref=None),
)
session.add(datasource)
session.flush()
return datasource