feat: implement governed datasource catalogue metadata
This commit is contained in:
@@ -11,6 +11,7 @@ from govoplan_core.core.datasources import (
|
||||
CAPABILITY_DATASOURCE_ORIGINS,
|
||||
DatasourceAccessError,
|
||||
DatasourceField,
|
||||
DatasourceGovernance,
|
||||
DatasourceOrigin,
|
||||
DatasourceOriginReadRequest,
|
||||
DatasourceOriginReadResult,
|
||||
@@ -22,6 +23,7 @@ from govoplan_core.core.datasources import (
|
||||
)
|
||||
from govoplan_core.db.base import Base, utcnow
|
||||
from govoplan_datasources.backend.db.models import (
|
||||
DatasourceGovernanceReferenceRecord,
|
||||
DatasourceMaterializationRecord,
|
||||
DatasourcePayloadRecord,
|
||||
DatasourcePayloadRowRecord,
|
||||
@@ -146,6 +148,7 @@ class DatasourceLifecycleTests(unittest.TestCase):
|
||||
self.engine,
|
||||
tables=[
|
||||
DatasourceRecord.__table__,
|
||||
DatasourceGovernanceReferenceRecord.__table__,
|
||||
DatasourcePayloadRecord.__table__,
|
||||
DatasourcePayloadRowRecord.__table__,
|
||||
DatasourceMaterializationRecord.__table__,
|
||||
@@ -170,6 +173,7 @@ class DatasourceLifecycleTests(unittest.TestCase):
|
||||
DatasourceMaterializationRecord.__table__,
|
||||
DatasourcePayloadRowRecord.__table__,
|
||||
DatasourcePayloadRecord.__table__,
|
||||
DatasourceGovernanceReferenceRecord.__table__,
|
||||
DatasourceRecord.__table__,
|
||||
],
|
||||
)
|
||||
@@ -261,6 +265,108 @@ class DatasourceLifecycleTests(unittest.TestCase):
|
||||
self.assertEqual(first_record.payload_id, frozen_record.payload_id)
|
||||
self.assertEqual([], first_record.rows)
|
||||
|
||||
def test_governance_is_queryable_and_snapshotted_per_materialization(self) -> None:
|
||||
stage = self.provider.create_stage(
|
||||
self.session,
|
||||
principal(),
|
||||
stage=DatasourceStageInput(
|
||||
name="Governed register",
|
||||
source_name="governed_register",
|
||||
kind="upload",
|
||||
mode="static",
|
||||
shape="tabular",
|
||||
rows=({"id": 1},),
|
||||
governance=DatasourceGovernance(
|
||||
owner_ref="function:data-owner",
|
||||
steward_ref="account:steward",
|
||||
responsible_organization_ref="organization:office-1",
|
||||
authority_mode="native_authoritative",
|
||||
legal_basis_refs=("policy:register-use",),
|
||||
purposes=("case_processing",),
|
||||
semantic_definition="Authoritative case register export.",
|
||||
official_keys=("id",),
|
||||
classification="restricted",
|
||||
publication_state="internal",
|
||||
quality_policy={"required_keys": ["id"]},
|
||||
affected_refs=("service:permit", "report:monthly"),
|
||||
dependency_refs=("dataflow:monthly-case-check",),
|
||||
),
|
||||
),
|
||||
)
|
||||
datasource, first = self.provider.promote_stage(
|
||||
self.session,
|
||||
principal(),
|
||||
stage_ref=stage.ref,
|
||||
)
|
||||
|
||||
self.assertEqual("function:data-owner", datasource.governance.owner_ref)
|
||||
self.assertEqual("restricted", first.governance.classification)
|
||||
self.assertEqual(
|
||||
[datasource.ref],
|
||||
[
|
||||
item.ref
|
||||
for item in self.provider.list_datasources(
|
||||
self.session,
|
||||
principal(),
|
||||
authority_mode="native_authoritative",
|
||||
classification="restricted",
|
||||
publication_state="internal",
|
||||
owner_ref="function:data-owner",
|
||||
responsible_organization_ref="organization:office-1",
|
||||
)
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
[datasource.ref],
|
||||
[
|
||||
item.ref
|
||||
for item in self.provider.list_datasources(
|
||||
self.session,
|
||||
principal(),
|
||||
affected_ref="service:permit",
|
||||
dependency_ref="dataflow:monthly-case-check",
|
||||
)
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
(),
|
||||
self.provider.list_datasources(
|
||||
self.session,
|
||||
principal("tenant-2"),
|
||||
affected_ref="service:permit",
|
||||
),
|
||||
)
|
||||
|
||||
changed = self.provider.update_datasource_governance(
|
||||
self.session,
|
||||
principal(),
|
||||
datasource_ref=datasource.ref,
|
||||
governance=DatasourceGovernance(
|
||||
owner_ref="function:new-owner",
|
||||
authority_mode="native_authoritative",
|
||||
purposes=("case_processing",),
|
||||
classification="confidential",
|
||||
publication_state="internal",
|
||||
),
|
||||
)
|
||||
history = self.provider.list_materializations(
|
||||
self.session,
|
||||
principal(),
|
||||
datasource_ref=datasource.ref,
|
||||
)
|
||||
|
||||
self.assertEqual("function:new-owner", changed.governance.owner_ref)
|
||||
self.assertEqual("function:data-owner", history[0].governance.owner_ref)
|
||||
self.assertEqual("restricted", history[0].governance.classification)
|
||||
self.assertEqual(
|
||||
(),
|
||||
self.provider.list_datasources(
|
||||
self.session,
|
||||
principal(),
|
||||
dependency_ref="dataflow:monthly-case-check",
|
||||
),
|
||||
)
|
||||
|
||||
def test_live_reads_origin_and_cached_refresh_is_explicit(self) -> None:
|
||||
live = self.provider.register_origin(
|
||||
self.session,
|
||||
|
||||
@@ -2,16 +2,26 @@ from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
from alembic import command
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from sqlalchemy import create_engine, inspect
|
||||
from sqlalchemy import MetaData, Table, create_engine, inspect, select
|
||||
|
||||
from govoplan_core.db.migrations import migrate_database
|
||||
from govoplan_core.db.migrations import alembic_config, migrate_database
|
||||
from govoplan_datasources.backend.manifest import get_manifest
|
||||
|
||||
|
||||
class DatasourceMigrationTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _config(url: str):
|
||||
return alembic_config(
|
||||
database_url=url,
|
||||
enabled_modules=("datasources",),
|
||||
manifest_factories=(get_manifest,),
|
||||
)
|
||||
|
||||
def test_baseline_creates_datasource_tables_and_head(self) -> None:
|
||||
with tempfile.TemporaryDirectory(prefix="govoplan-datasources-migration-") as directory:
|
||||
url = f"sqlite:///{Path(directory) / 'datasources.db'}"
|
||||
@@ -24,12 +34,29 @@ class DatasourceMigrationTests(unittest.TestCase):
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
self.assertIn(
|
||||
"d5f0a2b8c3e7",
|
||||
"b8d2f5a0c3e7",
|
||||
set(MigrationContext.configure(connection).get_current_heads()),
|
||||
)
|
||||
catalogue_columns = {
|
||||
item["name"]
|
||||
for item in inspect(connection).get_columns(
|
||||
"datasource_catalogue"
|
||||
)
|
||||
}
|
||||
self.assertTrue(
|
||||
{
|
||||
"authority_mode",
|
||||
"classification",
|
||||
"publication_state",
|
||||
"owner_ref",
|
||||
"quality_policy",
|
||||
"dependency_refs",
|
||||
}.issubset(catalogue_columns)
|
||||
)
|
||||
self.assertEqual(
|
||||
{
|
||||
"datasource_catalogue",
|
||||
"datasource_governance_references",
|
||||
"datasource_materializations",
|
||||
"datasource_payload_rows",
|
||||
"datasource_payloads",
|
||||
@@ -45,6 +72,72 @@ class DatasourceMigrationTests(unittest.TestCase):
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
def test_governance_reference_index_backfills_existing_catalogue_rows(
|
||||
self,
|
||||
) -> None:
|
||||
with tempfile.TemporaryDirectory(
|
||||
prefix="govoplan-datasources-governance-migration-"
|
||||
) as directory:
|
||||
url = f"sqlite:///{Path(directory) / 'datasources.db'}"
|
||||
config = self._config(url)
|
||||
command.upgrade(config, "a7c1e4d9b2f6")
|
||||
engine = create_engine(url)
|
||||
try:
|
||||
metadata = MetaData()
|
||||
catalogue = Table(
|
||||
"datasource_catalogue", metadata, autoload_with=engine
|
||||
)
|
||||
now = datetime.now(UTC)
|
||||
with engine.begin() as connection:
|
||||
connection.execute(
|
||||
catalogue.insert().values(
|
||||
id="datasource-1",
|
||||
tenant_id="tenant-1",
|
||||
source_name="governed",
|
||||
name="Governed",
|
||||
kind="upload",
|
||||
mode="static",
|
||||
shape="tabular",
|
||||
status="active",
|
||||
schema_version=1,
|
||||
schema=[],
|
||||
fingerprint="",
|
||||
provenance={},
|
||||
metadata={},
|
||||
affected_refs=["service:permit", "service:permit"],
|
||||
dependency_refs=["dataflow:monthly"],
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
|
||||
command.upgrade(config, "b8d2f5a0c3e7")
|
||||
|
||||
references = Table(
|
||||
"datasource_governance_references",
|
||||
MetaData(),
|
||||
autoload_with=engine,
|
||||
)
|
||||
with engine.connect() as connection:
|
||||
rows = connection.execute(
|
||||
select(
|
||||
references.c.relation,
|
||||
references.c.reference,
|
||||
).order_by(
|
||||
references.c.relation,
|
||||
references.c.reference,
|
||||
)
|
||||
).all()
|
||||
self.assertEqual(
|
||||
[
|
||||
("affected", "service:permit"),
|
||||
("depends_on", "dataflow:monthly"),
|
||||
],
|
||||
rows,
|
||||
)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user