457 lines
17 KiB
Python
457 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from govoplan_core.core.access import (
|
|
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
|
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
|
)
|
|
from govoplan_core.core.datasources import (
|
|
CAPABILITY_DATASOURCE_CATALOGUE,
|
|
CAPABILITY_DATASOURCE_LIFECYCLE,
|
|
CAPABILITY_DATASOURCE_ORIGINS,
|
|
CAPABILITY_DATASOURCE_PUBLICATION,
|
|
)
|
|
from govoplan_core.core.module_guards import (
|
|
drop_table_retirement_provider,
|
|
persistent_table_uninstall_guard,
|
|
)
|
|
from govoplan_core.core.modules import (
|
|
DocumentationTopic,
|
|
FrontendModule,
|
|
FrontendRoute,
|
|
MigrationSpec,
|
|
ModuleContext,
|
|
ModuleInterfaceProvider,
|
|
ModuleInterfaceRequirement,
|
|
ModuleManifest,
|
|
NavItem,
|
|
PermissionDefinition,
|
|
RoleTemplate,
|
|
)
|
|
from govoplan_core.core.provider_governance import (
|
|
ModuleArchitectureDeclaration,
|
|
ModuleArchitectureDocumentation,
|
|
ModuleMaturityEvidence,
|
|
)
|
|
from govoplan_core.core.views import ViewSurface
|
|
from govoplan_core.db.base import Base
|
|
from govoplan_datasources.backend.db import models as datasource_models
|
|
from govoplan_datasources.backend.service import (
|
|
ADMIN_SCOPE,
|
|
CATALOGUE_READ_SCOPE,
|
|
SOURCE_WRITE_SCOPE,
|
|
STAGE_WRITE_SCOPE,
|
|
SqlDatasourceProvider,
|
|
)
|
|
|
|
|
|
MODULE_ID = "datasources"
|
|
MODULE_NAME = "Datasources"
|
|
MODULE_VERSION = "0.1.14"
|
|
DATASOURCE_INTERFACE_VERSION = "0.1.0"
|
|
|
|
ARCHITECTURE = ModuleArchitectureDeclaration(
|
|
layer="data_reporting_integration",
|
|
kind="foundation",
|
|
maturity="vertical_slice",
|
|
evidence=(
|
|
ModuleMaturityEvidence(
|
|
kind="test",
|
|
reference="tests/test_lifecycle.py",
|
|
summary="Exercises staging, immutable materialization, publication, and typed governance behavior.",
|
|
),
|
|
ModuleMaturityEvidence(
|
|
kind="documentation",
|
|
reference="docs/CONCEPT.md",
|
|
summary="Defines datasource ownership and connector/dataflow boundaries.",
|
|
),
|
|
ModuleMaturityEvidence(
|
|
kind="migration",
|
|
reference="tests/test_migrations.py",
|
|
summary="Migrates and validates the governed catalogue schema.",
|
|
),
|
|
),
|
|
known_limits=(
|
|
"Governance references are stable provider-neutral refs; dedicated selectors depend on the owning optional modules.",
|
|
"Quality and freshness policies are stored and snapshotted but enforcement remains provider-specific.",
|
|
),
|
|
supported_authority_modes=(
|
|
"native_authoritative",
|
|
"external_authoritative",
|
|
"external_mirror",
|
|
"governed_sync",
|
|
"governance_overlay",
|
|
"linked_reference",
|
|
),
|
|
owned_concepts=(
|
|
"datasource catalogue identity",
|
|
"datasource governance metadata",
|
|
"staging and immutable materializations",
|
|
"datasource publication lifecycle",
|
|
),
|
|
non_owned_concepts=(
|
|
"external transport and credentials",
|
|
"dataflow transformation semantics",
|
|
"report presentation",
|
|
),
|
|
documentation=ModuleArchitectureDocumentation(
|
|
migration=("tests/test_migrations.py",),
|
|
recovery=("docs/CONCEPT.md",),
|
|
security=("docs/CONCEPT.md",),
|
|
operations=("docs/CONCEPT.md",),
|
|
),
|
|
)
|
|
|
|
|
|
def _permission(scope: str, label: str, description: str) -> PermissionDefinition:
|
|
module_id, resource, action = scope.split(":", 2)
|
|
return PermissionDefinition(
|
|
scope=scope,
|
|
label=label,
|
|
description=description,
|
|
category="Datasources",
|
|
level="tenant",
|
|
module_id=module_id,
|
|
resource=resource,
|
|
action=action,
|
|
)
|
|
|
|
|
|
PERMISSIONS = (
|
|
_permission(
|
|
CATALOGUE_READ_SCOPE,
|
|
"View datasources",
|
|
"Discover and preview policy-visible datasources and materializations.",
|
|
),
|
|
_permission(
|
|
SOURCE_WRITE_SCOPE,
|
|
"Manage datasources",
|
|
"Register, refresh, freeze, and retire governed datasources.",
|
|
),
|
|
_permission(
|
|
STAGE_WRITE_SCOPE,
|
|
"Stage datasource content",
|
|
"Upload, validate, inspect, and promote bounded datasource stages.",
|
|
),
|
|
_permission(
|
|
ADMIN_SCOPE,
|
|
"Administer datasources",
|
|
"Manage every tenant datasource, stage, materialization, and lifecycle policy.",
|
|
),
|
|
)
|
|
|
|
ROLE_TEMPLATES = (
|
|
RoleTemplate(
|
|
slug="datasource_manager",
|
|
name="Datasource manager",
|
|
description="Register, stage, refresh, freeze, and retire governed data.",
|
|
permissions=(
|
|
CATALOGUE_READ_SCOPE,
|
|
SOURCE_WRITE_SCOPE,
|
|
STAGE_WRITE_SCOPE,
|
|
),
|
|
),
|
|
RoleTemplate(
|
|
slug="datasource_reader",
|
|
name="Datasource reader",
|
|
description="Discover and preview policy-visible datasource states.",
|
|
permissions=(CATALOGUE_READ_SCOPE,),
|
|
),
|
|
)
|
|
|
|
|
|
def _router(context: ModuleContext):
|
|
from govoplan_datasources.backend.runtime import configure_runtime
|
|
|
|
configure_runtime(registry=context.registry)
|
|
from govoplan_datasources.backend.router import router
|
|
|
|
return router
|
|
|
|
|
|
def _provider(context: ModuleContext) -> SqlDatasourceProvider:
|
|
return SqlDatasourceProvider(registry=context.registry)
|
|
|
|
|
|
def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
|
|
return {
|
|
"datasources": (
|
|
session.query(datasource_models.DatasourceRecord)
|
|
.filter(
|
|
datasource_models.DatasourceRecord.tenant_id == tenant_id,
|
|
datasource_models.DatasourceRecord.deleted_at.is_(None),
|
|
)
|
|
.count()
|
|
),
|
|
"datasource_stages_ready": (
|
|
session.query(datasource_models.DatasourceStageRecord)
|
|
.filter(
|
|
datasource_models.DatasourceStageRecord.tenant_id == tenant_id,
|
|
datasource_models.DatasourceStageRecord.state == "ready",
|
|
)
|
|
.count()
|
|
),
|
|
"datasource_frozen_states": (
|
|
session.query(datasource_models.DatasourceMaterializationRecord)
|
|
.filter(
|
|
datasource_models.DatasourceMaterializationRecord.tenant_id
|
|
== tenant_id,
|
|
datasource_models.DatasourceMaterializationRecord.frozen_at.is_not(
|
|
None
|
|
),
|
|
)
|
|
.count()
|
|
),
|
|
}
|
|
|
|
|
|
manifest = ModuleManifest(
|
|
id=MODULE_ID,
|
|
name=MODULE_NAME,
|
|
version=MODULE_VERSION,
|
|
dependencies=(),
|
|
optional_dependencies=(
|
|
"access",
|
|
"audit",
|
|
"connectors",
|
|
"files",
|
|
"notifications",
|
|
"policy",
|
|
),
|
|
optional_capabilities=(
|
|
CAPABILITY_AUTH_PRINCIPAL_RESOLVER,
|
|
CAPABILITY_AUTH_PERMISSION_EVALUATOR,
|
|
CAPABILITY_DATASOURCE_ORIGINS,
|
|
),
|
|
provides_interfaces=(
|
|
ModuleInterfaceProvider(
|
|
name="datasources.catalogue",
|
|
version=DATASOURCE_INTERFACE_VERSION,
|
|
),
|
|
ModuleInterfaceProvider(
|
|
name="datasources.lifecycle",
|
|
version=DATASOURCE_INTERFACE_VERSION,
|
|
),
|
|
ModuleInterfaceProvider(
|
|
name="datasources.materializations",
|
|
version=DATASOURCE_INTERFACE_VERSION,
|
|
),
|
|
ModuleInterfaceProvider(
|
|
name="datasources.staging",
|
|
version=DATASOURCE_INTERFACE_VERSION,
|
|
),
|
|
ModuleInterfaceProvider(
|
|
name="datasources.publication",
|
|
version=DATASOURCE_INTERFACE_VERSION,
|
|
),
|
|
),
|
|
requires_interfaces=(
|
|
ModuleInterfaceRequirement(
|
|
name="connectors.datasource_origins",
|
|
version_min="0.1.0",
|
|
version_max_exclusive="1.0.0",
|
|
optional=True,
|
|
),
|
|
),
|
|
permissions=PERMISSIONS,
|
|
role_templates=ROLE_TEMPLATES,
|
|
nav_items=(
|
|
NavItem(
|
|
path="/datasources",
|
|
label="Datasources",
|
|
icon="database-zap",
|
|
required_any=(CATALOGUE_READ_SCOPE, ADMIN_SCOPE),
|
|
order=70,
|
|
),
|
|
),
|
|
frontend=FrontendModule(
|
|
module_id=MODULE_ID,
|
|
package_name="@govoplan/datasources-webui",
|
|
routes=(
|
|
FrontendRoute(
|
|
path="/datasources",
|
|
component="DatasourcesPage",
|
|
required_any=(CATALOGUE_READ_SCOPE, ADMIN_SCOPE),
|
|
order=70,
|
|
),
|
|
),
|
|
nav_items=(
|
|
NavItem(
|
|
path="/datasources",
|
|
label="Datasources",
|
|
icon="database-zap",
|
|
required_any=(CATALOGUE_READ_SCOPE, ADMIN_SCOPE),
|
|
order=70,
|
|
),
|
|
),
|
|
view_surfaces=(
|
|
ViewSurface(id="datasources.page", module_id=MODULE_ID, kind="route", label="Datasources", order=70),
|
|
ViewSurface(id="datasources.catalogue", module_id=MODULE_ID, kind="section", label="Datasource catalogue", order=10),
|
|
ViewSurface(id="datasources.staging", module_id=MODULE_ID, kind="section", label="Datasource staging", order=20),
|
|
ViewSurface(id="datasources.origins", module_id=MODULE_ID, kind="section", label="Datasource origins", order=30),
|
|
ViewSurface(id="datasources.governance", module_id=MODULE_ID, kind="action", label="Datasource governance", order=40),
|
|
ViewSurface(id="datasources.preview", module_id=MODULE_ID, kind="section", label="Datasource preview and materializations", order=50),
|
|
),
|
|
),
|
|
route_factory=_router,
|
|
capability_factories={
|
|
CAPABILITY_DATASOURCE_CATALOGUE: _provider,
|
|
CAPABILITY_DATASOURCE_LIFECYCLE: _provider,
|
|
CAPABILITY_DATASOURCE_PUBLICATION: _provider,
|
|
},
|
|
tenant_summary_providers=(_tenant_summary,),
|
|
migration_spec=MigrationSpec(
|
|
module_id=MODULE_ID,
|
|
metadata=Base.metadata,
|
|
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
|
retirement_supported=True,
|
|
retirement_provider=drop_table_retirement_provider(
|
|
datasource_models.DatasourcePublicationRecord,
|
|
datasource_models.DatasourceStageRecord,
|
|
datasource_models.DatasourceMaterializationRecord,
|
|
datasource_models.DatasourcePayloadRowRecord,
|
|
datasource_models.DatasourcePayloadRecord,
|
|
datasource_models.DatasourceRecord,
|
|
label="Datasources",
|
|
),
|
|
retirement_notes=(
|
|
"Destructive retirement drops datasource catalogue entries, immutable "
|
|
"materializations, and staging evidence after a database snapshot."
|
|
),
|
|
),
|
|
uninstall_guard_providers=(
|
|
persistent_table_uninstall_guard(
|
|
datasource_models.DatasourceRecord,
|
|
datasource_models.DatasourceMaterializationRecord,
|
|
datasource_models.DatasourcePayloadRecord,
|
|
datasource_models.DatasourcePayloadRowRecord,
|
|
datasource_models.DatasourceStageRecord,
|
|
datasource_models.DatasourcePublicationRecord,
|
|
label="Datasources",
|
|
),
|
|
),
|
|
architecture=ARCHITECTURE,
|
|
documentation=(
|
|
DocumentationTopic(
|
|
id="datasources.lifecycle",
|
|
title="Datasource lifecycle",
|
|
summary="Governed live, cached, and static data/register entries with staging, frozen states, provenance, and institutional ownership context.",
|
|
body=(
|
|
"Datasources is the provider-neutral catalogue consumed by Dataflow, "
|
|
"Workflow, Reporting, and policy-aware modules. Static data is staged "
|
|
"before promotion. Cached data refreshes connector origins into immutable "
|
|
"materializations. Live data is read through a connector and may be frozen "
|
|
"for reproducible evidence. Connectors owns protocols and credentials; "
|
|
"Datasources owns data identity, provenance, lifecycle, read semantics, "
|
|
"and the typed governance catalogue for authority, purpose, quality, "
|
|
"freshness, classification, correction, and dependent services, flows, "
|
|
"reports, controls, and decisions. Governance metadata visibility does not "
|
|
"grant access to protected rows."
|
|
),
|
|
layer="available",
|
|
documentation_types=("admin", "user"),
|
|
audience=("operator", "module_admin", "power_user", "product_owner"),
|
|
related_modules=(
|
|
"connectors",
|
|
"dataflow",
|
|
"workflow_engine",
|
|
"files",
|
|
"reporting",
|
|
"risk_compliance",
|
|
),
|
|
order=70,
|
|
metadata={
|
|
"seed": True,
|
|
"help_contexts": [
|
|
"datasources.page",
|
|
"datasources.catalogue",
|
|
"datasources.staging",
|
|
"datasources.origins",
|
|
"datasources.preview",
|
|
],
|
|
},
|
|
),
|
|
DocumentationTopic(
|
|
id="datasources.governance",
|
|
title="Datasource authority and governance",
|
|
summary="Explain who owns data meaning, authority, correction, privacy, quality, freshness, retention, and dependent uses.",
|
|
body=(
|
|
"Authority mode states whether GovOPlaN, an external system, a synchronized projection, an overlay, or a linked reference "
|
|
"controls the data. The authoritative source, owner, steward, responsible organization/function, schema owner, privacy "
|
|
"profile, retention policy, transfer agreement, legal basis, holds, correction procedure, purposes, official keys, and "
|
|
"known limits provide discoverable institutional context. Freshness and quality policies are typed JSON contracts retained "
|
|
"with materialization evidence; enforcement remains with the provider or consuming control that declares support. Metadata "
|
|
"visibility never grants row access."
|
|
),
|
|
layer="available",
|
|
documentation_types=("admin", "user"),
|
|
audience=("operator", "module_admin", "data_steward", "product_owner"),
|
|
related_modules=("policy", "organizations", "idm", "dataflow", "reporting", "risk_compliance"),
|
|
order=71,
|
|
metadata={
|
|
"seed": True,
|
|
"help_contexts": [
|
|
"datasources.governance",
|
|
"datasources.field.authority-mode",
|
|
"datasources.field.authoritative-source",
|
|
"datasources.field.classification",
|
|
"datasources.field.publication-state",
|
|
"datasources.field.freshness-policy",
|
|
"datasources.field.quality-policy",
|
|
],
|
|
},
|
|
),
|
|
DocumentationTopic(
|
|
id="datasources.reference.fields-and-consequences",
|
|
title="Datasource fields and lifecycle consequences",
|
|
summary="Live, cached, static, staging, promotion, refresh, freeze, and retirement semantics.",
|
|
body=(
|
|
"A Datasource key is the stable catalogue identity used by consumers. Live mode reads through an available origin; cached "
|
|
"mode refreshes an origin into immutable revisions; static mode promotes uploaded content from staging. Stages are bounded, "
|
|
"inspectable, and non-consumable until promoted. Promotion creates or updates a governed Datasource and appends an immutable "
|
|
"materialization. Refresh appends a new cached revision without rewriting older evidence. Freeze labels an immutable, "
|
|
"addressable state for reproducible execution. Retirement removes the Datasource from new definitions while retained "
|
|
"materialization references remain governed. Connector absence disables origin registration but leaves local catalogue and "
|
|
"staging behavior available."
|
|
),
|
|
layer="available",
|
|
documentation_types=("admin", "user"),
|
|
audience=("operator", "module_admin", "power_user", "product_owner"),
|
|
related_modules=("connectors", "dataflow", "workflow_engine", "reporting", "audit"),
|
|
order=72,
|
|
metadata={
|
|
"seed": True,
|
|
"help_contexts": [
|
|
"datasources.field.origin",
|
|
"datasources.field.key",
|
|
"datasources.field.mode",
|
|
"datasources.action.promote",
|
|
"datasources.action.refresh",
|
|
"datasources.action.freeze",
|
|
"datasources.action.retire",
|
|
],
|
|
"consequence_classes": {
|
|
"promote": "Creates or updates a governed Datasource and appends an immutable materialization revision.",
|
|
"refresh": "Reads the cached origin and appends a new immutable materialization revision.",
|
|
"freeze": "Creates a labelled immutable state for reproducible consumers and evidence.",
|
|
"retire": "Prevents new selection while retained materialization references remain governed.",
|
|
},
|
|
},
|
|
),
|
|
),
|
|
)
|
|
|
|
|
|
def get_manifest() -> ModuleManifest:
|
|
return manifest
|
|
|
|
|
|
__all__ = [
|
|
"DATASOURCE_INTERFACE_VERSION",
|
|
"MODULE_ID",
|
|
"MODULE_VERSION",
|
|
"get_manifest",
|
|
"manifest",
|
|
]
|