feat(connectors): add managed file and PostgreSQL origins
Module Package Release / publish-packages (push) Successful in 12s

This commit is contained in:
2026-08-21 19:29:58 +02:00
parent e6ab8291ec
commit 2c2b11f860
13 changed files with 2082 additions and 59 deletions
+42 -15
View File
@@ -18,6 +18,7 @@ from govoplan_core.core.modules import (
FrontendModule,
MigrationSpec,
ModuleInterfaceProvider,
ModuleInterfaceRequirement,
ModuleManifest,
PermissionDefinition,
RoleTemplate,
@@ -36,6 +37,7 @@ from govoplan_core.core.tabular_sources import (
CAPABILITY_CONNECTORS_TABULAR_SNAPSHOT_WRITER,
CAPABILITY_CONNECTORS_TABULAR_SOURCES,
)
from govoplan_core.core.files import CAPABILITY_FILES_TABULAR_CONTENT
from govoplan_core.core.sanctions import (
CAPABILITY_CONNECTORS_SANCTIONS_SNAPSHOTS,
)
@@ -81,7 +83,7 @@ from govoplan_connectors.backend.provider_state import (
MODULE_ID = "connectors"
MODULE_VERSION = "0.1.19"
MODULE_VERSION = "0.1.20"
TABULAR_SOURCE_INTERFACE_VERSION = "0.1.0"
DATASOURCE_ORIGIN_INTERFACE_VERSION = "0.1.0"
SANCTIONS_SNAPSHOT_INTERFACE_VERSION = "1.0.0"
@@ -98,6 +100,16 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
reference="tests/test_tabular_sources.py",
summary="Exercises tenant-safe immutable tabular snapshots and bounded reads.",
),
ModuleMaturityEvidence(
kind="test",
reference="tests/test_tabular_origin_provider.py",
summary="Exercises exact managed-file versions, reviewed refresh, live SQL projection, configuration drift, and tenant isolation.",
),
ModuleMaturityEvidence(
kind="test",
reference="tests/test_tabular_adapters.py",
summary="Exercises bounded CSV/XLSX parsing and the credential-governed read-only PostgreSQL adapter.",
),
ModuleMaturityEvidence(
kind="test",
reference="tests/test_sanctions_sources.py",
@@ -120,7 +132,7 @@ ARCHITECTURE = ModuleArchitectureDeclaration(
),
),
known_limits=(
"The executable generic datasource origin is an immutable tabular snapshot; database and arbitrary REST profiles remain future providers.",
"Tabular origins support immutable snapshots, exact managed CSV/XLSX versions, and read-only PostgreSQL tables; arbitrary REST and other database adapters remain future providers.",
"Feed publication renders a governed document but does not yet push it to an external publishing endpoint.",
"The generic governed runtime simulates deterministic mapping and validation; provider-specific live writes remain owned by explicit connector adapters.",
),
@@ -158,7 +170,7 @@ EXTERNAL_PROVIDERS = (
ExternalProviderDeclaration(
id=TABULAR_PROVIDER_ID,
module_id=MODULE_ID,
label="Immutable tabular snapshot provider",
label="Governed tabular source providers",
maturity="read",
operations=("discover", "search", "read", "preview", "dry_run"),
objects=(
@@ -170,10 +182,10 @@ EXTERNAL_PROVIDERS = (
),
),
behavior=ProviderBehaviorDeclaration(
revision_tokens="Source fingerprints and immutable snapshot ids are retained.",
concurrency="Reads may require the expected fingerprint; snapshots never mutate in place.",
freshness="Snapshot acquisition time and source timestamp are exposed.",
health="Import validation and source-read failures are explicit.",
revision_tokens="Source fingerprints, exact file versions, SQL configuration revisions, credential revisions, and discovery revisions are retained.",
concurrency="Reads may require the expected fingerprint; file versions remain pinned and SQL configuration or schema drift blocks reads until reviewed refresh.",
freshness="Snapshot acquisition time, exact/current file versions, and live SQL discovery health are exposed.",
health="Import validation, Files access/integrity, SQL authentication, configuration drift, schema drift, timeouts, and source unavailability are explicit without exposing secrets.",
max_read_items=1000,
idempotency="Feed imports accept a caller request key and replay the same committed immutable source without refetching.",
retry="Read-only acquisition may be retried only as a new deliberate request after a failed atomic operation.",
@@ -187,7 +199,7 @@ EXTERNAL_PROVIDERS = (
classifications=("internal", "confidential", "restricted"),
purposes=("governed import", "dataflow input", "evidence reconstruction"),
retention="Datasources or the consuming domain supplies retention and hold policy.",
secret_handling="Generic snapshots contain no connector credential; transport credentials stay in credential envelopes.",
secret_handling="Snapshots and managed-file sources contain no connector credential; PostgreSQL uses a scoped Core credential-envelope reference and never stores or returns the resolved secret.",
),
capability_names=(
CAPABILITY_CONNECTORS_TABULAR_SOURCES,
@@ -347,11 +359,13 @@ def _router(_context):
def _provider(_context) -> SqlTabularSourceProvider:
return SqlTabularSourceProvider()
return SqlTabularSourceProvider(registry=getattr(_context, "registry", None))
def _datasource_origin_provider(_context) -> ConnectorDatasourceOriginProvider:
return ConnectorDatasourceOriginProvider()
return ConnectorDatasourceOriginProvider(
SqlTabularSourceProvider(registry=getattr(_context, "registry", None))
)
def _sanctions_snapshot_provider(
@@ -451,6 +465,14 @@ manifest = ModuleManifest(
),
ModuleInterfaceProvider(name=CONNECTORS_DSAR_CAPABILITY, version="0.1.0"),
),
requires_interfaces=(
ModuleInterfaceRequirement(
name=CAPABILITY_FILES_TABULAR_CONTENT,
version_min="1.0.0",
version_max_exclusive="2.0.0",
optional=True,
),
),
permissions=PERMISSIONS,
role_templates=ROLE_TEMPLATES,
route_factory=_router,
@@ -646,11 +668,16 @@ manifest = ModuleManifest(
"fingerprints, and bounded reads. Dataflow stores only opaque source "
"references and expected fingerprints. Each source declares its live, "
"cached, file-backed, or static mode, structured health, and supported "
"projection, filter, aggregation, sorting, and pagination pushdown. The "
"first executable provider imports immutable JSON or CSV snapshots, "
"supports projection and pagination, and exposes them as Datasource "
"origins. Database and API providers can implement the same origin "
"contract without changing Datasources or Dataflow."
"projection, filter, aggregation, sorting, and pagination pushdown. "
"Immutable JSON/CSV snapshots remain available. Managed CSV/XLSX "
"sources use the optional Files capability, pin an exact authorized "
"version, apply archive and expansion limits, and require explicit "
"refresh before adopting a newer version. The PostgreSQL adapter uses "
"an active governed configuration and scoped Core credential envelope, "
"reflects simple schema/table identifiers, runs read-only bounded "
"projection and pagination, and blocks configuration, credential, or "
"schema drift until reviewed refresh. Credentials, endpoints, storage "
"keys, and raw file internals are never exposed through the origin."
),
layer="available",
documentation_types=("admin", "user"),