feat: consume governed datasources and shared graphs
This commit is contained in:
@@ -6,10 +6,10 @@ from sqlalchemy import select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.core.tabular_sources import (
|
||||
TabularReadRequest,
|
||||
TabularSourceError,
|
||||
tabular_source_provider,
|
||||
from govoplan_core.core.datasources import (
|
||||
DatasourceError,
|
||||
DatasourceReadRequest,
|
||||
datasource_catalogue,
|
||||
)
|
||||
from govoplan_core.db.base import utcnow
|
||||
from govoplan_dataflow.backend.db.models import (
|
||||
@@ -381,38 +381,41 @@ def preview_pipeline(
|
||||
started_at = utcnow()
|
||||
run: DataflowRun | None = None
|
||||
try:
|
||||
provider = tabular_source_provider(registry)
|
||||
provider = datasource_catalogue(registry)
|
||||
|
||||
def resolve_source(node: GraphNode, limit: int) -> ResolvedSource:
|
||||
if provider is None:
|
||||
raise PipelineExecutionError(
|
||||
"Connector-backed preview requires the Connectors tabular-source capability.",
|
||||
"Datasource-backed preview requires the Datasources catalogue capability.",
|
||||
node_id=node.id,
|
||||
)
|
||||
if principal is None:
|
||||
raise PipelineExecutionError(
|
||||
"Connector-backed preview requires a tenant API principal.",
|
||||
"Datasource-backed preview requires a tenant API principal.",
|
||||
node_id=node.id,
|
||||
)
|
||||
try:
|
||||
resolved = provider.read_source(
|
||||
resolved = provider.read_datasource(
|
||||
session,
|
||||
principal,
|
||||
request=TabularReadRequest(
|
||||
source_ref=str(node.config["source_ref"]),
|
||||
request=DatasourceReadRequest(
|
||||
datasource_ref=str(node.config["source_ref"]),
|
||||
consistency=str(
|
||||
node.config.get("consistency") or "current"
|
||||
), # type: ignore[arg-type]
|
||||
limit=limit,
|
||||
expected_fingerprint=_clean_optional(
|
||||
node.config.get("expected_fingerprint")
|
||||
),
|
||||
),
|
||||
)
|
||||
except TabularSourceError as exc:
|
||||
except DatasourceError as exc:
|
||||
raise PipelineExecutionError(str(exc), node_id=node.id) from exc
|
||||
return ResolvedSource(
|
||||
rows=tuple(dict(row) for row in resolved.rows),
|
||||
source_ref=resolved.source.ref,
|
||||
provider=resolved.source.provider,
|
||||
fingerprint=resolved.source.fingerprint,
|
||||
source_ref=resolved.datasource.ref,
|
||||
provider=resolved.datasource.provider or "datasources",
|
||||
fingerprint=resolved.datasource.fingerprint,
|
||||
total_rows=resolved.total_rows,
|
||||
truncated=resolved.truncated,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user