feat: consume governed datasources and shared graphs

src/govoplan_dataflow/backend/router.py

@@ -5,16 +5,19 @@ from sqlalchemy.orm import Session
 
 from govoplan_core.audit.logging import audit_event
 from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
+from govoplan_core.core.datasources import (
+    DatasourceAccessError,
+    DatasourceDescriptor,
+    DatasourceError,
+    DatasourceNotFoundError,
+    DatasourceStageInput,
+    DatasourceUnavailableError,
+    DatasourceValidationError,
+    datasource_catalogue,
+    datasource_lifecycle,
+)
 from govoplan_core.core.tabular_sources import (
-    TabularSnapshotInput,
-    TabularSource,
-    TabularSourceAccessError,
-    TabularSourceError,
-    TabularSourceNotFoundError,
-    TabularSourceValidationError,
     parse_tabular_csv,
-    tabular_snapshot_writer,
-    tabular_source_provider,
 )
 from govoplan_core.db.session import get_session
 from govoplan_dataflow.backend.manifest import ADMIN_SCOPE, READ_SCOPE, RUN_SCOPE, WRITE_SCOPE
@@ -93,12 +96,14 @@ def _http_error(exc: DataflowError) -> HTTPException:
     return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
 
 
-def _source_http_error(exc: TabularSourceError) -> HTTPException:
-    if isinstance(exc, TabularSourceAccessError):
+def _source_http_error(exc: DatasourceError) -> HTTPException:
+    if isinstance(exc, DatasourceAccessError):
         return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc))
-    if isinstance(exc, TabularSourceNotFoundError):
+    if isinstance(exc, DatasourceNotFoundError):
         return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
-    if isinstance(exc, TabularSourceValidationError):
+    if isinstance(exc, DatasourceUnavailableError):
+        return HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc))
+    if isinstance(exc, DatasourceValidationError):
         return HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
             detail=str(exc),
@@ -155,10 +160,10 @@ def _node_library_response() -> NodeLibraryResponse:
     )
 
 
-def _source_response(source: TabularSource) -> TabularSourceResponse:
+def _source_response(source: DatasourceDescriptor) -> TabularSourceResponse:
     return TabularSourceResponse(
         ref=source.ref,
-        provider=source.provider,
+        provider=source.provider or "datasources",
         source_name=source.source_name,
         name=source.name,
         description=source.description,
@@ -195,18 +200,18 @@ def api_list_sources(
 ) -> TabularSourceListResponse:
     _require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
     registry = get_registry()
-    provider = tabular_source_provider(registry)
-    writer = tabular_snapshot_writer(registry)
+    provider = datasource_catalogue(registry)
+    writer = datasource_lifecycle(registry)
     if provider is None:
         return TabularSourceListResponse(available=False, writable=False, sources=[])
     try:
-        sources = provider.list_sources(
+        sources = provider.list_datasources(
             session,
             principal,
             query=query,
             limit=100,
         )
-    except TabularSourceError as exc:
+    except DatasourceError as exc:
         raise _source_http_error(exc) from exc
     return TabularSourceListResponse(
         available=True,
@@ -226,11 +231,11 @@ def api_create_source_snapshot(
     principal: ApiPrincipal = Depends(get_api_principal),
 ) -> TabularSourceResponse:
     _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
-    writer = tabular_snapshot_writer(get_registry())
+    writer = datasource_lifecycle(get_registry())
     if writer is None:
         raise HTTPException(
             status_code=status.HTTP_409_CONFLICT,
-            detail="No tabular snapshot writer is available.",
+            detail="No datasource staging provider is available.",
         )
     try:
         rows = (
@@ -242,35 +247,49 @@ def api_create_source_snapshot(
             if payload.format == "csv"
             else tuple(payload.rows or ())
         )
-        source = writer.create_snapshot(
+        stage = writer.create_stage(
             session,
             principal,
-            snapshot=TabularSnapshotInput(
+            stage=DatasourceStageInput(
                 name=payload.name,
                 source_name=payload.source_name,
                 description=payload.description,
+                kind="upload",
+                mode="static",
+                shape="tabular",
                 rows=rows,
-                metadata={
+                provider="dataflow.upload",
+                provenance={
                     "created_via": "dataflow",
                     "source_format": payload.format,
                 },
+                metadata={
+                    "dataflow_convenience_import": True,
+                },
             ),
         )
-    except TabularSourceError as exc:
+        source, materialization = writer.promote_stage(
+            session,
+            principal,
+            stage_ref=stage.ref,
+        )
+    except DatasourceError as exc:
         raise _source_http_error(exc) from exc
     audit_event(
         session,
         tenant_id=principal.tenant_id,
         user_id=getattr(principal.user, "id", None),
         api_key_id=principal.api_key_id,
-        action="dataflow.source_snapshot.created",
-        object_type="tabular_source",
+        action="dataflow.datasource.created",
+        object_type="datasource",
         object_id=source.ref,
         details={
             "provider": source.provider,
             "source_name": source.source_name,
             "row_count": source.row_count,
             "fingerprint": source.fingerprint,
+            "stage_ref": stage.ref,
+            "materialization_ref": materialization.ref,
         },
     )
     response = _source_response(source)