feat: integrate datasource and workflow modules

tools/checks/check-datasource-composition.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Exercise the connector -> datasource -> dataflow capability path."""
+
+from __future__ import annotations
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from govoplan_connectors.backend.db.models import ConnectorTabularSource
+from govoplan_core.auth import ApiPrincipal
+from govoplan_core.core.access import PrincipalRef
+from govoplan_core.core.datasources import datasource_catalogue, datasource_lifecycle
+from govoplan_core.core.modules import ModuleContext
+from govoplan_core.core.tabular_sources import (
+    TabularSnapshotInput,
+    tabular_snapshot_writer,
+)
+from govoplan_core.db.base import Base
+from govoplan_core.server.registry import build_platform_registry
+from govoplan_dataflow.backend.schemas import (
+    GraphEdge,
+    GraphNode,
+    GraphPosition,
+    PipelineGraph,
+    PipelinePreviewRequest,
+)
+from govoplan_dataflow.backend.service import preview_pipeline
+from govoplan_datasources.backend.db.models import (
+    DatasourceMaterializationRecord,
+    DatasourceRecord,
+    DatasourceStageRecord,
+)
+
+
+def main() -> int:
+    registry = build_platform_registry(
+        ("connectors", "datasources", "dataflow", "workflow")
+    )
+    registry.configure_capability_context(
+        ModuleContext(registry=registry, settings=object())
+    )
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.create_all(
+        engine,
+        tables=[
+            ConnectorTabularSource.__table__,
+            DatasourceRecord.__table__,
+            DatasourceMaterializationRecord.__table__,
+            DatasourceStageRecord.__table__,
+        ],
+    )
+    session_factory = sessionmaker(bind=engine)
+    with session_factory() as session:
+        principal = _principal()
+        writer = tabular_snapshot_writer(registry)
+        lifecycle = datasource_lifecycle(registry)
+        catalogue = datasource_catalogue(registry)
+        if writer is None or lifecycle is None or catalogue is None:
+            raise RuntimeError("Datasource composition capabilities are incomplete.")
+
+        origin = writer.create_snapshot(
+            session,
+            principal,
+            snapshot=TabularSnapshotInput(
+                name="Monthly cases",
+                source_name="connector_monthly_cases",
+                rows=(
+                    {"id": 1, "amount": 5},
+                    {"id": 2, "amount": 15},
+                ),
+            ),
+        )
+        datasource = lifecycle.register_origin(
+            session,
+            principal,
+            origin_ref=origin.ref,
+            name="Monthly cases cache",
+            source_name="monthly_cases",
+            mode="cached",
+        )
+        result = preview_pipeline(
+            session,
+            tenant_id="tenant-1",
+            actor_id="account-1",
+            payload=PipelinePreviewRequest(
+                graph=_graph(
+                    datasource_ref=datasource.ref,
+                    fingerprint=datasource.fingerprint,
+                ),
+                row_limit=100,
+            ),
+            principal=principal,
+            registry=registry,
+        )
+        expected_rows = [
+            {"id": 1, "amount": 5},
+            {"id": 2, "amount": 15},
+        ]
+        if result.status != "succeeded":
+            raise RuntimeError(f"Dataflow preview failed: {result.diagnostics}")
+        if result.rows != expected_rows:
+            raise RuntimeError(f"Unexpected Dataflow rows: {result.rows!r}")
+        if result.source_fingerprints[0]["source_ref"] != datasource.ref:
+            raise RuntimeError("Dataflow lineage did not retain the datasource reference.")
+    engine.dispose()
+    print("Connector -> Datasources -> Dataflow composition passed.")
+    return 0
+
+
+def _principal() -> ApiPrincipal:
+    return ApiPrincipal(
+        principal=PrincipalRef(
+            account_id="account-1",
+            membership_id="membership-1",
+            tenant_id="tenant-1",
+            scopes=frozenset(
+                {
+                    "connectors:source:read",
+                    "connectors:source:write",
+                    "datasources:catalogue:read",
+                    "datasources:source:write",
+                    "datasources:stage:write",
+                    "dataflow:pipeline:run",
+                }
+            ),
+        ),
+        account=object(),
+        user=object(),
+    )
+
+
+def _graph(*, datasource_ref: str, fingerprint: str) -> PipelineGraph:
+    return PipelineGraph(
+        nodes=[
+            GraphNode(
+                id="source",
+                type="source.reference",
+                label="Cases",
+                position=GraphPosition(x=0, y=0),
+                config={
+                    "source_ref": datasource_ref,
+                    "source_name": "monthly_cases",
+                    "expected_fingerprint": fingerprint,
+                    "consistency": "current",
+                },
+            ),
+            GraphNode(
+                id="output",
+                type="output",
+                label="Output",
+                position=GraphPosition(x=200, y=0),
+                config={},
+            ),
+        ],
+        edges=[
+            GraphEdge(
+                id="source-output",
+                source="source",
+                target="output",
+            )
+        ],
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())