feat: integrate datasource and workflow modules

2026-07-28 12:45:05 +02:00
parent 603e07cec5
commit 163b35c0af
13 changed files with 262 additions and 8 deletions
--- a/tools/checks/check-datasource-composition.py
+++ b/tools/checks/check-datasource-composition.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""Exercise the connector -> datasource -> dataflow capability path."""
+
+from __future__ import annotations
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from govoplan_connectors.backend.db.models import ConnectorTabularSource
+from govoplan_core.auth import ApiPrincipal
+from govoplan_core.core.access import PrincipalRef
+from govoplan_core.core.datasources import datasource_catalogue, datasource_lifecycle
+from govoplan_core.core.modules import ModuleContext
+from govoplan_core.core.tabular_sources import (
+    TabularSnapshotInput,
+    tabular_snapshot_writer,
+)
+from govoplan_core.db.base import Base
+from govoplan_core.server.registry import build_platform_registry
+from govoplan_dataflow.backend.schemas import (
+    GraphEdge,
+    GraphNode,
+    GraphPosition,
+    PipelineGraph,
+    PipelinePreviewRequest,
+)
+from govoplan_dataflow.backend.service import preview_pipeline
+from govoplan_datasources.backend.db.models import (
+    DatasourceMaterializationRecord,
+    DatasourceRecord,
+    DatasourceStageRecord,
+)
+
+
+def main() -> int:
+    registry = build_platform_registry(
+        ("connectors", "datasources", "dataflow", "workflow")
+    )
+    registry.configure_capability_context(
+        ModuleContext(registry=registry, settings=object())
+    )
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.create_all(
+        engine,
+        tables=[
+            ConnectorTabularSource.__table__,
+            DatasourceRecord.__table__,
+            DatasourceMaterializationRecord.__table__,
+            DatasourceStageRecord.__table__,
+        ],
+    )
+    session_factory = sessionmaker(bind=engine)
+    with session_factory() as session:
+        principal = _principal()
+        writer = tabular_snapshot_writer(registry)
+        lifecycle = datasource_lifecycle(registry)
+        catalogue = datasource_catalogue(registry)
+        if writer is None or lifecycle is None or catalogue is None:
+            raise RuntimeError("Datasource composition capabilities are incomplete.")
+
+        origin = writer.create_snapshot(
+            session,
+            principal,
+            snapshot=TabularSnapshotInput(
+                name="Monthly cases",
+                source_name="connector_monthly_cases",
+                rows=(
+                    {"id": 1, "amount": 5},
+                    {"id": 2, "amount": 15},
+                ),
+            ),
+        )
+        datasource = lifecycle.register_origin(
+            session,
+            principal,
+            origin_ref=origin.ref,
+            name="Monthly cases cache",
+            source_name="monthly_cases",
+            mode="cached",
+        )
+        result = preview_pipeline(
+            session,
+            tenant_id="tenant-1",
+            actor_id="account-1",
+            payload=PipelinePreviewRequest(
+                graph=_graph(
+                    datasource_ref=datasource.ref,
+                    fingerprint=datasource.fingerprint,
+                ),
+                row_limit=100,
+            ),
+            principal=principal,
+            registry=registry,
+        )
+        expected_rows = [
+            {"id": 1, "amount": 5},
+            {"id": 2, "amount": 15},
+        ]
+        if result.status != "succeeded":
+            raise RuntimeError(f"Dataflow preview failed: {result.diagnostics}")
+        if result.rows != expected_rows:
+            raise RuntimeError(f"Unexpected Dataflow rows: {result.rows!r}")
+        if result.source_fingerprints[0]["source_ref"] != datasource.ref:
+            raise RuntimeError("Dataflow lineage did not retain the datasource reference.")
+    engine.dispose()
+    print("Connector -> Datasources -> Dataflow composition passed.")
+    return 0
+
+
+def _principal() -> ApiPrincipal:
+    return ApiPrincipal(
+        principal=PrincipalRef(
+            account_id="account-1",
+            membership_id="membership-1",
+            tenant_id="tenant-1",
+            scopes=frozenset(
+                {
+                    "connectors:source:read",
+                    "connectors:source:write",
+                    "datasources:catalogue:read",
+                    "datasources:source:write",
+                    "datasources:stage:write",
+                    "dataflow:pipeline:run",
+                }
+            ),
+        ),
+        account=object(),
+        user=object(),
+    )
+
+
+def _graph(*, datasource_ref: str, fingerprint: str) -> PipelineGraph:
+    return PipelineGraph(
+        nodes=[
+            GraphNode(
+                id="source",
+                type="source.reference",
+                label="Cases",
+                position=GraphPosition(x=0, y=0),
+                config={
+                    "source_ref": datasource_ref,
+                    "source_name": "monthly_cases",
+                    "expected_fingerprint": fingerprint,
+                    "consistency": "current",
+                },
+            ),
+            GraphNode(
+                id="output",
+                type="output",
+                label="Output",
+                position=GraphPosition(x=200, y=0),
+                config={},
+            ),
+        ],
+        edges=[
+            GraphEdge(
+                id="source-output",
+                source="source",
+                target="output",
+            )
+        ],
+    )
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/tools/checks/check-focused.sh
+++ b/tools/checks/check-focused.sh
@@ -70,7 +70,10 @@ PY
 "$META_ROOT/tools/checks/check_dependency_boundaries.py"
 "$PYTHON" -m unittest tests.test_module_system
 "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-connectors/tests
+"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-datasources/tests
 "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-dataflow/tests
+"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-workflow/tests
+"$PYTHON" "$META_ROOT/tools/checks/check-datasource-composition.py"
 "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests
 "$PYTHON" -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count

@@ -82,6 +85,9 @@ cd "$ROOT/webui"
 cd /mnt/DATA/git/govoplan-dataflow/webui
 "$NPM" run test:structure

+cd /mnt/DATA/git/govoplan-datasources/webui
+"$NPM" run typecheck
+
 cd /mnt/DATA/git/govoplan-mail/webui
 "$NPM" run test:mail-ui

--- a/tools/launch/launch-production-like-dev.sh
+++ b/tools/launch/launch-production-like-dev.sh
@@ -51,7 +51,7 @@ set +a

 export APP_ENV="${APP_ENV:-staging}"
 export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}"
-export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,connectors,dataflow,notifications,docs,ops}"
+export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,connectors,datasources,dataflow,workflow,notifications,docs,ops}"
 export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
 export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
 export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
--- a/tools/launch/production-like-dev.sh
+++ b/tools/launch/production-like-dev.sh
@@ -66,7 +66,7 @@ export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-po
 export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}"
 export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}"
 export CELERY_ENABLED="${CELERY_ENABLED:-true}"
-export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,connectors,dataflow,notifications,docs,ops}"
+export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,connectors,datasources,dataflow,workflow,notifications,docs,ops}"
 export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}"
 export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$META_ROOT/runtime/production-like/files}"
 export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}"
--- a/tools/release/push-release-tag.sh
+++ b/tools/release/push-release-tag.sh
@@ -89,6 +89,7 @@ PACKAGE_MODULE_REPOS=(
  "$PARENT/govoplan-contracts"
  "$PARENT/govoplan-dashboard"
  "$PARENT/govoplan-dataflow"
+  "$PARENT/govoplan-datasources"
  "$PARENT/govoplan-docs"
  "$PARENT/govoplan-facilities"
  "$PARENT/govoplan-files"
@@ -113,6 +114,7 @@ PACKAGE_MODULE_REPOS=(
  "$PARENT/govoplan-tenancy"
  "$PARENT/govoplan-transparency"
  "$PARENT/govoplan-evaluation"
+  "$PARENT/govoplan-workflow"
 )
 TAG_ONLY_MODULE_REPOS=(
  "$PARENT/govoplan-addresses"
@@ -136,7 +138,6 @@ TAG_ONLY_MODULE_REPOS=(
  "$PARENT/govoplan-soap"
  "$PARENT/govoplan-tasks"
  "$PARENT/govoplan-templates"
-  "$PARENT/govoplan-workflow"
  "$PARENT/govoplan-xoev"
  "$PARENT/govoplan-xrechnung"
  "$PARENT/govoplan-xta-osci"