From 163b35c0afbcb23814374f4612ed839addbfe2cc Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Tue, 28 Jul 2026 12:45:05 +0200 Subject: [PATCH] feat: integrate datasource and workflow modules --- .env.example | 2 +- README.md | 3 + docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md | 5 +- ...OURCE_AND_DEFINITION_GRAPH_ARCHITECTURE.md | 71 ++++++++ docs/REFERENCE_JOURNEY_PROGRAM.md | 5 +- docs/REPOSITORY_INDEX.md | 2 + repositories.json | 1 + requirements-dev.txt | 2 + tools/checks/check-datasource-composition.py | 166 ++++++++++++++++++ tools/checks/check-focused.sh | 6 + tools/launch/launch-production-like-dev.sh | 2 +- tools/launch/production-like-dev.sh | 2 +- tools/release/push-release-tag.sh | 3 +- 13 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 docs/DATASOURCE_AND_DEFINITION_GRAPH_ARCHITECTURE.md create mode 100644 tools/checks/check-datasource-composition.py diff --git a/.env.example b/.env.example index 5d95f1d..96cddcd 100644 --- a/.env.example +++ b/.env.example @@ -8,7 +8,7 @@ MASTER_KEY_B64= DATABASE_URL=postgresql+psycopg://govoplan:change-me@127.0.0.1:5432/govoplan GOVOPLAN_DATABASE_URL_PGTOOLS=postgresql://govoplan:change-me@127.0.0.1:5432/govoplan -ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,files,mail,campaigns,calendar,poll,scheduling,connectors,dataflow,notifications,docs,ops +ENABLED_MODULES=tenancy,organizations,identity,access,admin,dashboard,policy,audit,files,mail,campaigns,calendar,poll,scheduling,connectors,datasources,dataflow,workflow,notifications,docs,ops CELERY_ENABLED=true REDIS_URL=redis://127.0.0.1:6379/0 diff --git a/README.md b/README.md index cc7abd8..bd4a4ad 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,9 @@ Meta ownership and module install/contract boundaries are documented in `docs/META_REPO_SCAN.md` and `docs/MODULE_CONTRACTS_AND_INSTALLS.md`. Frontend layout principles for module pages are documented in `docs/FRONTEND_LAYOUT_PRINCIPLES.md`. +The provider-neutral datasource boundary and reusable Dataflow/Workflow graph +contract are documented in +`docs/DATASOURCE_AND_DEFINITION_GRAPH_ARCHITECTURE.md`. The cross-product destination, stakeholder visions, configuration archetypes, connected outcome stories, and capability horizons are documented in the [Connected Governance Platform Roadmap](docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md). diff --git a/docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md b/docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md index c5fbdbb..e773997 100644 --- a/docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md +++ b/docs/CONNECTED_GOVERNANCE_PLATFORM_ROADMAP.md @@ -1279,8 +1279,9 @@ them; they do not block the product vision today. - Which exact HIS/CampusOnline interfaces, student-statistics fields and official keys, accepted calculation, freeze/correction policy, privacy profile, and drill-down level should define the first analytical data product? -- After the first source-to-report proof, do repeated source/dataflow contracts - justify separate `govoplan-datasources` and `govoplan-dataflow` modules? +- Which database, REST, directory, and managed-file providers should follow the + implemented separation of `govoplan-connectors`, `govoplan-datasources`, and + `govoplan-dataflow`, and which quality/promotion policy should prove it first? - Which collaborative editor should be the first target, and should its first accepted experience emphasize concurrent editing, controlled check-out, or both? diff --git a/docs/DATASOURCE_AND_DEFINITION_GRAPH_ARCHITECTURE.md b/docs/DATASOURCE_AND_DEFINITION_GRAPH_ARCHITECTURE.md new file mode 100644 index 0000000..fc9b89b --- /dev/null +++ b/docs/DATASOURCE_AND_DEFINITION_GRAPH_ARCHITECTURE.md @@ -0,0 +1,71 @@ +# Datasource And Definition Graph Architecture + +## Two-Layer Data Boundary + +GovOPlaN separates governed data identity from external acquisition: + +| Layer | Owner | Responsibilities | +| --- | --- | --- | +| Datasource layer | `govoplan-datasources` | Catalogue, tenant visibility, live/cached/static mode, staging, immutable materializations, frozen states, schema, fingerprints, provenance, and bounded reads | +| Connector layer | `govoplan-connectors` and protocol/provider modules | External protocols, endpoints, connection profiles, credentials, discovery, provider health, source-side filtering, and query pushdown | + +Connectors publish versioned datasource origins. Datasources registers those +origins and presents one stable capability to Dataflow, Workflow, Reporting, +Risk Compliance, and other consumers. Consumers must not import connector +implementations or retain credentials. + +The initial provider path is: + +1. Connectors imports a bounded JSON/CSV snapshot and exposes it as an origin. +2. Datasources registers it as live or cached, or accepts a direct static upload + through staging. +3. Cached refreshes and static promotions append immutable materializations. +4. Any datasource may expose a frozen state for reproducible execution evidence. +5. Dataflow stores an opaque datasource reference, state policy, and expected + fingerprint. + +Database, REST/HTTP, LDAP/directory, managed file, watched-directory, feed, and +stream providers fit behind the same origin contract. Provider-specific +configuration remains in Connectors. + +## Shared Definition Graph + +Core owns domain-neutral graph primitives: + +- nodes, typed ports, edges, and configuration field descriptors; +- node libraries and category labels; +- graph size, connectivity, cycle, and node-count constraints; +- shared backend validation and frontend connection checks. + +Domain modules own their semantics: + +- Dataflow provides load, combine, filter, transform, and output nodes. Its + graph is acyclic and has one output. +- Workflow provides trigger, activity, review, decision, wait, module-action, + Dataflow, and outcome nodes. It permits governed loops and has exactly one + trigger plus one or more outcomes. + +This division permits a shared editor shell without making Workflow a special +kind of Dataflow or leaking either module into Core. + +## Current Implementation + +- Core graph and datasource contracts are versioned at `0.1.0`. +- Workflow exposes node-library discovery and definition validation APIs. +- Datasources exposes catalogue, origins, staging, promotion, preview, + materialization history, refresh, freeze, and retirement APIs. +- Datasources WebUI exposes all current lifecycle views. +- Connectors adapts existing tabular snapshots to datasource origins. +- Dataflow consumes only Datasources catalogue/lifecycle capabilities and can + request current, live, or latest-frozen state. + +## Next Slices + +1. Persist and version Workflow definitions, then reuse the graph UI shell. +2. Add SQL database and governed REST origin providers with credential-envelope + references and bounded pushdown. +3. Add managed-file and directory origins. +4. Add datasource quality rules, schema compatibility policy, retention, and + promotion approvals. +5. Publish Dataflow outputs as new materializations and expose run lifecycle to + Workflow. diff --git a/docs/REFERENCE_JOURNEY_PROGRAM.md b/docs/REFERENCE_JOURNEY_PROGRAM.md index ebf9730..20bb35f 100644 --- a/docs/REFERENCE_JOURNEY_PROGRAM.md +++ b/docs/REFERENCE_JOURNEY_PROGRAM.md @@ -445,8 +445,9 @@ that first needs them: - exact HIS/CampusOnline source endpoints, student-statistics keys and accepted calculation, freeze/correction policy, privacy profile, and permitted drill-down level; -- whether repeated data-source and dataflow contracts justify separate - `govoplan-datasources` and `govoplan-dataflow` modules after the Stage 3 proof; +- datasource provider selection and quality/promotion policy; the architecture + now separates `govoplan-datasources` lifecycle from `govoplan-connectors` + acquisition and `govoplan-dataflow` transformation; - first collaborative editor/provider and whether the first UX is concurrent editing, controlled check-out, or both; and - first Records/archive target and approval/signature assurance level. diff --git a/docs/REPOSITORY_INDEX.md b/docs/REPOSITORY_INDEX.md index 5c33139..fd042fb 100644 --- a/docs/REPOSITORY_INDEX.md +++ b/docs/REPOSITORY_INDEX.md @@ -29,6 +29,8 @@ Generated from `repositories.json`. Use that JSON file as the machine-readable s | `govoplan-consultation` | `domain` | `../govoplan-consultation` | [govoplan-consultation](https://git.add-ideas.de/GovOPlaN/govoplan-consultation) | | `govoplan-contracts` | `domain` | `../govoplan-contracts` | [govoplan-contracts](https://git.add-ideas.de/GovOPlaN/govoplan-contracts) | | `govoplan-dashboard` | `platform` | `../govoplan-dashboard` | [govoplan-dashboard](https://git.add-ideas.de/GovOPlaN/govoplan-dashboard) | +| `govoplan-dataflow` | `platform` | `../govoplan-dataflow` | [govoplan-dataflow](https://git.add-ideas.de/GovOPlaN/govoplan-dataflow) | +| `govoplan-datasources` | `platform` | `../govoplan-datasources` | [govoplan-datasources](https://git.add-ideas.de/GovOPlaN/govoplan-datasources) | | `govoplan-dms` | `domain` | `../govoplan-dms` | [govoplan-dms](https://git.add-ideas.de/GovOPlaN/govoplan-dms) | | `govoplan-dist-lists` | `domain` | `../govoplan-dist-lists` | [govoplan-dist-lists](https://git.add-ideas.de/GovOPlaN/govoplan-dist-lists) | | `govoplan-docs` | `platform` | `../govoplan-docs` | [govoplan-docs](https://git.add-ideas.de/GovOPlaN/govoplan-docs) | diff --git a/repositories.json b/repositories.json index e08fc2c..a5b16de 100644 --- a/repositories.json +++ b/repositories.json @@ -23,6 +23,7 @@ {"name": "govoplan-contracts", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:GovOPlaN/govoplan-contracts.git", "path": "govoplan-contracts"}, {"name": "govoplan-dashboard", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:GovOPlaN/govoplan-dashboard.git", "path": "govoplan-dashboard"}, {"name": "govoplan-dataflow", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:GovOPlaN/govoplan-dataflow.git", "path": "govoplan-dataflow"}, + {"name": "govoplan-datasources", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:GovOPlaN/govoplan-datasources.git", "path": "govoplan-datasources"}, {"name": "govoplan-dms", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:GovOPlaN/govoplan-dms.git", "path": "govoplan-dms"}, {"name": "govoplan-dist-lists", "category": "module", "subtype": "domain", "remote": "git@git.add-ideas.de:GovOPlaN/govoplan-dist-lists.git", "path": "govoplan-dist-lists"}, {"name": "govoplan-docs", "category": "module", "subtype": "platform", "remote": "git@git.add-ideas.de:GovOPlaN/govoplan-docs.git", "path": "govoplan-docs"}, diff --git a/requirements-dev.txt b/requirements-dev.txt index fa305c7..1442c41 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -20,7 +20,9 @@ -e ../govoplan-campaign -e ../govoplan-calendar -e ../govoplan-connectors +-e ../govoplan-datasources -e ../govoplan-dataflow +-e ../govoplan-workflow -e ../govoplan-poll -e ../govoplan-scheduling -e ../govoplan-notifications diff --git a/tools/checks/check-datasource-composition.py b/tools/checks/check-datasource-composition.py new file mode 100644 index 0000000..ae14377 --- /dev/null +++ b/tools/checks/check-datasource-composition.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Exercise the connector -> datasource -> dataflow capability path.""" + +from __future__ import annotations + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from govoplan_connectors.backend.db.models import ConnectorTabularSource +from govoplan_core.auth import ApiPrincipal +from govoplan_core.core.access import PrincipalRef +from govoplan_core.core.datasources import datasource_catalogue, datasource_lifecycle +from govoplan_core.core.modules import ModuleContext +from govoplan_core.core.tabular_sources import ( + TabularSnapshotInput, + tabular_snapshot_writer, +) +from govoplan_core.db.base import Base +from govoplan_core.server.registry import build_platform_registry +from govoplan_dataflow.backend.schemas import ( + GraphEdge, + GraphNode, + GraphPosition, + PipelineGraph, + PipelinePreviewRequest, +) +from govoplan_dataflow.backend.service import preview_pipeline +from govoplan_datasources.backend.db.models import ( + DatasourceMaterializationRecord, + DatasourceRecord, + DatasourceStageRecord, +) + + +def main() -> int: + registry = build_platform_registry( + ("connectors", "datasources", "dataflow", "workflow") + ) + registry.configure_capability_context( + ModuleContext(registry=registry, settings=object()) + ) + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all( + engine, + tables=[ + ConnectorTabularSource.__table__, + DatasourceRecord.__table__, + DatasourceMaterializationRecord.__table__, + DatasourceStageRecord.__table__, + ], + ) + session_factory = sessionmaker(bind=engine) + with session_factory() as session: + principal = _principal() + writer = tabular_snapshot_writer(registry) + lifecycle = datasource_lifecycle(registry) + catalogue = datasource_catalogue(registry) + if writer is None or lifecycle is None or catalogue is None: + raise RuntimeError("Datasource composition capabilities are incomplete.") + + origin = writer.create_snapshot( + session, + principal, + snapshot=TabularSnapshotInput( + name="Monthly cases", + source_name="connector_monthly_cases", + rows=( + {"id": 1, "amount": 5}, + {"id": 2, "amount": 15}, + ), + ), + ) + datasource = lifecycle.register_origin( + session, + principal, + origin_ref=origin.ref, + name="Monthly cases cache", + source_name="monthly_cases", + mode="cached", + ) + result = preview_pipeline( + session, + tenant_id="tenant-1", + actor_id="account-1", + payload=PipelinePreviewRequest( + graph=_graph( + datasource_ref=datasource.ref, + fingerprint=datasource.fingerprint, + ), + row_limit=100, + ), + principal=principal, + registry=registry, + ) + expected_rows = [ + {"id": 1, "amount": 5}, + {"id": 2, "amount": 15}, + ] + if result.status != "succeeded": + raise RuntimeError(f"Dataflow preview failed: {result.diagnostics}") + if result.rows != expected_rows: + raise RuntimeError(f"Unexpected Dataflow rows: {result.rows!r}") + if result.source_fingerprints[0]["source_ref"] != datasource.ref: + raise RuntimeError("Dataflow lineage did not retain the datasource reference.") + engine.dispose() + print("Connector -> Datasources -> Dataflow composition passed.") + return 0 + + +def _principal() -> ApiPrincipal: + return ApiPrincipal( + principal=PrincipalRef( + account_id="account-1", + membership_id="membership-1", + tenant_id="tenant-1", + scopes=frozenset( + { + "connectors:source:read", + "connectors:source:write", + "datasources:catalogue:read", + "datasources:source:write", + "datasources:stage:write", + "dataflow:pipeline:run", + } + ), + ), + account=object(), + user=object(), + ) + + +def _graph(*, datasource_ref: str, fingerprint: str) -> PipelineGraph: + return PipelineGraph( + nodes=[ + GraphNode( + id="source", + type="source.reference", + label="Cases", + position=GraphPosition(x=0, y=0), + config={ + "source_ref": datasource_ref, + "source_name": "monthly_cases", + "expected_fingerprint": fingerprint, + "consistency": "current", + }, + ), + GraphNode( + id="output", + type="output", + label="Output", + position=GraphPosition(x=200, y=0), + config={}, + ), + ], + edges=[ + GraphEdge( + id="source-output", + source="source", + target="output", + ) + ], + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/checks/check-focused.sh b/tools/checks/check-focused.sh index 3f44236..cfe0201 100644 --- a/tools/checks/check-focused.sh +++ b/tools/checks/check-focused.sh @@ -70,7 +70,10 @@ PY "$META_ROOT/tools/checks/check_dependency_boundaries.py" "$PYTHON" -m unittest tests.test_module_system "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-connectors/tests +"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-datasources/tests "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-dataflow/tests +"$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-workflow/tests +"$PYTHON" "$META_ROOT/tools/checks/check-datasource-composition.py" "$PYTHON" -m unittest discover -s /mnt/DATA/git/govoplan-mail/tests "$PYTHON" -m unittest tests.test_api_smoke.ApiSmokeTests.test_mailbox_message_listing_reports_total_count @@ -82,6 +85,9 @@ cd "$ROOT/webui" cd /mnt/DATA/git/govoplan-dataflow/webui "$NPM" run test:structure +cd /mnt/DATA/git/govoplan-datasources/webui +"$NPM" run typecheck + cd /mnt/DATA/git/govoplan-mail/webui "$NPM" run test:mail-ui diff --git a/tools/launch/launch-production-like-dev.sh b/tools/launch/launch-production-like-dev.sh index d81a65c..132913b 100644 --- a/tools/launch/launch-production-like-dev.sh +++ b/tools/launch/launch-production-like-dev.sh @@ -51,7 +51,7 @@ set +a export APP_ENV="${APP_ENV:-staging}" export GOVOPLAN_INSTALL_PROFILE="${GOVOPLAN_INSTALL_PROFILE:-production-like}" -export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,connectors,dataflow,notifications,docs,ops}" +export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,connectors,datasources,dataflow,workflow,notifications,docs,ops}" export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-postgresql+psycopg://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}" export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}" export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}" diff --git a/tools/launch/production-like-dev.sh b/tools/launch/production-like-dev.sh index dc9c1a5..c332d14 100644 --- a/tools/launch/production-like-dev.sh +++ b/tools/launch/production-like-dev.sh @@ -66,7 +66,7 @@ export DATABASE_URL="${DATABASE_URL:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL:-po export GOVOPLAN_DATABASE_URL_PGTOOLS="${GOVOPLAN_DATABASE_URL_PGTOOLS:-${GOVOPLAN_PRODUCTION_LIKE_DATABASE_URL_PGTOOLS:-postgresql://govoplan:govoplan-dev@127.0.0.1:55433/govoplan}}" export REDIS_URL="${REDIS_URL:-${GOVOPLAN_PRODUCTION_LIKE_REDIS_URL:-redis://127.0.0.1:56379/0}}" export CELERY_ENABLED="${CELERY_ENABLED:-true}" -export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,connectors,dataflow,notifications,docs,ops}" +export ENABLED_MODULES="${ENABLED_MODULES:-tenancy,organizations,identity,access,admin,dashboard,policy,audit,campaigns,files,mail,calendar,poll,scheduling,connectors,datasources,dataflow,workflow,notifications,docs,ops}" export FILE_STORAGE_BACKEND="${FILE_STORAGE_BACKEND:-local}" export FILE_STORAGE_LOCAL_ROOT="${FILE_STORAGE_LOCAL_ROOT:-$META_ROOT/runtime/production-like/files}" export DEV_AUTO_MIGRATE_ENABLED="${DEV_AUTO_MIGRATE_ENABLED:-false}" diff --git a/tools/release/push-release-tag.sh b/tools/release/push-release-tag.sh index 5ecc20d..9498079 100644 --- a/tools/release/push-release-tag.sh +++ b/tools/release/push-release-tag.sh @@ -89,6 +89,7 @@ PACKAGE_MODULE_REPOS=( "$PARENT/govoplan-contracts" "$PARENT/govoplan-dashboard" "$PARENT/govoplan-dataflow" + "$PARENT/govoplan-datasources" "$PARENT/govoplan-docs" "$PARENT/govoplan-facilities" "$PARENT/govoplan-files" @@ -113,6 +114,7 @@ PACKAGE_MODULE_REPOS=( "$PARENT/govoplan-tenancy" "$PARENT/govoplan-transparency" "$PARENT/govoplan-evaluation" + "$PARENT/govoplan-workflow" ) TAG_ONLY_MODULE_REPOS=( "$PARENT/govoplan-addresses" @@ -136,7 +138,6 @@ TAG_ONLY_MODULE_REPOS=( "$PARENT/govoplan-soap" "$PARENT/govoplan-tasks" "$PARENT/govoplan-templates" - "$PARENT/govoplan-workflow" "$PARENT/govoplan-xoev" "$PARENT/govoplan-xrechnung" "$PARENT/govoplan-xta-osci"