GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 7 Pull Requests Actions Packages Projects Releases Wiki Activity

Add governed reusable pipelines and triggers

Browse Source
This commit is contained in:
zemion
2026-07-28 15:04:23 +02:00
parent 6305ef9cef
commit 08ddc8d3e8
17 changed files with 4697 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_dataflow/backend/service.py
+309 -7
View File
@@ -1,13 +1,15 @@
from __future__ import annotations
from collections.abc import Mapping
import hashlib
import json
from dataclasses import dataclass
from sqlalchemy import select
from sqlalchemy import or_, select
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal
from govoplan_core.core.automation import AutomationInvocation
from govoplan_core.core.dataflows import (
DataflowRunConflictError,
DataflowRunDescriptor,
@@ -34,11 +36,16 @@ from govoplan_dataflow.backend.executor import (
ResolvedSource,
execute_preview,
)
from govoplan_dataflow.backend.governance import (
definition_governance_payload,
require_definition_action,
)
from govoplan_dataflow.backend.graph import canonical_graph_payload, definition_hash, validate_graph
from govoplan_dataflow.backend.schemas import (
DataflowDiagnostic,
GraphNode,
PipelineCreateRequest,
PipelineDeriveRequest,
PipelineDraftRequest,
PipelineGraph,
PipelinePreviewRequest,
@@ -87,7 +94,10 @@ def list_pipelines(session: Session, *, tenant_id: str) -> list[DataflowPipeline
session.scalars(
select(DataflowPipeline)
.where(
DataflowPipeline.tenant_id == tenant_id,
or_(
DataflowPipeline.tenant_id == tenant_id,
DataflowPipeline.tenant_id.is_(None),
),
DataflowPipeline.deleted_at.is_(None),
)
.order_by(DataflowPipeline.updated_at.desc(), DataflowPipeline.name)
@@ -104,7 +114,10 @@ def get_pipeline(
pipeline = session.scalar(
select(DataflowPipeline).where(
DataflowPipeline.id == pipeline_id,
DataflowPipeline.tenant_id == tenant_id,
or_(
DataflowPipeline.tenant_id == tenant_id,
DataflowPipeline.tenant_id.is_(None),
),
DataflowPipeline.deleted_at.is_(None),
)
)
@@ -145,8 +158,23 @@ def create_pipeline(
editor_mode=payload.editor_mode,
)
content_hash = definition_hash(definition.graph, definition.sql_text)
stored_tenant_id = None if payload.scope_type == "system" else tenant_id
scope_id = (
None
if payload.scope_type == "system"
else tenant_id
if payload.scope_type == "tenant"
else payload.scope_id
)
pipeline = DataflowPipeline(
tenant_id=tenant_id,
tenant_id=stored_tenant_id,
scope_type=payload.scope_type,
scope_id=scope_id,
definition_kind=payload.definition_kind,
inherit_to_lower_scopes=payload.inherit_to_lower_scopes,
allow_run=payload.allow_run,
allow_reuse=payload.allow_reuse,
allow_automation=payload.allow_automation,
name=payload.name.strip(),
description=_clean_optional(payload.description),
status=payload.status,
@@ -156,7 +184,7 @@ def create_pipeline(
metadata_={},
)
revision = DataflowPipelineRevision(
tenant_id=tenant_id,
tenant_id=stored_tenant_id,
revision=1,
schema_version=definition.graph.schema_version,
graph=canonical_graph_payload(definition.graph),
@@ -185,6 +213,21 @@ def update_pipeline(
f"Pipeline changed on the server; expected revision {payload.expected_revision}, "
f"current revision is {pipeline.current_revision}"
)
if (
payload.scope_type != pipeline.scope_type
or payload.scope_id != pipeline.scope_id
and not (
pipeline.scope_type == "tenant"
and payload.scope_id in {None, pipeline.scope_id}
)
):
raise DataflowConflictError(
"Definition scope is immutable; derive a scoped copy instead."
)
if payload.definition_kind != pipeline.definition_kind:
raise DataflowConflictError(
"Definition kind is immutable; derive a flow or template instead."
)
definition = normalize_definition(
graph=payload.graph,
sql_text=payload.sql_text,
@@ -195,12 +238,26 @@ def update_pipeline(
pipeline.name = payload.name.strip()
pipeline.description = _clean_optional(payload.description)
pipeline.status = payload.status
ancestor_limits = _ancestor_governance_limits(
pipeline.derivation_provenance
)
pipeline.inherit_to_lower_scopes = (
payload.inherit_to_lower_scopes
and ancestor_limits["inherit_to_lower_scopes"]
)
pipeline.allow_run = payload.allow_run and ancestor_limits["allow_run"]
pipeline.allow_reuse = (
payload.allow_reuse and ancestor_limits["allow_reuse"]
)
pipeline.allow_automation = (
payload.allow_automation and ancestor_limits["allow_automation"]
)
pipeline.updated_by = actor_id
if current.content_hash != content_hash or current.editor_mode != payload.editor_mode:
pipeline.current_revision += 1
pipeline.revisions.append(
DataflowPipelineRevision(
tenant_id=tenant_id,
tenant_id=pipeline.tenant_id,
revision=pipeline.current_revision,
schema_version=definition.graph.schema_version,
graph=canonical_graph_payload(definition.graph),
@@ -214,6 +271,110 @@ def update_pipeline(
return pipeline
def derive_pipeline(
session: Session,
*,
tenant_id: str,
actor_id: str | None,
principal: ApiPrincipal,
registry: object | None,
source_pipeline_id: str,
payload: PipelineDeriveRequest,
) -> DataflowPipeline:
source = get_pipeline(
session,
tenant_id=tenant_id,
pipeline_id=source_pipeline_id,
)
reuse_decision = require_definition_action(
source,
principal=principal,
registry=registry,
action="derive",
)
source_revision = get_pipeline_revision(
session,
pipeline=source,
revision=payload.source_revision,
)
stored_tenant_id = None if payload.scope_type == "system" else tenant_id
scope_id = (
None
if payload.scope_type == "system"
else tenant_id
if payload.scope_type == "tenant"
else payload.scope_id
)
source_limits = _effective_governance_limits(
source,
decision_details=reuse_decision.details,
)
effective_limits = {
"inherit_to_lower_scopes": (
source_limits["inherit_to_lower_scopes"]
and payload.inherit_to_lower_scopes
),
"allow_run": source_limits["allow_run"] and payload.allow_run,
"allow_reuse": source_limits["allow_reuse"] and payload.allow_reuse,
"allow_automation": (
source_limits["allow_automation"]
and payload.allow_automation
),
}
provenance = {
"source_ref": f"pipeline:{source.id}",
"source_scope": {
"scope_type": source.scope_type,
"scope_id": source.scope_id,
},
"source_definition_kind": source.definition_kind,
"source_revision": source_revision.revision,
"source_hash": source_revision.content_hash,
"source_effective_limits": effective_limits,
"policy_decision": reuse_decision.to_dict(),
"derived_by": actor_id,
"derived_at": utcnow().isoformat(),
}
pipeline = DataflowPipeline(
tenant_id=stored_tenant_id,
scope_type=payload.scope_type,
scope_id=scope_id,
definition_kind=payload.definition_kind,
inherit_to_lower_scopes=effective_limits[
"inherit_to_lower_scopes"
],
allow_run=effective_limits["allow_run"],
allow_reuse=effective_limits["allow_reuse"],
allow_automation=effective_limits["allow_automation"],
derived_from_pipeline_id=source.id,
derived_from_revision=source_revision.revision,
derived_from_hash=source_revision.content_hash,
derivation_provenance=provenance,
name=payload.name.strip(),
description=_clean_optional(payload.description),
status="draft",
current_revision=1,
created_by=actor_id,
updated_by=actor_id,
metadata_={},
)
pipeline.revisions.append(
DataflowPipelineRevision(
tenant_id=stored_tenant_id,
revision=1,
schema_version=source_revision.schema_version,
graph=dict(source_revision.graph),
sql_text=source_revision.sql_text,
editor_mode=source_revision.editor_mode,
content_hash=source_revision.content_hash,
created_by=actor_id,
)
)
session.add(pipeline)
session.flush()
return pipeline
def delete_pipeline(
session: Session,
*,
@@ -228,7 +389,13 @@ def delete_pipeline(
return pipeline
def pipeline_response(session: Session, pipeline: DataflowPipeline) -> PipelineResponse:
def pipeline_response(
session: Session,
pipeline: DataflowPipeline,
*,
principal: ApiPrincipal,
registry: object | None,
) -> PipelineResponse:
revision = get_pipeline_revision(session, pipeline=pipeline)
return PipelineResponse(
id=pipeline.id,
@@ -242,6 +409,11 @@ def pipeline_response(session: Session, pipeline: DataflowPipeline) -> PipelineR
created_at=pipeline.created_at,
updated_at=pipeline.updated_at,
revision=PipelineRevisionResponse.model_validate(revision),
governance=definition_governance_payload(
pipeline,
principal=principal,
registry=registry,
),
)
@@ -359,6 +531,20 @@ def preview_pipeline(
revision: DataflowPipelineRevision | None = None
if payload.pipeline_id:
pipeline = get_pipeline(session, tenant_id=tenant_id, pipeline_id=payload.pipeline_id)
if principal is None:
raise DataflowConflictError(
"Saved pipeline previews require a tenant API principal."
)
action = "edit" if pipeline.status == "draft" else "run"
try:
require_definition_action(
pipeline,
principal=principal,
registry=registry,
action=action,
)
except PermissionError as exc:
raise DataflowConflictError(str(exc)) from exc
revision = get_pipeline_revision(session, pipeline=pipeline, revision=payload.revision)
graph = PipelineGraph.model_validate(revision.graph)
sql_text = revision.sql_text
@@ -567,6 +753,20 @@ def start_pipeline_run(
tenant_id=tenant_id,
pipeline_id=pipeline_id,
)
action = (
"run"
if request.invocation.kind in {"manual", "api", "backfill"}
else "automate"
)
try:
require_definition_action(
pipeline,
principal=principal,
registry=registry,
action=action,
)
except PermissionError as exc:
raise DataflowConflictError(str(exc)) from exc
revision = get_pipeline_revision(
session,
pipeline=pipeline,
@@ -610,6 +810,17 @@ def start_pipeline_run(
idempotency_key=idempotency_key,
request_hash=request_hash,
request_=_run_request_payload(request),
invocation_kind=request.invocation.kind,
trigger_id=_strip_ref(
request.invocation.trigger_ref or "",
"dataflow-trigger:",
),
trigger_delivery_id=_strip_ref(
request.invocation.delivery_ref or "",
"dataflow-trigger-delivery:",
),
correlation_id=request.invocation.correlation_id,
causation_id=request.invocation.causation_id,
source_fingerprints=[],
result_schema=[],
diagnostics=[],
@@ -766,6 +977,17 @@ def pipeline_run_response(
output_publication_ref=run.output_publication_ref,
output_datasource_ref=run.output_datasource_ref,
output_materialization_ref=run.output_materialization_ref,
invocation_kind=run.invocation_kind,
trigger_ref=(
f"dataflow-trigger:{run.trigger_id}" if run.trigger_id else None
),
delivery_ref=(
f"dataflow-trigger-delivery:{run.trigger_delivery_id}"
if run.trigger_delivery_id
else None
),
correlation_id=run.correlation_id,
causation_id=run.causation_id,
error=run.error,
started_at=run.started_at,
finished_at=run.finished_at,
@@ -796,6 +1018,15 @@ def pipeline_run_descriptor(
output_publication_ref=run.output_publication_ref,
output_datasource_ref=run.output_datasource_ref,
output_materialization_ref=run.output_materialization_ref,
invocation_kind=run.invocation_kind,
trigger_ref=(
f"dataflow-trigger:{run.trigger_id}" if run.trigger_id else None
),
delivery_ref=(
f"dataflow-trigger-delivery:{run.trigger_delivery_id}"
if run.trigger_delivery_id
else None
),
error=run.error,
started_at=run.started_at,
finished_at=run.finished_at,
@@ -1000,6 +1231,28 @@ def _run_request_payload(request: DataflowRunRequest) -> dict[str, object]:
if target
else None
),
"invocation": _invocation_payload(request.invocation),
}
def _invocation_payload(
invocation: AutomationInvocation,
) -> dict[str, object]:
return {
"kind": invocation.kind,
"trigger_ref": invocation.trigger_ref,
"delivery_ref": invocation.delivery_ref,
"event_id": invocation.event_id,
"event_type": invocation.event_type,
"correlation_id": invocation.correlation_id,
"causation_id": invocation.causation_id,
"scheduled_for": (
invocation.scheduled_for.isoformat()
if invocation.scheduled_for
else None
),
"requested_by": invocation.requested_by,
"metadata": dict(invocation.metadata),
}
@@ -1020,6 +1273,54 @@ def _clean_optional(value: str | None) -> str | None:
return cleaned or None
def _ancestor_governance_limits(
provenance: Mapping[str, object],
) -> dict[str, bool]:
raw = provenance.get("source_effective_limits")
limits = raw if isinstance(raw, Mapping) else {}
return {
key: value if isinstance((value := limits.get(key)), bool) else True
for key in (
"inherit_to_lower_scopes",
"allow_run",
"allow_reuse",
"allow_automation",
)
}
def _effective_governance_limits(
pipeline: DataflowPipeline,
*,
decision_details: Mapping[str, object] | None = None,
) -> dict[str, bool]:
ancestor = _ancestor_governance_limits(
pipeline.derivation_provenance
)
effective = {
"inherit_to_lower_scopes": (
pipeline.inherit_to_lower_scopes
and ancestor["inherit_to_lower_scopes"]
),
"allow_run": pipeline.allow_run and ancestor["allow_run"],
"allow_reuse": pipeline.allow_reuse and ancestor["allow_reuse"],
"allow_automation": (
pipeline.allow_automation and ancestor["allow_automation"]
),
}
policy_limits = (
decision_details.get("effective_limits")
if decision_details is not None
else None
)
if isinstance(policy_limits, Mapping):
for key in effective:
value = policy_limits.get(key)
if isinstance(value, bool):
effective[key] = effective[key] and value
return effective
__all__ = [
"DataflowConflictError",
"DataflowError",
@@ -1029,6 +1330,7 @@ __all__ = [
"cancel_pipeline_run",
"compile_sql_draft",
"create_pipeline",
"derive_pipeline",
"delete_pipeline",
"get_pipeline",
"get_pipeline_revision",