GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Add governed reusable pipelines and triggers

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-28 15:04:23 +02:00
parent 6305ef9cef
commit 08ddc8d3e8
17 changed files with 4697 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

469
src/govoplan_dataflow/backend/router.py
View File

@@ -5,6 +5,7 @@ from sqlalchemy.orm import Session
from govoplan_core.audit.logging import audit_event
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
from govoplan_core.core.automation import AutomationInvocation
from govoplan_core.core.dataflows import (
DataflowPublicationTarget,
DataflowRunRequest,
@@ -24,13 +25,27 @@ from govoplan_core.core.tabular_sources import (
parse_tabular_csv,
)
from govoplan_core.db.session import get_session
from govoplan_dataflow.backend.manifest import ADMIN_SCOPE, READ_SCOPE, RUN_SCOPE, WRITE_SCOPE
from govoplan_dataflow.backend.governance import (
definition_decision,
normalize_definition_scope,
require_definition_action,
)
from govoplan_dataflow.backend.manifest import (
ADMIN_SCOPE,
READ_SCOPE,
RUN_SCOPE,
TRIGGER_DISPATCH_SCOPE,
TRIGGER_READ_SCOPE,
TRIGGER_WRITE_SCOPE,
WRITE_SCOPE,
)
from govoplan_dataflow.backend.node_library import CATEGORY_LABELS, NODE_LIBRARY
from govoplan_dataflow.backend.runtime import get_registry
from govoplan_dataflow.backend.schemas import (
NodeLibraryResponse,
NodeTypeDefinitionResponse,
PipelineCreateRequest,
PipelineDeriveRequest,
PipelineDeleteResponse,
PipelineDraftRequest,
PipelineListResponse,
@@ -47,6 +62,14 @@ from govoplan_dataflow.backend.schemas import (
TabularSourceColumnResponse,
TabularSourceListResponse,
TabularSourceResponse,
DataflowEventDeliveryRequest,
DataflowTriggerCreateRequest,
DataflowTriggerDeliveryListResponse,
DataflowTriggerDeleteResponse,
DataflowTriggerDispatchResponse,
DataflowTriggerListResponse,
DataflowTriggerResponse,
DataflowTriggerUpdateRequest,
)
from govoplan_dataflow.backend.service import (
DataflowConflictError,
@@ -55,6 +78,7 @@ from govoplan_dataflow.backend.service import (
DataflowValidationError,
compile_sql_draft,
create_pipeline,
derive_pipeline,
cancel_pipeline_run,
delete_pipeline,
get_pipeline,
@@ -69,6 +93,17 @@ from govoplan_dataflow.backend.service import (
update_pipeline,
validate_draft,
)
from govoplan_dataflow.backend.triggers import (
create_trigger,
delete_trigger,
delivery_response,
dispatch_due_triggers,
ingest_event_delivery,
list_deliveries,
list_triggers,
trigger_response,
update_trigger,
)
router = APIRouter(prefix="/dataflow", tags=["dataflow"])
@@ -108,6 +143,30 @@ def _http_error(exc: DataflowError) -> HTTPException:
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
def _governance_http_error(exc: PermissionError | ValueError) -> HTTPException:
return HTTPException(
status_code=(
status.HTTP_403_FORBIDDEN
if isinstance(exc, PermissionError)
else status.HTTP_422_UNPROCESSABLE_CONTENT
),
detail=str(exc),
)
def _pipeline_response(
session: Session,
pipeline,
principal: ApiPrincipal,
) -> PipelineResponse:
return pipeline_response(
session,
pipeline,
principal=principal,
registry=get_registry(),
)
def _source_http_error(exc: DatasourceError) -> HTTPException:
if isinstance(exc, DatasourceAccessError):
return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc))
@@ -317,8 +376,27 @@ def api_list_pipelines(
) -> PipelineListResponse:
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
pipelines = list_pipelines(session, tenant_id=principal.tenant_id)
registry = get_registry()
visible = [
pipeline
for pipeline in pipelines
if definition_decision(
pipeline,
principal=principal,
registry=registry,
action="view",
).allowed
]
return PipelineListResponse(
pipelines=[pipeline_response(session, pipeline) for pipeline in pipelines]
pipelines=[
pipeline_response(
session,
pipeline,
principal=principal,
registry=registry,
)
for pipeline in visible
]
)
@@ -330,12 +408,23 @@ def api_create_pipeline(
) -> PipelineResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
try:
tenant_id, scope_type, scope_id = normalize_definition_scope(
principal,
scope_type=payload.scope_type,
scope_id=payload.scope_id,
administrative=has_scope(principal, ADMIN_SCOPE),
)
payload = payload.model_copy(
update={"scope_type": scope_type, "scope_id": scope_id}
)
pipeline = create_pipeline(
session,
tenant_id=principal.tenant_id,
tenant_id=tenant_id or principal.tenant_id,
actor_id=_actor_id(principal),
payload=payload,
)
except (PermissionError, ValueError) as exc:
raise _governance_http_error(exc) from exc
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
@@ -348,7 +437,7 @@ def api_create_pipeline(
object_id=pipeline.id,
details={"revision": pipeline.current_revision, "status": pipeline.status},
)
response = pipeline_response(session, pipeline)
response = _pipeline_response(session, pipeline, principal)
session.commit()
return response
@@ -366,7 +455,15 @@ def api_get_pipeline(
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
)
return pipeline_response(session, pipeline)
require_definition_action(
pipeline,
principal=principal,
registry=get_registry(),
action="view",
)
return _pipeline_response(session, pipeline, principal)
except PermissionError as exc:
raise _governance_http_error(exc) from exc
except DataflowError as exc:
raise _http_error(exc) from exc
@@ -380,6 +477,17 @@ def api_update_pipeline(
) -> PipelineResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
try:
existing = get_pipeline(
session,
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
)
require_definition_action(
existing,
principal=principal,
registry=get_registry(),
action="edit",
)
pipeline = update_pipeline(
session,
tenant_id=principal.tenant_id,
@@ -387,6 +495,8 @@ def api_update_pipeline(
actor_id=_actor_id(principal),
payload=payload,
)
except PermissionError as exc:
raise _governance_http_error(exc) from exc
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
@@ -399,7 +509,7 @@ def api_update_pipeline(
object_id=pipeline.id,
details={"revision": pipeline.current_revision, "status": pipeline.status},
)
response = pipeline_response(session, pipeline)
response = _pipeline_response(session, pipeline, principal)
session.commit()
return response
@@ -412,12 +522,25 @@ def api_delete_pipeline(
) -> PipelineDeleteResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
try:
existing = get_pipeline(
session,
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
)
require_definition_action(
existing,
principal=principal,
registry=get_registry(),
action="edit",
)
pipeline = delete_pipeline(
session,
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
actor_id=_actor_id(principal),
)
except PermissionError as exc:
raise _governance_http_error(exc) from exc
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
@@ -434,6 +557,314 @@ def api_delete_pipeline(
return PipelineDeleteResponse(deleted=True, pipeline_id=pipeline.id)
@router.post(
"/pipelines/{pipeline_id}/derive",
response_model=PipelineResponse,
status_code=status.HTTP_201_CREATED,
)
def api_derive_pipeline(
pipeline_id: str,
payload: PipelineDeriveRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelineResponse:
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
try:
tenant_id, scope_type, scope_id = normalize_definition_scope(
principal,
scope_type=payload.scope_type,
scope_id=payload.scope_id,
administrative=has_scope(principal, ADMIN_SCOPE),
)
payload = payload.model_copy(
update={"scope_type": scope_type, "scope_id": scope_id}
)
pipeline = derive_pipeline(
session,
tenant_id=tenant_id or principal.tenant_id,
actor_id=_actor_id(principal),
principal=principal,
registry=get_registry(),
source_pipeline_id=pipeline_id,
payload=payload,
)
except (PermissionError, ValueError) as exc:
raise _governance_http_error(exc) from exc
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="dataflow.pipeline.derived",
object_type="dataflow_pipeline",
object_id=pipeline.id,
details={
"source_pipeline_id": pipeline.derived_from_pipeline_id,
"source_revision": pipeline.derived_from_revision,
"source_hash": pipeline.derived_from_hash,
"scope_type": pipeline.scope_type,
"scope_id": pipeline.scope_id,
},
)
response = _pipeline_response(session, pipeline, principal)
session.commit()
return response
@router.get(
"/pipelines/{pipeline_id}/triggers",
response_model=DataflowTriggerListResponse,
)
def api_list_triggers(
pipeline_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DataflowTriggerListResponse:
_require_any_scope(principal, TRIGGER_READ_SCOPE, ADMIN_SCOPE)
try:
pipeline = get_pipeline(
session,
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
)
require_definition_action(
pipeline,
principal=principal,
registry=get_registry(),
action="view",
)
items = list_triggers(
session,
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
)
return DataflowTriggerListResponse(
triggers=[trigger_response(session, item) for item in items]
)
except PermissionError as exc:
raise _governance_http_error(exc) from exc
except DataflowError as exc:
raise _http_error(exc) from exc
@router.post(
"/pipelines/{pipeline_id}/triggers",
response_model=DataflowTriggerResponse,
status_code=status.HTTP_201_CREATED,
)
def api_create_trigger(
pipeline_id: str,
payload: DataflowTriggerCreateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DataflowTriggerResponse:
_require_any_scope(principal, TRIGGER_WRITE_SCOPE, ADMIN_SCOPE)
try:
trigger = create_trigger(
session,
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
actor_id=_actor_id(principal),
principal=principal,
registry=get_registry(),
payload=payload,
)
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="dataflow.trigger.created",
object_type="dataflow_trigger",
object_id=trigger.id,
details={
"pipeline_id": trigger.pipeline_id,
"kind": trigger.kind,
"status": trigger.status,
"grant_scopes": list(trigger.grant_scopes),
},
)
response = trigger_response(session, trigger)
session.commit()
return response
@router.put(
"/triggers/{trigger_id}",
response_model=DataflowTriggerResponse,
)
def api_update_trigger(
trigger_id: str,
payload: DataflowTriggerUpdateRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DataflowTriggerResponse:
_require_any_scope(principal, TRIGGER_WRITE_SCOPE, ADMIN_SCOPE)
try:
trigger = update_trigger(
session,
tenant_id=principal.tenant_id,
trigger_id=trigger_id,
actor_id=_actor_id(principal),
principal=principal,
registry=get_registry(),
payload=payload,
)
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="dataflow.trigger.updated",
object_type="dataflow_trigger",
object_id=trigger.id,
details={
"pipeline_id": trigger.pipeline_id,
"kind": trigger.kind,
"status": trigger.status,
"revision": trigger.revision,
"grant_scopes": list(trigger.grant_scopes),
},
)
response = trigger_response(session, trigger)
session.commit()
return response
@router.delete(
"/triggers/{trigger_id}",
response_model=DataflowTriggerDeleteResponse,
)
def api_delete_trigger(
trigger_id: str,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DataflowTriggerDeleteResponse:
_require_any_scope(principal, TRIGGER_WRITE_SCOPE, ADMIN_SCOPE)
try:
trigger = delete_trigger(
session,
tenant_id=principal.tenant_id,
trigger_id=trigger_id,
)
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="dataflow.trigger.deleted",
object_type="dataflow_trigger",
object_id=trigger.id,
details={"pipeline_id": trigger.pipeline_id},
)
session.commit()
return DataflowTriggerDeleteResponse(
deleted=True,
trigger_id=trigger.id,
pipeline_id=trigger.pipeline_id,
)
@router.get(
"/triggers/{trigger_id}/deliveries",
response_model=DataflowTriggerDeliveryListResponse,
)
def api_list_trigger_deliveries(
trigger_id: str,
limit: int = 50,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DataflowTriggerDeliveryListResponse:
_require_any_scope(principal, TRIGGER_READ_SCOPE, ADMIN_SCOPE)
try:
items = list_deliveries(
session,
tenant_id=principal.tenant_id,
trigger_id=trigger_id,
limit=limit,
)
except DataflowError as exc:
raise _http_error(exc) from exc
return DataflowTriggerDeliveryListResponse(
deliveries=[delivery_response(item) for item in items]
)
@router.post(
"/triggers/events",
response_model=DataflowTriggerDispatchResponse,
)
def api_ingest_trigger_event(
payload: DataflowEventDeliveryRequest,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DataflowTriggerDispatchResponse:
_require_any_scope(principal, TRIGGER_DISPATCH_SCOPE, ADMIN_SCOPE)
try:
queued = ingest_event_delivery(
session,
tenant_id=principal.tenant_id,
event=payload,
)
except DataflowError as exc:
raise _http_error(exc) from exc
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="dataflow.trigger.event_ingested",
object_type="platform_event",
object_id=payload.event_id,
details={
"event_type": payload.type,
"module_id": payload.module_id,
"classification": payload.classification,
"queued": queued,
},
)
session.commit()
return DataflowTriggerDispatchResponse(queued=queued)
@router.post(
"/triggers/dispatch",
response_model=DataflowTriggerDispatchResponse,
)
def api_dispatch_triggers(
limit: int = 50,
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(get_api_principal),
) -> DataflowTriggerDispatchResponse:
_require_any_scope(principal, TRIGGER_DISPATCH_SCOPE, ADMIN_SCOPE)
result = dispatch_due_triggers(
session,
registry=get_registry(),
limit=limit,
tenant_id=principal.tenant_id,
)
audit_event(
session,
tenant_id=principal.tenant_id,
user_id=getattr(principal.user, "id", None),
api_key_id=principal.api_key_id,
action="dataflow.trigger.dispatched",
object_type="dataflow_trigger_dispatch",
object_id=principal.tenant_id,
details=result.model_dump(mode="json"),
)
session.commit()
return result
@router.get(
"/pipelines/{pipeline_id}/runs",
response_model=PipelineRunListResponse,
@@ -446,6 +877,17 @@ def api_list_pipeline_runs(
) -> PipelineRunListResponse:
_require_any_scope(principal, READ_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
try:
pipeline = get_pipeline(
session,
tenant_id=principal.tenant_id,
pipeline_id=pipeline_id,
)
require_definition_action(
pipeline,
principal=principal,
registry=get_registry(),
action="view",
)
runs = list_pipeline_runs(
session,
tenant_id=principal.tenant_id,
@@ -455,6 +897,8 @@ def api_list_pipeline_runs(
return PipelineRunListResponse(
runs=[pipeline_run_response(session, run) for run in runs]
)
except PermissionError as exc:
raise _governance_http_error(exc) from exc
except DataflowError as exc:
raise _http_error(exc) from exc
@@ -471,6 +915,8 @@ def api_start_pipeline_run(
principal: ApiPrincipal = Depends(get_api_principal),
) -> PipelineRunResponse:
_require_any_scope(principal, RUN_SCOPE, ADMIN_SCOPE)
if payload.invocation_kind == "backfill":
_require_any_scope(principal, ADMIN_SCOPE)
try:
pipeline = get_pipeline(
session,
@@ -503,6 +949,16 @@ def api_start_pipeline_run(
idempotency_key=payload.idempotency_key,
row_limit=payload.row_limit,
publication=publication,
invocation=AutomationInvocation(
kind=(
"backfill"
if payload.invocation_kind == "backfill"
else "api"
if principal.auth_method == "api_key"
else "manual"
),
requested_by=_actor_id(principal),
),
),
)
except DataflowError as exc:
@@ -523,6 +979,7 @@ def api_start_pipeline_run(
"pipeline_id": run.pipeline_id,
"revision_id": run.pipeline_revision_id,
"status": run.status,
"invocation_kind": run.invocation_kind,
"output_datasource_ref": run.output_datasource_ref,
"output_materialization_ref": run.output_materialization_ref,
},