from __future__ import annotations from fastapi import APIRouter, Depends, HTTPException, status from sqlalchemy.orm import Session from govoplan_core.audit.logging import audit_event from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope from govoplan_core.core.automation import AutomationInvocation from govoplan_core.core.dataflows import ( DataflowPublicationTarget, DataflowRunRequest, ) from govoplan_core.core.datasources import ( DatasourceAccessError, DatasourceDescriptor, DatasourceError, DatasourceNotFoundError, DatasourceStageInput, DatasourceUnavailableError, DatasourceValidationError, datasource_catalogue, datasource_lifecycle, ) from govoplan_core.core.tabular_sources import ( parse_tabular_csv, ) from govoplan_core.db.session import get_session from govoplan_dataflow.backend.governance import ( definition_decision, normalize_definition_scope, require_definition_action, ) from govoplan_dataflow.backend.manifest import ( ADMIN_SCOPE, READ_SCOPE, RUN_SCOPE, TRIGGER_DISPATCH_SCOPE, TRIGGER_READ_SCOPE, TRIGGER_WRITE_SCOPE, WRITE_SCOPE, ) from govoplan_dataflow.backend.node_library import CATEGORY_LABELS, NODE_LIBRARY from govoplan_dataflow.backend.runtime import get_registry from govoplan_dataflow.backend.schemas import ( NodeLibraryResponse, NodeTypeDefinitionResponse, PipelineCreateRequest, PipelineDeriveRequest, PipelineDeleteResponse, PipelineDraftRequest, PipelineListResponse, PipelinePreviewRequest, PipelinePreviewResponse, PipelineRunCreateRequest, PipelineRunListResponse, PipelineRunResponse, PipelineResponse, PipelineSqlResponse, PipelineUpdateRequest, PipelineValidationResponse, TabularSnapshotCreateRequest, TabularSourceColumnResponse, TabularSourceListResponse, TabularSourceResponse, DataflowEventDeliveryRequest, DataflowTriggerCreateRequest, DataflowTriggerDeliveryListResponse, DataflowTriggerDeleteResponse, DataflowTriggerDispatchResponse, DataflowTriggerListResponse, DataflowTriggerResponse, DataflowTriggerUpdateRequest, ) from govoplan_dataflow.backend.service import ( DataflowConflictError, DataflowError, DataflowNotFoundError, DataflowValidationError, compile_sql_draft, create_pipeline, derive_pipeline, cancel_pipeline_run, delete_pipeline, get_pipeline, get_pipeline_run, list_pipeline_runs, list_pipelines, pipeline_response, pipeline_run_response, preview_pipeline, render_graph_sql, start_pipeline_run, update_pipeline, validate_draft, ) from govoplan_dataflow.backend.triggers import ( create_trigger, delete_trigger, delivery_response, dispatch_due_triggers, ingest_event_delivery, list_deliveries, list_triggers, trigger_response, update_trigger, ) router = APIRouter(prefix="/dataflow", tags=["dataflow"]) def _require_any_scope(principal: ApiPrincipal, *scopes: str) -> None: if any(has_scope(principal, scope) for scope in scopes): return raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=f"Missing one of the required scopes: {', '.join(scopes)}", ) def _actor_id(principal: ApiPrincipal) -> str | None: return ( principal.account_id or principal.membership_id or getattr(principal.user, "id", None) or principal.identity_id ) def _http_error(exc: DataflowError) -> HTTPException: if isinstance(exc, DataflowNotFoundError): return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) if isinstance(exc, DataflowConflictError): return HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) if isinstance(exc, DataflowValidationError): return HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail={ "message": str(exc), "diagnostics": [item.model_dump(mode="json") for item in exc.diagnostics], }, ) return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) def _governance_http_error(exc: PermissionError | ValueError) -> HTTPException: return HTTPException( status_code=( status.HTTP_403_FORBIDDEN if isinstance(exc, PermissionError) else status.HTTP_422_UNPROCESSABLE_CONTENT ), detail=str(exc), ) def _pipeline_response( session: Session, pipeline, principal: ApiPrincipal, ) -> PipelineResponse: return pipeline_response( session, pipeline, principal=principal, registry=get_registry(), ) def _source_http_error(exc: DatasourceError) -> HTTPException: if isinstance(exc, DatasourceAccessError): return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)) if isinstance(exc, DatasourceNotFoundError): return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc)) if isinstance(exc, DatasourceUnavailableError): return HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) if isinstance(exc, DatasourceValidationError): return HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail=str(exc), ) return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)) def _node_library_response() -> NodeLibraryResponse: return NodeLibraryResponse( nodes=[ NodeTypeDefinitionResponse( type=definition.type, category=definition.category, category_label=CATEGORY_LABELS[definition.category], label=definition.label, description=definition.description, icon=definition.icon, input_ports=[ { "id": port.id, "label": port.label, "required": port.required, "multiple": port.multiple, "minimum_connections": port.minimum_connections, } for port in definition.input_ports ], output_ports=[ { "id": port.id, "label": port.label, "required": port.required, "multiple": port.multiple, "minimum_connections": port.minimum_connections, } for port in definition.output_ports ], config_fields=[ { "id": field.id, "label": field.label, "kind": field.kind, "required": field.required, "description": field.description, "options": list(field.options), } for field in definition.config_fields ], default_config=dict(definition.default_config), sql_support=definition.sql_support, ) for definition in NODE_LIBRARY ] ) def _source_response(source: DatasourceDescriptor) -> TabularSourceResponse: return TabularSourceResponse( ref=source.ref, provider=source.provider or "datasources", source_name=source.source_name, name=source.name, description=source.description, mode=source.mode, columns=[ TabularSourceColumnResponse( name=column.name, data_type=column.data_type, nullable=column.nullable, ) for column in source.schema ], schema_version=source.schema_version, fingerprint=source.fingerprint, row_count=source.row_count, byte_count=source.byte_count, updated_at=source.updated_at, capabilities=list(source.capabilities), ) @router.get("/node-types", response_model=NodeLibraryResponse) def api_node_types( principal: ApiPrincipal = Depends(get_api_principal), ) -> NodeLibraryResponse: _require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE) return _node_library_response() @router.get("/sources", response_model=TabularSourceListResponse) def api_list_sources( query: str = "", session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> TabularSourceListResponse: _require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE) registry = get_registry() provider = datasource_catalogue(registry) writer = datasource_lifecycle(registry) if provider is None: return TabularSourceListResponse(available=False, writable=False, sources=[]) try: sources = provider.list_datasources( session, principal, query=query, limit=100, ) except DatasourceError as exc: raise _source_http_error(exc) from exc return TabularSourceListResponse( available=True, writable=writer is not None, sources=[_source_response(source) for source in sources], ) @router.post( "/sources/snapshots", response_model=TabularSourceResponse, status_code=status.HTTP_201_CREATED, ) def api_create_source_snapshot( payload: TabularSnapshotCreateRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> TabularSourceResponse: _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE) writer = datasource_lifecycle(get_registry()) if writer is None: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail="No datasource staging provider is available.", ) try: rows = ( parse_tabular_csv( payload.csv_text or "", delimiter=payload.delimiter, max_rows=10_000, ) if payload.format == "csv" else tuple(payload.rows or ()) ) stage = writer.create_stage( session, principal, stage=DatasourceStageInput( name=payload.name, source_name=payload.source_name, description=payload.description, kind="upload", mode="static", shape="tabular", rows=rows, provider="dataflow.upload", provenance={ "created_via": "dataflow", "source_format": payload.format, }, metadata={ "dataflow_convenience_import": True, }, ), ) source, materialization = writer.promote_stage( session, principal, stage_ref=stage.ref, ) except DatasourceError as exc: raise _source_http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.datasource.created", object_type="datasource", object_id=source.ref, details={ "provider": source.provider, "source_name": source.source_name, "row_count": source.row_count, "fingerprint": source.fingerprint, "stage_ref": stage.ref, "materialization_ref": materialization.ref, }, ) response = _source_response(source) session.commit() return response @router.get("/pipelines", response_model=PipelineListResponse) def api_list_pipelines( session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineListResponse: _require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE) pipelines = list_pipelines(session, tenant_id=principal.tenant_id) registry = get_registry() visible = [ pipeline for pipeline in pipelines if definition_decision( pipeline, principal=principal, registry=registry, action="view", ).allowed ] return PipelineListResponse( pipelines=[ pipeline_response( session, pipeline, principal=principal, registry=registry, ) for pipeline in visible ] ) @router.post("/pipelines", response_model=PipelineResponse, status_code=status.HTTP_201_CREATED) def api_create_pipeline( payload: PipelineCreateRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineResponse: _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE) try: tenant_id, scope_type, scope_id = normalize_definition_scope( principal, scope_type=payload.scope_type, scope_id=payload.scope_id, administrative=has_scope(principal, ADMIN_SCOPE), ) payload = payload.model_copy( update={"scope_type": scope_type, "scope_id": scope_id} ) pipeline = create_pipeline( session, tenant_id=tenant_id or principal.tenant_id, actor_id=_actor_id(principal), payload=payload, ) except (PermissionError, ValueError) as exc: raise _governance_http_error(exc) from exc except DataflowError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.pipeline.created", object_type="dataflow_pipeline", object_id=pipeline.id, details={"revision": pipeline.current_revision, "status": pipeline.status}, ) response = _pipeline_response(session, pipeline, principal) session.commit() return response @router.get("/pipelines/{pipeline_id}", response_model=PipelineResponse) def api_get_pipeline( pipeline_id: str, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineResponse: _require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE) try: pipeline = get_pipeline( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, ) require_definition_action( pipeline, principal=principal, registry=get_registry(), action="view", ) return _pipeline_response(session, pipeline, principal) except PermissionError as exc: raise _governance_http_error(exc) from exc except DataflowError as exc: raise _http_error(exc) from exc @router.put("/pipelines/{pipeline_id}", response_model=PipelineResponse) def api_update_pipeline( pipeline_id: str, payload: PipelineUpdateRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineResponse: _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE) try: existing = get_pipeline( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, ) require_definition_action( existing, principal=principal, registry=get_registry(), action="edit", ) pipeline = update_pipeline( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, actor_id=_actor_id(principal), payload=payload, ) except PermissionError as exc: raise _governance_http_error(exc) from exc except DataflowError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.pipeline.updated", object_type="dataflow_pipeline", object_id=pipeline.id, details={"revision": pipeline.current_revision, "status": pipeline.status}, ) response = _pipeline_response(session, pipeline, principal) session.commit() return response @router.delete("/pipelines/{pipeline_id}", response_model=PipelineDeleteResponse) def api_delete_pipeline( pipeline_id: str, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineDeleteResponse: _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE) try: existing = get_pipeline( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, ) require_definition_action( existing, principal=principal, registry=get_registry(), action="edit", ) pipeline = delete_pipeline( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, actor_id=_actor_id(principal), ) except PermissionError as exc: raise _governance_http_error(exc) from exc except DataflowError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.pipeline.deleted", object_type="dataflow_pipeline", object_id=pipeline.id, details={"revision": pipeline.current_revision}, ) session.commit() return PipelineDeleteResponse(deleted=True, pipeline_id=pipeline.id) @router.post( "/pipelines/{pipeline_id}/derive", response_model=PipelineResponse, status_code=status.HTTP_201_CREATED, ) def api_derive_pipeline( pipeline_id: str, payload: PipelineDeriveRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineResponse: _require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE) try: tenant_id, scope_type, scope_id = normalize_definition_scope( principal, scope_type=payload.scope_type, scope_id=payload.scope_id, administrative=has_scope(principal, ADMIN_SCOPE), ) payload = payload.model_copy( update={"scope_type": scope_type, "scope_id": scope_id} ) pipeline = derive_pipeline( session, tenant_id=tenant_id or principal.tenant_id, actor_id=_actor_id(principal), principal=principal, registry=get_registry(), source_pipeline_id=pipeline_id, payload=payload, ) except (PermissionError, ValueError) as exc: raise _governance_http_error(exc) from exc except DataflowError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.pipeline.derived", object_type="dataflow_pipeline", object_id=pipeline.id, details={ "source_pipeline_id": pipeline.derived_from_pipeline_id, "source_revision": pipeline.derived_from_revision, "source_hash": pipeline.derived_from_hash, "scope_type": pipeline.scope_type, "scope_id": pipeline.scope_id, }, ) response = _pipeline_response(session, pipeline, principal) session.commit() return response @router.get( "/pipelines/{pipeline_id}/triggers", response_model=DataflowTriggerListResponse, ) def api_list_triggers( pipeline_id: str, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> DataflowTriggerListResponse: _require_any_scope(principal, TRIGGER_READ_SCOPE, ADMIN_SCOPE) try: pipeline = get_pipeline( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, ) require_definition_action( pipeline, principal=principal, registry=get_registry(), action="view", ) items = list_triggers( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, ) return DataflowTriggerListResponse( triggers=[trigger_response(session, item) for item in items] ) except PermissionError as exc: raise _governance_http_error(exc) from exc except DataflowError as exc: raise _http_error(exc) from exc @router.post( "/pipelines/{pipeline_id}/triggers", response_model=DataflowTriggerResponse, status_code=status.HTTP_201_CREATED, ) def api_create_trigger( pipeline_id: str, payload: DataflowTriggerCreateRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> DataflowTriggerResponse: _require_any_scope(principal, TRIGGER_WRITE_SCOPE, ADMIN_SCOPE) try: trigger = create_trigger( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, actor_id=_actor_id(principal), principal=principal, registry=get_registry(), payload=payload, ) except DataflowError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.trigger.created", object_type="dataflow_trigger", object_id=trigger.id, details={ "pipeline_id": trigger.pipeline_id, "kind": trigger.kind, "status": trigger.status, "grant_scopes": list(trigger.grant_scopes), }, ) response = trigger_response(session, trigger) session.commit() return response @router.put( "/triggers/{trigger_id}", response_model=DataflowTriggerResponse, ) def api_update_trigger( trigger_id: str, payload: DataflowTriggerUpdateRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> DataflowTriggerResponse: _require_any_scope(principal, TRIGGER_WRITE_SCOPE, ADMIN_SCOPE) try: trigger = update_trigger( session, tenant_id=principal.tenant_id, trigger_id=trigger_id, actor_id=_actor_id(principal), principal=principal, registry=get_registry(), payload=payload, ) except DataflowError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.trigger.updated", object_type="dataflow_trigger", object_id=trigger.id, details={ "pipeline_id": trigger.pipeline_id, "kind": trigger.kind, "status": trigger.status, "revision": trigger.revision, "grant_scopes": list(trigger.grant_scopes), }, ) response = trigger_response(session, trigger) session.commit() return response @router.delete( "/triggers/{trigger_id}", response_model=DataflowTriggerDeleteResponse, ) def api_delete_trigger( trigger_id: str, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> DataflowTriggerDeleteResponse: _require_any_scope(principal, TRIGGER_WRITE_SCOPE, ADMIN_SCOPE) try: trigger = delete_trigger( session, tenant_id=principal.tenant_id, trigger_id=trigger_id, ) except DataflowError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.trigger.deleted", object_type="dataflow_trigger", object_id=trigger.id, details={"pipeline_id": trigger.pipeline_id}, ) session.commit() return DataflowTriggerDeleteResponse( deleted=True, trigger_id=trigger.id, pipeline_id=trigger.pipeline_id, ) @router.get( "/triggers/{trigger_id}/deliveries", response_model=DataflowTriggerDeliveryListResponse, ) def api_list_trigger_deliveries( trigger_id: str, limit: int = 50, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> DataflowTriggerDeliveryListResponse: _require_any_scope(principal, TRIGGER_READ_SCOPE, ADMIN_SCOPE) try: items = list_deliveries( session, tenant_id=principal.tenant_id, trigger_id=trigger_id, limit=limit, ) except DataflowError as exc: raise _http_error(exc) from exc return DataflowTriggerDeliveryListResponse( deliveries=[delivery_response(item) for item in items] ) @router.post( "/triggers/events", response_model=DataflowTriggerDispatchResponse, ) def api_ingest_trigger_event( payload: DataflowEventDeliveryRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> DataflowTriggerDispatchResponse: _require_any_scope(principal, TRIGGER_DISPATCH_SCOPE, ADMIN_SCOPE) try: queued = ingest_event_delivery( session, tenant_id=principal.tenant_id, event=payload, ) except DataflowError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.trigger.event_ingested", object_type="platform_event", object_id=payload.event_id, details={ "event_type": payload.type, "module_id": payload.module_id, "classification": payload.classification, "queued": queued, }, ) session.commit() return DataflowTriggerDispatchResponse(queued=queued) @router.post( "/triggers/dispatch", response_model=DataflowTriggerDispatchResponse, ) def api_dispatch_triggers( limit: int = 50, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> DataflowTriggerDispatchResponse: _require_any_scope(principal, TRIGGER_DISPATCH_SCOPE, ADMIN_SCOPE) result = dispatch_due_triggers( session, registry=get_registry(), limit=limit, tenant_id=principal.tenant_id, ) audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.trigger.dispatched", object_type="dataflow_trigger_dispatch", object_id=principal.tenant_id, details=result.model_dump(mode="json"), ) session.commit() return result @router.get( "/pipelines/{pipeline_id}/runs", response_model=PipelineRunListResponse, ) def api_list_pipeline_runs( pipeline_id: str, limit: int = 50, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineRunListResponse: _require_any_scope(principal, READ_SCOPE, RUN_SCOPE, ADMIN_SCOPE) try: pipeline = get_pipeline( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, ) require_definition_action( pipeline, principal=principal, registry=get_registry(), action="view", ) runs = list_pipeline_runs( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, limit=limit, ) return PipelineRunListResponse( runs=[pipeline_run_response(session, run) for run in runs] ) except PermissionError as exc: raise _governance_http_error(exc) from exc except DataflowError as exc: raise _http_error(exc) from exc @router.post( "/pipelines/{pipeline_id}/runs", response_model=PipelineRunResponse, status_code=status.HTTP_201_CREATED, ) def api_start_pipeline_run( pipeline_id: str, payload: PipelineRunCreateRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineRunResponse: _require_any_scope(principal, RUN_SCOPE, ADMIN_SCOPE) if payload.invocation_kind == "backfill": _require_any_scope(principal, ADMIN_SCOPE) try: pipeline = get_pipeline( session, tenant_id=principal.tenant_id, pipeline_id=pipeline_id, ) publication = ( DataflowPublicationTarget( target_datasource_ref=payload.publication.target_datasource_ref, name=payload.publication.name, source_name=payload.publication.source_name, description=payload.publication.description, freeze=payload.publication.freeze, frozen_label=payload.publication.frozen_label, set_current=payload.publication.set_current, metadata=payload.publication.metadata, ) if payload.publication else None ) run, replayed = start_pipeline_run( session, tenant_id=principal.tenant_id, actor_id=_actor_id(principal), principal=principal, registry=get_registry(), request=DataflowRunRequest( pipeline_ref=f"pipeline:{pipeline.id}", revision=payload.revision or pipeline.current_revision, idempotency_key=payload.idempotency_key, row_limit=payload.row_limit, publication=publication, invocation=AutomationInvocation( kind=( "backfill" if payload.invocation_kind == "backfill" else "api" if principal.auth_method == "api_key" else "manual" ), requested_by=_actor_id(principal), ), ), ) except DataflowError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action=( "dataflow.run.replayed" if replayed else f"dataflow.run.{run.status}" ), object_type="dataflow_run", object_id=run.id, details={ "pipeline_id": run.pipeline_id, "revision_id": run.pipeline_revision_id, "status": run.status, "invocation_kind": run.invocation_kind, "output_datasource_ref": run.output_datasource_ref, "output_materialization_ref": run.output_materialization_ref, }, ) response = pipeline_run_response(session, run, replayed=replayed) session.commit() return response @router.get("/runs/{run_id}", response_model=PipelineRunResponse) def api_get_pipeline_run( run_id: str, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineRunResponse: _require_any_scope(principal, READ_SCOPE, RUN_SCOPE, ADMIN_SCOPE) try: run = get_pipeline_run( session, tenant_id=principal.tenant_id, run_ref=f"dataflow-run:{run_id}", ) return pipeline_run_response(session, run) except DataflowError as exc: raise _http_error(exc) from exc @router.post("/runs/{run_id}/cancel", response_model=PipelineRunResponse) def api_cancel_pipeline_run( run_id: str, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineRunResponse: _require_any_scope(principal, RUN_SCOPE, ADMIN_SCOPE) try: run = cancel_pipeline_run( session, tenant_id=principal.tenant_id, run_ref=f"dataflow-run:{run_id}", ) except DataflowError as exc: raise _http_error(exc) from exc audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.run.cancelled", object_type="dataflow_run", object_id=run.id, details={"pipeline_id": run.pipeline_id}, ) response = pipeline_run_response(session, run) session.commit() return response @router.post("/validate", response_model=PipelineValidationResponse) def api_validate_pipeline( payload: PipelineDraftRequest, principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineValidationResponse: _require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE) return validate_draft(payload) @router.post("/sql/compile", response_model=PipelineSqlResponse) def api_compile_pipeline_sql( payload: PipelineDraftRequest, principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineSqlResponse: _require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE) return compile_sql_draft(payload) @router.post("/sql/render", response_model=PipelineSqlResponse) def api_render_pipeline_sql( payload: PipelineDraftRequest, principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelineSqlResponse: _require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE) return render_graph_sql(payload) @router.post("/preview", response_model=PipelinePreviewResponse) def api_preview_pipeline( payload: PipelinePreviewRequest, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(get_api_principal), ) -> PipelinePreviewResponse: _require_any_scope(principal, RUN_SCOPE, ADMIN_SCOPE) try: response = preview_pipeline( session, tenant_id=principal.tenant_id, actor_id=_actor_id(principal), principal=principal, registry=get_registry(), payload=payload, ) except DataflowError as exc: raise _http_error(exc) from exc if response.pipeline_id: audit_event( session, tenant_id=principal.tenant_id, user_id=getattr(principal.user, "id", None), api_key_id=principal.api_key_id, action="dataflow.pipeline.previewed", object_type="dataflow_pipeline", object_id=response.pipeline_id, details={ "revision": response.revision, "run_id": response.run_id, "status": response.status, "output_row_count": response.total_rows, }, ) session.commit() return response __all__ = ["router"]