feat: run pinned pipelines and publish outputs
This commit is contained in:
@@ -5,6 +5,10 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.audit.logging import audit_event
|
||||
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
|
||||
from govoplan_core.core.dataflows import (
|
||||
DataflowPublicationTarget,
|
||||
DataflowRunRequest,
|
||||
)
|
||||
from govoplan_core.core.datasources import (
|
||||
DatasourceAccessError,
|
||||
DatasourceDescriptor,
|
||||
@@ -32,6 +36,9 @@ from govoplan_dataflow.backend.schemas import (
|
||||
PipelineListResponse,
|
||||
PipelinePreviewRequest,
|
||||
PipelinePreviewResponse,
|
||||
PipelineRunCreateRequest,
|
||||
PipelineRunListResponse,
|
||||
PipelineRunResponse,
|
||||
PipelineResponse,
|
||||
PipelineSqlResponse,
|
||||
PipelineUpdateRequest,
|
||||
@@ -48,12 +55,17 @@ from govoplan_dataflow.backend.service import (
|
||||
DataflowValidationError,
|
||||
compile_sql_draft,
|
||||
create_pipeline,
|
||||
cancel_pipeline_run,
|
||||
delete_pipeline,
|
||||
get_pipeline,
|
||||
get_pipeline_run,
|
||||
list_pipeline_runs,
|
||||
list_pipelines,
|
||||
pipeline_response,
|
||||
pipeline_run_response,
|
||||
preview_pipeline,
|
||||
render_graph_sql,
|
||||
start_pipeline_run,
|
||||
update_pipeline,
|
||||
validate_draft,
|
||||
)
|
||||
@@ -167,6 +179,7 @@ def _source_response(source: DatasourceDescriptor) -> TabularSourceResponse:
|
||||
source_name=source.source_name,
|
||||
name=source.name,
|
||||
description=source.description,
|
||||
mode=source.mode,
|
||||
columns=[
|
||||
TabularSourceColumnResponse(
|
||||
name=column.name,
|
||||
@@ -421,6 +434,152 @@ def api_delete_pipeline(
|
||||
return PipelineDeleteResponse(deleted=True, pipeline_id=pipeline.id)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/pipelines/{pipeline_id}/runs",
|
||||
response_model=PipelineRunListResponse,
|
||||
)
|
||||
def api_list_pipeline_runs(
|
||||
pipeline_id: str,
|
||||
limit: int = 50,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> PipelineRunListResponse:
|
||||
_require_any_scope(principal, READ_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
||||
try:
|
||||
runs = list_pipeline_runs(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
pipeline_id=pipeline_id,
|
||||
limit=limit,
|
||||
)
|
||||
return PipelineRunListResponse(
|
||||
runs=[pipeline_run_response(session, run) for run in runs]
|
||||
)
|
||||
except DataflowError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post(
|
||||
"/pipelines/{pipeline_id}/runs",
|
||||
response_model=PipelineRunResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def api_start_pipeline_run(
|
||||
pipeline_id: str,
|
||||
payload: PipelineRunCreateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> PipelineRunResponse:
|
||||
_require_any_scope(principal, RUN_SCOPE, ADMIN_SCOPE)
|
||||
try:
|
||||
pipeline = get_pipeline(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
pipeline_id=pipeline_id,
|
||||
)
|
||||
publication = (
|
||||
DataflowPublicationTarget(
|
||||
target_datasource_ref=payload.publication.target_datasource_ref,
|
||||
name=payload.publication.name,
|
||||
source_name=payload.publication.source_name,
|
||||
description=payload.publication.description,
|
||||
freeze=payload.publication.freeze,
|
||||
frozen_label=payload.publication.frozen_label,
|
||||
set_current=payload.publication.set_current,
|
||||
metadata=payload.publication.metadata,
|
||||
)
|
||||
if payload.publication
|
||||
else None
|
||||
)
|
||||
run, replayed = start_pipeline_run(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
actor_id=_actor_id(principal),
|
||||
principal=principal,
|
||||
registry=get_registry(),
|
||||
request=DataflowRunRequest(
|
||||
pipeline_ref=f"pipeline:{pipeline.id}",
|
||||
revision=payload.revision or pipeline.current_revision,
|
||||
idempotency_key=payload.idempotency_key,
|
||||
row_limit=payload.row_limit,
|
||||
publication=publication,
|
||||
),
|
||||
)
|
||||
except DataflowError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
audit_event(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=getattr(principal.user, "id", None),
|
||||
api_key_id=principal.api_key_id,
|
||||
action=(
|
||||
"dataflow.run.replayed"
|
||||
if replayed
|
||||
else f"dataflow.run.{run.status}"
|
||||
),
|
||||
object_type="dataflow_run",
|
||||
object_id=run.id,
|
||||
details={
|
||||
"pipeline_id": run.pipeline_id,
|
||||
"revision_id": run.pipeline_revision_id,
|
||||
"status": run.status,
|
||||
"output_datasource_ref": run.output_datasource_ref,
|
||||
"output_materialization_ref": run.output_materialization_ref,
|
||||
},
|
||||
)
|
||||
response = pipeline_run_response(session, run, replayed=replayed)
|
||||
session.commit()
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/runs/{run_id}", response_model=PipelineRunResponse)
|
||||
def api_get_pipeline_run(
|
||||
run_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> PipelineRunResponse:
|
||||
_require_any_scope(principal, READ_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
||||
try:
|
||||
run = get_pipeline_run(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
run_ref=f"dataflow-run:{run_id}",
|
||||
)
|
||||
return pipeline_run_response(session, run)
|
||||
except DataflowError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
|
||||
|
||||
@router.post("/runs/{run_id}/cancel", response_model=PipelineRunResponse)
|
||||
def api_cancel_pipeline_run(
|
||||
run_id: str,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> PipelineRunResponse:
|
||||
_require_any_scope(principal, RUN_SCOPE, ADMIN_SCOPE)
|
||||
try:
|
||||
run = cancel_pipeline_run(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
run_ref=f"dataflow-run:{run_id}",
|
||||
)
|
||||
except DataflowError as exc:
|
||||
raise _http_error(exc) from exc
|
||||
audit_event(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=getattr(principal.user, "id", None),
|
||||
api_key_id=principal.api_key_id,
|
||||
action="dataflow.run.cancelled",
|
||||
object_type="dataflow_run",
|
||||
object_id=run.id,
|
||||
details={"pipeline_id": run.pipeline_id},
|
||||
)
|
||||
response = pipeline_run_response(session, run)
|
||||
session.commit()
|
||||
return response
|
||||
|
||||
|
||||
@router.post("/validate", response_model=PipelineValidationResponse)
|
||||
def api_validate_pipeline(
|
||||
payload: PipelineDraftRequest,
|
||||
|
||||
Reference in New Issue
Block a user