1106 lines
35 KiB
Python
1106 lines
35 KiB
Python
from __future__ import annotations
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, status
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_core.audit.logging import audit_event
|
|
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
|
|
from govoplan_core.core.automation import AutomationInvocation
|
|
from govoplan_core.core.dataflows import (
|
|
DataflowPublicationTarget,
|
|
DataflowRunRequest,
|
|
)
|
|
from govoplan_core.core.datasources import (
|
|
DatasourceAccessError,
|
|
DatasourceDescriptor,
|
|
DatasourceError,
|
|
DatasourceNotFoundError,
|
|
DatasourceStageInput,
|
|
DatasourceUnavailableError,
|
|
DatasourceValidationError,
|
|
datasource_catalogue,
|
|
datasource_lifecycle,
|
|
)
|
|
from govoplan_core.core.tabular_sources import (
|
|
parse_tabular_csv,
|
|
)
|
|
from govoplan_core.db.session import get_session
|
|
from govoplan_dataflow.backend.governance import (
|
|
definition_decision,
|
|
normalize_definition_scope,
|
|
require_definition_action,
|
|
)
|
|
from govoplan_dataflow.backend.manifest import (
|
|
ADMIN_SCOPE,
|
|
READ_SCOPE,
|
|
RUN_SCOPE,
|
|
TRIGGER_DISPATCH_SCOPE,
|
|
TRIGGER_READ_SCOPE,
|
|
TRIGGER_WRITE_SCOPE,
|
|
WRITE_SCOPE,
|
|
)
|
|
from govoplan_dataflow.backend.node_library import CATEGORY_LABELS, NODE_LIBRARY
|
|
from govoplan_dataflow.backend.runtime import get_registry
|
|
from govoplan_dataflow.backend.schemas import (
|
|
NodeLibraryResponse,
|
|
NodeTypeDefinitionResponse,
|
|
PipelineCreateRequest,
|
|
PipelineDeriveRequest,
|
|
PipelineDeleteResponse,
|
|
PipelineDraftRequest,
|
|
PipelineListResponse,
|
|
PipelinePreviewRequest,
|
|
PipelinePreviewResponse,
|
|
PipelineRunCreateRequest,
|
|
PipelineRunListResponse,
|
|
PipelineRunResponse,
|
|
PipelineResponse,
|
|
PipelineSqlResponse,
|
|
PipelineUpdateRequest,
|
|
PipelineValidationResponse,
|
|
TabularSnapshotCreateRequest,
|
|
TabularSourceColumnResponse,
|
|
TabularSourceListResponse,
|
|
TabularSourceResponse,
|
|
DataflowEventDeliveryRequest,
|
|
DataflowTriggerCreateRequest,
|
|
DataflowTriggerDeliveryListResponse,
|
|
DataflowTriggerDeleteResponse,
|
|
DataflowTriggerDispatchResponse,
|
|
DataflowTriggerListResponse,
|
|
DataflowTriggerResponse,
|
|
DataflowTriggerUpdateRequest,
|
|
)
|
|
from govoplan_dataflow.backend.service import (
|
|
DataflowConflictError,
|
|
DataflowError,
|
|
DataflowNotFoundError,
|
|
DataflowValidationError,
|
|
compile_sql_draft,
|
|
create_pipeline,
|
|
derive_pipeline,
|
|
cancel_pipeline_run,
|
|
delete_pipeline,
|
|
get_pipeline,
|
|
get_pipeline_run,
|
|
list_pipeline_runs,
|
|
list_pipelines,
|
|
pipeline_response,
|
|
pipeline_run_response,
|
|
preview_pipeline,
|
|
render_graph_sql,
|
|
start_pipeline_run,
|
|
update_pipeline,
|
|
validate_draft,
|
|
)
|
|
from govoplan_dataflow.backend.triggers import (
|
|
create_trigger,
|
|
delete_trigger,
|
|
delivery_response,
|
|
dispatch_due_triggers,
|
|
ingest_event_delivery,
|
|
list_deliveries,
|
|
list_triggers,
|
|
trigger_response,
|
|
update_trigger,
|
|
)
|
|
|
|
|
|
router = APIRouter(prefix="/dataflow", tags=["dataflow"])
|
|
|
|
|
|
def _require_any_scope(principal: ApiPrincipal, *scopes: str) -> None:
|
|
if any(has_scope(principal, scope) for scope in scopes):
|
|
return
|
|
raise HTTPException(
|
|
status_code=status.HTTP_403_FORBIDDEN,
|
|
detail=f"Missing one of the required scopes: {', '.join(scopes)}",
|
|
)
|
|
|
|
|
|
def _actor_id(principal: ApiPrincipal) -> str | None:
|
|
return (
|
|
principal.account_id
|
|
or principal.membership_id
|
|
or getattr(principal.user, "id", None)
|
|
or principal.identity_id
|
|
)
|
|
|
|
|
|
def _http_error(exc: DataflowError) -> HTTPException:
|
|
if isinstance(exc, DataflowNotFoundError):
|
|
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
|
if isinstance(exc, DataflowConflictError):
|
|
return HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc))
|
|
if isinstance(exc, DataflowValidationError):
|
|
return HTTPException(
|
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
detail={
|
|
"message": str(exc),
|
|
"diagnostics": [item.model_dump(mode="json") for item in exc.diagnostics],
|
|
},
|
|
)
|
|
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
|
|
|
|
|
def _governance_http_error(exc: PermissionError | ValueError) -> HTTPException:
|
|
return HTTPException(
|
|
status_code=(
|
|
status.HTTP_403_FORBIDDEN
|
|
if isinstance(exc, PermissionError)
|
|
else status.HTTP_422_UNPROCESSABLE_CONTENT
|
|
),
|
|
detail=str(exc),
|
|
)
|
|
|
|
|
|
def _pipeline_response(
|
|
session: Session,
|
|
pipeline,
|
|
principal: ApiPrincipal,
|
|
) -> PipelineResponse:
|
|
return pipeline_response(
|
|
session,
|
|
pipeline,
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
)
|
|
|
|
|
|
def _source_http_error(exc: DatasourceError) -> HTTPException:
|
|
if isinstance(exc, DatasourceAccessError):
|
|
return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc))
|
|
if isinstance(exc, DatasourceNotFoundError):
|
|
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
|
if isinstance(exc, DatasourceUnavailableError):
|
|
return HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc))
|
|
if isinstance(exc, DatasourceValidationError):
|
|
return HTTPException(
|
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
|
detail=str(exc),
|
|
)
|
|
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
|
|
|
|
|
def _node_library_response() -> NodeLibraryResponse:
|
|
return NodeLibraryResponse(
|
|
nodes=[
|
|
NodeTypeDefinitionResponse(
|
|
type=definition.type,
|
|
category=definition.category,
|
|
category_label=CATEGORY_LABELS[definition.category],
|
|
label=definition.label,
|
|
description=definition.description,
|
|
icon=definition.icon,
|
|
input_ports=[
|
|
{
|
|
"id": port.id,
|
|
"label": port.label,
|
|
"required": port.required,
|
|
"multiple": port.multiple,
|
|
"minimum_connections": port.minimum_connections,
|
|
}
|
|
for port in definition.input_ports
|
|
],
|
|
output_ports=[
|
|
{
|
|
"id": port.id,
|
|
"label": port.label,
|
|
"required": port.required,
|
|
"multiple": port.multiple,
|
|
"minimum_connections": port.minimum_connections,
|
|
}
|
|
for port in definition.output_ports
|
|
],
|
|
config_fields=[
|
|
{
|
|
"id": field.id,
|
|
"label": field.label,
|
|
"kind": field.kind,
|
|
"required": field.required,
|
|
"description": field.description,
|
|
"options": list(field.options),
|
|
}
|
|
for field in definition.config_fields
|
|
],
|
|
default_config=dict(definition.default_config),
|
|
sql_support=definition.sql_support,
|
|
)
|
|
for definition in NODE_LIBRARY
|
|
]
|
|
)
|
|
|
|
|
|
def _source_response(source: DatasourceDescriptor) -> TabularSourceResponse:
|
|
return TabularSourceResponse(
|
|
ref=source.ref,
|
|
provider=source.provider or "datasources",
|
|
source_name=source.source_name,
|
|
name=source.name,
|
|
description=source.description,
|
|
mode=source.mode,
|
|
columns=[
|
|
TabularSourceColumnResponse(
|
|
name=column.name,
|
|
data_type=column.data_type,
|
|
nullable=column.nullable,
|
|
)
|
|
for column in source.schema
|
|
],
|
|
schema_version=source.schema_version,
|
|
fingerprint=source.fingerprint,
|
|
row_count=source.row_count,
|
|
byte_count=source.byte_count,
|
|
updated_at=source.updated_at,
|
|
capabilities=list(source.capabilities),
|
|
)
|
|
|
|
|
|
@router.get("/node-types", response_model=NodeLibraryResponse)
|
|
def api_node_types(
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> NodeLibraryResponse:
|
|
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
|
return _node_library_response()
|
|
|
|
|
|
@router.get("/sources", response_model=TabularSourceListResponse)
|
|
def api_list_sources(
|
|
query: str = "",
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> TabularSourceListResponse:
|
|
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
|
registry = get_registry()
|
|
provider = datasource_catalogue(registry)
|
|
writer = datasource_lifecycle(registry)
|
|
if provider is None:
|
|
return TabularSourceListResponse(available=False, writable=False, sources=[])
|
|
try:
|
|
sources = provider.list_datasources(
|
|
session,
|
|
principal,
|
|
query=query,
|
|
limit=100,
|
|
)
|
|
except DatasourceError as exc:
|
|
raise _source_http_error(exc) from exc
|
|
return TabularSourceListResponse(
|
|
available=True,
|
|
writable=writer is not None,
|
|
sources=[_source_response(source) for source in sources],
|
|
)
|
|
|
|
|
|
@router.post(
|
|
"/sources/snapshots",
|
|
response_model=TabularSourceResponse,
|
|
status_code=status.HTTP_201_CREATED,
|
|
)
|
|
def api_create_source_snapshot(
|
|
payload: TabularSnapshotCreateRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> TabularSourceResponse:
|
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
|
writer = datasource_lifecycle(get_registry())
|
|
if writer is None:
|
|
raise HTTPException(
|
|
status_code=status.HTTP_409_CONFLICT,
|
|
detail="No datasource staging provider is available.",
|
|
)
|
|
try:
|
|
rows = (
|
|
parse_tabular_csv(
|
|
payload.csv_text or "",
|
|
delimiter=payload.delimiter,
|
|
max_rows=10_000,
|
|
)
|
|
if payload.format == "csv"
|
|
else tuple(payload.rows or ())
|
|
)
|
|
stage = writer.create_stage(
|
|
session,
|
|
principal,
|
|
stage=DatasourceStageInput(
|
|
name=payload.name,
|
|
source_name=payload.source_name,
|
|
description=payload.description,
|
|
kind="upload",
|
|
mode="static",
|
|
shape="tabular",
|
|
rows=rows,
|
|
provider="dataflow.upload",
|
|
provenance={
|
|
"created_via": "dataflow",
|
|
"source_format": payload.format,
|
|
},
|
|
metadata={
|
|
"dataflow_convenience_import": True,
|
|
},
|
|
),
|
|
)
|
|
source, materialization = writer.promote_stage(
|
|
session,
|
|
principal,
|
|
stage_ref=stage.ref,
|
|
)
|
|
except DatasourceError as exc:
|
|
raise _source_http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.datasource.created",
|
|
object_type="datasource",
|
|
object_id=source.ref,
|
|
details={
|
|
"provider": source.provider,
|
|
"source_name": source.source_name,
|
|
"row_count": source.row_count,
|
|
"fingerprint": source.fingerprint,
|
|
"stage_ref": stage.ref,
|
|
"materialization_ref": materialization.ref,
|
|
},
|
|
)
|
|
response = _source_response(source)
|
|
session.commit()
|
|
return response
|
|
|
|
|
|
@router.get("/pipelines", response_model=PipelineListResponse)
|
|
def api_list_pipelines(
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineListResponse:
|
|
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
|
pipelines = list_pipelines(session, tenant_id=principal.tenant_id)
|
|
registry = get_registry()
|
|
visible = [
|
|
pipeline
|
|
for pipeline in pipelines
|
|
if definition_decision(
|
|
pipeline,
|
|
principal=principal,
|
|
registry=registry,
|
|
action="view",
|
|
).allowed
|
|
]
|
|
return PipelineListResponse(
|
|
pipelines=[
|
|
pipeline_response(
|
|
session,
|
|
pipeline,
|
|
principal=principal,
|
|
registry=registry,
|
|
)
|
|
for pipeline in visible
|
|
]
|
|
)
|
|
|
|
|
|
@router.post("/pipelines", response_model=PipelineResponse, status_code=status.HTTP_201_CREATED)
|
|
def api_create_pipeline(
|
|
payload: PipelineCreateRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineResponse:
|
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
tenant_id, scope_type, scope_id = normalize_definition_scope(
|
|
principal,
|
|
scope_type=payload.scope_type,
|
|
scope_id=payload.scope_id,
|
|
administrative=has_scope(principal, ADMIN_SCOPE),
|
|
)
|
|
payload = payload.model_copy(
|
|
update={"scope_type": scope_type, "scope_id": scope_id}
|
|
)
|
|
pipeline = create_pipeline(
|
|
session,
|
|
tenant_id=tenant_id or principal.tenant_id,
|
|
actor_id=_actor_id(principal),
|
|
payload=payload,
|
|
)
|
|
except (PermissionError, ValueError) as exc:
|
|
raise _governance_http_error(exc) from exc
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.pipeline.created",
|
|
object_type="dataflow_pipeline",
|
|
object_id=pipeline.id,
|
|
details={"revision": pipeline.current_revision, "status": pipeline.status},
|
|
)
|
|
response = _pipeline_response(session, pipeline, principal)
|
|
session.commit()
|
|
return response
|
|
|
|
|
|
@router.get("/pipelines/{pipeline_id}", response_model=PipelineResponse)
|
|
def api_get_pipeline(
|
|
pipeline_id: str,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineResponse:
|
|
_require_any_scope(principal, READ_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
pipeline = get_pipeline(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
)
|
|
require_definition_action(
|
|
pipeline,
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
action="view",
|
|
)
|
|
return _pipeline_response(session, pipeline, principal)
|
|
except PermissionError as exc:
|
|
raise _governance_http_error(exc) from exc
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
|
|
|
|
@router.put("/pipelines/{pipeline_id}", response_model=PipelineResponse)
|
|
def api_update_pipeline(
|
|
pipeline_id: str,
|
|
payload: PipelineUpdateRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineResponse:
|
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
existing = get_pipeline(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
)
|
|
require_definition_action(
|
|
existing,
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
action="edit",
|
|
)
|
|
pipeline = update_pipeline(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
actor_id=_actor_id(principal),
|
|
payload=payload,
|
|
)
|
|
except PermissionError as exc:
|
|
raise _governance_http_error(exc) from exc
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.pipeline.updated",
|
|
object_type="dataflow_pipeline",
|
|
object_id=pipeline.id,
|
|
details={"revision": pipeline.current_revision, "status": pipeline.status},
|
|
)
|
|
response = _pipeline_response(session, pipeline, principal)
|
|
session.commit()
|
|
return response
|
|
|
|
|
|
@router.delete("/pipelines/{pipeline_id}", response_model=PipelineDeleteResponse)
|
|
def api_delete_pipeline(
|
|
pipeline_id: str,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineDeleteResponse:
|
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
existing = get_pipeline(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
)
|
|
require_definition_action(
|
|
existing,
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
action="edit",
|
|
)
|
|
pipeline = delete_pipeline(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
actor_id=_actor_id(principal),
|
|
)
|
|
except PermissionError as exc:
|
|
raise _governance_http_error(exc) from exc
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.pipeline.deleted",
|
|
object_type="dataflow_pipeline",
|
|
object_id=pipeline.id,
|
|
details={"revision": pipeline.current_revision},
|
|
)
|
|
session.commit()
|
|
return PipelineDeleteResponse(deleted=True, pipeline_id=pipeline.id)
|
|
|
|
|
|
@router.post(
|
|
"/pipelines/{pipeline_id}/derive",
|
|
response_model=PipelineResponse,
|
|
status_code=status.HTTP_201_CREATED,
|
|
)
|
|
def api_derive_pipeline(
|
|
pipeline_id: str,
|
|
payload: PipelineDeriveRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineResponse:
|
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
tenant_id, scope_type, scope_id = normalize_definition_scope(
|
|
principal,
|
|
scope_type=payload.scope_type,
|
|
scope_id=payload.scope_id,
|
|
administrative=has_scope(principal, ADMIN_SCOPE),
|
|
)
|
|
payload = payload.model_copy(
|
|
update={"scope_type": scope_type, "scope_id": scope_id}
|
|
)
|
|
pipeline = derive_pipeline(
|
|
session,
|
|
tenant_id=tenant_id or principal.tenant_id,
|
|
actor_id=_actor_id(principal),
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
source_pipeline_id=pipeline_id,
|
|
payload=payload,
|
|
)
|
|
except (PermissionError, ValueError) as exc:
|
|
raise _governance_http_error(exc) from exc
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.pipeline.derived",
|
|
object_type="dataflow_pipeline",
|
|
object_id=pipeline.id,
|
|
details={
|
|
"source_pipeline_id": pipeline.derived_from_pipeline_id,
|
|
"source_revision": pipeline.derived_from_revision,
|
|
"source_hash": pipeline.derived_from_hash,
|
|
"scope_type": pipeline.scope_type,
|
|
"scope_id": pipeline.scope_id,
|
|
},
|
|
)
|
|
response = _pipeline_response(session, pipeline, principal)
|
|
session.commit()
|
|
return response
|
|
|
|
|
|
@router.get(
|
|
"/pipelines/{pipeline_id}/triggers",
|
|
response_model=DataflowTriggerListResponse,
|
|
)
|
|
def api_list_triggers(
|
|
pipeline_id: str,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> DataflowTriggerListResponse:
|
|
_require_any_scope(principal, TRIGGER_READ_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
pipeline = get_pipeline(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
)
|
|
require_definition_action(
|
|
pipeline,
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
action="view",
|
|
)
|
|
items = list_triggers(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
)
|
|
return DataflowTriggerListResponse(
|
|
triggers=[trigger_response(session, item) for item in items]
|
|
)
|
|
except PermissionError as exc:
|
|
raise _governance_http_error(exc) from exc
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
|
|
|
|
@router.post(
|
|
"/pipelines/{pipeline_id}/triggers",
|
|
response_model=DataflowTriggerResponse,
|
|
status_code=status.HTTP_201_CREATED,
|
|
)
|
|
def api_create_trigger(
|
|
pipeline_id: str,
|
|
payload: DataflowTriggerCreateRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> DataflowTriggerResponse:
|
|
_require_any_scope(principal, TRIGGER_WRITE_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
trigger = create_trigger(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
actor_id=_actor_id(principal),
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
payload=payload,
|
|
)
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.trigger.created",
|
|
object_type="dataflow_trigger",
|
|
object_id=trigger.id,
|
|
details={
|
|
"pipeline_id": trigger.pipeline_id,
|
|
"kind": trigger.kind,
|
|
"status": trigger.status,
|
|
"grant_scopes": list(trigger.grant_scopes),
|
|
},
|
|
)
|
|
response = trigger_response(session, trigger)
|
|
session.commit()
|
|
return response
|
|
|
|
|
|
@router.put(
|
|
"/triggers/{trigger_id}",
|
|
response_model=DataflowTriggerResponse,
|
|
)
|
|
def api_update_trigger(
|
|
trigger_id: str,
|
|
payload: DataflowTriggerUpdateRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> DataflowTriggerResponse:
|
|
_require_any_scope(principal, TRIGGER_WRITE_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
trigger = update_trigger(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
trigger_id=trigger_id,
|
|
actor_id=_actor_id(principal),
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
payload=payload,
|
|
)
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.trigger.updated",
|
|
object_type="dataflow_trigger",
|
|
object_id=trigger.id,
|
|
details={
|
|
"pipeline_id": trigger.pipeline_id,
|
|
"kind": trigger.kind,
|
|
"status": trigger.status,
|
|
"revision": trigger.revision,
|
|
"grant_scopes": list(trigger.grant_scopes),
|
|
},
|
|
)
|
|
response = trigger_response(session, trigger)
|
|
session.commit()
|
|
return response
|
|
|
|
|
|
@router.delete(
|
|
"/triggers/{trigger_id}",
|
|
response_model=DataflowTriggerDeleteResponse,
|
|
)
|
|
def api_delete_trigger(
|
|
trigger_id: str,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> DataflowTriggerDeleteResponse:
|
|
_require_any_scope(principal, TRIGGER_WRITE_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
trigger = delete_trigger(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
trigger_id=trigger_id,
|
|
)
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.trigger.deleted",
|
|
object_type="dataflow_trigger",
|
|
object_id=trigger.id,
|
|
details={"pipeline_id": trigger.pipeline_id},
|
|
)
|
|
session.commit()
|
|
return DataflowTriggerDeleteResponse(
|
|
deleted=True,
|
|
trigger_id=trigger.id,
|
|
pipeline_id=trigger.pipeline_id,
|
|
)
|
|
|
|
|
|
@router.get(
|
|
"/triggers/{trigger_id}/deliveries",
|
|
response_model=DataflowTriggerDeliveryListResponse,
|
|
)
|
|
def api_list_trigger_deliveries(
|
|
trigger_id: str,
|
|
limit: int = 50,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> DataflowTriggerDeliveryListResponse:
|
|
_require_any_scope(principal, TRIGGER_READ_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
items = list_deliveries(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
trigger_id=trigger_id,
|
|
limit=limit,
|
|
)
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
return DataflowTriggerDeliveryListResponse(
|
|
deliveries=[delivery_response(item) for item in items]
|
|
)
|
|
|
|
|
|
@router.post(
|
|
"/triggers/events",
|
|
response_model=DataflowTriggerDispatchResponse,
|
|
)
|
|
def api_ingest_trigger_event(
|
|
payload: DataflowEventDeliveryRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> DataflowTriggerDispatchResponse:
|
|
_require_any_scope(principal, TRIGGER_DISPATCH_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
queued = ingest_event_delivery(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
event=payload,
|
|
)
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.trigger.event_ingested",
|
|
object_type="platform_event",
|
|
object_id=payload.event_id,
|
|
details={
|
|
"event_type": payload.type,
|
|
"module_id": payload.module_id,
|
|
"classification": payload.classification,
|
|
"queued": queued,
|
|
},
|
|
)
|
|
session.commit()
|
|
return DataflowTriggerDispatchResponse(queued=queued)
|
|
|
|
|
|
@router.post(
|
|
"/triggers/dispatch",
|
|
response_model=DataflowTriggerDispatchResponse,
|
|
)
|
|
def api_dispatch_triggers(
|
|
limit: int = 50,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> DataflowTriggerDispatchResponse:
|
|
_require_any_scope(principal, TRIGGER_DISPATCH_SCOPE, ADMIN_SCOPE)
|
|
result = dispatch_due_triggers(
|
|
session,
|
|
registry=get_registry(),
|
|
limit=limit,
|
|
tenant_id=principal.tenant_id,
|
|
)
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.trigger.dispatched",
|
|
object_type="dataflow_trigger_dispatch",
|
|
object_id=principal.tenant_id,
|
|
details=result.model_dump(mode="json"),
|
|
)
|
|
session.commit()
|
|
return result
|
|
|
|
|
|
@router.get(
|
|
"/pipelines/{pipeline_id}/runs",
|
|
response_model=PipelineRunListResponse,
|
|
)
|
|
def api_list_pipeline_runs(
|
|
pipeline_id: str,
|
|
limit: int = 50,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineRunListResponse:
|
|
_require_any_scope(principal, READ_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
pipeline = get_pipeline(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
)
|
|
require_definition_action(
|
|
pipeline,
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
action="view",
|
|
)
|
|
runs = list_pipeline_runs(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
limit=limit,
|
|
)
|
|
return PipelineRunListResponse(
|
|
runs=[pipeline_run_response(session, run) for run in runs]
|
|
)
|
|
except PermissionError as exc:
|
|
raise _governance_http_error(exc) from exc
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
|
|
|
|
@router.post(
|
|
"/pipelines/{pipeline_id}/runs",
|
|
response_model=PipelineRunResponse,
|
|
status_code=status.HTTP_201_CREATED,
|
|
)
|
|
def api_start_pipeline_run(
|
|
pipeline_id: str,
|
|
payload: PipelineRunCreateRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineRunResponse:
|
|
_require_any_scope(principal, RUN_SCOPE, ADMIN_SCOPE)
|
|
if payload.invocation_kind == "backfill":
|
|
_require_any_scope(principal, ADMIN_SCOPE)
|
|
try:
|
|
pipeline = get_pipeline(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
)
|
|
publication = (
|
|
DataflowPublicationTarget(
|
|
target_datasource_ref=payload.publication.target_datasource_ref,
|
|
name=payload.publication.name,
|
|
source_name=payload.publication.source_name,
|
|
description=payload.publication.description,
|
|
freeze=payload.publication.freeze,
|
|
frozen_label=payload.publication.frozen_label,
|
|
set_current=payload.publication.set_current,
|
|
metadata=payload.publication.metadata,
|
|
)
|
|
if payload.publication
|
|
else None
|
|
)
|
|
run, replayed = start_pipeline_run(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
actor_id=_actor_id(principal),
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
request=DataflowRunRequest(
|
|
pipeline_ref=f"pipeline:{pipeline.id}",
|
|
revision=payload.revision or pipeline.current_revision,
|
|
idempotency_key=payload.idempotency_key,
|
|
row_limit=payload.row_limit,
|
|
publication=publication,
|
|
invocation=AutomationInvocation(
|
|
kind=(
|
|
"backfill"
|
|
if payload.invocation_kind == "backfill"
|
|
else "api"
|
|
if principal.auth_method == "api_key"
|
|
else "manual"
|
|
),
|
|
requested_by=_actor_id(principal),
|
|
),
|
|
),
|
|
)
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action=(
|
|
"dataflow.run.replayed"
|
|
if replayed
|
|
else f"dataflow.run.{run.status}"
|
|
),
|
|
object_type="dataflow_run",
|
|
object_id=run.id,
|
|
details={
|
|
"pipeline_id": run.pipeline_id,
|
|
"revision_id": run.pipeline_revision_id,
|
|
"status": run.status,
|
|
"invocation_kind": run.invocation_kind,
|
|
"output_datasource_ref": run.output_datasource_ref,
|
|
"output_materialization_ref": run.output_materialization_ref,
|
|
},
|
|
)
|
|
response = pipeline_run_response(session, run, replayed=replayed)
|
|
session.commit()
|
|
return response
|
|
|
|
|
|
@router.get("/runs/{run_id}", response_model=PipelineRunResponse)
|
|
def api_get_pipeline_run(
|
|
run_id: str,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineRunResponse:
|
|
_require_any_scope(principal, READ_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
run = get_pipeline_run(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
run_ref=f"dataflow-run:{run_id}",
|
|
)
|
|
return pipeline_run_response(session, run)
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
|
|
|
|
@router.post("/runs/{run_id}/cancel", response_model=PipelineRunResponse)
|
|
def api_cancel_pipeline_run(
|
|
run_id: str,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineRunResponse:
|
|
_require_any_scope(principal, RUN_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
run = cancel_pipeline_run(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
run_ref=f"dataflow-run:{run_id}",
|
|
)
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.run.cancelled",
|
|
object_type="dataflow_run",
|
|
object_id=run.id,
|
|
details={"pipeline_id": run.pipeline_id},
|
|
)
|
|
response = pipeline_run_response(session, run)
|
|
session.commit()
|
|
return response
|
|
|
|
|
|
@router.post("/validate", response_model=PipelineValidationResponse)
|
|
def api_validate_pipeline(
|
|
payload: PipelineDraftRequest,
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineValidationResponse:
|
|
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
|
return validate_draft(payload)
|
|
|
|
|
|
@router.post("/sql/compile", response_model=PipelineSqlResponse)
|
|
def api_compile_pipeline_sql(
|
|
payload: PipelineDraftRequest,
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineSqlResponse:
|
|
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
|
return compile_sql_draft(payload)
|
|
|
|
|
|
@router.post("/sql/render", response_model=PipelineSqlResponse)
|
|
def api_render_pipeline_sql(
|
|
payload: PipelineDraftRequest,
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelineSqlResponse:
|
|
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
|
return render_graph_sql(payload)
|
|
|
|
|
|
@router.post("/preview", response_model=PipelinePreviewResponse)
|
|
def api_preview_pipeline(
|
|
payload: PipelinePreviewRequest,
|
|
session: Session = Depends(get_session),
|
|
principal: ApiPrincipal = Depends(get_api_principal),
|
|
) -> PipelinePreviewResponse:
|
|
_require_any_scope(principal, RUN_SCOPE, ADMIN_SCOPE)
|
|
try:
|
|
response = preview_pipeline(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
actor_id=_actor_id(principal),
|
|
principal=principal,
|
|
registry=get_registry(),
|
|
payload=payload,
|
|
)
|
|
except DataflowError as exc:
|
|
raise _http_error(exc) from exc
|
|
if response.pipeline_id:
|
|
audit_event(
|
|
session,
|
|
tenant_id=principal.tenant_id,
|
|
user_id=getattr(principal.user, "id", None),
|
|
api_key_id=principal.api_key_id,
|
|
action="dataflow.pipeline.previewed",
|
|
object_type="dataflow_pipeline",
|
|
object_id=response.pipeline_id,
|
|
details={
|
|
"revision": response.revision,
|
|
"run_id": response.run_id,
|
|
"status": response.status,
|
|
"output_row_count": response.total_rows,
|
|
},
|
|
)
|
|
session.commit()
|
|
return response
|
|
|
|
|
|
__all__ = ["router"]
|