Expand governed Dataflow editor and node library
This commit is contained in:
@@ -5,9 +5,24 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.audit.logging import audit_event
|
||||
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
|
||||
from govoplan_core.core.tabular_sources import (
|
||||
TabularSnapshotInput,
|
||||
TabularSource,
|
||||
TabularSourceAccessError,
|
||||
TabularSourceError,
|
||||
TabularSourceNotFoundError,
|
||||
TabularSourceValidationError,
|
||||
parse_tabular_csv,
|
||||
tabular_snapshot_writer,
|
||||
tabular_source_provider,
|
||||
)
|
||||
from govoplan_core.db.session import get_session
|
||||
from govoplan_dataflow.backend.manifest import ADMIN_SCOPE, READ_SCOPE, RUN_SCOPE, WRITE_SCOPE
|
||||
from govoplan_dataflow.backend.node_library import CATEGORY_LABELS, NODE_LIBRARY
|
||||
from govoplan_dataflow.backend.runtime import get_registry
|
||||
from govoplan_dataflow.backend.schemas import (
|
||||
NodeLibraryResponse,
|
||||
NodeTypeDefinitionResponse,
|
||||
PipelineCreateRequest,
|
||||
PipelineDeleteResponse,
|
||||
PipelineDraftRequest,
|
||||
@@ -18,6 +33,10 @@ from govoplan_dataflow.backend.schemas import (
|
||||
PipelineSqlResponse,
|
||||
PipelineUpdateRequest,
|
||||
PipelineValidationResponse,
|
||||
TabularSnapshotCreateRequest,
|
||||
TabularSourceColumnResponse,
|
||||
TabularSourceListResponse,
|
||||
TabularSourceResponse,
|
||||
)
|
||||
from govoplan_dataflow.backend.service import (
|
||||
DataflowConflictError,
|
||||
@@ -74,6 +93,191 @@ def _http_error(exc: DataflowError) -> HTTPException:
|
||||
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||
|
||||
|
||||
def _source_http_error(exc: TabularSourceError) -> HTTPException:
|
||||
if isinstance(exc, TabularSourceAccessError):
|
||||
return HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=str(exc))
|
||||
if isinstance(exc, TabularSourceNotFoundError):
|
||||
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
||||
if isinstance(exc, TabularSourceValidationError):
|
||||
return HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||
detail=str(exc),
|
||||
)
|
||||
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||
|
||||
|
||||
def _node_library_response() -> NodeLibraryResponse:
|
||||
return NodeLibraryResponse(
|
||||
nodes=[
|
||||
NodeTypeDefinitionResponse(
|
||||
type=definition.type,
|
||||
category=definition.category,
|
||||
category_label=CATEGORY_LABELS[definition.category],
|
||||
label=definition.label,
|
||||
description=definition.description,
|
||||
icon=definition.icon,
|
||||
input_ports=[
|
||||
{
|
||||
"id": port.id,
|
||||
"label": port.label,
|
||||
"required": port.required,
|
||||
"multiple": port.multiple,
|
||||
"minimum_connections": port.minimum_connections,
|
||||
}
|
||||
for port in definition.input_ports
|
||||
],
|
||||
output_ports=[
|
||||
{
|
||||
"id": port.id,
|
||||
"label": port.label,
|
||||
"required": port.required,
|
||||
"multiple": port.multiple,
|
||||
"minimum_connections": port.minimum_connections,
|
||||
}
|
||||
for port in definition.output_ports
|
||||
],
|
||||
config_fields=[
|
||||
{
|
||||
"id": field.id,
|
||||
"label": field.label,
|
||||
"kind": field.kind,
|
||||
"required": field.required,
|
||||
"description": field.description,
|
||||
"options": list(field.options),
|
||||
}
|
||||
for field in definition.config_fields
|
||||
],
|
||||
default_config=dict(definition.default_config),
|
||||
sql_support=definition.sql_support,
|
||||
)
|
||||
for definition in NODE_LIBRARY
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _source_response(source: TabularSource) -> TabularSourceResponse:
|
||||
return TabularSourceResponse(
|
||||
ref=source.ref,
|
||||
provider=source.provider,
|
||||
source_name=source.source_name,
|
||||
name=source.name,
|
||||
description=source.description,
|
||||
columns=[
|
||||
TabularSourceColumnResponse(
|
||||
name=column.name,
|
||||
data_type=column.data_type,
|
||||
nullable=column.nullable,
|
||||
)
|
||||
for column in source.schema
|
||||
],
|
||||
schema_version=source.schema_version,
|
||||
fingerprint=source.fingerprint,
|
||||
row_count=source.row_count,
|
||||
byte_count=source.byte_count,
|
||||
updated_at=source.updated_at,
|
||||
capabilities=list(source.capabilities),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/node-types", response_model=NodeLibraryResponse)
|
||||
def api_node_types(
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> NodeLibraryResponse:
|
||||
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
||||
return _node_library_response()
|
||||
|
||||
|
||||
@router.get("/sources", response_model=TabularSourceListResponse)
|
||||
def api_list_sources(
|
||||
query: str = "",
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> TabularSourceListResponse:
|
||||
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
||||
registry = get_registry()
|
||||
provider = tabular_source_provider(registry)
|
||||
writer = tabular_snapshot_writer(registry)
|
||||
if provider is None:
|
||||
return TabularSourceListResponse(available=False, writable=False, sources=[])
|
||||
try:
|
||||
sources = provider.list_sources(
|
||||
session,
|
||||
principal,
|
||||
query=query,
|
||||
limit=100,
|
||||
)
|
||||
except TabularSourceError as exc:
|
||||
raise _source_http_error(exc) from exc
|
||||
return TabularSourceListResponse(
|
||||
available=True,
|
||||
writable=writer is not None,
|
||||
sources=[_source_response(source) for source in sources],
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/sources/snapshots",
|
||||
response_model=TabularSourceResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
def api_create_source_snapshot(
|
||||
payload: TabularSnapshotCreateRequest,
|
||||
session: Session = Depends(get_session),
|
||||
principal: ApiPrincipal = Depends(get_api_principal),
|
||||
) -> TabularSourceResponse:
|
||||
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
||||
writer = tabular_snapshot_writer(get_registry())
|
||||
if writer is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_409_CONFLICT,
|
||||
detail="No tabular snapshot writer is available.",
|
||||
)
|
||||
try:
|
||||
rows = (
|
||||
parse_tabular_csv(
|
||||
payload.csv_text or "",
|
||||
delimiter=payload.delimiter,
|
||||
max_rows=10_000,
|
||||
)
|
||||
if payload.format == "csv"
|
||||
else tuple(payload.rows or ())
|
||||
)
|
||||
source = writer.create_snapshot(
|
||||
session,
|
||||
principal,
|
||||
snapshot=TabularSnapshotInput(
|
||||
name=payload.name,
|
||||
source_name=payload.source_name,
|
||||
description=payload.description,
|
||||
rows=rows,
|
||||
metadata={
|
||||
"created_via": "dataflow",
|
||||
"source_format": payload.format,
|
||||
},
|
||||
),
|
||||
)
|
||||
except TabularSourceError as exc:
|
||||
raise _source_http_error(exc) from exc
|
||||
audit_event(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
user_id=getattr(principal.user, "id", None),
|
||||
api_key_id=principal.api_key_id,
|
||||
action="dataflow.source_snapshot.created",
|
||||
object_type="tabular_source",
|
||||
object_id=source.ref,
|
||||
details={
|
||||
"provider": source.provider,
|
||||
"source_name": source.source_name,
|
||||
"row_count": source.row_count,
|
||||
"fingerprint": source.fingerprint,
|
||||
},
|
||||
)
|
||||
response = _source_response(source)
|
||||
session.commit()
|
||||
return response
|
||||
|
||||
|
||||
@router.get("/pipelines", response_model=PipelineListResponse)
|
||||
def api_list_pipelines(
|
||||
session: Session = Depends(get_session),
|
||||
@@ -237,6 +441,8 @@ def api_preview_pipeline(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
actor_id=_actor_id(principal),
|
||||
principal=principal,
|
||||
registry=get_registry(),
|
||||
payload=payload,
|
||||
)
|
||||
except DataflowError as exc:
|
||||
|
||||
Reference in New Issue
Block a user