import { apiFetch, apiReferenceOptionProvider, type ApiSettings, type ReferenceOptionProvider } from "@govoplan/core-webui"; import type { DefinitionGraphNodeType } from "@govoplan/core-webui/definition-graph"; export type PipelineStatus = "draft" | "active" | "archived"; export type DefinitionScopeType = "system" | "tenant" | "group" | "user"; export type DefinitionKind = "flow" | "template"; export type EditorMode = "graph" | "sql"; export type NodeCategory = "load" | "combine" | "filter" | "transform" | "quality" | "output"; export type GraphPosition = { x: number; y: number }; export type PipelineGraphNode = { id: string; type: string; label: string; position: GraphPosition; config: Record; }; export type PipelineGraphEdge = { id: string; source: string; target: string; source_port?: string; target_port?: string; }; export type PipelineGraph = { schema_version: 1; nodes: PipelineGraphNode[]; edges: PipelineGraphEdge[]; }; export type DataflowDiagnostic = { severity: "info" | "warning" | "error"; code: string; message: string; node_id?: string | null; field?: string | null; source_location?: { start_line: number; start_column: number; end_line: number; end_column: number; start_offset?: number | null; end_offset?: number | null; } | null; }; export type PipelineRevision = { id: string; revision: number; schema_version: number; graph: PipelineGraph; sql_text?: string | null; editor_mode: EditorMode; content_hash: string; created_by?: string | null; created_at: string; }; export type Pipeline = { id: string; tenant_id: string | null; name: string; description?: string | null; status: PipelineStatus; current_revision: number; created_by?: string | null; updated_by?: string | null; created_at: string; updated_at: string; revision: PipelineRevision; governance: PipelineGovernance; }; export type DefinitionActionDecision = { allowed: boolean; reason?: string | null; source_path: Array>; requirements: string[]; details: Record; }; export type PipelineGovernance = { scope_type: DefinitionScopeType; scope_id?: string | null; definition_kind: DefinitionKind; inherit_to_lower_scopes: boolean; allow_run: boolean; allow_reuse: boolean; allow_automation: boolean; derived_from_pipeline_id?: string | null; derived_from_revision?: number | null; derived_from_hash?: string | null; source_available: boolean; source_name?: string | null; source_current_revision?: number | null; source_current_hash?: string | null; update_available: boolean; derivation_provenance: Record; actions: Record; }; export type PipelinePayload = { name: string; description?: string | null; status: PipelineStatus; graph: PipelineGraph; sql_text?: string | null; editor_mode: EditorMode; scope_type: DefinitionScopeType; scope_id?: string | null; definition_kind: DefinitionKind; inherit_to_lower_scopes: boolean; allow_run: boolean; allow_reuse: boolean; allow_automation: boolean; }; export type PipelineValidation = { valid: boolean; graph?: PipelineGraph | null; sql_text?: string | null; diagnostics: DataflowDiagnostic[]; }; export type PreviewColumn = { name: string; type: string; nullable: boolean; }; export type NodePreviewDiagnostic = { node_id: string; status: "succeeded" | "failed" | "skipped"; input_rows: number; output_rows: number; duration_ms: number; columns: PreviewColumn[]; messages: string[]; }; export type NodePreviewResult = { node_id: string; columns: PreviewColumn[]; rows: Record[]; total_rows: number; truncated: boolean; }; export type PipelinePreview = { run_id?: string | null; pipeline_id?: string | null; revision?: number | null; status: "succeeded" | "failed"; columns: PreviewColumn[]; rows: Record[]; total_rows: number; truncated: boolean; diagnostics: DataflowDiagnostic[]; node_diagnostics: NodePreviewDiagnostic[]; node_preview?: NodePreviewResult | null; source_fingerprints: Record[]; input_row_count: number; definition_hash: string; executor_version: string; }; export type NodePortDefinition = { id: string; label: string; required: boolean; multiple: boolean; minimum_connections: number; }; export type NodeConfigFieldDefinition = { id: string; label: string; kind: string; required: boolean; description?: string | null; options: Array<[string, string]>; }; export type NodeTypeDefinition = DefinitionGraphNodeType & { category: NodeCategory; input_ports: NodePortDefinition[]; output_ports: NodePortDefinition[]; config_fields: NodeConfigFieldDefinition[]; sql_support: "full" | "partial" | "none"; }; export type TabularSourceColumn = { name: string; data_type: string; nullable: boolean; }; export type TabularSource = { ref: string; provider: string; source_name: string; name: string; description?: string | null; mode: "live" | "cached" | "static"; columns: TabularSourceColumn[]; schema_version: string; fingerprint: string; decisions_included: boolean; row_count?: number | null; byte_count?: number | null; updated_at?: string | null; capabilities: string[]; }; export type TabularSourceCatalogue = { available: boolean; writable: boolean; sources: TabularSource[]; }; export type ReconciliationDecisionAction = "accept" | "reject" | "correct" | "defer"; export type ReconciliationDecision = { ref: string; id: string; revision: number; key_hash: string; input_hash: string; action: ReconciliationDecisionAction; reason: string; correction?: Record | null; actor_ref: string; decided_at: string; }; export type ReconciliationDecisionSet = { ref: string; id: string; pipeline_id: string; name: string; node_id?: string | null; resource_revision: number; etag: string; fingerprint: string; current_decisions: ReconciliationDecision[]; history: ReconciliationDecision[]; created_by?: string | null; updated_by?: string | null; created_at: string; updated_at: string; }; export type PipelineRun = { ref: string; pipeline_id: string; revision: number; run_type: string; status: "queued" | "retrying" | "running" | "succeeded" | "failed" | "cancelled" | "outcome_unknown"; idempotency_key?: string | null; execution_backend: "auto" | "reference" | "duckdb"; environment: "development" | "staging" | "production"; definition_hash: string; executor_version: string; source_fingerprints: Record[]; result_schema: PreviewColumn[]; diagnostics: DataflowDiagnostic[]; input_row_count: number; output_row_count: number; output_publication_ref?: string | null; output_datasource_ref?: string | null; output_materialization_ref?: string | null; invocation_kind: string; trigger_ref?: string | null; delivery_ref?: string | null; correlation_id?: string | null; causation_id?: string | null; attempts: number; max_attempts: number; available_at?: string | null; claimed_at?: string | null; lease_expires_at?: string | null; cancellation_requested_at?: string | null; progress_percent: number; progress_phase: string; retention_until?: string | null; purged_at?: string | null; error?: string | null; started_at?: string | null; finished_at?: string | null; created_by?: string | null; created_at: string; recovery_operation_id?: string | null; recovery_operation_type?: string | null; recovery_mode?: string | null; recovery_status?: string | null; recovery_requires_attention: boolean; recovery_explanation?: string | null; replayed: boolean; }; export type PipelineDeployment = { id: string; pipeline_id: string; revision: number; environment: "development" | "staging" | "production"; source_environment: "development" | "staging" | "production"; status: string; provenance: Record; promoted_by?: string | null; created_at: string; updated_at: string; }; export type DataflowTriggerKind = "once" | "interval" | "event"; export type DataflowTrigger = { id: string; tenant_id: string; pipeline_id: string; pipeline_revision: number; name: string; kind: DataflowTriggerKind; status: "enabled" | "disabled" | "completed" | "blocked"; revision: number; schedule?: { run_at?: string | null; anchor_at?: string | null; interval_seconds?: number | null; timezone: string; } | null; event?: { event_type: string; module_id?: string | null; filters: Record; } | null; row_limit: number; catch_up_policy: "skip" | "coalesce"; max_concurrent_runs: number; next_fire_at?: string | null; last_fire_at?: string | null; last_status?: string | null; last_error?: string | null; grant_scopes: string[]; authorization_provenance: Record; created_at: string; updated_at: string; }; export type DataflowTriggerPayload = { name: string; kind: DataflowTriggerKind; revision?: number | null; enabled: boolean; schedule?: DataflowTrigger["schedule"]; event?: DataflowTrigger["event"]; row_limit: number; catch_up_policy: "skip" | "coalesce"; max_concurrent_runs: number; }; export async function listDataflowNodeTypes(settings: ApiSettings): Promise { const response = await apiFetch<{ nodes: NodeTypeDefinition[] }>( settings, "/api/v1/dataflow/node-types" ); return response.nodes; } export function listDataflowSources( settings: ApiSettings, query = "" ): Promise { const suffix = query.trim() ? `?query=${encodeURIComponent(query.trim())}` : ""; return apiFetch(settings, `/api/v1/dataflow/sources${suffix}`); } export function createDataflowSourceSnapshot( settings: ApiSettings, payload: { name: string; source_name: string; description?: string | null; } & ( { format: "json"; rows: Record[]; } | { format: "csv"; csv_text: string; delimiter: string; csv_value_mode?: "text" | "legacy_typed"; } ) ): Promise { return apiFetch(settings, "/api/v1/dataflow/sources/snapshots", { method: "POST", body: JSON.stringify(payload) }); } export async function listDataflowDecisionSets( settings: ApiSettings, pipelineId: string ): Promise { const response = await apiFetch<{ decision_sets: ReconciliationDecisionSet[] }>( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/decision-sets` ); return response.decision_sets; } export function createDataflowDecisionSet( settings: ApiSettings, pipelineId: string, payload: { name: string; node_id?: string | null } ): Promise { return apiFetch( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/decision-sets`, { method: "POST", body: JSON.stringify(payload) } ); } export function getDataflowDecisionSet( settings: ApiSettings, decisionSetId: string ): Promise { return apiFetch( settings, `/api/v1/dataflow/decision-sets/${encodeURIComponent(decisionSetId)}` ); } export function recordDataflowDecision( settings: ApiSettings, decisionSetId: string, payload: { base_revision: number; key_hash: string; input_hash: string; action: ReconciliationDecisionAction; reason: string; correction?: Record | null; } ): Promise { return apiFetch( settings, `/api/v1/dataflow/decision-sets/${encodeURIComponent(decisionSetId)}/decisions`, { method: "POST", body: JSON.stringify(payload) } ); } export async function listDataflowPipelines(settings: ApiSettings): Promise { const response = await apiFetch<{ pipelines: Pipeline[] }>(settings, "/api/v1/dataflow/pipelines"); return response.pipelines; } export function createDataflowPipeline( settings: ApiSettings, payload: PipelinePayload ): Promise { return apiFetch(settings, "/api/v1/dataflow/pipelines", { method: "POST", body: JSON.stringify(payload) }); } export function updateDataflowPipeline( settings: ApiSettings, pipelineId: string, payload: PipelinePayload & { expected_revision: number } ): Promise { return apiFetch(settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}`, { method: "PUT", body: JSON.stringify(payload) }); } export function deleteDataflowPipeline(settings: ApiSettings, pipelineId: string): Promise<{ deleted: boolean }> { return apiFetch<{ deleted: boolean }>( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}`, { method: "DELETE" } ); } export function deriveDataflowPipeline( settings: ApiSettings, pipelineId: string, payload: { name: string; description?: string | null; source_revision?: number | null; scope_type: DefinitionScopeType; scope_id?: string | null; definition_kind: DefinitionKind; inherit_to_lower_scopes: boolean; allow_run: boolean; allow_reuse: boolean; allow_automation: boolean; } ): Promise { return apiFetch( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/derive`, { method: "POST", body: JSON.stringify(payload) } ); } export function rebaseDataflowPipeline( settings: ApiSettings, pipelineId: string, payload: { expected_revision: number; source_revision: number; source_hash: string; reason: string; } ): Promise { return apiFetch( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/rebase`, { method: "POST", body: JSON.stringify(payload) } ); } export function dataflowScopeReferenceProvider( settings: ApiSettings, scopeType: "user" | "group" ): ReferenceOptionProvider { return apiReferenceOptionProvider( settings, "/api/v1/dataflow/scope-targets", { scope_type: scopeType } ); } export async function listDataflowTriggers( settings: ApiSettings, pipelineId: string ): Promise { const response = await apiFetch<{ triggers: DataflowTrigger[] }>( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/triggers` ); return response.triggers; } export function createDataflowTrigger( settings: ApiSettings, pipelineId: string, payload: DataflowTriggerPayload ): Promise { return apiFetch( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/triggers`, { method: "POST", body: JSON.stringify(payload) } ); } export function updateDataflowTrigger( settings: ApiSettings, triggerId: string, payload: DataflowTriggerPayload & { expected_revision: number } ): Promise { return apiFetch( settings, `/api/v1/dataflow/triggers/${encodeURIComponent(triggerId)}`, { method: "PUT", body: JSON.stringify(payload) } ); } export function deleteDataflowTrigger( settings: ApiSettings, triggerId: string ): Promise<{ deleted: boolean }> { return apiFetch( settings, `/api/v1/dataflow/triggers/${encodeURIComponent(triggerId)}`, { method: "DELETE" } ); } export function validateDataflowPipeline( settings: ApiSettings, payload: { pipeline_id?: string | null; graph?: PipelineGraph; sql_text?: string; source_nodes?: PipelineGraphNode[] } ): Promise { return apiFetch(settings, "/api/v1/dataflow/validate", { method: "POST", body: JSON.stringify(payload) }); } export function compileDataflowSql( settings: ApiSettings, payload: { graph?: PipelineGraph; sql_text: string; source_nodes?: PipelineGraphNode[] } ): Promise { return apiFetch(settings, "/api/v1/dataflow/sql/compile", { method: "POST", body: JSON.stringify(payload) }); } export function renderDataflowSql( settings: ApiSettings, graph: PipelineGraph ): Promise { return apiFetch(settings, "/api/v1/dataflow/sql/render", { method: "POST", body: JSON.stringify({ graph }) }); } export function previewDataflowPipeline( settings: ApiSettings, payload: { pipeline_id?: string; revision?: number; graph?: PipelineGraph; sql_text?: string; source_nodes?: PipelineGraphNode[]; preview_node_id?: string; row_limit?: number; execution_backend?: "reference" | "duckdb" | "auto"; } ): Promise { return apiFetch(settings, "/api/v1/dataflow/preview", { method: "POST", body: JSON.stringify(payload) }); } export async function listDataflowPipelineRuns( settings: ApiSettings, pipelineId: string ): Promise { const response = await apiFetch<{ runs: PipelineRun[] }>( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/runs?limit=25` ); return response.runs; } export function runDataflowPipeline( settings: ApiSettings, pipelineId: string, payload: { revision: number; idempotency_key: string; row_limit?: number; execution_backend?: "reference" | "duckdb" | "auto"; environment?: "development" | "staging" | "production"; max_attempts?: number; retention_days?: number; publication?: { target_datasource_ref?: string | null; name?: string | null; source_name?: string | null; description?: string | null; freeze?: boolean; frozen_label?: string | null; set_current?: boolean; metadata?: Record; } | null; } ): Promise { return apiFetch( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/runs`, { method: "POST", body: JSON.stringify(payload) } ); } export function cancelDataflowPipelineRun( settings: ApiSettings, runRef: string ): Promise { const runId = runRef.replace(/^dataflow-run:/, ""); return apiFetch( settings, `/api/v1/dataflow/runs/${encodeURIComponent(runId)}/cancel`, { method: "POST" } ); } export async function listDataflowPipelineDeployments( settings: ApiSettings, pipelineId: string ): Promise { const response = await apiFetch<{ deployments: PipelineDeployment[] }>( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/deployments` ); return response.deployments; } export function promoteDataflowPipeline( settings: ApiSettings, pipelineId: string, payload: { revision: number; source_environment: "development" | "staging"; target_environment: "staging" | "production"; } ): Promise { return apiFetch( settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/promotions`, { method: "POST", body: JSON.stringify(payload) } ); }