govoplan-dataflow/webui/src/api/dataflow.ts

import {
  apiFetch,
  apiReferenceOptionProvider,
  type ApiSettings,
  type ReferenceOptionProvider
} from "@govoplan/core-webui";
import type { DefinitionGraphNodeType } from "@govoplan/core-webui/definition-graph";

export type PipelineStatus = "draft" | "active" | "archived";
export type DefinitionScopeType = "system" | "tenant" | "group" | "user";
export type DefinitionKind = "flow" | "template";
export type EditorMode = "graph" | "sql";
export type NodeCategory = "load" | "combine" | "filter" | "transform" | "quality" | "output";

export type GraphPosition = { x: number; y: number };
export type PipelineGraphNode = {
  id: string;
  type: string;
  label: string;
  position: GraphPosition;
  config: Record<string, unknown>;
};
export type PipelineGraphEdge = {
  id: string;
  source: string;
  target: string;
  source_port?: string;
  target_port?: string;
};
export type PipelineGraph = {
  schema_version: 1;
  nodes: PipelineGraphNode[];
  edges: PipelineGraphEdge[];
};

export type DataflowDiagnostic = {
  severity: "info" | "warning" | "error";
  code: string;
  message: string;
  node_id?: string | null;
  field?: string | null;
  source_location?: {
    start_line: number;
    start_column: number;
    end_line: number;
    end_column: number;
    start_offset?: number | null;
    end_offset?: number | null;
  } | null;
};

export type PipelineRevision = {
  id: string;
  revision: number;
  schema_version: number;
  graph: PipelineGraph;
  sql_text?: string | null;
  editor_mode: EditorMode;
  content_hash: string;
  created_by?: string | null;
  created_at: string;
};

export type Pipeline = {
  id: string;
  tenant_id: string | null;
  name: string;
  description?: string | null;
  status: PipelineStatus;
  current_revision: number;
  created_by?: string | null;
  updated_by?: string | null;
  created_at: string;
  updated_at: string;
  revision: PipelineRevision;
  governance: PipelineGovernance;
};

export type DefinitionActionDecision = {
  allowed: boolean;
  reason?: string | null;
  source_path: Array<Record<string, unknown>>;
  requirements: string[];
  details: Record<string, unknown>;
};

export type PipelineGovernance = {
  scope_type: DefinitionScopeType;
  scope_id?: string | null;
  definition_kind: DefinitionKind;
  inherit_to_lower_scopes: boolean;
  allow_run: boolean;
  allow_reuse: boolean;
  allow_automation: boolean;
  derived_from_pipeline_id?: string | null;
  derived_from_revision?: number | null;
  derived_from_hash?: string | null;
  source_available: boolean;
  source_name?: string | null;
  source_current_revision?: number | null;
  source_current_hash?: string | null;
  update_available: boolean;
  derivation_provenance: Record<string, unknown>;
  actions: Record<string, DefinitionActionDecision>;
};

export type PipelinePayload = {
  name: string;
  description?: string | null;
  status: PipelineStatus;
  graph: PipelineGraph;
  sql_text?: string | null;
  editor_mode: EditorMode;
  scope_type: DefinitionScopeType;
  scope_id?: string | null;
  definition_kind: DefinitionKind;
  inherit_to_lower_scopes: boolean;
  allow_run: boolean;
  allow_reuse: boolean;
  allow_automation: boolean;
};

export type PipelineValidation = {
  valid: boolean;
  graph?: PipelineGraph | null;
  sql_text?: string | null;
  diagnostics: DataflowDiagnostic[];
};

export type PreviewColumn = {
  name: string;
  type: string;
  nullable: boolean;
};

export type NodePreviewDiagnostic = {
  node_id: string;
  status: "succeeded" | "failed" | "skipped";
  input_rows: number;
  output_rows: number;
  duration_ms: number;
  columns: PreviewColumn[];
  messages: string[];
};

export type NodePreviewResult = {
  node_id: string;
  columns: PreviewColumn[];
  rows: Record<string, unknown>[];
  total_rows: number;
  truncated: boolean;
};

export type PipelinePreview = {
  run_id?: string | null;
  pipeline_id?: string | null;
  revision?: number | null;
  status: "succeeded" | "failed";
  columns: PreviewColumn[];
  rows: Record<string, unknown>[];
  total_rows: number;
  truncated: boolean;
  diagnostics: DataflowDiagnostic[];
  node_diagnostics: NodePreviewDiagnostic[];
  node_preview?: NodePreviewResult | null;
  source_fingerprints: Record<string, unknown>[];
  input_row_count: number;
  definition_hash: string;
  executor_version: string;
};

export type NodePortDefinition = {
  id: string;
  label: string;
  required: boolean;
  multiple: boolean;
  minimum_connections: number;
};

export type NodeConfigFieldDefinition = {
  id: string;
  label: string;
  kind: string;
  required: boolean;
  description?: string | null;
  options: Array<[string, string]>;
};

export type NodeTypeDefinition = DefinitionGraphNodeType & {
  category: NodeCategory;
  input_ports: NodePortDefinition[];
  output_ports: NodePortDefinition[];
  config_fields: NodeConfigFieldDefinition[];
  sql_support: "full" | "partial" | "none";
};

export type TabularSourceColumn = {
  name: string;
  data_type: string;
  nullable: boolean;
};

export type TabularSource = {
  ref: string;
  provider: string;
  source_name: string;
  name: string;
  description?: string | null;
  mode: "live" | "cached" | "static";
  columns: TabularSourceColumn[];
  schema_version: string;
  fingerprint: string;
  decisions_included: boolean;
  row_count?: number | null;
  byte_count?: number | null;
  updated_at?: string | null;
  capabilities: string[];
};

export type TabularSourceCatalogue = {
  available: boolean;
  writable: boolean;
  sources: TabularSource[];
};

export type ReconciliationDecisionAction = "accept" | "reject" | "correct" | "defer";

export type ReconciliationDecision = {
  ref: string;
  id: string;
  revision: number;
  key_hash: string;
  input_hash: string;
  action: ReconciliationDecisionAction;
  reason: string;
  correction?: Record<string, unknown> | null;
  actor_ref: string;
  decided_at: string;
};

export type ReconciliationDecisionSet = {
  ref: string;
  id: string;
  pipeline_id: string;
  name: string;
  node_id?: string | null;
  resource_revision: number;
  etag: string;
  fingerprint: string;
  current_decisions: ReconciliationDecision[];
  history: ReconciliationDecision[];
  created_by?: string | null;
  updated_by?: string | null;
  created_at: string;
  updated_at: string;
};

export type PipelineRun = {
  ref: string;
  pipeline_id: string;
  revision: number;
  run_type: string;
  status: "queued" | "retrying" | "running" | "succeeded" | "failed" | "cancelled" | "outcome_unknown";
  idempotency_key?: string | null;
  execution_backend: "auto" | "reference" | "duckdb";
  environment: "development" | "staging" | "production";
  definition_hash: string;
  executor_version: string;
  source_fingerprints: Record<string, unknown>[];
  result_schema: PreviewColumn[];
  diagnostics: DataflowDiagnostic[];
  input_row_count: number;
  output_row_count: number;
  output_publication_ref?: string | null;
  output_datasource_ref?: string | null;
  output_materialization_ref?: string | null;
  invocation_kind: string;
  trigger_ref?: string | null;
  delivery_ref?: string | null;
  correlation_id?: string | null;
  causation_id?: string | null;
  attempts: number;
  max_attempts: number;
  available_at?: string | null;
  claimed_at?: string | null;
  lease_expires_at?: string | null;
  cancellation_requested_at?: string | null;
  progress_percent: number;
  progress_phase: string;
  retention_until?: string | null;
  purged_at?: string | null;
  error?: string | null;
  started_at?: string | null;
  finished_at?: string | null;
  created_by?: string | null;
  created_at: string;
  recovery_operation_id?: string | null;
  recovery_operation_type?: string | null;
  recovery_mode?: string | null;
  recovery_status?: string | null;
  recovery_requires_attention: boolean;
  recovery_explanation?: string | null;
  replayed: boolean;
};

export type PipelineDeployment = {
  id: string;
  pipeline_id: string;
  revision: number;
  environment: "development" | "staging" | "production";
  source_environment: "development" | "staging" | "production";
  status: string;
  provenance: Record<string, unknown>;
  promoted_by?: string | null;
  created_at: string;
  updated_at: string;
};

export type DataflowTriggerKind = "once" | "interval" | "event";
export type DataflowTrigger = {
  id: string;
  tenant_id: string;
  pipeline_id: string;
  pipeline_revision: number;
  name: string;
  kind: DataflowTriggerKind;
  status: "enabled" | "disabled" | "completed" | "blocked";
  revision: number;
  schedule?: {
    run_at?: string | null;
    anchor_at?: string | null;
    interval_seconds?: number | null;
    timezone: string;
  } | null;
  event?: {
    event_type: string;
    module_id?: string | null;
    filters: Record<string, string | number | boolean | null>;
  } | null;
  row_limit: number;
  catch_up_policy: "skip" | "coalesce";
  max_concurrent_runs: number;
  next_fire_at?: string | null;
  last_fire_at?: string | null;
  last_status?: string | null;
  last_error?: string | null;
  grant_scopes: string[];
  authorization_provenance: Record<string, unknown>;
  created_at: string;
  updated_at: string;
};

export type DataflowTriggerPayload = {
  name: string;
  kind: DataflowTriggerKind;
  revision?: number | null;
  enabled: boolean;
  schedule?: DataflowTrigger["schedule"];
  event?: DataflowTrigger["event"];
  row_limit: number;
  catch_up_policy: "skip" | "coalesce";
  max_concurrent_runs: number;
};

export async function listDataflowNodeTypes(settings: ApiSettings): Promise<NodeTypeDefinition[]> {
  const response = await apiFetch<{ nodes: NodeTypeDefinition[] }>(
    settings,
    "/api/v1/dataflow/node-types"
  );
  return response.nodes;
}

export function listDataflowSources(
  settings: ApiSettings,
  query = ""
): Promise<TabularSourceCatalogue> {
  const suffix = query.trim() ? `?query=${encodeURIComponent(query.trim())}` : "";
  return apiFetch<TabularSourceCatalogue>(settings, `/api/v1/dataflow/sources${suffix}`);
}

export function createDataflowSourceSnapshot(
  settings: ApiSettings,
  payload: {
    name: string;
    source_name: string;
    description?: string | null;
  } & (
    {
      format: "json";
      rows: Record<string, unknown>[];
    } | {
      format: "csv";
      csv_text: string;
      delimiter: string;
      csv_value_mode?: "text" | "legacy_typed";
    }
  )
): Promise<TabularSource> {
  return apiFetch<TabularSource>(settings, "/api/v1/dataflow/sources/snapshots", {
    method: "POST",
    body: JSON.stringify(payload)
  });
}

export async function listDataflowDecisionSets(
  settings: ApiSettings,
  pipelineId: string
): Promise<ReconciliationDecisionSet[]> {
  const response = await apiFetch<{ decision_sets: ReconciliationDecisionSet[] }>(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/decision-sets`
  );
  return response.decision_sets;
}

export function createDataflowDecisionSet(
  settings: ApiSettings,
  pipelineId: string,
  payload: { name: string; node_id?: string | null }
): Promise<ReconciliationDecisionSet> {
  return apiFetch(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/decision-sets`,
    { method: "POST", body: JSON.stringify(payload) }
  );
}

export function getDataflowDecisionSet(
  settings: ApiSettings,
  decisionSetId: string
): Promise<ReconciliationDecisionSet> {
  return apiFetch(
    settings,
    `/api/v1/dataflow/decision-sets/${encodeURIComponent(decisionSetId)}`
  );
}

export function recordDataflowDecision(
  settings: ApiSettings,
  decisionSetId: string,
  payload: {
    base_revision: number;
    key_hash: string;
    input_hash: string;
    action: ReconciliationDecisionAction;
    reason: string;
    correction?: Record<string, unknown> | null;
  }
): Promise<ReconciliationDecisionSet> {
  return apiFetch(
    settings,
    `/api/v1/dataflow/decision-sets/${encodeURIComponent(decisionSetId)}/decisions`,
    { method: "POST", body: JSON.stringify(payload) }
  );
}

export async function listDataflowPipelines(settings: ApiSettings): Promise<Pipeline[]> {
  const response = await apiFetch<{ pipelines: Pipeline[] }>(settings, "/api/v1/dataflow/pipelines");
  return response.pipelines;
}

export function createDataflowPipeline(
  settings: ApiSettings,
  payload: PipelinePayload
): Promise<Pipeline> {
  return apiFetch<Pipeline>(settings, "/api/v1/dataflow/pipelines", {
    method: "POST",
    body: JSON.stringify(payload)
  });
}

export function updateDataflowPipeline(
  settings: ApiSettings,
  pipelineId: string,
  payload: PipelinePayload & { expected_revision: number }
): Promise<Pipeline> {
  return apiFetch<Pipeline>(settings, `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}`, {
    method: "PUT",
    body: JSON.stringify(payload)
  });
}

export function deleteDataflowPipeline(settings: ApiSettings, pipelineId: string): Promise<{ deleted: boolean }> {
  return apiFetch<{ deleted: boolean }>(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}`,
    { method: "DELETE" }
  );
}

export function deriveDataflowPipeline(
  settings: ApiSettings,
  pipelineId: string,
  payload: {
    name: string;
    description?: string | null;
    source_revision?: number | null;
    scope_type: DefinitionScopeType;
    scope_id?: string | null;
    definition_kind: DefinitionKind;
    inherit_to_lower_scopes: boolean;
    allow_run: boolean;
    allow_reuse: boolean;
    allow_automation: boolean;
  }
): Promise<Pipeline> {
  return apiFetch(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/derive`,
    { method: "POST", body: JSON.stringify(payload) }
  );
}

export function rebaseDataflowPipeline(
  settings: ApiSettings,
  pipelineId: string,
  payload: {
    expected_revision: number;
    source_revision: number;
    source_hash: string;
    reason: string;
  }
): Promise<Pipeline> {
  return apiFetch(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/rebase`,
    { method: "POST", body: JSON.stringify(payload) }
  );
}

export function dataflowScopeReferenceProvider(
  settings: ApiSettings,
  scopeType: "user" | "group"
): ReferenceOptionProvider {
  return apiReferenceOptionProvider(
    settings,
    "/api/v1/dataflow/scope-targets",
    { scope_type: scopeType }
  );
}

export async function listDataflowTriggers(
  settings: ApiSettings,
  pipelineId: string
): Promise<DataflowTrigger[]> {
  const response = await apiFetch<{ triggers: DataflowTrigger[] }>(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/triggers`
  );
  return response.triggers;
}

export function createDataflowTrigger(
  settings: ApiSettings,
  pipelineId: string,
  payload: DataflowTriggerPayload
): Promise<DataflowTrigger> {
  return apiFetch(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/triggers`,
    { method: "POST", body: JSON.stringify(payload) }
  );
}

export function updateDataflowTrigger(
  settings: ApiSettings,
  triggerId: string,
  payload: DataflowTriggerPayload & { expected_revision: number }
): Promise<DataflowTrigger> {
  return apiFetch(
    settings,
    `/api/v1/dataflow/triggers/${encodeURIComponent(triggerId)}`,
    { method: "PUT", body: JSON.stringify(payload) }
  );
}

export function deleteDataflowTrigger(
  settings: ApiSettings,
  triggerId: string
): Promise<{ deleted: boolean }> {
  return apiFetch(
    settings,
    `/api/v1/dataflow/triggers/${encodeURIComponent(triggerId)}`,
    { method: "DELETE" }
  );
}

export function validateDataflowPipeline(
  settings: ApiSettings,
  payload: { pipeline_id?: string | null; graph?: PipelineGraph; sql_text?: string; source_nodes?: PipelineGraphNode[] }
): Promise<PipelineValidation> {
  return apiFetch<PipelineValidation>(settings, "/api/v1/dataflow/validate", {
    method: "POST",
    body: JSON.stringify(payload)
  });
}

export function compileDataflowSql(
  settings: ApiSettings,
  payload: { graph?: PipelineGraph; sql_text: string; source_nodes?: PipelineGraphNode[] }
): Promise<PipelineValidation> {
  return apiFetch<PipelineValidation>(settings, "/api/v1/dataflow/sql/compile", {
    method: "POST",
    body: JSON.stringify(payload)
  });
}

export function renderDataflowSql(
  settings: ApiSettings,
  graph: PipelineGraph
): Promise<PipelineValidation> {
  return apiFetch<PipelineValidation>(settings, "/api/v1/dataflow/sql/render", {
    method: "POST",
    body: JSON.stringify({ graph })
  });
}

export function previewDataflowPipeline(
  settings: ApiSettings,
  payload: {
    pipeline_id?: string;
    revision?: number;
    graph?: PipelineGraph;
    sql_text?: string;
    source_nodes?: PipelineGraphNode[];
    preview_node_id?: string;
    row_limit?: number;
    execution_backend?: "reference" | "duckdb" | "auto";
  }
): Promise<PipelinePreview> {
  return apiFetch<PipelinePreview>(settings, "/api/v1/dataflow/preview", {
    method: "POST",
    body: JSON.stringify(payload)
  });
}

export async function listDataflowPipelineRuns(
  settings: ApiSettings,
  pipelineId: string
): Promise<PipelineRun[]> {
  const response = await apiFetch<{ runs: PipelineRun[] }>(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/runs?limit=25`
  );
  return response.runs;
}

export function runDataflowPipeline(
  settings: ApiSettings,
  pipelineId: string,
  payload: {
    revision: number;
    idempotency_key: string;
    row_limit?: number;
    execution_backend?: "reference" | "duckdb" | "auto";
    environment?: "development" | "staging" | "production";
    max_attempts?: number;
    retention_days?: number;
    publication?: {
      target_datasource_ref?: string | null;
      name?: string | null;
      source_name?: string | null;
      description?: string | null;
      freeze?: boolean;
      frozen_label?: string | null;
      set_current?: boolean;
      metadata?: Record<string, unknown>;
    } | null;
  }
): Promise<PipelineRun> {
  return apiFetch(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/runs`,
    {
      method: "POST",
      body: JSON.stringify(payload)
    }
  );
}

export function cancelDataflowPipelineRun(
  settings: ApiSettings,
  runRef: string
): Promise<PipelineRun> {
  const runId = runRef.replace(/^dataflow-run:/, "");
  return apiFetch(
    settings,
    `/api/v1/dataflow/runs/${encodeURIComponent(runId)}/cancel`,
    { method: "POST" }
  );
}

export async function listDataflowPipelineDeployments(
  settings: ApiSettings,
  pipelineId: string
): Promise<PipelineDeployment[]> {
  const response = await apiFetch<{ deployments: PipelineDeployment[] }>(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/deployments`
  );
  return response.deployments;
}

export function promoteDataflowPipeline(
  settings: ApiSettings,
  pipelineId: string,
  payload: {
    revision: number;
    source_environment: "development" | "staging";
    target_environment: "staging" | "production";
  }
): Promise<PipelineDeployment> {
  return apiFetch(
    settings,
    `/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/promotions`,
    {
      method: "POST",
      body: JSON.stringify(payload)
    }
  );
}