GovOPlaN/govoplan-dataflow
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Add governed reusable pipelines and triggers

Browse Source
This commit is contained in:
zemion
2026-07-28 15:04:23 +02:00
parent 6305ef9cef
commit 08ddc8d3e8
17 changed files with 4697 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
webui/src/api/dataflow.ts
+154 -1
View File
@@ -2,6 +2,8 @@ import { apiFetch, type ApiSettings } from "@govoplan/core-webui";
import type { DefinitionGraphNodeType } from "@govoplan/core-webui/definition-graph";
export type PipelineStatus = "draft" | "active" | "archived";
export type DefinitionScopeType = "system" | "tenant" | "group" | "user";
export type DefinitionKind = "flow" | "template";
export type EditorMode = "graph" | "sql";
export type NodeCategory = "load" | "combine" | "filter" | "transform" | "output";
@@ -48,7 +50,7 @@ export type PipelineRevision = {
export type Pipeline = {
id: string;
tenant_id: string;
tenant_id: string | null;
name: string;
description?: string | null;
status: PipelineStatus;
@@ -58,6 +60,30 @@ export type Pipeline = {
created_at: string;
updated_at: string;
revision: PipelineRevision;
governance: PipelineGovernance;
};
export type DefinitionActionDecision = {
allowed: boolean;
reason?: string | null;
source_path: Array<Record<string, unknown>>;
requirements: string[];
details: Record<string, unknown>;
};
export type PipelineGovernance = {
scope_type: DefinitionScopeType;
scope_id?: string | null;
definition_kind: DefinitionKind;
inherit_to_lower_scopes: boolean;
allow_run: boolean;
allow_reuse: boolean;
allow_automation: boolean;
derived_from_pipeline_id?: string | null;
derived_from_revision?: number | null;
derived_from_hash?: string | null;
derivation_provenance: Record<string, unknown>;
actions: Record<string, DefinitionActionDecision>;
};
export type PipelinePayload = {
@@ -67,6 +93,13 @@ export type PipelinePayload = {
graph: PipelineGraph;
sql_text?: string | null;
editor_mode: EditorMode;
scope_type: DefinitionScopeType;
scope_id?: string | null;
definition_kind: DefinitionKind;
inherit_to_lower_scopes: boolean;
allow_run: boolean;
allow_reuse: boolean;
allow_automation: boolean;
};
export type PipelineValidation = {
@@ -179,6 +212,11 @@ export type PipelineRun = {
output_publication_ref?: string | null;
output_datasource_ref?: string | null;
output_materialization_ref?: string | null;
invocation_kind: string;
trigger_ref?: string | null;
delivery_ref?: string | null;
correlation_id?: string | null;
causation_id?: string | null;
error?: string | null;
started_at: string;
finished_at?: string | null;
@@ -187,6 +225,52 @@ export type PipelineRun = {
replayed: boolean;
};
export type DataflowTriggerKind = "once" | "interval" | "event";
export type DataflowTrigger = {
id: string;
tenant_id: string;
pipeline_id: string;
pipeline_revision: number;
name: string;
kind: DataflowTriggerKind;
status: "enabled" | "disabled" | "completed" | "blocked";
revision: number;
schedule?: {
run_at?: string | null;
anchor_at?: string | null;
interval_seconds?: number | null;
timezone: string;
} | null;
event?: {
event_type: string;
module_id?: string | null;
filters: Record<string, string | number | boolean | null>;
} | null;
row_limit: number;
catch_up_policy: "skip" | "coalesce";
max_concurrent_runs: number;
next_fire_at?: string | null;
last_fire_at?: string | null;
last_status?: string | null;
last_error?: string | null;
grant_scopes: string[];
authorization_provenance: Record<string, unknown>;
created_at: string;
updated_at: string;
};
export type DataflowTriggerPayload = {
name: string;
kind: DataflowTriggerKind;
revision?: number | null;
enabled: boolean;
schedule?: DataflowTrigger["schedule"];
event?: DataflowTrigger["event"];
row_limit: number;
catch_up_policy: "skip" | "coalesce";
max_concurrent_runs: number;
};
export async function listDataflowNodeTypes(settings: ApiSettings): Promise<NodeTypeDefinition[]> {
const response = await apiFetch<{ nodes: NodeTypeDefinition[] }>(
settings,
@@ -260,6 +344,75 @@ export function deleteDataflowPipeline(settings: ApiSettings, pipelineId: string
);
}
export function deriveDataflowPipeline(
settings: ApiSettings,
pipelineId: string,
payload: {
name: string;
description?: string | null;
source_revision?: number | null;
scope_type: DefinitionScopeType;
scope_id?: string | null;
definition_kind: DefinitionKind;
inherit_to_lower_scopes: boolean;
allow_run: boolean;
allow_reuse: boolean;
allow_automation: boolean;
}
): Promise<Pipeline> {
return apiFetch(
settings,
`/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/derive`,
{ method: "POST", body: JSON.stringify(payload) }
);
}
export async function listDataflowTriggers(
settings: ApiSettings,
pipelineId: string
): Promise<DataflowTrigger[]> {
const response = await apiFetch<{ triggers: DataflowTrigger[] }>(
settings,
`/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/triggers`
);
return response.triggers;
}
export function createDataflowTrigger(
settings: ApiSettings,
pipelineId: string,
payload: DataflowTriggerPayload
): Promise<DataflowTrigger> {
return apiFetch(
settings,
`/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/triggers`,
{ method: "POST", body: JSON.stringify(payload) }
);
}
export function updateDataflowTrigger(
settings: ApiSettings,
triggerId: string,
payload: DataflowTriggerPayload & { expected_revision: number }
): Promise<DataflowTrigger> {
return apiFetch(
settings,
`/api/v1/dataflow/triggers/${encodeURIComponent(triggerId)}`,
{ method: "PUT", body: JSON.stringify(payload) }
);
}
export function deleteDataflowTrigger(
settings: ApiSettings,
triggerId: string
): Promise<{ deleted: boolean }> {
return apiFetch(
settings,
`/api/v1/dataflow/triggers/${encodeURIComponent(triggerId)}`,
{ method: "DELETE" }
);
}
export function validateDataflowPipeline(
settings: ApiSettings,
payload: { graph?: PipelineGraph; sql_text?: string; source_nodes?: PipelineGraphNode[] }