562 lines
16 KiB
Python
562 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import math
|
|
from datetime import datetime
|
|
from typing import Any, Literal
|
|
|
|
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
|
|
|
|
|
PipelineStatus = Literal["draft", "active", "archived"]
|
|
DefinitionScopeType = Literal["system", "tenant", "group", "user"]
|
|
DefinitionKind = Literal["flow", "template"]
|
|
EditorMode = Literal["graph", "sql"]
|
|
DiagnosticSeverity = Literal["info", "warning", "error"]
|
|
TriggerKind = Literal["once", "interval", "event"]
|
|
TriggerStatus = Literal["enabled", "disabled", "completed", "blocked"]
|
|
TriggerDeliveryStatus = Literal[
|
|
"queued",
|
|
"running",
|
|
"succeeded",
|
|
"failed",
|
|
"blocked",
|
|
"skipped",
|
|
]
|
|
|
|
|
|
class GraphPosition(BaseModel):
|
|
x: float
|
|
y: float
|
|
|
|
@field_validator("x", "y")
|
|
@classmethod
|
|
def finite_coordinate(cls, value: float) -> float:
|
|
if not math.isfinite(value):
|
|
raise ValueError("Graph coordinates must be finite")
|
|
return value
|
|
|
|
|
|
class GraphNode(BaseModel):
|
|
id: str = Field(min_length=1, max_length=100, pattern=r"^[A-Za-z0-9_.:-]+$")
|
|
type: str = Field(min_length=1, max_length=80)
|
|
label: str = Field(min_length=1, max_length=200)
|
|
position: GraphPosition
|
|
config: dict[str, Any] = Field(default_factory=dict)
|
|
|
|
|
|
class GraphEdge(BaseModel):
|
|
id: str = Field(min_length=1, max_length=120, pattern=r"^[A-Za-z0-9_.:-]+$")
|
|
source: str = Field(min_length=1, max_length=100)
|
|
target: str = Field(min_length=1, max_length=100)
|
|
source_port: str = Field(default="output", min_length=1, max_length=80)
|
|
target_port: str = Field(default="input", min_length=1, max_length=80)
|
|
|
|
|
|
class PipelineGraph(BaseModel):
|
|
schema_version: Literal[1] = 1
|
|
nodes: list[GraphNode] = Field(default_factory=list, max_length=100)
|
|
edges: list[GraphEdge] = Field(default_factory=list, max_length=200)
|
|
|
|
|
|
class DataflowDiagnostic(BaseModel):
|
|
severity: DiagnosticSeverity
|
|
code: str
|
|
message: str
|
|
node_id: str | None = None
|
|
field: str | None = None
|
|
|
|
|
|
class PipelineRevisionResponse(BaseModel):
|
|
model_config = ConfigDict(from_attributes=True)
|
|
|
|
id: str
|
|
revision: int
|
|
schema_version: int
|
|
graph: PipelineGraph
|
|
sql_text: str | None
|
|
editor_mode: EditorMode
|
|
content_hash: str
|
|
created_by: str | None
|
|
created_at: datetime
|
|
|
|
|
|
class DefinitionActionDecisionResponse(BaseModel):
|
|
allowed: bool
|
|
reason: str | None = None
|
|
source_path: list[dict[str, Any]] = Field(default_factory=list)
|
|
requirements: list[str] = Field(default_factory=list)
|
|
details: dict[str, Any] = Field(default_factory=dict)
|
|
|
|
|
|
class PipelineGovernanceResponse(BaseModel):
|
|
scope_type: DefinitionScopeType
|
|
scope_id: str | None
|
|
definition_kind: DefinitionKind
|
|
inherit_to_lower_scopes: bool
|
|
allow_run: bool
|
|
allow_reuse: bool
|
|
allow_automation: bool
|
|
derived_from_pipeline_id: str | None
|
|
derived_from_revision: int | None
|
|
derived_from_hash: str | None
|
|
derivation_provenance: dict[str, Any] = Field(default_factory=dict)
|
|
actions: dict[str, DefinitionActionDecisionResponse]
|
|
|
|
|
|
class PipelineResponse(BaseModel):
|
|
id: str
|
|
tenant_id: str | None
|
|
name: str
|
|
description: str | None
|
|
status: PipelineStatus
|
|
current_revision: int
|
|
created_by: str | None
|
|
updated_by: str | None
|
|
created_at: datetime
|
|
updated_at: datetime
|
|
revision: PipelineRevisionResponse
|
|
governance: PipelineGovernanceResponse
|
|
|
|
|
|
class PipelineListResponse(BaseModel):
|
|
pipelines: list[PipelineResponse]
|
|
|
|
|
|
class PipelineCreateRequest(BaseModel):
|
|
name: str = Field(min_length=1, max_length=300)
|
|
description: str | None = Field(default=None, max_length=4000)
|
|
status: PipelineStatus = "draft"
|
|
graph: PipelineGraph
|
|
sql_text: str | None = Field(default=None, max_length=100_000)
|
|
editor_mode: EditorMode = "graph"
|
|
scope_type: DefinitionScopeType = "tenant"
|
|
scope_id: str | None = Field(default=None, max_length=36)
|
|
definition_kind: DefinitionKind = "flow"
|
|
inherit_to_lower_scopes: bool = False
|
|
allow_run: bool = True
|
|
allow_reuse: bool = False
|
|
allow_automation: bool = False
|
|
|
|
@model_validator(mode="after")
|
|
def validate_definition_kind(self) -> "PipelineCreateRequest":
|
|
if self.definition_kind == "template" and self.status == "active":
|
|
raise ValueError("Templates cannot be active.")
|
|
return self
|
|
|
|
|
|
class PipelineUpdateRequest(PipelineCreateRequest):
|
|
expected_revision: int = Field(ge=1)
|
|
|
|
|
|
class PipelineDeriveRequest(BaseModel):
|
|
name: str = Field(min_length=1, max_length=300)
|
|
description: str | None = Field(default=None, max_length=4_000)
|
|
source_revision: int | None = Field(default=None, ge=1)
|
|
scope_type: DefinitionScopeType = "tenant"
|
|
scope_id: str | None = Field(default=None, max_length=36)
|
|
definition_kind: DefinitionKind = "flow"
|
|
inherit_to_lower_scopes: bool = False
|
|
allow_run: bool = True
|
|
allow_reuse: bool = False
|
|
allow_automation: bool = False
|
|
|
|
|
|
class PipelineDraftRequest(BaseModel):
|
|
graph: PipelineGraph | None = None
|
|
sql_text: str | None = Field(default=None, max_length=100_000)
|
|
source_nodes: list[GraphNode] = Field(default_factory=list, max_length=20)
|
|
|
|
|
|
class PipelineValidationResponse(BaseModel):
|
|
valid: bool
|
|
graph: PipelineGraph | None
|
|
sql_text: str | None
|
|
diagnostics: list[DataflowDiagnostic]
|
|
|
|
|
|
class PipelineSqlResponse(PipelineValidationResponse):
|
|
pass
|
|
|
|
|
|
class PipelinePreviewRequest(BaseModel):
|
|
pipeline_id: str | None = None
|
|
revision: int | None = Field(default=None, ge=1)
|
|
graph: PipelineGraph | None = None
|
|
sql_text: str | None = Field(default=None, max_length=100_000)
|
|
source_nodes: list[GraphNode] = Field(default_factory=list, max_length=20)
|
|
preview_node_id: str | None = Field(
|
|
default=None,
|
|
min_length=1,
|
|
max_length=100,
|
|
pattern=r"^[A-Za-z0-9_.:-]+$",
|
|
)
|
|
row_limit: int = Field(default=100, ge=1, le=500)
|
|
|
|
|
|
class PreviewColumn(BaseModel):
|
|
name: str
|
|
type: str
|
|
nullable: bool = True
|
|
|
|
|
|
class NodePreviewDiagnostic(BaseModel):
|
|
node_id: str
|
|
status: Literal["succeeded", "failed", "skipped"]
|
|
input_rows: int = 0
|
|
output_rows: int = 0
|
|
duration_ms: float = 0
|
|
columns: list[PreviewColumn] = Field(default_factory=list)
|
|
messages: list[str] = Field(default_factory=list)
|
|
|
|
|
|
class NodePreviewResult(BaseModel):
|
|
node_id: str
|
|
columns: list[PreviewColumn]
|
|
rows: list[dict[str, Any]]
|
|
total_rows: int
|
|
truncated: bool
|
|
|
|
|
|
class PipelinePreviewResponse(BaseModel):
|
|
run_id: str | None
|
|
pipeline_id: str | None
|
|
revision: int | None
|
|
status: Literal["succeeded", "failed"]
|
|
columns: list[PreviewColumn]
|
|
rows: list[dict[str, Any]]
|
|
total_rows: int
|
|
truncated: bool
|
|
diagnostics: list[DataflowDiagnostic]
|
|
node_diagnostics: list[NodePreviewDiagnostic]
|
|
node_preview: NodePreviewResult | None = None
|
|
source_fingerprints: list[dict[str, Any]]
|
|
input_row_count: int
|
|
definition_hash: str
|
|
executor_version: str
|
|
|
|
|
|
class PipelineRunPublicationRequest(BaseModel):
|
|
target_datasource_ref: str | None = Field(default=None, max_length=500)
|
|
name: str | None = Field(default=None, min_length=1, max_length=300)
|
|
source_name: str | None = Field(
|
|
default=None,
|
|
min_length=1,
|
|
max_length=120,
|
|
pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
|
)
|
|
description: str | None = Field(default=None, max_length=4_000)
|
|
freeze: bool = False
|
|
frozen_label: str | None = Field(default=None, max_length=300)
|
|
set_current: bool = True
|
|
metadata: dict[str, Any] = Field(default_factory=dict)
|
|
|
|
@model_validator(mode="after")
|
|
def validate_target(self) -> "PipelineRunPublicationRequest":
|
|
if self.target_datasource_ref:
|
|
return self
|
|
if not self.name or not self.source_name:
|
|
raise ValueError(
|
|
"New publication targets require name and source_name."
|
|
)
|
|
return self
|
|
|
|
|
|
class PipelineRunCreateRequest(BaseModel):
|
|
revision: int | None = Field(default=None, ge=1)
|
|
idempotency_key: str = Field(min_length=1, max_length=255)
|
|
row_limit: int = Field(default=500, ge=1, le=500)
|
|
publication: PipelineRunPublicationRequest | None = None
|
|
invocation_kind: Literal["manual", "api", "backfill"] = "manual"
|
|
|
|
|
|
class PipelineRunResponse(BaseModel):
|
|
ref: str
|
|
pipeline_id: str
|
|
revision: int
|
|
run_type: str
|
|
status: Literal["running", "succeeded", "failed", "cancelled"]
|
|
idempotency_key: str | None
|
|
definition_hash: str
|
|
executor_version: str
|
|
source_fingerprints: list[dict[str, Any]]
|
|
result_schema: list[PreviewColumn]
|
|
diagnostics: list[DataflowDiagnostic]
|
|
input_row_count: int
|
|
output_row_count: int
|
|
output_publication_ref: str | None
|
|
output_datasource_ref: str | None
|
|
output_materialization_ref: str | None
|
|
invocation_kind: str
|
|
trigger_ref: str | None
|
|
delivery_ref: str | None
|
|
correlation_id: str | None
|
|
causation_id: str | None
|
|
error: str | None
|
|
started_at: datetime
|
|
finished_at: datetime | None
|
|
created_by: str | None
|
|
created_at: datetime
|
|
replayed: bool = False
|
|
|
|
|
|
class PipelineRunListResponse(BaseModel):
|
|
runs: list[PipelineRunResponse]
|
|
|
|
|
|
JsonScalar = str | int | float | bool | None
|
|
|
|
|
|
class DataflowTriggerSchedule(BaseModel):
|
|
run_at: datetime | None = None
|
|
anchor_at: datetime | None = None
|
|
interval_seconds: int | None = Field(default=None, ge=60, le=31_536_000)
|
|
timezone: str = Field(default="UTC", min_length=1, max_length=100)
|
|
|
|
|
|
class DataflowTriggerEvent(BaseModel):
|
|
event_type: str = Field(
|
|
min_length=1,
|
|
max_length=200,
|
|
pattern=r"^[A-Za-z0-9_.:-]+$",
|
|
)
|
|
module_id: str | None = Field(
|
|
default=None,
|
|
max_length=100,
|
|
pattern=r"^[a-z][a-z0-9_]*$",
|
|
)
|
|
filters: dict[str, JsonScalar] = Field(default_factory=dict)
|
|
|
|
@field_validator("filters")
|
|
@classmethod
|
|
def bounded_filters(
|
|
cls,
|
|
value: dict[str, JsonScalar],
|
|
) -> dict[str, JsonScalar]:
|
|
if len(value) > 20:
|
|
raise ValueError("Event triggers support at most 20 filters.")
|
|
for key in value:
|
|
if not key or len(key) > 120 or "." in key:
|
|
raise ValueError(
|
|
"Event filter keys must be direct payload fields."
|
|
)
|
|
return value
|
|
|
|
|
|
class DataflowTriggerCreateRequest(BaseModel):
|
|
name: str = Field(min_length=1, max_length=300)
|
|
kind: TriggerKind
|
|
revision: int | None = Field(default=None, ge=1)
|
|
enabled: bool = False
|
|
schedule: DataflowTriggerSchedule | None = None
|
|
event: DataflowTriggerEvent | None = None
|
|
publication: PipelineRunPublicationRequest | None = None
|
|
row_limit: int = Field(default=500, ge=1, le=500)
|
|
catch_up_policy: Literal["skip", "coalesce"] = "coalesce"
|
|
max_concurrent_runs: int = Field(default=1, ge=1, le=10)
|
|
|
|
@model_validator(mode="after")
|
|
def validate_trigger(self) -> "DataflowTriggerCreateRequest":
|
|
if self.kind == "once":
|
|
if self.schedule is None or self.schedule.run_at is None:
|
|
raise ValueError("One-time triggers require schedule.run_at.")
|
|
elif self.kind == "interval":
|
|
if (
|
|
self.schedule is None
|
|
or self.schedule.interval_seconds is None
|
|
):
|
|
raise ValueError(
|
|
"Interval triggers require schedule.interval_seconds."
|
|
)
|
|
elif self.kind == "event" and self.event is None:
|
|
raise ValueError("Event triggers require event configuration.")
|
|
return self
|
|
|
|
|
|
class DataflowTriggerUpdateRequest(DataflowTriggerCreateRequest):
|
|
expected_revision: int = Field(ge=1)
|
|
|
|
|
|
class DataflowTriggerResponse(BaseModel):
|
|
id: str
|
|
tenant_id: str
|
|
pipeline_id: str
|
|
pipeline_revision: int
|
|
name: str
|
|
kind: TriggerKind
|
|
status: TriggerStatus
|
|
revision: int
|
|
schedule: DataflowTriggerSchedule | None
|
|
event: DataflowTriggerEvent | None
|
|
publication: PipelineRunPublicationRequest | None
|
|
row_limit: int
|
|
catch_up_policy: Literal["skip", "coalesce"]
|
|
max_concurrent_runs: int
|
|
next_fire_at: datetime | None
|
|
last_fire_at: datetime | None
|
|
last_status: str | None
|
|
last_error: str | None
|
|
grant_scopes: list[str]
|
|
authorization_provenance: dict[str, Any] = Field(default_factory=dict)
|
|
created_by: str | None
|
|
updated_by: str | None
|
|
created_at: datetime
|
|
updated_at: datetime
|
|
|
|
|
|
class DataflowTriggerListResponse(BaseModel):
|
|
triggers: list[DataflowTriggerResponse]
|
|
|
|
|
|
class DataflowTriggerDeliveryResponse(BaseModel):
|
|
id: str
|
|
trigger_id: str
|
|
invocation_kind: str
|
|
status: TriggerDeliveryStatus
|
|
scheduled_for: datetime | None
|
|
event_id: str | None
|
|
run_id: str | None
|
|
attempts: int
|
|
authorization_provenance: dict[str, Any] = Field(default_factory=dict)
|
|
error: str | None
|
|
created_at: datetime
|
|
finished_at: datetime | None
|
|
|
|
|
|
class DataflowTriggerDeliveryListResponse(BaseModel):
|
|
deliveries: list[DataflowTriggerDeliveryResponse]
|
|
|
|
|
|
class DataflowEventDeliveryRequest(BaseModel):
|
|
event_id: str = Field(min_length=1, max_length=128)
|
|
type: str = Field(
|
|
min_length=1,
|
|
max_length=200,
|
|
pattern=r"^[A-Za-z0-9_.:-]+$",
|
|
)
|
|
module_id: str = Field(
|
|
min_length=1,
|
|
max_length=100,
|
|
pattern=r"^[a-z][a-z0-9_]*$",
|
|
)
|
|
payload: dict[str, Any] = Field(default_factory=dict)
|
|
occurred_at: datetime
|
|
correlation_id: str | None = Field(default=None, max_length=128)
|
|
causation_id: str | None = Field(default=None, max_length=128)
|
|
classification: Literal[
|
|
"public",
|
|
"internal",
|
|
"confidential",
|
|
"restricted",
|
|
] = "internal"
|
|
|
|
|
|
class DataflowTriggerDispatchResponse(BaseModel):
|
|
queued: int = 0
|
|
processed: int = 0
|
|
succeeded: int = 0
|
|
failed: int = 0
|
|
blocked: int = 0
|
|
skipped: int = 0
|
|
|
|
|
|
class DataflowTriggerDeleteResponse(BaseModel):
|
|
deleted: bool
|
|
trigger_id: str
|
|
pipeline_id: str
|
|
|
|
|
|
class PipelineDeleteResponse(BaseModel):
|
|
deleted: bool
|
|
pipeline_id: str
|
|
|
|
|
|
class NodePortDefinitionResponse(BaseModel):
|
|
id: str
|
|
label: str
|
|
required: bool
|
|
multiple: bool
|
|
minimum_connections: int
|
|
|
|
|
|
class NodeConfigFieldResponse(BaseModel):
|
|
id: str
|
|
label: str
|
|
kind: str
|
|
required: bool
|
|
description: str | None
|
|
options: list[tuple[str, str]]
|
|
|
|
|
|
class NodeTypeDefinitionResponse(BaseModel):
|
|
type: str
|
|
category: str
|
|
category_label: str
|
|
label: str
|
|
description: str
|
|
icon: str
|
|
input_ports: list[NodePortDefinitionResponse]
|
|
output_ports: list[NodePortDefinitionResponse]
|
|
config_fields: list[NodeConfigFieldResponse]
|
|
default_config: dict[str, Any]
|
|
sql_support: str
|
|
|
|
|
|
class NodeLibraryResponse(BaseModel):
|
|
nodes: list[NodeTypeDefinitionResponse]
|
|
|
|
|
|
class TabularSourceColumnResponse(BaseModel):
|
|
name: str
|
|
data_type: str
|
|
nullable: bool
|
|
|
|
|
|
class TabularSourceResponse(BaseModel):
|
|
ref: str
|
|
provider: str
|
|
source_name: str
|
|
name: str
|
|
description: str | None
|
|
mode: Literal["live", "cached", "static"]
|
|
columns: list[TabularSourceColumnResponse]
|
|
schema_version: str
|
|
fingerprint: str
|
|
row_count: int | None
|
|
byte_count: int | None
|
|
updated_at: datetime | None
|
|
capabilities: list[str]
|
|
|
|
|
|
class TabularSourceListResponse(BaseModel):
|
|
available: bool
|
|
writable: bool
|
|
sources: list[TabularSourceResponse]
|
|
|
|
|
|
class TabularSnapshotCreateRequest(BaseModel):
|
|
name: str = Field(min_length=1, max_length=300)
|
|
source_name: str = Field(
|
|
min_length=1,
|
|
max_length=120,
|
|
pattern=r"^[A-Za-z_][A-Za-z0-9_]*$",
|
|
)
|
|
description: str | None = Field(default=None, max_length=4000)
|
|
format: Literal["json", "csv"] = "json"
|
|
rows: list[dict[str, Any]] | None = Field(default=None, max_length=10_000)
|
|
csv_text: str | None = Field(default=None, max_length=5_000_000)
|
|
delimiter: str = Field(default=",", min_length=1, max_length=1)
|
|
|
|
@model_validator(mode="after")
|
|
def validate_format_payload(self) -> TabularSnapshotCreateRequest:
|
|
if self.format == "json":
|
|
if self.rows is None:
|
|
raise ValueError("JSON snapshots require rows.")
|
|
if self.csv_text is not None:
|
|
raise ValueError("JSON snapshots cannot include CSV text.")
|
|
else:
|
|
if not self.csv_text:
|
|
raise ValueError("CSV snapshots require CSV text.")
|
|
if self.rows is not None:
|
|
raise ValueError("CSV snapshots cannot include JSON rows.")
|
|
return self
|