GovOPlaN/govoplan-dataflow
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

Add governed reusable pipelines and triggers

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-28 15:04:23 +02:00
parent 6305ef9cef
commit 08ddc8d3e8
17 changed files with 4697 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
src/govoplan_dataflow/backend/schemas.py
View File

@@ -8,8 +8,20 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida
PipelineStatus = Literal["draft", "active", "archived"]
DefinitionScopeType = Literal["system", "tenant", "group", "user"]
DefinitionKind = Literal["flow", "template"]
EditorMode = Literal["graph", "sql"]
DiagnosticSeverity = Literal["info", "warning", "error"]
TriggerKind = Literal["once", "interval", "event"]
TriggerStatus = Literal["enabled", "disabled", "completed", "blocked"]
TriggerDeliveryStatus = Literal[
"queued",
"running",
"succeeded",
"failed",
"blocked",
"skipped",
]
class GraphPosition(BaseModel):
@@ -68,9 +80,32 @@ class PipelineRevisionResponse(BaseModel):
created_at: datetime
class DefinitionActionDecisionResponse(BaseModel):
allowed: bool
reason: str | None = None
source_path: list[dict[str, Any]] = Field(default_factory=list)
requirements: list[str] = Field(default_factory=list)
details: dict[str, Any] = Field(default_factory=dict)
class PipelineGovernanceResponse(BaseModel):
scope_type: DefinitionScopeType
scope_id: str | None
definition_kind: DefinitionKind
inherit_to_lower_scopes: bool
allow_run: bool
allow_reuse: bool
allow_automation: bool
derived_from_pipeline_id: str | None
derived_from_revision: int | None
derived_from_hash: str | None
derivation_provenance: dict[str, Any] = Field(default_factory=dict)
actions: dict[str, DefinitionActionDecisionResponse]
class PipelineResponse(BaseModel):
id: str
tenant_id: str
tenant_id: str | None
name: str
description: str | None
status: PipelineStatus
@@ -80,6 +115,7 @@ class PipelineResponse(BaseModel):
created_at: datetime
updated_at: datetime
revision: PipelineRevisionResponse
governance: PipelineGovernanceResponse
class PipelineListResponse(BaseModel):
@@ -93,12 +129,38 @@ class PipelineCreateRequest(BaseModel):
graph: PipelineGraph
sql_text: str | None = Field(default=None, max_length=100_000)
editor_mode: EditorMode = "graph"
scope_type: DefinitionScopeType = "tenant"
scope_id: str | None = Field(default=None, max_length=36)
definition_kind: DefinitionKind = "flow"
inherit_to_lower_scopes: bool = False
allow_run: bool = True
allow_reuse: bool = False
allow_automation: bool = False
@model_validator(mode="after")
def validate_definition_kind(self) -> "PipelineCreateRequest":
if self.definition_kind == "template" and self.status == "active":
raise ValueError("Templates cannot be active.")
return self
class PipelineUpdateRequest(PipelineCreateRequest):
expected_revision: int = Field(ge=1)
class PipelineDeriveRequest(BaseModel):
name: str = Field(min_length=1, max_length=300)
description: str | None = Field(default=None, max_length=4_000)
source_revision: int | None = Field(default=None, ge=1)
scope_type: DefinitionScopeType = "tenant"
scope_id: str | None = Field(default=None, max_length=36)
definition_kind: DefinitionKind = "flow"
inherit_to_lower_scopes: bool = False
allow_run: bool = True
allow_reuse: bool = False
allow_automation: bool = False
class PipelineDraftRequest(BaseModel):
graph: PipelineGraph | None = None
sql_text: str | None = Field(default=None, max_length=100_000)
@@ -189,6 +251,7 @@ class PipelineRunCreateRequest(BaseModel):
idempotency_key: str = Field(min_length=1, max_length=255)
row_limit: int = Field(default=500, ge=1, le=500)
publication: PipelineRunPublicationRequest | None = None
invocation_kind: Literal["manual", "api", "backfill"] = "manual"
class PipelineRunResponse(BaseModel):
@@ -208,6 +271,11 @@ class PipelineRunResponse(BaseModel):
output_publication_ref: str | None
output_datasource_ref: str | None
output_materialization_ref: str | None
invocation_kind: str
trigger_ref: str | None
delivery_ref: str | None
correlation_id: str | None
causation_id: str | None
error: str | None
started_at: datetime
finished_at: datetime | None
@@ -220,6 +288,168 @@ class PipelineRunListResponse(BaseModel):
runs: list[PipelineRunResponse]
JsonScalar = str | int | float | bool | None
class DataflowTriggerSchedule(BaseModel):
run_at: datetime | None = None
anchor_at: datetime | None = None
interval_seconds: int | None = Field(default=None, ge=60, le=31_536_000)
timezone: str = Field(default="UTC", min_length=1, max_length=100)
class DataflowTriggerEvent(BaseModel):
event_type: str = Field(
min_length=1,
max_length=200,
pattern=r"^[A-Za-z0-9_.:-]+$",
)
module_id: str | None = Field(
default=None,
max_length=100,
pattern=r"^[a-z][a-z0-9_]*$",
)
filters: dict[str, JsonScalar] = Field(default_factory=dict)
@field_validator("filters")
@classmethod
def bounded_filters(
cls,
value: dict[str, JsonScalar],
) -> dict[str, JsonScalar]:
if len(value) > 20:
raise ValueError("Event triggers support at most 20 filters.")
for key in value:
if not key or len(key) > 120 or "." in key:
raise ValueError(
"Event filter keys must be direct payload fields."
)
return value
class DataflowTriggerCreateRequest(BaseModel):
name: str = Field(min_length=1, max_length=300)
kind: TriggerKind
revision: int | None = Field(default=None, ge=1)
enabled: bool = False
schedule: DataflowTriggerSchedule | None = None
event: DataflowTriggerEvent | None = None
publication: PipelineRunPublicationRequest | None = None
row_limit: int = Field(default=500, ge=1, le=500)
catch_up_policy: Literal["skip", "coalesce"] = "coalesce"
max_concurrent_runs: int = Field(default=1, ge=1, le=10)
@model_validator(mode="after")
def validate_trigger(self) -> "DataflowTriggerCreateRequest":
if self.kind == "once":
if self.schedule is None or self.schedule.run_at is None:
raise ValueError("One-time triggers require schedule.run_at.")
elif self.kind == "interval":
if (
self.schedule is None
or self.schedule.interval_seconds is None
):
raise ValueError(
"Interval triggers require schedule.interval_seconds."
)
elif self.kind == "event" and self.event is None:
raise ValueError("Event triggers require event configuration.")
return self
class DataflowTriggerUpdateRequest(DataflowTriggerCreateRequest):
expected_revision: int = Field(ge=1)
class DataflowTriggerResponse(BaseModel):
id: str
tenant_id: str
pipeline_id: str
pipeline_revision: int
name: str
kind: TriggerKind
status: TriggerStatus
revision: int
schedule: DataflowTriggerSchedule | None
event: DataflowTriggerEvent | None
publication: PipelineRunPublicationRequest | None
row_limit: int
catch_up_policy: Literal["skip", "coalesce"]
max_concurrent_runs: int
next_fire_at: datetime | None
last_fire_at: datetime | None
last_status: str | None
last_error: str | None
grant_scopes: list[str]
authorization_provenance: dict[str, Any] = Field(default_factory=dict)
created_by: str | None
updated_by: str | None
created_at: datetime
updated_at: datetime
class DataflowTriggerListResponse(BaseModel):
triggers: list[DataflowTriggerResponse]
class DataflowTriggerDeliveryResponse(BaseModel):
id: str
trigger_id: str
invocation_kind: str
status: TriggerDeliveryStatus
scheduled_for: datetime | None
event_id: str | None
run_id: str | None
attempts: int
authorization_provenance: dict[str, Any] = Field(default_factory=dict)
error: str | None
created_at: datetime
finished_at: datetime | None
class DataflowTriggerDeliveryListResponse(BaseModel):
deliveries: list[DataflowTriggerDeliveryResponse]
class DataflowEventDeliveryRequest(BaseModel):
event_id: str = Field(min_length=1, max_length=128)
type: str = Field(
min_length=1,
max_length=200,
pattern=r"^[A-Za-z0-9_.:-]+$",
)
module_id: str = Field(
min_length=1,
max_length=100,
pattern=r"^[a-z][a-z0-9_]*$",
)
payload: dict[str, Any] = Field(default_factory=dict)
occurred_at: datetime
correlation_id: str | None = Field(default=None, max_length=128)
causation_id: str | None = Field(default=None, max_length=128)
classification: Literal[
"public",
"internal",
"confidential",
"restricted",
] = "internal"
class DataflowTriggerDispatchResponse(BaseModel):
queued: int = 0
processed: int = 0
succeeded: int = 0
failed: int = 0
blocked: int = 0
skipped: int = 0
class DataflowTriggerDeleteResponse(BaseModel):
deleted: bool
trigger_id: str
pipeline_id: str
class PipelineDeleteResponse(BaseModel):
deleted: bool
pipeline_id: str