feat: run dataflows through durable workers

src/govoplan_dataflow/backend/schemas.py

@@ -22,6 +22,16 @@ TriggerDeliveryStatus = Literal[
     "blocked",
     "skipped",
 ]
+DataflowEnvironment = Literal["development", "staging", "production"]
+DataflowExecutionBackend = Literal["auto", "reference", "duckdb"]
+DataflowRunStatus = Literal[
+    "queued",
+    "retrying",
+    "running",
+    "succeeded",
+    "failed",
+    "cancelled",
+]
 
 
 class GraphPosition(BaseModel):
@@ -267,7 +277,11 @@ class PipelineRunPublicationRequest(BaseModel):
 class PipelineRunCreateRequest(BaseModel):
     revision: int | None = Field(default=None, ge=1)
     idempotency_key: str = Field(min_length=1, max_length=255)
-    row_limit: int = Field(default=500, ge=1, le=500)
+    row_limit: int = Field(default=500, ge=1, le=10_000)
+    execution_backend: DataflowExecutionBackend = "auto"
+    environment: DataflowEnvironment = "development"
+    max_attempts: int = Field(default=3, ge=1, le=5)
+    retention_days: int = Field(default=30, ge=1, le=365)
     publication: PipelineRunPublicationRequest | None = None
     invocation_kind: Literal["manual", "api", "backfill"] = "manual"
 
@@ -277,8 +291,10 @@ class PipelineRunResponse(BaseModel):
     pipeline_id: str
     revision: int
     run_type: str
-    status: Literal["running", "succeeded", "failed", "cancelled"]
+    status: DataflowRunStatus
     idempotency_key: str | None
+    execution_backend: str
+    environment: DataflowEnvironment
     definition_hash: str
     executor_version: str
     source_fingerprints: list[dict[str, Any]]
@@ -294,8 +310,18 @@ class PipelineRunResponse(BaseModel):
     delivery_ref: str | None
     correlation_id: str | None
     causation_id: str | None
+    attempts: int
+    max_attempts: int
+    available_at: datetime | None
+    claimed_at: datetime | None
+    lease_expires_at: datetime | None
+    cancellation_requested_at: datetime | None
+    progress_percent: int
+    progress_phase: str
+    retention_until: datetime | None
+    purged_at: datetime | None
     error: str | None
-    started_at: datetime
+    started_at: datetime | None
     finished_at: datetime | None
     created_by: str | None
     created_at: datetime
@@ -306,6 +332,51 @@ class PipelineRunListResponse(BaseModel):
     runs: list[PipelineRunResponse]
 
 
+class PipelineRunMetricsResponse(BaseModel):
+    statuses: dict[str, int]
+    active: int
+    completed_last_24_hours: int
+    failed_last_24_hours: int
+    average_duration_seconds: float | None
+    oldest_queued_at: datetime | None
+
+
+class PipelinePromotionRequest(BaseModel):
+    revision: int | None = Field(default=None, ge=1)
+    source_environment: DataflowEnvironment = "development"
+    target_environment: Literal["staging", "production"]
+
+    @model_validator(mode="after")
+    def validate_transition(self) -> "PipelinePromotionRequest":
+        allowed = {
+            ("development", "staging"),
+            ("staging", "production"),
+        }
+        if (self.source_environment, self.target_environment) not in allowed:
+            raise ValueError(
+                "Promotions must move from development to staging or "
+                "from staging to production."
+            )
+        return self
+
+
+class PipelineDeploymentResponse(BaseModel):
+    id: str
+    pipeline_id: str
+    revision: int
+    environment: DataflowEnvironment
+    source_environment: DataflowEnvironment
+    status: str
+    provenance: dict[str, Any]
+    promoted_by: str | None
+    created_at: datetime
+    updated_at: datetime
+
+
+class PipelineDeploymentListResponse(BaseModel):
+    deployments: list[PipelineDeploymentResponse]
+
+
 JsonScalar = str | int | float | bool | None
 
 
@@ -353,7 +424,11 @@ class DataflowTriggerCreateRequest(BaseModel):
     schedule: DataflowTriggerSchedule | None = None
     event: DataflowTriggerEvent | None = None
     publication: PipelineRunPublicationRequest | None = None
-    row_limit: int = Field(default=500, ge=1, le=500)
+    row_limit: int = Field(default=500, ge=1, le=10_000)
+    environment: DataflowEnvironment = "development"
+    execution_backend: DataflowExecutionBackend = "auto"
+    max_attempts: int = Field(default=3, ge=1, le=5)
+    retention_days: int = Field(default=30, ge=1, le=365)
     catch_up_policy: Literal["skip", "coalesce"] = "coalesce"
     max_concurrent_runs: int = Field(default=1, ge=1, le=10)
 
@@ -392,6 +467,10 @@ class DataflowTriggerResponse(BaseModel):
     event: DataflowTriggerEvent | None
     publication: PipelineRunPublicationRequest | None
     row_limit: int
+    environment: DataflowEnvironment
+    execution_backend: DataflowExecutionBackend
+    max_attempts: int
+    retention_days: int
     catch_up_policy: Literal["skip", "coalesce"]
     max_concurrent_runs: int
     next_fire_at: datetime | None