feat: run dataflows through durable workers

2026-07-30 02:33:02 +02:00
parent d61e9d8942
commit 5af02933ef
16 changed files with 2306 additions and 67 deletions
@@ -1,5 +1,15 @@
 from __future__ import annotations

-from govoplan_dataflow.backend.db.models import DataflowPipeline, DataflowPipelineRevision, DataflowRun
+from govoplan_dataflow.backend.db.models import (
+    DataflowPipeline,
+    DataflowPipelineDeployment,
+    DataflowPipelineRevision,
+    DataflowRun,
+)

-__all__ = ["DataflowPipeline", "DataflowPipelineRevision", "DataflowRun"]
+__all__ = [
+    "DataflowPipeline",
+    "DataflowPipelineDeployment",
+    "DataflowPipelineRevision",
+    "DataflowRun",
+]
@@ -111,6 +111,11 @@ class DataflowPipeline(Base, TimestampMixin):
        cascade="all, delete-orphan",
        order_by="DataflowRun.created_at",
    )
+    deployments: Mapped[list["DataflowPipelineDeployment"]] = relationship(
+        back_populates="pipeline",
+        cascade="all, delete-orphan",
+        order_by="DataflowPipelineDeployment.environment",
+    )
    triggers: Mapped[list["DataflowTrigger"]] = relationship(
        back_populates="pipeline",
        cascade="all, delete-orphan",
@@ -159,6 +164,12 @@ class DataflowRun(Base, TimestampMixin):
        ),
        Index("ix_dataflow_runs_tenant_status", "tenant_id", "status"),
        Index("ix_dataflow_runs_pipeline_created", "pipeline_id", "created_at"),
+        Index(
+            "ix_dataflow_runs_worker_queue",
+            "status",
+            "available_at",
+            "created_at",
+        ),
    )

    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
@@ -175,6 +186,18 @@ class DataflowRun(Base, TimestampMixin):
    )
    run_type: Mapped[str] = mapped_column(String(30), default="preview", nullable=False, index=True)
    status: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
+    execution_backend: Mapped[str] = mapped_column(
+        String(30),
+        default="auto",
+        nullable=False,
+        index=True,
+    )
+    environment: Mapped[str] = mapped_column(
+        String(30),
+        default="development",
+        nullable=False,
+        index=True,
+    )
    executor_version: Mapped[str] = mapped_column(String(40), nullable=False)
    definition_hash: Mapped[str] = mapped_column(String(64), nullable=False)
    idempotency_key: Mapped[str | None] = mapped_column(
@@ -232,7 +255,73 @@ class DataflowRun(Base, TimestampMixin):
        String(500),
        nullable=True,
    )
-    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
+    attempts: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    max_attempts: Mapped[int] = mapped_column(Integer, default=3, nullable=False)
+    queued_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+    )
+    available_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+        index=True,
+    )
+    claimed_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+    )
+    lease_expires_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+        index=True,
+    )
+    heartbeat_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+    )
+    worker_id: Mapped[str | None] = mapped_column(
+        String(255),
+        nullable=True,
+        index=True,
+    )
+    cancellation_requested_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+    )
+    progress_percent: Mapped[int] = mapped_column(
+        Integer,
+        default=0,
+        nullable=False,
+    )
+    progress_phase: Mapped[str] = mapped_column(
+        String(80),
+        default="queued",
+        nullable=False,
+    )
+    retention_until: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+        index=True,
+    )
+    purged_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+    )
+    authorization_: Mapped[dict[str, Any]] = mapped_column(
+        "authorization",
+        JSON,
+        default=dict,
+        nullable=False,
+    )
+    resource_budget: Mapped[dict[str, Any]] = mapped_column(
+        JSON,
+        default=dict,
+        nullable=False,
+    )
+    started_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+    )
    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
    error: Mapped[str | None] = mapped_column(Text)
    created_by: Mapped[str | None] = mapped_column(String(255), nullable=True, index=True)
@@ -240,6 +329,56 @@ class DataflowRun(Base, TimestampMixin):
    pipeline: Mapped[DataflowPipeline] = relationship(back_populates="runs")


+class DataflowPipelineDeployment(Base, TimestampMixin):
+    __tablename__ = "dataflow_pipeline_deployments"
+    __table_args__ = (
+        UniqueConstraint(
+            "tenant_id",
+            "pipeline_id",
+            "environment",
+            name="uq_dataflow_pipeline_deployment_environment",
+        ),
+        Index(
+            "ix_dataflow_deployments_tenant_environment",
+            "tenant_id",
+            "environment",
+        ),
+    )
+
+    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
+    tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
+    pipeline_id: Mapped[str] = mapped_column(
+        ForeignKey("dataflow_pipelines.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    pipeline_revision_id: Mapped[str] = mapped_column(
+        ForeignKey("dataflow_pipeline_revisions.id", ondelete="RESTRICT"),
+        nullable=False,
+        index=True,
+    )
+    environment: Mapped[str] = mapped_column(String(30), nullable=False, index=True)
+    source_environment: Mapped[str] = mapped_column(String(30), nullable=False)
+    status: Mapped[str] = mapped_column(
+        String(30),
+        default="active",
+        nullable=False,
+        index=True,
+    )
+    provenance: Mapped[dict[str, Any]] = mapped_column(
+        JSON,
+        default=dict,
+        nullable=False,
+    )
+    promoted_by: Mapped[str | None] = mapped_column(
+        String(255),
+        nullable=True,
+        index=True,
+    )
+
+    pipeline: Mapped[DataflowPipeline] = relationship(back_populates="deployments")
+
+
 class DataflowTrigger(Base, TimestampMixin):
    __tablename__ = "dataflow_triggers"
    __table_args__ = (