Add governed reusable pipelines and triggers

2026-07-28 15:04:23 +02:00
parent 6305ef9cef
commit 08ddc8d3e8
17 changed files with 4697 additions and 46 deletions
@@ -4,7 +4,17 @@ import uuid
 from datetime import datetime
 from typing import Any

-from sqlalchemy import DateTime, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint
+from sqlalchemy import (
+    Boolean,
+    DateTime,
+    ForeignKey,
+    Index,
+    Integer,
+    JSON,
+    String,
+    Text,
+    UniqueConstraint,
+)
 from sqlalchemy.orm import Mapped, mapped_column, relationship

 from govoplan_core.db.base import Base, TimestampMixin
@@ -22,7 +32,66 @@ class DataflowPipeline(Base, TimestampMixin):
    )

    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
-    tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
+    tenant_id: Mapped[str | None] = mapped_column(
+        String(36),
+        nullable=True,
+        index=True,
+    )
+    scope_type: Mapped[str] = mapped_column(
+        String(20),
+        default="tenant",
+        nullable=False,
+        index=True,
+    )
+    scope_id: Mapped[str | None] = mapped_column(
+        String(36),
+        nullable=True,
+        index=True,
+    )
+    definition_kind: Mapped[str] = mapped_column(
+        String(20),
+        default="flow",
+        nullable=False,
+        index=True,
+    )
+    inherit_to_lower_scopes: Mapped[bool] = mapped_column(
+        Boolean,
+        default=False,
+        nullable=False,
+    )
+    allow_run: Mapped[bool] = mapped_column(
+        Boolean,
+        default=True,
+        nullable=False,
+    )
+    allow_reuse: Mapped[bool] = mapped_column(
+        Boolean,
+        default=False,
+        nullable=False,
+    )
+    allow_automation: Mapped[bool] = mapped_column(
+        Boolean,
+        default=False,
+        nullable=False,
+    )
+    derived_from_pipeline_id: Mapped[str | None] = mapped_column(
+        String(36),
+        nullable=True,
+        index=True,
+    )
+    derived_from_revision: Mapped[int | None] = mapped_column(
+        Integer,
+        nullable=True,
+    )
+    derived_from_hash: Mapped[str | None] = mapped_column(
+        String(64),
+        nullable=True,
+    )
+    derivation_provenance: Mapped[dict[str, Any]] = mapped_column(
+        JSON,
+        default=dict,
+        nullable=False,
+    )
    name: Mapped[str] = mapped_column(String(300), nullable=False)
    description: Mapped[str | None] = mapped_column(Text)
    status: Mapped[str] = mapped_column(String(32), default="draft", nullable=False, index=True)
@@ -42,6 +111,11 @@ class DataflowPipeline(Base, TimestampMixin):
        cascade="all, delete-orphan",
        order_by="DataflowRun.created_at",
    )
+    triggers: Mapped[list["DataflowTrigger"]] = relationship(
+        back_populates="pipeline",
+        cascade="all, delete-orphan",
+        order_by="DataflowTrigger.created_at",
+    )


 class DataflowPipelineRevision(Base, TimestampMixin):
@@ -53,7 +127,11 @@ class DataflowPipelineRevision(Base, TimestampMixin):
    )

    id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
-    tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
+    tenant_id: Mapped[str | None] = mapped_column(
+        String(36),
+        nullable=True,
+        index=True,
+    )
    pipeline_id: Mapped[str] = mapped_column(
        ForeignKey("dataflow_pipelines.id", ondelete="CASCADE"),
        nullable=False,
@@ -111,6 +189,32 @@ class DataflowRun(Base, TimestampMixin):
        default=dict,
        nullable=False,
    )
+    invocation_kind: Mapped[str] = mapped_column(
+        String(30),
+        default="manual",
+        nullable=False,
+        index=True,
+    )
+    trigger_id: Mapped[str | None] = mapped_column(
+        String(36),
+        nullable=True,
+        index=True,
+    )
+    trigger_delivery_id: Mapped[str | None] = mapped_column(
+        String(36),
+        nullable=True,
+        index=True,
+    )
+    correlation_id: Mapped[str | None] = mapped_column(
+        String(128),
+        nullable=True,
+        index=True,
+    )
+    causation_id: Mapped[str | None] = mapped_column(
+        String(128),
+        nullable=True,
+        index=True,
+    )
    source_fingerprints: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
    result_schema: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
    diagnostics: Mapped[list[dict[str, Any]]] = mapped_column(JSON, default=list, nullable=False)
@@ -136,9 +240,231 @@ class DataflowRun(Base, TimestampMixin):
    pipeline: Mapped[DataflowPipeline] = relationship(back_populates="runs")


+class DataflowTrigger(Base, TimestampMixin):
+    __tablename__ = "dataflow_triggers"
+    __table_args__ = (
+        Index(
+            "ix_dataflow_triggers_due",
+            "status",
+            "next_fire_at",
+        ),
+        Index(
+            "ix_dataflow_triggers_tenant_pipeline",
+            "tenant_id",
+            "pipeline_id",
+        ),
+    )
+
+    id: Mapped[str] = mapped_column(
+        String(36),
+        primary_key=True,
+        default=new_uuid,
+    )
+    tenant_id: Mapped[str] = mapped_column(
+        String(36),
+        nullable=False,
+        index=True,
+    )
+    pipeline_id: Mapped[str] = mapped_column(
+        ForeignKey("dataflow_pipelines.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    pipeline_revision_id: Mapped[str] = mapped_column(
+        ForeignKey("dataflow_pipeline_revisions.id", ondelete="RESTRICT"),
+        nullable=False,
+        index=True,
+    )
+    name: Mapped[str] = mapped_column(String(300), nullable=False)
+    kind: Mapped[str] = mapped_column(
+        String(30),
+        nullable=False,
+        index=True,
+    )
+    status: Mapped[str] = mapped_column(
+        String(30),
+        default="disabled",
+        nullable=False,
+        index=True,
+    )
+    revision: Mapped[int] = mapped_column(Integer, default=1, nullable=False)
+    config_: Mapped[dict[str, Any]] = mapped_column(
+        "config",
+        JSON,
+        default=dict,
+        nullable=False,
+    )
+    publication_: Mapped[dict[str, Any] | None] = mapped_column(
+        "publication",
+        JSON,
+        nullable=True,
+    )
+    row_limit: Mapped[int] = mapped_column(
+        Integer,
+        default=500,
+        nullable=False,
+    )
+    catch_up_policy: Mapped[str] = mapped_column(
+        String(20),
+        default="coalesce",
+        nullable=False,
+    )
+    max_concurrent_runs: Mapped[int] = mapped_column(
+        Integer,
+        default=1,
+        nullable=False,
+    )
+    next_fire_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+        index=True,
+    )
+    last_fire_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+    )
+    last_status: Mapped[str | None] = mapped_column(String(30), nullable=True)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    authorization_account_id: Mapped[str] = mapped_column(
+        String(36),
+        nullable=False,
+    )
+    authorization_membership_id: Mapped[str] = mapped_column(
+        String(36),
+        nullable=False,
+    )
+    authorization_ref: Mapped[str] = mapped_column(
+        String(255),
+        nullable=False,
+        unique=True,
+    )
+    grant_scopes: Mapped[list[str]] = mapped_column(
+        JSON,
+        default=list,
+        nullable=False,
+    )
+    created_by: Mapped[str | None] = mapped_column(
+        String(255),
+        nullable=True,
+        index=True,
+    )
+    updated_by: Mapped[str | None] = mapped_column(
+        String(255),
+        nullable=True,
+        index=True,
+    )
+
+    pipeline: Mapped[DataflowPipeline] = relationship(
+        back_populates="triggers"
+    )
+    deliveries: Mapped[list["DataflowTriggerDelivery"]] = relationship(
+        back_populates="trigger",
+        cascade="all, delete-orphan",
+        order_by="DataflowTriggerDelivery.created_at",
+    )
+
+
+class DataflowTriggerDelivery(Base, TimestampMixin):
+    __tablename__ = "dataflow_trigger_deliveries"
+    __table_args__ = (
+        UniqueConstraint(
+            "trigger_id",
+            "source_key",
+            name="uq_dataflow_trigger_delivery_source",
+        ),
+        Index(
+            "ix_dataflow_trigger_deliveries_queue",
+            "status",
+            "created_at",
+        ),
+        Index(
+            "ix_dataflow_trigger_deliveries_tenant_trigger",
+            "tenant_id",
+            "trigger_id",
+        ),
+    )
+
+    id: Mapped[str] = mapped_column(
+        String(36),
+        primary_key=True,
+        default=new_uuid,
+    )
+    tenant_id: Mapped[str] = mapped_column(
+        String(36),
+        nullable=False,
+        index=True,
+    )
+    trigger_id: Mapped[str] = mapped_column(
+        ForeignKey("dataflow_triggers.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    pipeline_id: Mapped[str] = mapped_column(
+        ForeignKey("dataflow_pipelines.id", ondelete="CASCADE"),
+        nullable=False,
+        index=True,
+    )
+    pipeline_revision_id: Mapped[str] = mapped_column(
+        ForeignKey("dataflow_pipeline_revisions.id", ondelete="RESTRICT"),
+        nullable=False,
+        index=True,
+    )
+    source_key: Mapped[str] = mapped_column(String(500), nullable=False)
+    invocation_kind: Mapped[str] = mapped_column(
+        String(30),
+        nullable=False,
+        index=True,
+    )
+    status: Mapped[str] = mapped_column(
+        String(30),
+        default="queued",
+        nullable=False,
+        index=True,
+    )
+    scheduled_for: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+    )
+    event_: Mapped[dict[str, Any] | None] = mapped_column(
+        "event",
+        JSON,
+        nullable=True,
+    )
+    run_id: Mapped[str | None] = mapped_column(
+        ForeignKey("dataflow_runs.id", ondelete="SET NULL"),
+        nullable=True,
+        index=True,
+    )
+    attempts: Mapped[int] = mapped_column(
+        Integer,
+        default=0,
+        nullable=False,
+    )
+    authorization_provenance: Mapped[dict[str, Any]] = mapped_column(
+        JSON,
+        default=dict,
+        nullable=False,
+    )
+    error: Mapped[str | None] = mapped_column(Text, nullable=True)
+    claimed_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+    )
+    finished_at: Mapped[datetime | None] = mapped_column(
+        DateTime(timezone=True),
+        nullable=True,
+    )
+
+    trigger: Mapped[DataflowTrigger] = relationship(
+        back_populates="deliveries"
+    )
+
+
 __all__ = [
    "DataflowPipeline",
    "DataflowPipelineRevision",
    "DataflowRun",
+    "DataflowTrigger",
+    "DataflowTriggerDelivery",
    "new_uuid",
 ]