feat: run dataflows through durable workers

src/govoplan_dataflow/backend/service.py

@@ -4,13 +4,15 @@ from collections.abc import Mapping
 import hashlib
 import json
 from dataclasses import dataclass
+from datetime import datetime, timedelta
 
-from sqlalchemy import or_, select
+from sqlalchemy import func, or_, select
 from sqlalchemy.orm import Session
 
 from govoplan_core.auth import ApiPrincipal
 from govoplan_core.core.automation import AutomationInvocation
 from govoplan_core.core.dataflows import (
+    DataflowPublicationTarget,
     DataflowRunConflictError,
     DataflowRunDescriptor,
     DataflowRunError,
@@ -21,6 +23,7 @@ from govoplan_core.core.datasources import (
     DatasourceError,
     DatasourcePublicationRequest,
     DatasourceReadRequest,
+    DatasourceUnavailableError,
     datasource_catalogue,
     datasource_publication,
 )
@@ -34,6 +37,7 @@ from govoplan_dataflow.backend.backends import (
 from govoplan_dataflow.backend.batches import TypedBatch
 from govoplan_dataflow.backend.db.models import (
     DataflowPipeline,
+    DataflowPipelineDeployment,
     DataflowPipelineRevision,
     DataflowRun,
 )
@@ -65,6 +69,8 @@ from govoplan_dataflow.backend.schemas import (
     PipelineGraph,
     PipelinePreviewRequest,
     PipelinePreviewResponse,
+    PipelineDeploymentResponse,
+    PipelinePromotionRequest,
     PipelineResponse,
     PipelineRevisionResponse,
     PipelineRunResponse,
@@ -98,6 +104,13 @@ class DataflowValidationError(DataflowError):
         self.diagnostics = diagnostics
 
 
+MAX_PENDING_RUNS_PER_TENANT = 100
+MAX_PRODUCTION_ROWS = 10_000
+RUN_SCOPE = "dataflow:pipeline:run"
+DATASOURCE_READ_SCOPE = "datasources:catalogue:read"
+DATASOURCE_WRITE_SCOPE = "datasources:source:write"
+
+
 @dataclass(frozen=True)
 class NormalizedDefinition:
     graph: PipelineGraph
@@ -694,6 +707,7 @@ def _execute_pipeline_preview(
     backend: str,
     row_limit: int,
     preview_node_id: str | None,
+    budget: ExecutionBudget | None = None,
 ) -> tuple[PipelineExecutionResult, str]:
     source_resolver = _preview_source_resolver(
         session=session,
@@ -713,13 +727,14 @@ def _execute_pipeline_preview(
     sources = _typed_backend_sources(
         graph,
         source_resolver=source_resolver,
+        source_limit=max(MAX_SOURCE_ROWS, row_limit),
     )
     try:
         result = execute_typed_graph(
             graph,
             backend=backend,
             sources=sources,
-            budget=ExecutionBudget(max_output_rows=row_limit),
+            budget=budget or ExecutionBudget(max_output_rows=row_limit),
             preview_node_id=preview_node_id,
         )
     except BackendExecutionError as exc:
@@ -727,6 +742,7 @@ def _execute_pipeline_preview(
             str(exc),
             node_id=exc.node_id,
             diagnostics=tuple(exc.diagnostics),
+            retryable=exc.code == "backend.capacity",
         ) from exc
     columns = [
         PreviewColumn(
@@ -802,12 +818,13 @@ def _typed_backend_sources(
     graph: PipelineGraph,
     *,
     source_resolver,
+    source_limit: int = MAX_SOURCE_ROWS,
 ) -> dict[str, BackendSource]:
     sources: dict[str, BackendSource] = {}
     for node in graph.nodes:
         if node.type != "source.reference":
             continue
-        resolved = source_resolver(node, MAX_SOURCE_ROWS)
+        resolved = source_resolver(node, source_limit)
         sources[node.id] = BackendSource(
             node_id=node.id,
             batch=TypedBatch.from_rows(resolved.rows),
@@ -863,6 +880,137 @@ def list_pipeline_runs(
     return list(session.scalars(statement))
 
 
+def list_pipeline_deployments(
+    session: Session,
+    *,
+    tenant_id: str,
+    pipeline_id: str,
+) -> list[DataflowPipelineDeployment]:
+    get_pipeline(
+        session,
+        tenant_id=tenant_id,
+        pipeline_id=pipeline_id,
+    )
+    return list(
+        session.scalars(
+            select(DataflowPipelineDeployment)
+            .where(
+                DataflowPipelineDeployment.tenant_id == tenant_id,
+                DataflowPipelineDeployment.pipeline_id == pipeline_id,
+                DataflowPipelineDeployment.status == "active",
+            )
+            .order_by(DataflowPipelineDeployment.environment)
+        )
+    )
+
+
+def promote_pipeline(
+    session: Session,
+    *,
+    tenant_id: str,
+    pipeline_id: str,
+    actor_id: str | None,
+    principal: ApiPrincipal,
+    registry: object | None,
+    payload: PipelinePromotionRequest,
+) -> DataflowPipelineDeployment:
+    pipeline = get_pipeline(
+        session,
+        tenant_id=tenant_id,
+        pipeline_id=pipeline_id,
+    )
+    try:
+        require_definition_action(
+            pipeline,
+            principal=principal,
+            registry=registry,
+            action="edit",
+        )
+    except PermissionError as exc:
+        raise DataflowConflictError(str(exc)) from exc
+    revision = get_pipeline_revision(
+        session,
+        pipeline=pipeline,
+        revision=payload.revision,
+    )
+    if payload.source_environment == "staging":
+        source = session.scalar(
+            select(DataflowPipelineDeployment).where(
+                DataflowPipelineDeployment.tenant_id == tenant_id,
+                DataflowPipelineDeployment.pipeline_id == pipeline.id,
+                DataflowPipelineDeployment.environment == "staging",
+                DataflowPipelineDeployment.status == "active",
+            )
+        )
+        if source is None or source.pipeline_revision_id != revision.id:
+            raise DataflowConflictError(
+                "Only the revision currently promoted to staging can be "
+                "promoted to production."
+            )
+    deployment = session.scalar(
+        select(DataflowPipelineDeployment).where(
+            DataflowPipelineDeployment.tenant_id == tenant_id,
+            DataflowPipelineDeployment.pipeline_id == pipeline.id,
+            DataflowPipelineDeployment.environment
+            == payload.target_environment,
+        )
+    )
+    previous_revision_id = (
+        deployment.pipeline_revision_id if deployment is not None else None
+    )
+    if deployment is None:
+        deployment = DataflowPipelineDeployment(
+            tenant_id=tenant_id,
+            pipeline_id=pipeline.id,
+            pipeline_revision_id=revision.id,
+            environment=payload.target_environment,
+            source_environment=payload.source_environment,
+            status="active",
+            promoted_by=actor_id,
+        )
+        session.add(deployment)
+    deployment.pipeline_revision_id = revision.id
+    deployment.source_environment = payload.source_environment
+    deployment.status = "active"
+    deployment.promoted_by = actor_id
+    deployment.provenance = {
+        "pipeline_ref": f"pipeline:{pipeline.id}",
+        "revision": revision.revision,
+        "definition_hash": revision.content_hash,
+        "source_environment": payload.source_environment,
+        "target_environment": payload.target_environment,
+        "previous_revision_id": previous_revision_id,
+        "promoted_by": actor_id,
+        "promoted_at": utcnow().isoformat(),
+    }
+    session.flush()
+    return deployment
+
+
+def pipeline_deployment_response(
+    session: Session,
+    deployment: DataflowPipelineDeployment,
+) -> PipelineDeploymentResponse:
+    revision = session.get(
+        DataflowPipelineRevision,
+        deployment.pipeline_revision_id,
+    )
+    if revision is None:
+        raise DataflowNotFoundError("Dataflow pipeline revision not found")
+    return PipelineDeploymentResponse(
+        id=deployment.id,
+        pipeline_id=deployment.pipeline_id,
+        revision=revision.revision,
+        environment=deployment.environment,  # type: ignore[arg-type]
+        source_environment=deployment.source_environment,  # type: ignore[arg-type]
+        status=deployment.status,
+        provenance=dict(deployment.provenance),
+        promoted_by=deployment.promoted_by,
+        created_at=deployment.created_at,
+        updated_at=deployment.updated_at,
+    )
+
+
 def get_pipeline_run(
     session: Session,
     *,
@@ -891,6 +1039,7 @@ def start_pipeline_run(
     principal: ApiPrincipal,
     registry: object | None,
     request: DataflowRunRequest,
+    defer_execution: bool = False,
 ) -> tuple[DataflowRun, bool]:
     pipeline, revision = _run_definition(
         session,
@@ -909,6 +1058,8 @@ def start_pipeline_run(
     )
     if existing is not None:
         return existing, True
+    if defer_execution:
+        _require_run_queue_capacity(session, tenant_id=tenant_id)
     run = _new_pipeline_run(
         tenant_id=tenant_id,
         actor_id=actor_id,
@@ -917,9 +1068,20 @@ def start_pipeline_run(
         request=request,
         idempotency_key=idempotency_key,
         request_hash=request_hash,
+        principal=principal,
+        defer_execution=defer_execution,
     )
     session.add(run)
     session.flush()
+    run.authorization_ = {
+        **dict(run.authorization_),
+        "authorization_ref": (
+            request.invocation.trigger_ref or f"dataflow-run:{run.id}"
+        ),
+    }
+    if defer_execution:
+        session.flush()
+        return run, False
     _execute_pipeline_run(
         session,
         run=run,
@@ -968,6 +1130,13 @@ def _run_definition(
         pipeline=pipeline,
         revision=request.revision,
     )
+    _require_deployed_revision(
+        session,
+        tenant_id=tenant_id,
+        pipeline=pipeline,
+        revision=revision,
+        environment=request.environment,
+    )
     return pipeline, revision
 
 
@@ -977,13 +1146,77 @@ def _validated_run_identity(request: DataflowRunRequest) -> tuple[str, str]:
         raise DataflowConflictError(
             "A Dataflow run idempotency key of at most 255 characters is required."
         )
-    if request.row_limit < 1 or request.row_limit > 500:
+    if request.row_limit < 1 or request.row_limit > MAX_PRODUCTION_ROWS:
         raise DataflowConflictError(
-            "The bounded Dataflow runner supports between 1 and 500 output rows."
+            "The bounded Dataflow runner supports between 1 and 10,000 "
+            "output rows."
+        )
+    if request.execution_backend not in {"auto", "reference", "duckdb"}:
+        raise DataflowConflictError("Unknown Dataflow execution backend.")
+    if request.environment not in {"development", "staging", "production"}:
+        raise DataflowConflictError("Unknown Dataflow environment.")
+    if request.max_attempts < 1 or request.max_attempts > 5:
+        raise DataflowConflictError(
+            "Dataflow runs support between one and five attempts."
+        )
+    if request.retention_days < 1 or request.retention_days > 365:
+        raise DataflowConflictError(
+            "Dataflow run retention must be between one and 365 days."
         )
     return idempotency_key, _run_request_hash(request)
 
 
+def _require_run_queue_capacity(
+    session: Session,
+    *,
+    tenant_id: str,
+) -> None:
+    pending = int(
+        session.scalar(
+            select(func.count())
+            .select_from(DataflowRun)
+            .where(
+                DataflowRun.tenant_id == tenant_id,
+                DataflowRun.status.in_(("queued", "retrying", "running")),
+            )
+        )
+        or 0
+    )
+    if pending >= MAX_PENDING_RUNS_PER_TENANT:
+        raise DataflowConflictError(
+            "The tenant Dataflow queue is full; wait for an active run to "
+            "finish before submitting more work."
+        )
+
+
+def _require_deployed_revision(
+    session: Session,
+    *,
+    tenant_id: str,
+    pipeline: DataflowPipeline,
+    revision: DataflowPipelineRevision,
+    environment: str,
+) -> None:
+    if environment == "development":
+        return
+    deployment = session.scalar(
+        select(DataflowPipelineDeployment).where(
+            DataflowPipelineDeployment.tenant_id == tenant_id,
+            DataflowPipelineDeployment.pipeline_id == pipeline.id,
+            DataflowPipelineDeployment.environment == environment,
+            DataflowPipelineDeployment.status == "active",
+        )
+    )
+    if (
+        deployment is None
+        or deployment.pipeline_revision_id != revision.id
+    ):
+        raise DataflowConflictError(
+            f"Pipeline revision {revision.revision} is not promoted to "
+            f"{environment}."
+        )
+
+
 def _existing_pipeline_run(
     session: Session,
     *,
@@ -1017,13 +1250,19 @@ def _new_pipeline_run(
     request: DataflowRunRequest,
     idempotency_key: str,
     request_hash: str,
+    principal: ApiPrincipal,
+    defer_execution: bool,
 ) -> DataflowRun:
+    now = utcnow()
+    budget = _run_resource_budget(request)
     return DataflowRun(
         tenant_id=tenant_id,
         pipeline_id=pipeline.id,
         pipeline_revision_id=revision.id,
         run_type="published" if request.publication else "run",
-        status="running",
+        status="queued" if defer_execution else "running",
+        execution_backend=request.execution_backend,
+        environment=request.environment,
         executor_version=EXECUTOR_VERSION,
         definition_hash=revision.content_hash,
         idempotency_key=idempotency_key,
@@ -1045,11 +1284,73 @@ def _new_pipeline_run(
         diagnostics=[],
         input_row_count=0,
         output_row_count=0,
-        started_at=utcnow(),
+        attempts=0 if defer_execution else 1,
+        max_attempts=request.max_attempts,
+        queued_at=now if defer_execution else None,
+        available_at=now if defer_execution else None,
+        progress_percent=0 if defer_execution else 10,
+        progress_phase="queued" if defer_execution else "executing",
+        retention_until=now + timedelta(days=request.retention_days),
+        authorization_=_run_authorization_payload(
+            principal,
+            request=request,
+            revision=revision,
+        ),
+        resource_budget=budget,
+        started_at=None if defer_execution else now,
         created_by=actor_id,
     )
 
 
+def _run_authorization_payload(
+    principal: ApiPrincipal,
+    *,
+    request: DataflowRunRequest,
+    revision: DataflowPipelineRevision,
+) -> dict[str, object]:
+    principal_ref = principal.to_platform_principal()
+    graph = PipelineGraph.model_validate(revision.graph)
+    scopes = {RUN_SCOPE}
+    if any(
+        node.type == "source.reference"
+        or bool(node.config.get("source_ref"))
+        for node in graph.nodes
+    ):
+        scopes.add(DATASOURCE_READ_SCOPE)
+    if request.publication is not None:
+        scopes.add(DATASOURCE_WRITE_SCOPE)
+    return {
+        "contract_version": "1",
+        "subject_kind": (
+            "service_account"
+            if principal_ref.service_account_id
+            else "delegated_user"
+        ),
+        "account_id": principal_ref.account_id,
+        "membership_id": principal_ref.membership_id,
+        "service_account_id": principal_ref.service_account_id,
+        "grant_scopes": sorted(scopes),
+        "submitted_principal": principal_ref.to_dict(),
+        "authorization_ref": request.invocation.trigger_ref,
+        "resolution": "rechecked_by_worker",
+    }
+
+
+def _run_resource_budget(
+    request: DataflowRunRequest,
+) -> dict[str, object]:
+    production = request.environment in {"staging", "production"}
+    return {
+        "max_output_rows": request.row_limit,
+        "max_batch_bytes": 64 * 1024 * 1024 if production else 8 * 1024 * 1024,
+        "max_wall_seconds": 30.0 if production else 10.0,
+        "max_memory_bytes": (
+            512 * 1024 * 1024 if production else 256 * 1024 * 1024
+        ),
+        "max_concurrency": 1,
+    }
+
+
 def _execute_pipeline_run(
     session: Session,
     *,
@@ -1059,18 +1360,40 @@ def _execute_pipeline_run(
     request: DataflowRunRequest,
     principal: ApiPrincipal,
     registry: object | None,
-) -> None:
+) -> bool:
     try:
-        result = execute_preview(
+        if run.cancellation_requested_at is not None:
+            _mark_pipeline_run_cancelled(run)
+            return False
+        execution_backend = run.execution_backend
+        if run.environment in {"staging", "production"}:
+            if execution_backend == "reference":
+                raise PipelineExecutionError(
+                    "Staging and production runs require the isolated "
+                    "DuckDB execution backend."
+                )
+            execution_backend = "duckdb"
+        run.progress_percent = 20
+        run.progress_phase = "reading_sources"
+        result, executor_version = _execute_pipeline_preview(
             PipelineGraph.model_validate(revision.graph),
+            session=session,
+            principal=principal,
+            registry=registry,
+            backend=execution_backend,
             row_limit=request.row_limit,
-            source_resolver=_datasource_source_resolver(
-                session=session,
-                principal=principal,
-                registry=registry,
-            ),
+            preview_node_id=None,
+            budget=_execution_budget(run),
         )
+        run.executor_version = executor_version
         _apply_pipeline_result(run, result)
+        run.progress_percent = 80
+        run.progress_phase = "publishing" if request.publication else "finalizing"
+        session.flush()
+        session.expire(run, ["cancellation_requested_at"])
+        if run.cancellation_requested_at is not None:
+            _mark_pipeline_run_cancelled(run)
+            return False
         if request.publication:
             _ensure_publishable(result)
             _publish_pipeline_result(
@@ -1086,8 +1409,27 @@ def _execute_pipeline_run(
         run.status = "succeeded"
         run.finished_at = utcnow()
         run.error = None
+        run.progress_percent = 100
+        run.progress_phase = "completed"
+        return False
     except (DatasourceError, PipelineExecutionError) as exc:
         _mark_pipeline_run_failed(run, exc)
+        return isinstance(exc, DatasourceUnavailableError) or bool(
+            getattr(exc, "retryable", False)
+        )
+
+
+def _execution_budget(run: DataflowRun) -> ExecutionBudget:
+    value = dict(run.resource_budget)
+    return ExecutionBudget(
+        max_output_rows=int(value.get("max_output_rows") or 500),
+        max_batch_bytes=int(value.get("max_batch_bytes") or 8 * 1024 * 1024),
+        max_wall_seconds=float(value.get("max_wall_seconds") or 10.0),
+        max_memory_bytes=int(
+            value.get("max_memory_bytes") or 256 * 1024 * 1024
+        ),
+        max_concurrency=int(value.get("max_concurrency") or 1),
+    )
 
 
 def _apply_pipeline_result(
@@ -1176,6 +1518,7 @@ def _mark_pipeline_run_failed(
     run.status = "failed"
     run.finished_at = utcnow()
     run.error = str(exc)
+    run.progress_phase = "failed"
     diagnostics = list(getattr(exc, "diagnostics", ()))
     diagnostics.append(
         DataflowDiagnostic(
@@ -1194,6 +1537,13 @@ def _mark_pipeline_run_failed(
     run.input_row_count = int(getattr(exc, "input_row_count", 0))
 
 
+def _mark_pipeline_run_cancelled(run: DataflowRun) -> None:
+    run.status = "cancelled"
+    run.finished_at = utcnow()
+    run.error = "Cancelled by request."
+    run.progress_phase = "cancelled"
+
+
 def cancel_pipeline_run(
     session: Session,
     *,
@@ -1205,13 +1555,15 @@ def cancel_pipeline_run(
         tenant_id=tenant_id,
         run_ref=run_ref,
     )
-    if run.status not in {"queued", "running"}:
+    if run.status not in {"queued", "retrying", "running"}:
         raise DataflowConflictError(
             f"Dataflow run is already {run.status} and cannot be cancelled."
         )
-    run.status = "cancelled"
-    run.finished_at = utcnow()
-    run.error = "Cancelled by request."
+    run.cancellation_requested_at = utcnow()
+    if run.status in {"queued", "retrying"}:
+        _mark_pipeline_run_cancelled(run)
+    else:
+        run.progress_phase = "cancellation_requested"
     session.flush()
     return run
 
@@ -1232,6 +1584,8 @@ def pipeline_run_response(
         run_type=run.run_type,
         status=run.status,  # type: ignore[arg-type]
         idempotency_key=run.idempotency_key,
+        execution_backend=run.execution_backend,
+        environment=run.environment,  # type: ignore[arg-type]
         definition_hash=run.definition_hash,
         executor_version=run.executor_version,
         source_fingerprints=list(run.source_fingerprints),
@@ -1253,6 +1607,16 @@ def pipeline_run_response(
         ),
         correlation_id=run.correlation_id,
         causation_id=run.causation_id,
+        attempts=run.attempts,
+        max_attempts=run.max_attempts,
+        available_at=run.available_at,
+        claimed_at=run.claimed_at,
+        lease_expires_at=run.lease_expires_at,
+        cancellation_requested_at=run.cancellation_requested_at,
+        progress_percent=run.progress_percent,
+        progress_phase=run.progress_phase,
+        retention_until=run.retention_until,
+        purged_at=run.purged_at,
         error=run.error,
         started_at=run.started_at,
         finished_at=run.finished_at,
@@ -1298,6 +1662,12 @@ def pipeline_run_descriptor(
         replayed=replayed,
         metadata={
             "run_type": run.run_type,
+            "execution_backend": run.execution_backend,
+            "environment": run.environment,
+            "attempts": run.attempts,
+            "max_attempts": run.max_attempts,
+            "progress_percent": run.progress_percent,
+            "progress_phase": run.progress_phase,
             "source_fingerprints": list(run.source_fingerprints),
             "diagnostics": list(run.diagnostics),
         },
@@ -1323,6 +1693,7 @@ class SqlDataflowRunLifecycleProvider:
             principal=api_principal,
             registry=self._registry,
             request=request,
+            defer_execution=True,
         )
         return pipeline_run_descriptor(db, run, replayed=replayed)
 
@@ -1421,30 +1792,66 @@ def _datasource_source_resolver(
                 "catalogue capability.",
                 node_id=node.id,
             )
+        rows: list[dict[str, object]] = []
+        resolved = None
+        offset = 0
+        expected_fingerprint = _clean_optional(
+            node.config.get("expected_fingerprint")
+        )
         try:
-            resolved = provider.read_datasource(
-                session,
-                principal,
-                request=DatasourceReadRequest(
-                    datasource_ref=str(node.config["source_ref"]),
-                    consistency=str(
-                        node.config.get("consistency") or "current"
-                    ),  # type: ignore[arg-type]
-                    limit=limit,
-                    expected_fingerprint=_clean_optional(
-                        node.config.get("expected_fingerprint")
+            while offset < limit:
+                page = provider.read_datasource(
+                    session,
+                    principal,
+                    request=DatasourceReadRequest(
+                        datasource_ref=str(node.config["source_ref"]),
+                        consistency=str(
+                            node.config.get("consistency") or "current"
+                        ),  # type: ignore[arg-type]
+                        limit=min(500, limit - offset),
+                        offset=offset,
+                        expected_fingerprint=expected_fingerprint,
                     ),
-                ),
-            )
+                )
+                if resolved is None:
+                    resolved = page
+                    expected_fingerprint = page.datasource.fingerprint
+                elif (
+                    page.datasource.fingerprint
+                    != resolved.datasource.fingerprint
+                ):
+                    raise PipelineExecutionError(
+                        "Datasource changed while the run was reading it.",
+                        node_id=node.id,
+                        retryable=True,
+                    )
+                rows.extend(dict(row) for row in page.rows)
+                offset += len(page.rows)
+                if not page.truncated or not page.rows:
+                    resolved = page
+                    break
+                resolved = page
+        except DatasourceUnavailableError as exc:
+            raise PipelineExecutionError(
+                str(exc),
+                node_id=node.id,
+                retryable=True,
+            ) from exc
         except DatasourceError as exc:
             raise PipelineExecutionError(str(exc), node_id=node.id) from exc
+        if resolved is None:
+            raise PipelineExecutionError(
+                "Datasource returned no result.",
+                node_id=node.id,
+                retryable=True,
+            )
         return ResolvedSource(
-            rows=tuple(dict(row) for row in resolved.rows),
+            rows=tuple(rows),
             source_ref=resolved.datasource.ref,
             provider=resolved.datasource.provider or "datasources",
             fingerprint=resolved.datasource.fingerprint,
             total_rows=resolved.total_rows,
-            truncated=resolved.truncated,
+            truncated=len(rows) < resolved.total_rows,
         )
 
     return resolve_source
@@ -1482,6 +1889,10 @@ def _run_request_payload(request: DataflowRunRequest) -> dict[str, object]:
         "pipeline_ref": request.pipeline_ref,
         "revision": request.revision,
         "row_limit": request.row_limit,
+        "execution_backend": request.execution_backend,
+        "environment": request.environment,
+        "max_attempts": request.max_attempts,
+        "retention_days": request.retention_days,
         "publication": (
             {
                 "target_datasource_ref": target.target_datasource_ref,
@@ -1521,6 +1932,113 @@ def _invocation_payload(
     }
 
 
+def pipeline_run_request(run: DataflowRun) -> DataflowRunRequest:
+    value = dict(run.request_)
+    publication_value = value.get("publication")
+    publication = (
+        DataflowPublicationTarget(
+            target_datasource_ref=_mapping_optional_text(
+                publication_value,
+                "target_datasource_ref",
+            ),
+            name=_mapping_optional_text(publication_value, "name"),
+            source_name=_mapping_optional_text(
+                publication_value,
+                "source_name",
+            ),
+            description=_mapping_optional_text(
+                publication_value,
+                "description",
+            ),
+            freeze=bool(publication_value.get("freeze", False)),
+            frozen_label=_mapping_optional_text(
+                publication_value,
+                "frozen_label",
+            ),
+            set_current=bool(publication_value.get("set_current", True)),
+            metadata=(
+                dict(publication_value.get("metadata") or {})
+                if isinstance(publication_value.get("metadata"), Mapping)
+                else {}
+            ),
+        )
+        if isinstance(publication_value, Mapping)
+        else None
+    )
+    invocation_value = value.get("invocation")
+    invocation_mapping = (
+        invocation_value if isinstance(invocation_value, Mapping) else {}
+    )
+    metadata = invocation_mapping.get("metadata")
+    return DataflowRunRequest(
+        pipeline_ref=str(value.get("pipeline_ref") or f"pipeline:{run.pipeline_id}"),
+        revision=int(value.get("revision") or 1),
+        idempotency_key=str(value.get("idempotency_key") or run.idempotency_key or run.id),
+        row_limit=int(value.get("row_limit") or 500),
+        execution_backend=str(
+            value.get("execution_backend") or run.execution_backend
+        ),
+        environment=str(value.get("environment") or run.environment),
+        max_attempts=int(value.get("max_attempts") or run.max_attempts),
+        retention_days=int(value.get("retention_days") or 30),
+        publication=publication,
+        invocation=AutomationInvocation(
+            kind=str(
+                invocation_mapping.get("kind") or run.invocation_kind
+            ),  # type: ignore[arg-type]
+            trigger_ref=_mapping_optional_text(
+                invocation_mapping,
+                "trigger_ref",
+            ),
+            delivery_ref=_mapping_optional_text(
+                invocation_mapping,
+                "delivery_ref",
+            ),
+            event_id=_mapping_optional_text(invocation_mapping, "event_id"),
+            event_type=_mapping_optional_text(
+                invocation_mapping,
+                "event_type",
+            ),
+            correlation_id=_mapping_optional_text(
+                invocation_mapping,
+                "correlation_id",
+            ),
+            causation_id=_mapping_optional_text(
+                invocation_mapping,
+                "causation_id",
+            ),
+            scheduled_for=_optional_datetime(
+                invocation_mapping.get("scheduled_for")
+            ),
+            requested_by=_mapping_optional_text(
+                invocation_mapping,
+                "requested_by",
+            ),
+            metadata=dict(metadata) if isinstance(metadata, Mapping) else {},
+        ),
+    )
+
+
+def _mapping_optional_text(
+    value: object,
+    key: str,
+) -> str | None:
+    if not isinstance(value, Mapping):
+        return None
+    return _clean_optional(value.get(key))
+
+
+def _optional_datetime(value: object) -> datetime | None:
+    if isinstance(value, datetime):
+        return value
+    if isinstance(value, str) and value.strip():
+        try:
+            return datetime.fromisoformat(value)
+        except ValueError:
+            return None
+    return None
+
+
 def _run_request_hash(request: DataflowRunRequest) -> str:
     encoded = json.dumps(
         _run_request_payload(request),
@@ -1531,10 +2049,10 @@ def _run_request_hash(request: DataflowRunRequest) -> str:
     return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
 
 
-def _clean_optional(value: str | None) -> str | None:
+def _clean_optional(value: object | None) -> str | None:
     if value is None:
         return None
-    cleaned = value.strip()
+    cleaned = str(value).strip()
     return cleaned or None
 
 
@@ -1600,12 +2118,16 @@ __all__ = [
     "get_pipeline",
     "get_pipeline_revision",
     "get_pipeline_run",
+    "list_pipeline_deployments",
     "list_pipeline_runs",
     "list_pipelines",
     "normalize_definition",
     "pipeline_response",
+    "pipeline_deployment_response",
     "pipeline_run_descriptor",
+    "pipeline_run_request",
     "pipeline_run_response",
+    "promote_pipeline",
     "preview_pipeline",
     "render_graph_sql",
     "start_pipeline_run",