feat(dataflow): govern reusable definition updates

src/govoplan_dataflow/backend/service.py

@@ -60,6 +60,7 @@ from govoplan_dataflow.backend.graph import (
     preserve_compatible_graph_layout,
     validate_graph,
 )
+from govoplan_dataflow.backend.ir import graph_to_ir
 from govoplan_dataflow.backend.schemas import (
     DataflowDiagnostic,
     GraphNode,
@@ -72,6 +73,7 @@ from govoplan_dataflow.backend.schemas import (
     PipelinePreviewResponse,
     PipelineDeploymentResponse,
     PipelinePromotionRequest,
+    PipelineRebaseRequest,
     PipelineResponse,
     PipelineRevisionResponse,
     PipelineRunResponse,
@@ -187,15 +189,177 @@ def get_pipeline_revision(
     return item
 
 
+def _resolve_reusable_subflows(
+    session: Session,
+    *,
+    tenant_id: str,
+    graph: PipelineGraph,
+    principal: ApiPrincipal | None,
+    registry: object | None,
+    target_pipeline_id: str | None,
+    ancestry: tuple[str, ...] = (),
+) -> PipelineGraph:
+    if not any(node.type == "subflow" for node in graph.nodes):
+        return graph
+    if principal is None:
+        raise DataflowConflictError(
+            "Reusable subflows require a tenant principal and current Policy "
+            "decision."
+        )
+    resolved_nodes: list[GraphNode] = []
+    for node in graph.nodes:
+        if node.type != "subflow":
+            resolved_nodes.append(node)
+            continue
+        source_id = _pipeline_id_from_ref(node.config.get("template_ref"))
+        if source_id == target_pipeline_id:
+            raise DataflowConflictError(
+                "A pipeline cannot reference itself as a reusable subflow."
+            )
+        source = get_pipeline(
+            session,
+            tenant_id=tenant_id,
+            pipeline_id=source_id,
+        )
+        reuse_decision = require_definition_action(
+            source,
+            principal=principal,
+            registry=registry,
+            action="reuse",
+        )
+        source_revision_number = _subflow_revision(
+            node.config.get("template_version")
+        )
+        source_revision = get_pipeline_revision(
+            session,
+            pipeline=source,
+            revision=source_revision_number,
+        )
+        reference_key = f"{source.id}:{source_revision.revision}"
+        if reference_key in ancestry:
+            raise DataflowConflictError(
+                "Reusable subflow references contain a cycle at "
+                f"pipeline:{source.id} revision {source_revision.revision}."
+            )
+        nested = _resolve_reusable_subflows(
+            session,
+            tenant_id=tenant_id,
+            graph=PipelineGraph.model_validate(source_revision.graph),
+            principal=principal,
+            registry=registry,
+            target_pipeline_id=target_pipeline_id,
+            ancestry=(*ancestry, reference_key),
+        )
+        input_nodes = [
+            item
+            for item in nested.nodes
+            if item.type == "source.inline"
+            and item.config.get("input_binding") is True
+        ]
+        if len(input_nodes) != 1:
+            raise DataflowConflictError(
+                "A referenced reusable definition must declare exactly one "
+                "inline template input binding."
+            )
+        typed = graph_to_ir(nested)
+        typed_by_id = {item.id: item for item in typed.nodes}
+        output_nodes = [item for item in nested.nodes if item.type == "output"]
+        if len(output_nodes) != 1:
+            raise DataflowConflictError(
+                "A referenced reusable definition must have exactly one "
+                "typed output."
+            )
+        input_contract = _typed_contract(
+            typed_by_id[input_nodes[0].id].output_schema,
+            label="input",
+        )
+        output_contract = _typed_contract(
+            typed_by_id[output_nodes[0].id].output_schema,
+            label="output",
+        )
+        config = {
+            **node.config,
+            "template_ref": f"pipeline:{source.id}",
+            "template_version": str(source_revision.revision),
+            "template_hash": source_revision.content_hash,
+            "graph": canonical_graph_payload(nested),
+            "input_schema": input_contract,
+            "output_schema": output_contract,
+            "reference_provenance": {
+                "source_scope": {
+                    "scope_type": source.scope_type,
+                    "scope_id": source.scope_id,
+                },
+                "source_definition_kind": source.definition_kind,
+                "policy_decision": reuse_decision.to_dict(),
+            },
+        }
+        resolved_nodes.append(
+            node.model_copy(update={"config": config}, deep=True)
+        )
+    return graph.model_copy(update={"nodes": resolved_nodes}, deep=True)
+
+
+def _pipeline_id_from_ref(value: object) -> str:
+    text = str(value or "").strip()
+    if not text.startswith("pipeline:") or len(text) <= len("pipeline:"):
+        raise DataflowConflictError(
+            "Reusable subflows require a canonical pipeline reference."
+        )
+    return text.removeprefix("pipeline:")
+
+
+def _subflow_revision(value: object) -> int:
+    try:
+        revision = int(str(value).strip())
+    except (TypeError, ValueError) as exc:
+        raise DataflowConflictError(
+            "Reusable subflows require a valid immutable source revision."
+        ) from exc
+    if revision < 1:
+        raise DataflowConflictError(
+            "Reusable subflow revisions must be positive."
+        )
+    return revision
+
+
+def _typed_contract(schema: object, *, label: str) -> list[dict[str, object]]:
+    fields = tuple(getattr(schema, "fields", ()))
+    if not fields or any(getattr(item, "type", "unknown") == "unknown" for item in fields):
+        raise DataflowConflictError(
+            f"The reusable definition needs a closed typed {label} contract. "
+            "Provide representative typed rows at its template input."
+        )
+    return [
+        {
+            "name": str(item.name),
+            "type": str(item.type),
+            "nullable": bool(item.nullable),
+        }
+        for item in fields
+    ]
+
+
 def create_pipeline(
     session: Session,
     *,
     tenant_id: str,
     actor_id: str | None,
     payload: PipelineCreateRequest,
+    principal: ApiPrincipal | None = None,
+    registry: object | None = None,
 ) -> DataflowPipeline:
-    definition = normalize_definition(
+    pipeline_id = new_uuid()
+    graph = _resolve_reusable_subflows(
+        session,
+        tenant_id=tenant_id,
         graph=payload.graph,
+        principal=principal,
+        registry=registry,
+        target_pipeline_id=pipeline_id,
+    )
+    definition = normalize_definition(
+        graph=graph,
         sql_text=payload.sql_text,
         editor_mode=payload.editor_mode,
     )
@@ -209,6 +373,7 @@ def create_pipeline(
         else payload.scope_id
     )
     pipeline = DataflowPipeline(
+        id=pipeline_id,
         tenant_id=stored_tenant_id,
         scope_type=payload.scope_type,
         scope_id=scope_id,
@@ -248,6 +413,8 @@ def update_pipeline(
     pipeline_id: str,
     actor_id: str | None,
     payload: PipelineUpdateRequest,
+    principal: ApiPrincipal | None = None,
+    registry: object | None = None,
 ) -> DataflowPipeline:
     pipeline = get_pipeline(session, tenant_id=tenant_id, pipeline_id=pipeline_id)
     if payload.expected_revision != pipeline.current_revision:
@@ -270,8 +437,16 @@ def update_pipeline(
         raise DataflowConflictError(
             "Definition kind is immutable; derive a flow or template instead."
         )
-    definition = normalize_definition(
+    graph = _resolve_reusable_subflows(
+        session,
+        tenant_id=tenant_id,
         graph=payload.graph,
+        principal=principal,
+        registry=registry,
+        target_pipeline_id=pipeline.id,
+    )
+    definition = normalize_definition(
+        graph=graph,
         sql_text=payload.sql_text,
         editor_mode=payload.editor_mode,
     )
@@ -417,6 +592,195 @@ def derive_pipeline(
     return pipeline
 
 
+def pipeline_source_update_status(
+    session: Session,
+    *,
+    tenant_id: str,
+    pipeline: DataflowPipeline,
+) -> dict[str, object]:
+    source_id = pipeline.derived_from_pipeline_id
+    if not source_id:
+        return {
+            "source_available": False,
+            "source_name": None,
+            "source_current_revision": None,
+            "source_current_hash": None,
+            "update_available": False,
+        }
+    source = session.scalar(
+        select(DataflowPipeline).where(
+            DataflowPipeline.id == source_id,
+            or_(
+                DataflowPipeline.tenant_id == tenant_id,
+                DataflowPipeline.tenant_id.is_(None),
+            ),
+            DataflowPipeline.deleted_at.is_(None),
+        )
+    )
+    if source is None:
+        return {
+            "source_available": False,
+            "source_name": None,
+            "source_current_revision": None,
+            "source_current_hash": None,
+            "update_available": False,
+        }
+    revision = get_pipeline_revision(session, pipeline=source)
+    return {
+        "source_available": True,
+        "source_name": source.name,
+        "source_current_revision": revision.revision,
+        "source_current_hash": revision.content_hash,
+        "update_available": (
+            revision.revision != pipeline.derived_from_revision
+            or revision.content_hash != pipeline.derived_from_hash
+        ),
+    }
+
+
+def rebase_pipeline(
+    session: Session,
+    *,
+    tenant_id: str,
+    pipeline_id: str,
+    actor_id: str | None,
+    principal: ApiPrincipal,
+    registry: object | None,
+    payload: PipelineRebaseRequest,
+) -> DataflowPipeline:
+    pipeline = get_pipeline(
+        session,
+        tenant_id=tenant_id,
+        pipeline_id=pipeline_id,
+    )
+    if payload.expected_revision != pipeline.current_revision:
+        raise DataflowConflictError(
+            "Derived pipeline changed on the server; expected revision "
+            f"{payload.expected_revision}, current revision is "
+            f"{pipeline.current_revision}."
+        )
+    source_id = pipeline.derived_from_pipeline_id
+    if not source_id:
+        raise DataflowConflictError(
+            "Only a pipeline derived from another definition can adopt a "
+            "source update."
+        )
+    source = get_pipeline(
+        session,
+        tenant_id=tenant_id,
+        pipeline_id=source_id,
+    )
+    reuse_decision = require_definition_action(
+        source,
+        principal=principal,
+        registry=registry,
+        action="derive",
+    )
+    source_revision = get_pipeline_revision(
+        session,
+        pipeline=source,
+        revision=payload.source_revision,
+    )
+    if source_revision.content_hash != payload.source_hash:
+        raise DataflowConflictError(
+            "The reviewed source hash no longer matches the requested "
+            "revision; reload before adopting the update."
+        )
+    if (
+        pipeline.derived_from_revision is not None
+        and source_revision.revision <= pipeline.derived_from_revision
+    ):
+        raise DataflowConflictError(
+            "A source update must use a revision newer than the currently "
+            "pinned revision."
+        )
+
+    previous_child_revision = pipeline.current_revision
+    previous_child = get_pipeline_revision(session, pipeline=pipeline)
+    previous_source_revision = pipeline.derived_from_revision
+    previous_source_hash = pipeline.derived_from_hash
+    source_limits = _effective_governance_limits(
+        source,
+        decision_details=reuse_decision.details,
+    )
+    effective_limits = {
+        "inherit_to_lower_scopes": (
+            pipeline.inherit_to_lower_scopes
+            and source_limits["inherit_to_lower_scopes"]
+        ),
+        "allow_run": pipeline.allow_run and source_limits["allow_run"],
+        "allow_reuse": pipeline.allow_reuse and source_limits["allow_reuse"],
+        "allow_automation": (
+            pipeline.allow_automation and source_limits["allow_automation"]
+        ),
+    }
+    next_child_revision = previous_child_revision + 1
+    rebased_at = utcnow()
+    history_value = pipeline.derivation_provenance.get("rebase_history", [])
+    history = list(history_value) if isinstance(history_value, list) else []
+    history.append(
+        {
+            "child_revision_before": previous_child_revision,
+            "child_hash_before": previous_child.content_hash,
+            "child_revision_after": next_child_revision,
+            "source_revision_before": previous_source_revision,
+            "source_hash_before": previous_source_hash,
+            "source_revision_after": source_revision.revision,
+            "source_hash_after": source_revision.content_hash,
+            "policy_decision": reuse_decision.to_dict(),
+            "reason": payload.reason.strip(),
+            "rebased_by": actor_id,
+            "rebased_at": rebased_at.isoformat(),
+        }
+    )
+    provenance = dict(pipeline.derivation_provenance)
+    provenance.update(
+        {
+            "source_ref": f"pipeline:{source.id}",
+            "source_scope": {
+                "scope_type": source.scope_type,
+                "scope_id": source.scope_id,
+            },
+            "source_definition_kind": source.definition_kind,
+            "source_revision": source_revision.revision,
+            "source_hash": source_revision.content_hash,
+            "source_effective_limits": effective_limits,
+            "policy_decision": reuse_decision.to_dict(),
+            "last_rebased_by": actor_id,
+            "last_rebased_at": rebased_at.isoformat(),
+            "last_rebase_reason": payload.reason.strip(),
+            "rebase_history": history,
+        }
+    )
+
+    pipeline.current_revision = next_child_revision
+    pipeline.status = "draft"
+    pipeline.inherit_to_lower_scopes = effective_limits[
+        "inherit_to_lower_scopes"
+    ]
+    pipeline.allow_run = effective_limits["allow_run"]
+    pipeline.allow_reuse = effective_limits["allow_reuse"]
+    pipeline.allow_automation = effective_limits["allow_automation"]
+    pipeline.derived_from_revision = source_revision.revision
+    pipeline.derived_from_hash = source_revision.content_hash
+    pipeline.derivation_provenance = provenance
+    pipeline.updated_by = actor_id
+    pipeline.revisions.append(
+        DataflowPipelineRevision(
+            tenant_id=pipeline.tenant_id,
+            revision=next_child_revision,
+            schema_version=source_revision.schema_version,
+            graph=json.loads(json.dumps(source_revision.graph)),
+            sql_text=source_revision.sql_text,
+            editor_mode=source_revision.editor_mode,
+            content_hash=source_revision.content_hash,
+            created_by=actor_id,
+        )
+    )
+    session.flush()
+    return pipeline
+
+
 def delete_pipeline(
     session: Session,
     *,
@@ -455,11 +819,36 @@ def pipeline_response(
             pipeline,
             principal=principal,
             registry=registry,
+            source_update=pipeline_source_update_status(
+                session,
+                tenant_id=principal.tenant_id,
+                pipeline=pipeline,
+            ),
         ),
     )
 
 
-def validate_draft(payload: PipelineDraftRequest) -> PipelineValidationResponse:
+def validate_draft(
+    payload: PipelineDraftRequest,
+    *,
+    session: Session | None = None,
+    tenant_id: str | None = None,
+    principal: ApiPrincipal | None = None,
+    registry: object | None = None,
+) -> PipelineValidationResponse:
+    if payload.graph is not None and session is not None and tenant_id is not None:
+        payload = payload.model_copy(
+            update={
+                "graph": _resolve_reusable_subflows(
+                    session,
+                    tenant_id=tenant_id,
+                    graph=payload.graph,
+                    principal=principal,
+                    registry=registry,
+                    target_pipeline_id=payload.pipeline_id,
+                )
+            }
+        )
     if payload.sql_text and payload.sql_text.strip():
         try:
             graph, sql_text, diagnostics = compile_sql(
@@ -598,7 +987,13 @@ def preview_pipeline(
             sql_text=payload.sql_text,
             source_nodes=payload.source_nodes,
         )
-        validated = validate_draft(draft)
+        validated = validate_draft(
+            draft,
+            session=session,
+            tenant_id=tenant_id,
+            principal=principal,
+            registry=registry,
+        )
         if not validated.valid or validated.graph is None:
             return PipelinePreviewResponse(
                 run_id=None,
@@ -2290,11 +2685,13 @@ __all__ = [
     "list_pipelines",
     "normalize_definition",
     "pipeline_response",
+    "pipeline_source_update_status",
     "pipeline_deployment_response",
     "pipeline_run_descriptor",
     "pipeline_run_request",
     "pipeline_run_response",
     "promote_pipeline",
+    "rebase_pipeline",
     "preview_pipeline",
     "render_graph_sql",
     "start_pipeline_run",