GovOPlaN/govoplan-dataflow
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(security): isolate reference execution and bound source inputs

Browse Source
This commit is contained in:
zemion
2026-09-08 07:47:18 +02:00
parent 618f10fe89
commit 4175262b8b
6 changed files with 574 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_dataflow/backend/backends/base.py
+11 -2
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from dataclasses import dataclass, field
import math
from typing import Any, Mapping, Protocol, runtime_checkable
from govoplan_dataflow.backend.batches import TypedBatch
@@ -33,8 +34,8 @@ class ExecutionBudget:
raise ValueError("Execution output row limit must be positive.")
if self.max_batch_bytes < 1:
raise ValueError("Execution byte limit must be positive.")
if self.max_wall_seconds <= 0:
raise ValueError("Execution time limit must be positive.")
if not math.isfinite(self.max_wall_seconds) or self.max_wall_seconds <= 0:
raise ValueError("Execution time limit must be finite and positive.")
if self.max_memory_bytes < 64 * 1024 * 1024:
raise ValueError("Execution memory limit must be at least 64 MiB.")
if self.max_concurrency < 1:
@@ -106,11 +107,19 @@ class BackendExecutionError(RuntimeError):
code: str = "backend.execution",
node_id: str | None = None,
diagnostics: tuple[DataflowDiagnostic, ...] = (),
node_diagnostics: tuple[NodePreviewDiagnostic, ...] = (),
source_fingerprints: tuple[dict[str, Any], ...] = (),
input_row_count: int = 0,
node_preview: NodePreviewResult | None = None,
) -> None:
super().__init__(message)
self.code = code
self.node_id = node_id
self.diagnostics = diagnostics
self.node_diagnostics = node_diagnostics
self.source_fingerprints = source_fingerprints
self.input_row_count = input_row_count
self.node_preview = node_preview
@runtime_checkable
src/govoplan_dataflow/backend/backends/reference.py
+167 -2
View File
@@ -1,10 +1,25 @@
from __future__ import annotations
from dataclasses import asdict
import hashlib
import json
import math
from govoplan_core.security.bounded_process import (
ProcessBudgetError,
ProcessLimits,
run_bounded_operation,
)
from govoplan_core.security.worker_payload import (
decode_worker_payload,
encode_worker_payload,
)
from govoplan_dataflow.backend.backends.base import (
BackendExecutionError,
BackendExecutionRequest,
BackendExecutionResult,
BackendSource,
ExecutionBudget,
canonical_result_schema,
)
from govoplan_dataflow.backend.batches import TypedBatch
@@ -14,8 +29,17 @@ from govoplan_dataflow.backend.executor import (
ResolvedSource,
execute_preview,
)
from govoplan_dataflow.backend.ir import IrExecutionResult, ir_to_graph
from govoplan_dataflow.backend.schemas import GraphNode
from govoplan_dataflow.backend.ir import IrExecutionResult, IrSchema, TypedGraphIr, ir_to_graph
from govoplan_dataflow.backend.planner import ExecutionPlan
from govoplan_dataflow.backend.schemas import (
DataflowDiagnostic,
GraphNode,
NodePreviewDiagnostic,
NodePreviewResult,
)
_TRANSPORT_BYTES = 32 * 1024 * 1024
class ReferenceExecutionBackend:
@@ -32,6 +56,52 @@ class ReferenceExecutionBackend:
self,
request: BackendExecutionRequest,
) -> BackendExecutionResult:
_validate_source_batches(request)
try:
limits = ProcessLimits(
wall_seconds=request.budget.max_wall_seconds,
cpu_seconds=max(1, math.ceil(request.budget.max_wall_seconds)),
memory_bytes=request.budget.max_memory_bytes,
input_bytes=_TRANSPORT_BYTES,
output_bytes=_TRANSPORT_BYTES,
)
payload = encode_worker_payload(_request_payload(request), max_bytes=_TRANSPORT_BYTES)
response = decode_worker_payload(
run_bounded_operation(_execute_reference_worker, payload, limits=limits),
max_bytes=_TRANSPORT_BYTES,
)
except ProcessBudgetError as exc:
raise BackendExecutionError(str(exc), code=f"backend.process.{exc.code}") from exc
except ValueError as exc:
raise BackendExecutionError(
"Reference execution exceeds its supported process/transport budget.",
code="backend.budget",
) from exc
if "error" in response:
error = response["error"]
raise BackendExecutionError(
error["message"], code=error["code"], node_id=error["node_id"],
diagnostics=tuple(DataflowDiagnostic.model_validate(item) for item in error["diagnostics"]),
node_diagnostics=tuple(NodePreviewDiagnostic.model_validate(item) for item in error["node_diagnostics"]),
source_fingerprints=error["source_fingerprints"],
input_row_count=error["input_row_count"],
node_preview=NodePreviewResult.model_validate(error["node_preview"]) if error["node_preview"] else None,
)
batch = _batch_from_payload(response["batch"])
batch.ensure_within(
max_rows=request.budget.max_output_rows,
max_bytes=request.budget.max_batch_bytes,
)
return BackendExecutionResult(
contract=IrExecutionResult.model_validate(response["contract"]),
batch=batch,
node_diagnostics=tuple(NodePreviewDiagnostic.model_validate(item) for item in response["node_diagnostics"]),
node_preview=NodePreviewResult.model_validate(response["node_preview"]) if response["node_preview"] else None,
metadata=response["metadata"],
)
def _execute_in_process(self, request: BackendExecutionRequest) -> BackendExecutionResult:
"""Pure reference evaluation, called only inside the disposable worker."""
_validate_source_batches(request)
try:
result = execute_preview(
@@ -50,6 +120,10 @@ class ReferenceExecutionBackend:
code="backend.reference",
node_id=exc.node_id,
diagnostics=tuple(exc.diagnostics),
node_diagnostics=tuple(exc.node_diagnostics),
source_fingerprints=tuple(exc.source_fingerprints),
input_row_count=exc.input_row_count,
node_preview=exc.node_preview,
) from exc
observed_batch = TypedBatch.from_rows(result.rows)
batch = TypedBatch.from_rows(
@@ -97,6 +171,90 @@ class ReferenceExecutionBackend:
)
def _batch_payload(batch: TypedBatch) -> dict:
return {
"schema": batch.schema.model_dump(mode="python"),
"columns": dict(batch.columns),
"row_count": batch.row_count,
"byte_count": batch.byte_count,
}
def _batch_from_payload(value: dict) -> TypedBatch:
return TypedBatch(
schema=IrSchema.model_validate(value["schema"]), columns=value["columns"],
row_count=value["row_count"], byte_count=value["byte_count"],
)
def _request_payload(request: BackendExecutionRequest) -> dict:
plan = request.plan
return {
"plan": {
"graph": plan.graph.model_dump(mode="python"),
"ordered_node_ids": plan.ordered_node_ids,
"diagnostics": tuple(item.model_dump(mode="python") for item in plan.diagnostics),
"generated_sql": plan.generated_sql,
"sql_diagnostics": tuple(item.model_dump(mode="python") for item in plan.sql_diagnostics),
"semantic_hash": plan.semantic_hash,
},
"budget": asdict(request.budget),
"preview_node_id": request.preview_node_id,
"sources": {
key: {
"node_id": source.node_id, "batch": _batch_payload(source.batch),
"source_ref": source.source_ref, "provider": source.provider,
"fingerprint": source.fingerprint, "total_rows": source.total_rows,
"truncated": source.truncated, "source_name": source.source_name,
"kind": source.kind,
}
for key, source in request.sources.items()
},
}
def _execute_reference_worker(payload: bytes) -> bytes:
"""Data-only worker boundary; no database, provider callback or principal."""
value = decode_worker_payload(payload, max_bytes=_TRANSPORT_BYTES)
plan = value["plan"]
request = BackendExecutionRequest(
plan=ExecutionPlan(
graph=TypedGraphIr.model_validate(plan["graph"]),
ordered_node_ids=plan["ordered_node_ids"],
diagnostics=tuple(DataflowDiagnostic.model_validate(item) for item in plan["diagnostics"]),
generated_sql=plan["generated_sql"],
sql_diagnostics=tuple(DataflowDiagnostic.model_validate(item) for item in plan["sql_diagnostics"]),
semantic_hash=plan["semantic_hash"],
),
budget=ExecutionBudget(**value["budget"]),
preview_node_id=value["preview_node_id"],
sources={
key: BackendSource(**{**source, "batch": _batch_from_payload(source["batch"])})
for key, source in value["sources"].items()
},
)
try:
result = ReferenceExecutionBackend()._execute_in_process(request)
except BackendExecutionError as exc:
response = {"error": {
"message": str(exc), "code": exc.code, "node_id": exc.node_id,
"diagnostics": tuple(item.model_dump(mode="python") for item in exc.diagnostics),
"node_diagnostics": tuple(item.model_dump(mode="python") for item in exc.node_diagnostics),
"source_fingerprints": exc.source_fingerprints,
"input_row_count": exc.input_row_count,
"node_preview": exc.node_preview.model_dump(mode="python") if exc.node_preview else None,
}}
else:
response = {
"contract": result.contract.model_dump(mode="python"),
"batch": _batch_payload(result.batch),
"node_diagnostics": tuple(item.model_dump(mode="python") for item in result.node_diagnostics),
"node_preview": result.node_preview.model_dump(mode="python") if result.node_preview else None,
"metadata": dict(result.metadata),
}
return encode_worker_payload(response, max_bytes=_TRANSPORT_BYTES)
def _validate_source_batches(request: BackendExecutionRequest) -> None:
for source in request.sources.values():
try:
@@ -139,6 +297,7 @@ def _source_for_node(
node: GraphNode,
) -> BackendSource | None:
candidates = (
reference_source_key(node),
node.id,
str(node.config.get("source_ref") or ""),
str(node.config.get("source_name") or ""),
@@ -153,4 +312,10 @@ def _source_for_node(
)
def reference_source_key(node: GraphNode) -> str:
"""Nested graphs may reuse node IDs; bind resolved data to the full config."""
content = json.dumps(node.config, sort_keys=True, separators=(",", ":"), default=str)
return "reference-config:" + hashlib.sha256(content.encode("utf-8")).hexdigest()
__all__ = ["ReferenceExecutionBackend"]
src/govoplan_dataflow/backend/manifest.py
+34
View File
@@ -154,6 +154,40 @@ ROLE_TEMPLATES = (
)
DOCUMENTATION = localize_documentation_topics((
DocumentationTopic(
id="dataflow.reference-worker-limits",
title="Reference execution process limits",
summary="Contain expensive expressions and intermediate allocations without changing datasource authority.",
body=(
"Reference previews and reference development runs evaluate in a fresh disposable process, including regex and aggregate intermediate allocations. "
"The existing row, node and per-result byte checks remain. The process additionally enforces the request's wall-clock and memory budgets "
"(default preview: 2 seconds and 256 MiB virtual address space), rounded-up CPU seconds, no regular-file output, and 32 MiB per data-only input/result transport. "
"Supported process budgets are at most 600 seconds and 8 GiB; unsupported controls or exceeded limits fail with structured backend.process diagnostics, never inline fallback. "
"Datasource authorization and bounded source reads remain in the parent, including nested subflow sources; sessions and credentials are not passed to the child. "
"Reference source collection also checks a cumulative 32 MiB typed-data budget before constructing another columnar copy; providers retain their separate per-read limits. "
"Completed node diagnostics survive ordinary evaluation errors; a killed worker returns no partial rows or invented node progress. "
"GOVOPLAN_ISOLATED_PROCESS_CONCURRENCY limits shared isolated-work admission per API/worker process, default 1; busy capacity is retryable. "
"It is not a fleet-wide quota or arbitrary-code sandbox. Cancellation checks before/after reference runs remain; hard wall limits stop an unresponsive expression. "
"Staging/production still require the separate DuckDB backend and are not converted to reference execution."
),
layer="static", documentation_types=("user", "admin"), audience=("user", "module_admin", "operator"), order=6,
translations={"de": {
"title": "Prozessgrenzen der Referenzausführung",
"summary": "Aufwendige Ausdrücke und Zwischenspeicher begrenzen, ohne Datenquellenrechte zu verändern.",
"body": (
"Referenz-Vorschauen und Referenz-Entwicklungsläufe werten Ausdrücke in einem frischen, kurzlebigen Prozess aus, einschließlich regulärer Ausdrücke und großer Zwischenergebnisse. "
"Bestehende Zeilen-, Knoten- und Ergebnis-Bytegrenzen bleiben bestehen. Zusätzlich gelten das Laufzeit- und Speicherbudget der Anfrage "
"(Vorschau standardmäßig 2 Sekunden und 256 MiB virtueller Adressraum), aufgerundete CPU-Sekunden, keine regulären Ausgabedateien und je 32 MiB für den reinen Datentransport. "
"Prozessbudgets unterstützen höchstens 600 Sekunden und 8 GiB. Fehlende Betriebssystemkontrollen oder überschrittene Grenzen erzeugen strukturierte backend.process-Diagnosen, ohne Ausweichbetrieb im Hauptprozess. "
"Datenquellenrechte und begrenzte Quellabrufe werden im Hauptprozess geprüft, auch für verschachtelte Teilflüsse; Sitzungen und Zugangsdaten gelangen nicht in den Kindprozess. "
"Referenz-Quellabrufe prüfen außerdem zusammen höchstens 32 MiB typisierte Daten, bevor eine weitere spaltenweise Kopie entsteht; getrennte Abrufgrenzen der Anbieter bleiben bestehen. "
"Gewöhnliche Auswertungsfehler behalten bereits abgeschlossene Knotendiagnosen. Ein gestoppter Prozess liefert keine Teilzeilen und keinen erfundenen Knotenfortschritt. "
"GOVOPLAN_ISOLATED_PROCESS_CONCURRENCY begrenzt gemeinsam genutzte isolierte Arbeit je API-/Worker-Prozess, standardmäßig 1; bei Auslastung ist ein erneuter Versuch möglich. "
"Dies ist weder eine systemweite Quote noch eine Sandbox für beliebigen Code. Abbruchprüfungen vor und nach Referenzläufen bleiben erhalten; harte Laufzeitgrenzen stoppen hängende Ausdrücke. "
"Staging und Produktion benötigen weiterhin das gesonderte DuckDB-Backend und wechseln nicht zur Referenzausführung."
),
}},
),
DocumentationTopic(
id="dataflow.workspace-layout",
title="Dataflow workspace actions",
src/govoplan_dataflow/backend/service.py
+51 -15
View File
@@ -28,6 +28,7 @@ from govoplan_core.core.datasources import (
datasource_publication,
)
from govoplan_core.db.base import utcnow
from govoplan_core.security.worker_payload import WorkerPayloadError, encode_worker_payload
from govoplan_dataflow.backend.backends import (
BackendExecutionError,
BackendSource,
@@ -35,6 +36,7 @@ from govoplan_dataflow.backend.backends import (
execute_typed_graph,
)
from govoplan_dataflow.backend.batches import TypedBatch
from govoplan_dataflow.backend.backends.reference import reference_source_key
from govoplan_dataflow.backend.db.models import (
DataflowPipeline,
DataflowPipelineDeployment,
@@ -48,7 +50,6 @@ from govoplan_dataflow.backend.executor import (
PipelineExecutionError,
PipelineExecutionResult,
ResolvedSource,
execute_preview,
)
from govoplan_dataflow.backend.governance import (
definition_governance_payload,
@@ -1122,20 +1123,11 @@ def _execute_pipeline_preview(
principal=principal,
registry=registry,
)
if backend == "reference":
return (
execute_preview(
graph,
row_limit=row_limit,
source_resolver=source_resolver,
preview_node_id=preview_node_id,
),
EXECUTOR_VERSION,
)
sources = _typed_backend_sources(
graph,
source_resolver=source_resolver,
source_limit=max(MAX_SOURCE_ROWS, row_limit),
source_limit=MAX_SOURCE_ROWS if backend == "reference" else max(MAX_SOURCE_ROWS, row_limit),
include_subflows=backend == "reference",
)
try:
result = execute_typed_graph(
@@ -1149,8 +1141,14 @@ def _execute_pipeline_preview(
raise PipelineExecutionError(
str(exc),
node_id=exc.node_id,
diagnostics=tuple(exc.diagnostics),
retryable=exc.code == "backend.capacity",
diagnostics=(*exc.diagnostics, DataflowDiagnostic(
severity="error", code=exc.code, message=str(exc), node_id=exc.node_id,
)),
node_diagnostics=exc.node_diagnostics,
source_fingerprints=exc.source_fingerprints,
input_row_count=exc.input_row_count,
node_preview=exc.node_preview,
retryable=exc.code in {"backend.capacity", "backend.process.busy"},
) from exc
columns = [
PreviewColumn(
@@ -1227,13 +1225,51 @@ def _typed_backend_sources(
*,
source_resolver,
source_limit: int = MAX_SOURCE_ROWS,
include_subflows: bool = False,
_depth: int = 0,
_remaining_source_bytes: list[int] | None = None,
) -> dict[str, BackendSource]:
if _depth > 5:
raise PipelineExecutionError("Subflows are limited to five nested levels.")
if _remaining_source_bytes is None:
_remaining_source_bytes = [32 * 1024 * 1024]
sources: dict[str, BackendSource] = {}
for node in graph.nodes:
if include_subflows and node.type == "subflow":
from govoplan_dataflow.backend.subflows import substitute_parameters
parameters = node.config.get("parameters")
nested = PipelineGraph.model_validate(substitute_parameters(
node.config.get("graph"), parameters if isinstance(parameters, dict) else {},
))
sources.update(_typed_backend_sources(
nested, source_resolver=source_resolver, source_limit=source_limit,
include_subflows=True, _depth=_depth + 1,
_remaining_source_bytes=_remaining_source_bytes,
))
if node.type != "source.reference":
continue
if include_subflows and _remaining_source_bytes[0] <= 0:
raise PipelineExecutionError(
"Combined source data exceeds the 32 MiB transfer budget.", node_id=node.id,
)
resolved = source_resolver(node, source_limit)
sources[node.id] = BackendSource(
if include_subflows:
try:
# Check before constructing another columnar copy. The provider
# still owns bounds on its individual authorized read; do not keep
# accumulating individually valid batches before the worker gate.
encoded_size = len(encode_worker_payload(
tuple(dict(row) for row in resolved.rows), max_bytes=_remaining_source_bytes[0],
))
except WorkerPayloadError as exc:
raise PipelineExecutionError(
"Combined source data exceeds the 32 MiB transfer budget or contains unsupported values.",
node_id=node.id,
) from exc
_remaining_source_bytes[0] -= encoded_size
key = reference_source_key(node) if include_subflows else node.id
sources[key] = BackendSource(
node_id=node.id,
batch=TypedBatch.from_rows(resolved.rows),
source_ref=resolved.source_ref,