Pin reporting datasets to published Dataflow runs

This commit is contained in:
2026-08-04 12:48:37 +02:00
parent aebe94ecc2
commit 4e0238f701
6 changed files with 84 additions and 1 deletions
+4
View File
@@ -27,6 +27,10 @@ published by a source-owning module. Do not expose another module's ORM or an
unbounded SQL connection as a report source. Configure an explicit schema,
freshness policy, source fingerprint expectations, purpose, privacy,
retention, and a row-policy provider where source access alone is not enough.
For a Dataflow source, `source_run_ref` optionally pins one successful run that
published an immutable Datasource materialization. Reporting passes this pin
through rather than re-executing the pipeline. The current principal must still
be authorized for the Dataflow definition and the exact Datasource output.
On PostgreSQL installations, Reporting compiles bounded semantic filters,
grouping, measures, calculated measures, ordering, offsets, and limits into a
+5
View File
@@ -28,6 +28,11 @@ the exact definition and output. Warnings explain freshness, inferred schema,
or provider diagnostics. A failed quality gate records a failed execution and
does not publish a result.
A report configured against an exact published Dataflow run reads that run's
immutable Datasource materialization. It does not rerun the flow with current
inputs. The evidence identifies the Dataflow run and materialization, and access
to both is checked again when the report runs.
The **Effective access** explanation states when dimensions, measures, source
rows, or actions were removed by Policy. A result with no hidden elements says
so explicitly; catalogue visibility never grants access to protected detail.
@@ -501,6 +501,7 @@ def _read_dataset(
request=DataflowDatasetRequest(
pipeline_ref=dataset.source_ref,
revision=dataset.source_revision or 0,
run_ref=dataset.source_run_ref,
parameters=source_parameters,
row_limit=2_000,
expected_definition_hash=dataset.definition_hash,
+1 -1
View File
@@ -488,7 +488,7 @@ manifest = ModuleManifest(
"schedules, exports, and publication providers replace unchecked SQL in the "
"presentation layer. PostgreSQL executes bounded semantic plans when available. "
"Signed drill contexts reauthorize contributor rows, and Files/Mail publication "
"adapters retain idempotent evidence. Dataflow and module read models remain source owners."
"adapters retain idempotent evidence. A dataset may pin one successful published Dataflow run, which is read from its exact Datasource materialization after both source boundaries reauthorize the current principal. Dataflow and module read models remain source owners."
),
layer="available",
documentation_types=("admin", "user"),
@@ -101,6 +101,7 @@ class DatasetDefinition(BaseModel):
source_kind: Literal["dataflow", "read_model", "static"]
source_ref: str = Field(min_length=1, max_length=500)
source_revision: int | None = Field(default=None, ge=1)
source_run_ref: str | None = Field(default=None, min_length=1, max_length=500)
definition_hash: str | None = Field(default=None, min_length=1, max_length=128)
source_parameters: dict[str, Any] = Field(default_factory=dict)
static_rows: list[dict[str, Any]] = Field(default_factory=list, max_length=2_000)
@@ -130,6 +131,8 @@ class DatasetDefinition(BaseModel):
def validate_source_pin(self) -> "DatasetDefinition":
if self.source_kind == "dataflow" and self.source_revision is None:
raise ValueError("Dataflow datasets require a pinned source revision.")
if self.source_run_ref is not None and self.source_kind != "dataflow":
raise ValueError("Only Dataflow datasets can pin a source run.")
if self.source_kind == "static" and not self.static_rows:
raise ValueError("Static analytical datasets require static_rows.")
names = [item.name for item in self.fields]
+70
View File
@@ -8,6 +8,10 @@ from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from govoplan_core.db.base import Base
from govoplan_core.core.dataflows import (
CAPABILITY_DATAFLOW_DATASET_OUTPUT,
DataflowDatasetResult,
)
from govoplan_core.core.files import (
CAPABILITY_FILES_ARTIFACT_STORE,
ManagedArtifactRef,
@@ -132,6 +136,34 @@ class ArtifactStore:
)
class DataflowOutput:
def __init__(self, rows: list[dict[str, object]]) -> None:
self.rows = tuple(dict(item) for item in rows)
self.last_request = None
def list_outputs(self, *_args, **_kwargs):
return ()
def read_output(self, _session, _principal, *, request):
self.last_request = request
return DataflowDatasetResult(
pipeline_ref=request.pipeline_ref,
revision=request.revision,
definition_hash=request.expected_definition_hash or "pipeline-hash",
rows=self.rows,
total_rows=len(self.rows),
truncated=False,
output_hash="d" * 64,
executor_version="duckdb-v1",
run_ref=request.run_ref,
source_fingerprints=(
{"node_id": "source", "fingerprint": "source-v1"},
),
generated_at=NOW,
provenance={"immutable_run": bool(request.run_ref)},
)
class ReportingServiceTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite+pysqlite:///:memory:")
@@ -352,6 +384,44 @@ class ReportingServiceTests(unittest.TestCase):
)
self.assertTrue(raised.exception.execution_id)
def test_dataflow_dataset_can_pin_an_exact_published_run(self) -> None:
payload = dataset_payload()
rows = list(payload.pop("static_rows"))
payload.update(
{
"source_kind": "dataflow",
"source_ref": "pipeline:monthly-comparison",
"source_revision": 4,
"source_run_ref": "dataflow-run:published-july",
"definition_hash": "pipeline-hash",
}
)
self._create("dataset", "dataset-1", payload)
self._create("semantic_model", "semantic-1", semantic_payload())
self._create("report", "report-1", report_payload())
provider = DataflowOutput(rows)
registry = CapabilityRegistry()
registry.providers[CAPABILITY_DATAFLOW_DATASET_OUTPUT] = provider
result = execute_report(
self.session,
self.principal,
registry=registry,
report_id="report-1",
report_revision=1,
parameters={},
query=None,
idempotency_key="published-dataflow-run",
)
self.assertEqual("succeeded", result["status"])
self.assertIsNotNone(provider.last_request)
self.assertEqual(
"dataflow-run:published-july",
provider.last_request.run_ref,
)
self.assertTrue(result["provenance"]["source"]["immutable_run"])
def test_restricted_access_and_service_scope_guards(self) -> None:
self._create_report_graph(
report_access={