Pin reporting datasets to published Dataflow runs

This commit is contained in:
2026-08-04 12:48:37 +02:00
parent aebe94ecc2
commit 4e0238f701
6 changed files with 84 additions and 1 deletions
@@ -501,6 +501,7 @@ def _read_dataset(
request=DataflowDatasetRequest(
pipeline_ref=dataset.source_ref,
revision=dataset.source_revision or 0,
run_ref=dataset.source_run_ref,
parameters=source_parameters,
row_limit=2_000,
expected_definition_hash=dataset.definition_hash,
+1 -1
View File
@@ -488,7 +488,7 @@ manifest = ModuleManifest(
"schedules, exports, and publication providers replace unchecked SQL in the "
"presentation layer. PostgreSQL executes bounded semantic plans when available. "
"Signed drill contexts reauthorize contributor rows, and Files/Mail publication "
"adapters retain idempotent evidence. Dataflow and module read models remain source owners."
"adapters retain idempotent evidence. A dataset may pin one successful published Dataflow run, which is read from its exact Datasource materialization after both source boundaries reauthorize the current principal. Dataflow and module read models remain source owners."
),
layer="available",
documentation_types=("admin", "user"),
@@ -101,6 +101,7 @@ class DatasetDefinition(BaseModel):
source_kind: Literal["dataflow", "read_model", "static"]
source_ref: str = Field(min_length=1, max_length=500)
source_revision: int | None = Field(default=None, ge=1)
source_run_ref: str | None = Field(default=None, min_length=1, max_length=500)
definition_hash: str | None = Field(default=None, min_length=1, max_length=128)
source_parameters: dict[str, Any] = Field(default_factory=dict)
static_rows: list[dict[str, Any]] = Field(default_factory=list, max_length=2_000)
@@ -130,6 +131,8 @@ class DatasetDefinition(BaseModel):
def validate_source_pin(self) -> "DatasetDefinition":
if self.source_kind == "dataflow" and self.source_revision is None:
raise ValueError("Dataflow datasets require a pinned source revision.")
if self.source_run_ref is not None and self.source_kind != "dataflow":
raise ValueError("Only Dataflow datasets can pin a source run.")
if self.source_kind == "static" and not self.static_rows:
raise ValueError("Static analytical datasets require static_rows.")
names = [item.name for item in self.fields]