Serve exact published runs as datasets
This commit is contained in:
@@ -11,6 +11,11 @@ from govoplan_core.core.dataflows import (
|
||||
DataflowRunConflictError,
|
||||
DataflowRunUnavailableError,
|
||||
)
|
||||
from govoplan_core.core.datasources import (
|
||||
DatasourceError,
|
||||
DatasourceReadRequest,
|
||||
datasource_catalogue,
|
||||
)
|
||||
from govoplan_core.security.time import utc_now
|
||||
from govoplan_dataflow.backend.backends.base import ExecutionBudget
|
||||
from govoplan_dataflow.backend.executor import PipelineExecutionError
|
||||
@@ -20,6 +25,7 @@ from govoplan_dataflow.backend.service import (
|
||||
_execute_pipeline_preview,
|
||||
get_pipeline,
|
||||
get_pipeline_revision,
|
||||
get_pipeline_run,
|
||||
list_pipelines,
|
||||
)
|
||||
from govoplan_dataflow.backend.subflows import substitute_parameters
|
||||
@@ -110,6 +116,17 @@ class SqlDataflowDatasetOutputProvider:
|
||||
raise DataflowRunConflictError(
|
||||
"The pinned Dataflow definition hash no longer matches the requested revision."
|
||||
)
|
||||
if request.run_ref:
|
||||
return _read_published_run_output(
|
||||
typed_session,
|
||||
typed_principal,
|
||||
registry=self.registry,
|
||||
request=request,
|
||||
pipeline=pipeline,
|
||||
revision=revision,
|
||||
policy_decision=decision.to_dict(),
|
||||
row_limit=row_limit,
|
||||
)
|
||||
graph = PipelineGraph.model_validate(
|
||||
substitute_parameters(revision.graph, dict(request.parameters))
|
||||
)
|
||||
@@ -179,6 +196,143 @@ def dataset_output_provider(context: object | None = None):
|
||||
return SqlDataflowDatasetOutputProvider(getattr(context, "registry", None))
|
||||
|
||||
|
||||
def _read_published_run_output(
|
||||
session,
|
||||
principal: ApiPrincipal,
|
||||
*,
|
||||
registry: object | None,
|
||||
request: DataflowDatasetRequest,
|
||||
pipeline,
|
||||
revision,
|
||||
policy_decision: dict[str, object],
|
||||
row_limit: int,
|
||||
) -> DataflowDatasetResult:
|
||||
if request.parameters:
|
||||
raise DataflowRunConflictError(
|
||||
"An immutable published run cannot be evaluated with new parameters."
|
||||
)
|
||||
run = get_pipeline_run(
|
||||
session,
|
||||
tenant_id=principal.tenant_id,
|
||||
run_ref=request.run_ref or "",
|
||||
)
|
||||
if run.pipeline_id != pipeline.id or run.pipeline_revision_id != revision.id:
|
||||
raise DataflowRunConflictError(
|
||||
"The published run does not belong to the pinned Dataflow revision."
|
||||
)
|
||||
if run.definition_hash != revision.content_hash:
|
||||
raise DataflowRunConflictError(
|
||||
"The published run definition evidence does not match the pinned revision."
|
||||
)
|
||||
if run.status != "succeeded":
|
||||
raise DataflowRunUnavailableError(
|
||||
"Only a successful Dataflow run can be used as an immutable dataset."
|
||||
)
|
||||
if not run.output_datasource_ref or not run.output_materialization_ref:
|
||||
raise DataflowRunUnavailableError(
|
||||
"The successful Dataflow run has no immutable Datasource publication."
|
||||
)
|
||||
provider = datasource_catalogue(registry)
|
||||
if provider is None:
|
||||
raise DataflowRunUnavailableError(
|
||||
"The Datasource catalogue required by this published run is not enabled."
|
||||
)
|
||||
|
||||
rows: list[dict[str, object]] = []
|
||||
materialization = None
|
||||
total_rows = 0
|
||||
try:
|
||||
while len(rows) < row_limit:
|
||||
remaining = row_limit - len(rows)
|
||||
page = provider.read_datasource(
|
||||
session,
|
||||
principal,
|
||||
request=DatasourceReadRequest(
|
||||
datasource_ref=run.output_datasource_ref,
|
||||
materialization_ref=run.output_materialization_ref,
|
||||
limit=min(500, remaining),
|
||||
offset=len(rows),
|
||||
expected_fingerprint=(
|
||||
materialization.fingerprint
|
||||
if materialization is not None
|
||||
else None
|
||||
),
|
||||
),
|
||||
)
|
||||
if (
|
||||
page.materialization is None
|
||||
or page.materialization.ref != run.output_materialization_ref
|
||||
):
|
||||
raise DataflowRunConflictError(
|
||||
"The Datasource provider returned a different output materialization."
|
||||
)
|
||||
if len(page.rows) > remaining:
|
||||
raise DataflowRunConflictError(
|
||||
"The Datasource provider exceeded the requested output window."
|
||||
)
|
||||
materialization = page.materialization
|
||||
total_rows = page.total_rows
|
||||
rows.extend(dict(item) for item in page.rows)
|
||||
if not page.truncated:
|
||||
break
|
||||
if not page.rows:
|
||||
raise DataflowRunUnavailableError(
|
||||
"The Datasource provider made no progress while reading the published output."
|
||||
)
|
||||
except DatasourceError as exc:
|
||||
raise DataflowRunUnavailableError(
|
||||
"The immutable Datasource output is unavailable to the current principal."
|
||||
) from exc
|
||||
|
||||
source_fingerprints = tuple(dict(item) for item in run.source_fingerprints)
|
||||
if request.expected_source_fingerprints and not _fingerprints_match(
|
||||
request.expected_source_fingerprints,
|
||||
source_fingerprints,
|
||||
):
|
||||
raise DataflowRunConflictError(
|
||||
"Dataflow source fingerprints differ from the pinned run evidence."
|
||||
)
|
||||
output_hash = hashlib.sha256(
|
||||
json.dumps(
|
||||
rows,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
default=str,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
if materialization is None:
|
||||
raise DataflowRunUnavailableError(
|
||||
"The Datasource provider returned no materialization evidence."
|
||||
)
|
||||
return DataflowDatasetResult(
|
||||
pipeline_ref=pipeline.id,
|
||||
revision=revision.revision,
|
||||
definition_hash=revision.content_hash,
|
||||
rows=tuple(rows),
|
||||
total_rows=total_rows,
|
||||
truncated=len(rows) < total_rows,
|
||||
output_hash=output_hash,
|
||||
executor_version=run.executor_version,
|
||||
run_ref=f"dataflow-run:{run.id}",
|
||||
source_fingerprints=source_fingerprints,
|
||||
diagnostics=tuple(dict(item) for item in run.diagnostics),
|
||||
generated_at=run.finished_at or materialization.created_at,
|
||||
provenance={
|
||||
"module": "dataflow",
|
||||
"scope_type": pipeline.scope_type,
|
||||
"scope_id": pipeline.scope_id,
|
||||
"policy_decision": policy_decision,
|
||||
"immutable_run": True,
|
||||
"publication_ref": run.output_publication_ref,
|
||||
"datasource_ref": run.output_datasource_ref,
|
||||
"materialization_ref": run.output_materialization_ref,
|
||||
"materialization_fingerprint": materialization.fingerprint,
|
||||
"governance": materialization.governance.to_dict(),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _fingerprints_match(expected, actual) -> bool:
|
||||
def normalized(values):
|
||||
return sorted(
|
||||
|
||||
@@ -270,6 +270,7 @@ DOCUMENTATION = (
|
||||
"states whether work stopped, completed, failed, or requires operator reconciliation. Scheduled, event, "
|
||||
"and queued execution is partitioned by tenant module entitlement before a run is claimed. Disabling "
|
||||
"Dataflow stops new admission and leaves accepted runs available for an explicit operator decision."
|
||||
" Reporting may pin a successful published run; Dataflow then rechecks run authority and Datasource access and reads only the exact recorded materialization without reparameterizing or re-executing it."
|
||||
),
|
||||
layer="available",
|
||||
documentation_types=("admin", "user"),
|
||||
|
||||
Reference in New Issue
Block a user