Add governed Dataflow audience outputs
This commit is contained in:
@@ -0,0 +1,200 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from govoplan_core.auth import ApiPrincipal
|
||||
from govoplan_core.core.dataflows import (
|
||||
DataflowDatasetDescriptor,
|
||||
DataflowDatasetRequest,
|
||||
DataflowDatasetResult,
|
||||
DataflowRunConflictError,
|
||||
DataflowRunUnavailableError,
|
||||
)
|
||||
from govoplan_core.security.time import utc_now
|
||||
from govoplan_dataflow.backend.backends.base import ExecutionBudget
|
||||
from govoplan_dataflow.backend.executor import PipelineExecutionError
|
||||
from govoplan_dataflow.backend.governance import require_definition_action
|
||||
from govoplan_dataflow.backend.schemas import PipelineGraph
|
||||
from govoplan_dataflow.backend.service import (
|
||||
_execute_pipeline_preview,
|
||||
get_pipeline,
|
||||
get_pipeline_revision,
|
||||
list_pipelines,
|
||||
)
|
||||
from govoplan_dataflow.backend.subflows import substitute_parameters
|
||||
|
||||
|
||||
class SqlDataflowDatasetOutputProvider:
|
||||
def __init__(self, registry: object | None = None) -> None:
|
||||
self.registry = registry
|
||||
|
||||
def list_outputs(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
query: str = "",
|
||||
limit: int = 100,
|
||||
) -> tuple[DataflowDatasetDescriptor, ...]:
|
||||
typed_session, typed_principal = _contracts(session, principal)
|
||||
normalized_query = query.strip().casefold()
|
||||
result: list[DataflowDatasetDescriptor] = []
|
||||
for pipeline in list_pipelines(
|
||||
typed_session,
|
||||
tenant_id=typed_principal.tenant_id,
|
||||
):
|
||||
if normalized_query and normalized_query not in pipeline.name.casefold():
|
||||
continue
|
||||
try:
|
||||
decision = require_definition_action(
|
||||
pipeline,
|
||||
principal=typed_principal,
|
||||
registry=self.registry,
|
||||
action="run",
|
||||
)
|
||||
except PermissionError:
|
||||
continue
|
||||
revision = get_pipeline_revision(typed_session, pipeline=pipeline)
|
||||
result.append(
|
||||
DataflowDatasetDescriptor(
|
||||
pipeline_ref=pipeline.id,
|
||||
name=pipeline.name,
|
||||
description=pipeline.description,
|
||||
revision=revision.revision,
|
||||
definition_hash=revision.content_hash,
|
||||
status=pipeline.status,
|
||||
updated_at=pipeline.updated_at,
|
||||
parameters=dict(pipeline.metadata_.get("parameters") or {}),
|
||||
provenance={
|
||||
"module": "dataflow",
|
||||
"scope_type": pipeline.scope_type,
|
||||
"scope_id": pipeline.scope_id,
|
||||
"policy_decision": decision.to_dict(),
|
||||
},
|
||||
)
|
||||
)
|
||||
if len(result) >= max(1, min(limit, 200)):
|
||||
break
|
||||
return tuple(result)
|
||||
|
||||
def read_output(
|
||||
self,
|
||||
session: object,
|
||||
principal: object,
|
||||
*,
|
||||
request: DataflowDatasetRequest,
|
||||
) -> DataflowDatasetResult:
|
||||
typed_session, typed_principal = _contracts(session, principal)
|
||||
row_limit = max(1, min(request.row_limit, 2_000))
|
||||
pipeline = get_pipeline(
|
||||
typed_session,
|
||||
tenant_id=typed_principal.tenant_id,
|
||||
pipeline_id=request.pipeline_ref,
|
||||
)
|
||||
decision = require_definition_action(
|
||||
pipeline,
|
||||
principal=typed_principal,
|
||||
registry=self.registry,
|
||||
action="run",
|
||||
)
|
||||
revision = get_pipeline_revision(
|
||||
typed_session,
|
||||
pipeline=pipeline,
|
||||
revision=request.revision,
|
||||
)
|
||||
if (
|
||||
request.expected_definition_hash
|
||||
and request.expected_definition_hash != revision.content_hash
|
||||
):
|
||||
raise DataflowRunConflictError(
|
||||
"The pinned Dataflow definition hash no longer matches the requested revision."
|
||||
)
|
||||
graph = PipelineGraph.model_validate(
|
||||
substitute_parameters(revision.graph, dict(request.parameters))
|
||||
)
|
||||
try:
|
||||
result, executor_version = _execute_pipeline_preview(
|
||||
graph,
|
||||
session=typed_session,
|
||||
principal=typed_principal,
|
||||
registry=self.registry,
|
||||
backend="auto",
|
||||
row_limit=row_limit,
|
||||
preview_node_id=None,
|
||||
budget=ExecutionBudget(
|
||||
max_output_rows=row_limit,
|
||||
max_batch_bytes=4_000_000,
|
||||
max_wall_seconds=5.0,
|
||||
),
|
||||
)
|
||||
except PipelineExecutionError as exc:
|
||||
raise DataflowRunUnavailableError(
|
||||
f"The pinned Dataflow output could not be evaluated: {exc}"
|
||||
) from exc
|
||||
source_fingerprints = tuple(
|
||||
dict(item) for item in result.source_fingerprints
|
||||
)
|
||||
if request.expected_source_fingerprints and not _fingerprints_match(
|
||||
request.expected_source_fingerprints,
|
||||
source_fingerprints,
|
||||
):
|
||||
raise DataflowRunConflictError(
|
||||
"Dataflow source fingerprints differ from the pinned source state."
|
||||
)
|
||||
output_hash = hashlib.sha256(
|
||||
json.dumps(
|
||||
result.rows,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
default=str,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
return DataflowDatasetResult(
|
||||
pipeline_ref=pipeline.id,
|
||||
revision=revision.revision,
|
||||
definition_hash=revision.content_hash,
|
||||
rows=tuple(dict(item) for item in result.rows),
|
||||
total_rows=result.total_rows,
|
||||
truncated=result.truncated,
|
||||
output_hash=output_hash,
|
||||
executor_version=executor_version,
|
||||
source_fingerprints=source_fingerprints,
|
||||
diagnostics=tuple(
|
||||
item.model_dump(mode="json") for item in result.diagnostics
|
||||
),
|
||||
generated_at=utc_now(),
|
||||
provenance={
|
||||
"module": "dataflow",
|
||||
"scope_type": pipeline.scope_type,
|
||||
"scope_id": pipeline.scope_id,
|
||||
"policy_decision": decision.to_dict(),
|
||||
"parameters": dict(request.parameters),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def dataset_output_provider(context: object | None = None):
|
||||
return SqlDataflowDatasetOutputProvider(getattr(context, "registry", None))
|
||||
|
||||
|
||||
def _fingerprints_match(expected, actual) -> bool:
|
||||
def normalized(values):
|
||||
return sorted(
|
||||
json.dumps(dict(item), sort_keys=True, separators=(",", ":"), default=str)
|
||||
for item in values
|
||||
)
|
||||
|
||||
return normalized(expected) == normalized(actual)
|
||||
|
||||
|
||||
def _contracts(session: object, principal: object):
|
||||
if not hasattr(session, "scalar") or not hasattr(session, "scalars"):
|
||||
raise TypeError("Dataflow dataset output requires a SQLAlchemy session.")
|
||||
if not isinstance(principal, ApiPrincipal):
|
||||
raise TypeError("Dataflow dataset output requires an API principal.")
|
||||
return session, principal
|
||||
|
||||
|
||||
__all__ = ["SqlDataflowDatasetOutputProvider", "dataset_output_provider"]
|
||||
@@ -11,6 +11,7 @@ from govoplan_core.core.access import (
|
||||
CAPABILITY_AUTH_AUTOMATION_PRINCIPAL_PROVIDER,
|
||||
)
|
||||
from govoplan_core.core.dataflows import (
|
||||
CAPABILITY_DATAFLOW_DATASET_OUTPUT,
|
||||
CAPABILITY_DATAFLOW_RUN_LIFECYCLE,
|
||||
CAPABILITY_DATAFLOW_RUN_WORKER,
|
||||
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER,
|
||||
@@ -279,7 +280,7 @@ manifest = ModuleManifest(
|
||||
ModuleInterfaceProvider(name="dataflow.pipeline_preview", version=MODULE_VERSION),
|
||||
ModuleInterfaceProvider(name="dataflow.run_lifecycle", version=MODULE_VERSION),
|
||||
ModuleInterfaceProvider(name="dataflow.run_worker", version=MODULE_VERSION),
|
||||
ModuleInterfaceProvider(name="dataflow.dataset_output", version=MODULE_VERSION),
|
||||
ModuleInterfaceProvider(name=CAPABILITY_DATAFLOW_DATASET_OUTPUT, version=MODULE_VERSION),
|
||||
ModuleInterfaceProvider(
|
||||
name="dataflow.trigger_dispatcher",
|
||||
version=MODULE_VERSION,
|
||||
@@ -378,6 +379,10 @@ manifest = ModuleManifest(
|
||||
),
|
||||
route_factory=_dataflow_router,
|
||||
capability_factories={
|
||||
CAPABILITY_DATAFLOW_DATASET_OUTPUT: lambda context: __import__(
|
||||
"govoplan_dataflow.backend.dataset_output",
|
||||
fromlist=["dataset_output_provider"],
|
||||
).dataset_output_provider(context),
|
||||
CAPABILITY_DATAFLOW_RUN_LIFECYCLE: _run_provider,
|
||||
CAPABILITY_DATAFLOW_RUN_WORKER: _run_worker,
|
||||
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER: _trigger_provider,
|
||||
|
||||
Reference in New Issue
Block a user