Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a86220db27 | ||
|
|
6767905cbb | ||
|
|
d12e0bce7d | ||
|
|
21da2e1ad4 | ||
|
|
20d134898a | ||
|
|
f908ddbf9d | ||
|
|
b14c693bde | ||
|
|
3dae80b6b8 | ||
|
|
6e11bdd30d | ||
|
|
62cdba2c5e |
@@ -19,6 +19,21 @@ without storing the previewed row contents.
|
|||||||
- **Risk Compliance:** sanctions matching policy, review, dispositions, and
|
- **Risk Compliance:** sanctions matching policy, review, dispositions, and
|
||||||
legal evidence.
|
legal evidence.
|
||||||
|
|
||||||
|
## Data-subject requests
|
||||||
|
|
||||||
|
Dataflow publishes `privacy.dsar.dataflow` for exact pipeline, revision,
|
||||||
|
reconciliation, run, deployment, trigger, and delivery references and for
|
||||||
|
minimized operator or automation-authority attribution. It never exports
|
||||||
|
graphs, SQL, request/event payloads, reconciliation corrections, authorization
|
||||||
|
snapshots, provenance bodies, errors, source details, hashes, credentials, or
|
||||||
|
output rows. Authoritative input modules locate and correct subject facts;
|
||||||
|
Dataflow does not guess identity by scanning arbitrary transformations.
|
||||||
|
|
||||||
|
Exact terminal run and delivery detail can be minimized idempotently, and
|
||||||
|
subject-linked automation authority can be disabled and revoked. Definitions,
|
||||||
|
active work, decisions, deployments, broad pipeline packages, published
|
||||||
|
Datasource outputs, and institutional attribution require review or retention.
|
||||||
|
|
||||||
## Node Library
|
## Node Library
|
||||||
|
|
||||||
The canonical backend catalogue is exposed to the WebUI and groups executable
|
The canonical backend catalogue is exposed to the WebUI and groups executable
|
||||||
@@ -152,6 +167,16 @@ logical row disappeared. It never silently applies a correction to business
|
|||||||
data; a downstream governed transform or Workflow handoff must interpret the
|
data; a downstream governed transform or Workflow handoff must interpret the
|
||||||
recorded action.
|
recorded action.
|
||||||
|
|
||||||
|
For saved reconciliation pipelines, the preview results provide a review
|
||||||
|
dialog for those rows. Reviewers create a tenant-owned decision set and append
|
||||||
|
accept, reject, correct, or defer decisions with a mandatory reason. Writes use
|
||||||
|
optimistic concurrency; updating a decision creates another immutable revision
|
||||||
|
rather than replacing history. The current projection appears in the ordinary
|
||||||
|
Dataflow source catalogue as `dataflow-decision-set:<id>` and carries a content
|
||||||
|
fingerprint. A changed input hash is therefore shown as stale and cannot be
|
||||||
|
silently reused. Corrections remain annotations until an explicit downstream
|
||||||
|
transform applies them.
|
||||||
|
|
||||||
Reporting consumers may either evaluate a pinned pipeline revision or pin one
|
Reporting consumers may either evaluate a pinned pipeline revision or pin one
|
||||||
successful published run. An exact run pin is immutable: it cannot be supplied
|
successful published run. An exact run pin is immutable: it cannot be supplied
|
||||||
new parameters, and Dataflow reads only the recorded Datasource materialization
|
new parameters, and Dataflow reads only the recorded Datasource materialization
|
||||||
|
|||||||
+2
-2
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "govoplan-dataflow"
|
name = "govoplan-dataflow"
|
||||||
version = "0.1.15"
|
version = "0.1.19"
|
||||||
description = "Governed graphical and SQL data pipelines for GovOPlaN."
|
description = "Governed graphical and SQL data pipelines for GovOPlaN."
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
license = "AGPL-3.0-or-later"
|
license = "AGPL-3.0-or-later"
|
||||||
authors = [{ name = "GovOPlaN" }]
|
authors = [{ name = "GovOPlaN" }]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"govoplan-core>=0.1.15",
|
"govoplan-core>=0.1.18",
|
||||||
"sqlglot>=30.14,<31",
|
"sqlglot>=30.14,<31",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
__version__ = "0.1.15"
|
__version__ = "0.1.19"
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from sqlalchemy import (
|
|||||||
)
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
|
from govoplan_core.core.concurrency import strong_resource_etag
|
||||||
from govoplan_core.db.base import Base, TimestampMixin
|
from govoplan_core.db.base import Base, TimestampMixin
|
||||||
|
|
||||||
|
|
||||||
@@ -121,6 +122,11 @@ class DataflowPipeline(Base, TimestampMixin):
|
|||||||
cascade="all, delete-orphan",
|
cascade="all, delete-orphan",
|
||||||
order_by="DataflowTrigger.created_at",
|
order_by="DataflowTrigger.created_at",
|
||||||
)
|
)
|
||||||
|
decision_sets: Mapped[list["DataflowReconciliationDecisionSet"]] = relationship(
|
||||||
|
back_populates="pipeline",
|
||||||
|
cascade="all, delete-orphan",
|
||||||
|
order_by="DataflowReconciliationDecisionSet.name",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DataflowPipelineRevision(Base, TimestampMixin):
|
class DataflowPipelineRevision(Base, TimestampMixin):
|
||||||
@@ -153,6 +159,99 @@ class DataflowPipelineRevision(Base, TimestampMixin):
|
|||||||
pipeline: Mapped[DataflowPipeline] = relationship(back_populates="revisions")
|
pipeline: Mapped[DataflowPipeline] = relationship(back_populates="revisions")
|
||||||
|
|
||||||
|
|
||||||
|
class DataflowReconciliationDecisionSet(Base, TimestampMixin):
|
||||||
|
__tablename__ = "dataflow_reconciliation_decision_sets"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"pipeline_id",
|
||||||
|
"name",
|
||||||
|
name="uq_dataflow_decision_sets_pipeline_name",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_dataflow_decision_sets_tenant_pipeline",
|
||||||
|
"tenant_id",
|
||||||
|
"pipeline_id",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
pipeline_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("dataflow_pipelines.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||||
|
node_id: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
||||||
|
resource_revision: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
default=1,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
|
updated_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
|
|
||||||
|
pipeline: Mapped[DataflowPipeline] = relationship(back_populates="decision_sets")
|
||||||
|
decisions: Mapped[list["DataflowReconciliationDecision"]] = relationship(
|
||||||
|
back_populates="decision_set",
|
||||||
|
cascade="all, delete-orphan",
|
||||||
|
order_by="DataflowReconciliationDecision.revision",
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def strong_etag(self) -> str:
|
||||||
|
return strong_resource_etag(
|
||||||
|
"dataflow_reconciliation_decision_set",
|
||||||
|
self.id,
|
||||||
|
self.resource_revision,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DataflowReconciliationDecision(Base, TimestampMixin):
|
||||||
|
__tablename__ = "dataflow_reconciliation_decisions"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"decision_set_id",
|
||||||
|
"revision",
|
||||||
|
name="uq_dataflow_reconciliation_decision_revision",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_dataflow_reconciliation_decisions_current",
|
||||||
|
"tenant_id",
|
||||||
|
"decision_set_id",
|
||||||
|
"key_hash",
|
||||||
|
"revision",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
decision_set_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey(
|
||||||
|
"dataflow_reconciliation_decision_sets.id",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
key_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
|
input_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
action: Mapped[str] = mapped_column(String(20), nullable=False)
|
||||||
|
reason: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
correction: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||||
|
actor_ref: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
decided_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
decision_set: Mapped[DataflowReconciliationDecisionSet] = relationship(
|
||||||
|
back_populates="decisions"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DataflowRun(Base, TimestampMixin):
|
class DataflowRun(Base, TimestampMixin):
|
||||||
__tablename__ = "dataflow_runs"
|
__tablename__ = "dataflow_runs"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
|||||||
@@ -0,0 +1,835 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.dsar import (
|
||||||
|
DsarErasureActionRef,
|
||||||
|
DsarExecutionResultRef,
|
||||||
|
DsarRecordRef,
|
||||||
|
DsarSubjectRef,
|
||||||
|
dsar_capability_name,
|
||||||
|
)
|
||||||
|
from govoplan_dataflow.backend.db.models import (
|
||||||
|
DataflowPipeline,
|
||||||
|
DataflowPipelineDeployment,
|
||||||
|
DataflowPipelineRevision,
|
||||||
|
DataflowReconciliationDecision,
|
||||||
|
DataflowReconciliationDecisionSet,
|
||||||
|
DataflowRun,
|
||||||
|
DataflowTrigger,
|
||||||
|
DataflowTriggerDelivery,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
DATAFLOW_DSAR_CAPABILITY = dsar_capability_name("dataflow")
|
||||||
|
_MAX_RECORDS = 5_000
|
||||||
|
_CONFLICT = object()
|
||||||
|
_DIRECT_ALIASES = {
|
||||||
|
"pipeline_id": ("dataflow.pipeline",),
|
||||||
|
"revision_id": ("dataflow.pipeline_revision", "dataflow.revision"),
|
||||||
|
"decision_set_id": ("dataflow.decision_set",),
|
||||||
|
"decision_id": ("dataflow.decision",),
|
||||||
|
"run_id": ("dataflow.run",),
|
||||||
|
"deployment_id": ("dataflow.deployment",),
|
||||||
|
"trigger_id": ("dataflow.trigger",),
|
||||||
|
"delivery_id": ("dataflow.trigger_delivery", "dataflow.delivery"),
|
||||||
|
}
|
||||||
|
_RESOURCE_MODELS = {
|
||||||
|
"dataflow_pipeline": DataflowPipeline,
|
||||||
|
"dataflow_pipeline_revision": DataflowPipelineRevision,
|
||||||
|
"dataflow_decision_set": DataflowReconciliationDecisionSet,
|
||||||
|
"dataflow_decision": DataflowReconciliationDecision,
|
||||||
|
"dataflow_run": DataflowRun,
|
||||||
|
"dataflow_deployment": DataflowPipelineDeployment,
|
||||||
|
"dataflow_trigger": DataflowTrigger,
|
||||||
|
"dataflow_trigger_delivery": DataflowTriggerDelivery,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _Selectors:
|
||||||
|
account_id: str | None
|
||||||
|
identity_id: str | None
|
||||||
|
membership_id: str | None
|
||||||
|
direct: dict[str, str]
|
||||||
|
|
||||||
|
@property
|
||||||
|
def actor_ids(self) -> tuple[str, ...]:
|
||||||
|
return tuple(
|
||||||
|
value
|
||||||
|
for value in (self.account_id, self.identity_id, self.membership_id)
|
||||||
|
if value
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class _Match:
|
||||||
|
resource_type: str
|
||||||
|
row: Any
|
||||||
|
category: str
|
||||||
|
|
||||||
|
|
||||||
|
class DataflowDsarProvider:
|
||||||
|
provider_id = "dataflow"
|
||||||
|
module_id = "dataflow"
|
||||||
|
|
||||||
|
def search_subject(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
) -> Sequence[DsarRecordRef]:
|
||||||
|
db = _session(session)
|
||||||
|
selectors = _selectors(subject)
|
||||||
|
if selectors is None or not (selectors.actor_ids or selectors.direct):
|
||||||
|
return ()
|
||||||
|
direct = _direct_matches(db, tenant_id=tenant_id, selectors=selectors)
|
||||||
|
if direct is None:
|
||||||
|
return ()
|
||||||
|
if direct:
|
||||||
|
if selectors.actor_ids and not all(
|
||||||
|
_correlates(
|
||||||
|
db,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
match=match,
|
||||||
|
actor_ids=selectors.actor_ids,
|
||||||
|
)
|
||||||
|
for match in direct
|
||||||
|
):
|
||||||
|
return ()
|
||||||
|
matches = direct
|
||||||
|
else:
|
||||||
|
matches = _canonical_matches(
|
||||||
|
db,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
actor_ids=selectors.actor_ids,
|
||||||
|
)
|
||||||
|
|
||||||
|
records: list[DsarRecordRef] = []
|
||||||
|
seen: set[tuple[str, str]] = set()
|
||||||
|
for match in matches:
|
||||||
|
key = (match.resource_type, str(match.row.id))
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
if len(records) >= _MAX_RECORDS:
|
||||||
|
raise ValueError(
|
||||||
|
"Dataflow DSAR result limit exceeded; narrow the selectors."
|
||||||
|
)
|
||||||
|
seen.add(key)
|
||||||
|
records.append(_record(match))
|
||||||
|
return tuple(records)
|
||||||
|
|
||||||
|
def plan_erasure(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
records: Sequence[DsarRecordRef],
|
||||||
|
) -> Sequence[DsarErasureActionRef]:
|
||||||
|
del tenant_id
|
||||||
|
_session(session)
|
||||||
|
if _selectors(subject) is None:
|
||||||
|
raise ValueError("Dataflow DSAR subject selectors conflict.")
|
||||||
|
actions: list[DsarErasureActionRef] = []
|
||||||
|
for record in records:
|
||||||
|
_validate_record(record)
|
||||||
|
kind = _planned_kind(record)
|
||||||
|
executable = kind in {"anonymize", "revoke"}
|
||||||
|
actions.append(
|
||||||
|
DsarErasureActionRef(
|
||||||
|
action_id=(
|
||||||
|
f"dataflow:{kind}:{record.resource_type}:{record.resource_id}"
|
||||||
|
),
|
||||||
|
provider_id=self.provider_id,
|
||||||
|
module_id=self.module_id,
|
||||||
|
kind=kind,
|
||||||
|
resource_type=record.resource_type,
|
||||||
|
resource_id=record.resource_id,
|
||||||
|
title=(
|
||||||
|
f"Minimize {record.title}"
|
||||||
|
if kind == "anonymize"
|
||||||
|
else f"{kind.replace('_', ' ').title()} {record.title}"
|
||||||
|
),
|
||||||
|
rationale=_rationale(record, kind=kind),
|
||||||
|
executable=executable,
|
||||||
|
irreversible=kind == "anonymize",
|
||||||
|
metadata={"record_category": record.category},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(actions)
|
||||||
|
|
||||||
|
def execute_erasure(
|
||||||
|
self,
|
||||||
|
session: object,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
subject: DsarSubjectRef,
|
||||||
|
actions: Sequence[DsarErasureActionRef],
|
||||||
|
request_id: str,
|
||||||
|
) -> Sequence[DsarExecutionResultRef]:
|
||||||
|
db = _session(session)
|
||||||
|
selectors = _selectors(subject)
|
||||||
|
if selectors is None:
|
||||||
|
raise ValueError("Dataflow DSAR subject selectors conflict.")
|
||||||
|
results: list[DsarExecutionResultRef] = []
|
||||||
|
for action in actions:
|
||||||
|
_validate_action(action)
|
||||||
|
if not action.executable:
|
||||||
|
results.append(
|
||||||
|
DsarExecutionResultRef(
|
||||||
|
action_id=action.action_id,
|
||||||
|
status="blocked",
|
||||||
|
summary=(
|
||||||
|
"Review pipeline dependencies, published outputs, "
|
||||||
|
"retention, and institutional evidence."
|
||||||
|
),
|
||||||
|
evidence={"request_id": request_id},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
model = _RESOURCE_MODELS[action.resource_type]
|
||||||
|
row = (
|
||||||
|
db.query(model)
|
||||||
|
.filter(model.tenant_id == tenant_id, model.id == action.resource_id)
|
||||||
|
.with_for_update()
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
if row is None:
|
||||||
|
status = "unchanged"
|
||||||
|
summary = "Dataflow row was already absent or minimized."
|
||||||
|
else:
|
||||||
|
match = _Match(action.resource_type, row, "execution")
|
||||||
|
if not (
|
||||||
|
_directly_targets(selectors, match)
|
||||||
|
or _correlates(
|
||||||
|
db,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
match=match,
|
||||||
|
actor_ids=selectors.actor_ids,
|
||||||
|
)
|
||||||
|
or _already_revoked(action.resource_type, row)
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"Dataflow DSAR action is not corroborated by the subject."
|
||||||
|
)
|
||||||
|
status, summary = _execute_action(
|
||||||
|
db,
|
||||||
|
resource_type=action.resource_type,
|
||||||
|
row=row,
|
||||||
|
kind=action.kind,
|
||||||
|
)
|
||||||
|
results.append(
|
||||||
|
DsarExecutionResultRef(
|
||||||
|
action_id=action.action_id,
|
||||||
|
status=status,
|
||||||
|
summary=summary,
|
||||||
|
evidence={"request_id": request_id},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return tuple(results)
|
||||||
|
|
||||||
|
|
||||||
|
def _direct_matches(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
selectors: _Selectors,
|
||||||
|
) -> list[_Match] | None:
|
||||||
|
matches: list[_Match] = []
|
||||||
|
for selector, raw_value in selectors.direct.items():
|
||||||
|
value = _strip_prefix(raw_value)
|
||||||
|
if selector == "pipeline_id":
|
||||||
|
pipeline = _one(session, DataflowPipeline, tenant_id, value)
|
||||||
|
if pipeline is None:
|
||||||
|
return None
|
||||||
|
current = _pipeline_package(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
pipeline=pipeline,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
model, resource_type = {
|
||||||
|
"revision_id": (
|
||||||
|
DataflowPipelineRevision,
|
||||||
|
"dataflow_pipeline_revision",
|
||||||
|
),
|
||||||
|
"decision_set_id": (
|
||||||
|
DataflowReconciliationDecisionSet,
|
||||||
|
"dataflow_decision_set",
|
||||||
|
),
|
||||||
|
"decision_id": (
|
||||||
|
DataflowReconciliationDecision,
|
||||||
|
"dataflow_decision",
|
||||||
|
),
|
||||||
|
"run_id": (DataflowRun, "dataflow_run"),
|
||||||
|
"deployment_id": (
|
||||||
|
DataflowPipelineDeployment,
|
||||||
|
"dataflow_deployment",
|
||||||
|
),
|
||||||
|
"trigger_id": (DataflowTrigger, "dataflow_trigger"),
|
||||||
|
"delivery_id": (
|
||||||
|
DataflowTriggerDelivery,
|
||||||
|
"dataflow_trigger_delivery",
|
||||||
|
),
|
||||||
|
}[selector]
|
||||||
|
row = _one(session, model, tenant_id, value)
|
||||||
|
if row is None:
|
||||||
|
return None
|
||||||
|
current = [_Match(resource_type, row, _direct_category(resource_type, row))]
|
||||||
|
matches.extend(current)
|
||||||
|
if len(matches) > _MAX_RECORDS:
|
||||||
|
raise ValueError(
|
||||||
|
"Dataflow DSAR result limit exceeded; narrow the selectors."
|
||||||
|
)
|
||||||
|
return matches
|
||||||
|
|
||||||
|
|
||||||
|
def _pipeline_package(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
pipeline: DataflowPipeline,
|
||||||
|
) -> list[_Match]:
|
||||||
|
matches = [_Match("dataflow_pipeline", pipeline, "dataflow_configuration")]
|
||||||
|
specs = (
|
||||||
|
(DataflowPipelineRevision, "dataflow_pipeline_revision"),
|
||||||
|
(DataflowReconciliationDecisionSet, "dataflow_decision_set"),
|
||||||
|
(DataflowRun, "dataflow_run"),
|
||||||
|
(DataflowPipelineDeployment, "dataflow_deployment"),
|
||||||
|
(DataflowTrigger, "dataflow_trigger"),
|
||||||
|
(DataflowTriggerDelivery, "dataflow_trigger_delivery"),
|
||||||
|
)
|
||||||
|
decision_sets: list[DataflowReconciliationDecisionSet] = []
|
||||||
|
for model, resource_type in specs:
|
||||||
|
rows = (
|
||||||
|
session.query(model)
|
||||||
|
.filter(model.tenant_id == tenant_id, model.pipeline_id == pipeline.id)
|
||||||
|
.order_by(model.id)
|
||||||
|
.limit(_MAX_RECORDS + 1)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
if model is DataflowReconciliationDecisionSet:
|
||||||
|
decision_sets = rows
|
||||||
|
matches.extend(
|
||||||
|
_Match(resource_type, row, "pipeline_related_state") for row in rows
|
||||||
|
)
|
||||||
|
decision_set_ids = {row.id for row in decision_sets}
|
||||||
|
if decision_set_ids:
|
||||||
|
decisions = (
|
||||||
|
session.query(DataflowReconciliationDecision)
|
||||||
|
.filter(
|
||||||
|
DataflowReconciliationDecision.tenant_id == tenant_id,
|
||||||
|
DataflowReconciliationDecision.decision_set_id.in_(decision_set_ids),
|
||||||
|
)
|
||||||
|
.order_by(DataflowReconciliationDecision.id)
|
||||||
|
.limit(_MAX_RECORDS + 1)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
matches.extend(
|
||||||
|
_Match("dataflow_decision", row, "pipeline_related_state")
|
||||||
|
for row in decisions
|
||||||
|
)
|
||||||
|
return matches
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_matches(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
actor_ids: tuple[str, ...],
|
||||||
|
) -> list[_Match]:
|
||||||
|
if not actor_ids:
|
||||||
|
return []
|
||||||
|
triggers = (
|
||||||
|
session.query(DataflowTrigger)
|
||||||
|
.filter(
|
||||||
|
DataflowTrigger.tenant_id == tenant_id,
|
||||||
|
or_(
|
||||||
|
DataflowTrigger.authorization_account_id.in_(actor_ids),
|
||||||
|
DataflowTrigger.authorization_membership_id.in_(actor_ids),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.order_by(DataflowTrigger.id)
|
||||||
|
.limit(_MAX_RECORDS + 1)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
matches = [
|
||||||
|
_Match("dataflow_trigger", row, "dataflow_automation_authority")
|
||||||
|
for row in triggers
|
||||||
|
]
|
||||||
|
specs = (
|
||||||
|
(
|
||||||
|
DataflowPipeline,
|
||||||
|
or_(
|
||||||
|
DataflowPipeline.created_by.in_(actor_ids),
|
||||||
|
DataflowPipeline.updated_by.in_(actor_ids),
|
||||||
|
),
|
||||||
|
"dataflow_pipeline",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
DataflowPipelineRevision,
|
||||||
|
DataflowPipelineRevision.created_by.in_(actor_ids),
|
||||||
|
"dataflow_pipeline_revision",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
DataflowReconciliationDecisionSet,
|
||||||
|
or_(
|
||||||
|
DataflowReconciliationDecisionSet.created_by.in_(actor_ids),
|
||||||
|
DataflowReconciliationDecisionSet.updated_by.in_(actor_ids),
|
||||||
|
),
|
||||||
|
"dataflow_decision_set",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
DataflowReconciliationDecision,
|
||||||
|
DataflowReconciliationDecision.actor_ref.in_(actor_ids),
|
||||||
|
"dataflow_decision",
|
||||||
|
),
|
||||||
|
(DataflowRun, DataflowRun.created_by.in_(actor_ids), "dataflow_run"),
|
||||||
|
(
|
||||||
|
DataflowPipelineDeployment,
|
||||||
|
DataflowPipelineDeployment.promoted_by.in_(actor_ids),
|
||||||
|
"dataflow_deployment",
|
||||||
|
),
|
||||||
|
(
|
||||||
|
DataflowTrigger,
|
||||||
|
or_(
|
||||||
|
DataflowTrigger.created_by.in_(actor_ids),
|
||||||
|
DataflowTrigger.updated_by.in_(actor_ids),
|
||||||
|
),
|
||||||
|
"dataflow_trigger",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for model, condition, resource_type in specs:
|
||||||
|
rows = (
|
||||||
|
session.query(model)
|
||||||
|
.filter(model.tenant_id == tenant_id, condition)
|
||||||
|
.order_by(model.id)
|
||||||
|
.limit(_MAX_RECORDS + 1)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
matches.extend(
|
||||||
|
_Match(resource_type, row, "dataflow_operator_attribution") for row in rows
|
||||||
|
)
|
||||||
|
if len(matches) > _MAX_RECORDS:
|
||||||
|
raise ValueError(
|
||||||
|
"Dataflow DSAR result limit exceeded; narrow the selectors."
|
||||||
|
)
|
||||||
|
return matches
|
||||||
|
|
||||||
|
|
||||||
|
def _one(session: Session, model: Any, tenant_id: str, row_id: str) -> Any | None:
|
||||||
|
return (
|
||||||
|
session.query(model)
|
||||||
|
.filter(model.tenant_id == tenant_id, model.id == row_id)
|
||||||
|
.one_or_none()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _direct_category(resource_type: str, row: Any) -> str:
|
||||||
|
if resource_type == "dataflow_run":
|
||||||
|
return (
|
||||||
|
"terminal_dataflow_run"
|
||||||
|
if row.status in {"succeeded", "failed", "cancelled", "outcome_unknown"}
|
||||||
|
else "active_dataflow_run"
|
||||||
|
)
|
||||||
|
if resource_type == "dataflow_trigger_delivery":
|
||||||
|
return (
|
||||||
|
"terminal_trigger_delivery"
|
||||||
|
if row.status in {"succeeded", "failed", "skipped", "cancelled"}
|
||||||
|
else "active_trigger_delivery"
|
||||||
|
)
|
||||||
|
return "dataflow_configuration"
|
||||||
|
|
||||||
|
|
||||||
|
def _correlates(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
match: _Match,
|
||||||
|
actor_ids: tuple[str, ...],
|
||||||
|
) -> bool:
|
||||||
|
row = match.row
|
||||||
|
if any(
|
||||||
|
str(getattr(row, field, "") or "") in actor_ids
|
||||||
|
for field in (
|
||||||
|
"created_by",
|
||||||
|
"updated_by",
|
||||||
|
"promoted_by",
|
||||||
|
"actor_ref",
|
||||||
|
"authorization_account_id",
|
||||||
|
"authorization_membership_id",
|
||||||
|
)
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
if match.resource_type == "dataflow_trigger_delivery":
|
||||||
|
trigger = session.get(DataflowTrigger, row.trigger_id)
|
||||||
|
if (
|
||||||
|
trigger
|
||||||
|
and trigger.tenant_id == tenant_id
|
||||||
|
and any(
|
||||||
|
str(getattr(trigger, field, "") or "") in actor_ids
|
||||||
|
for field in (
|
||||||
|
"created_by",
|
||||||
|
"updated_by",
|
||||||
|
"authorization_account_id",
|
||||||
|
"authorization_membership_id",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
pipeline_id = getattr(row, "pipeline_id", None)
|
||||||
|
if not pipeline_id and match.resource_type == "dataflow_decision":
|
||||||
|
decision_set = session.get(
|
||||||
|
DataflowReconciliationDecisionSet,
|
||||||
|
row.decision_set_id,
|
||||||
|
)
|
||||||
|
pipeline_id = decision_set.pipeline_id if decision_set else None
|
||||||
|
if pipeline_id:
|
||||||
|
pipeline = session.get(DataflowPipeline, pipeline_id)
|
||||||
|
return bool(
|
||||||
|
pipeline
|
||||||
|
and pipeline.tenant_id == tenant_id
|
||||||
|
and (pipeline.created_by in actor_ids or pipeline.updated_by in actor_ids)
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _directly_targets(selectors: _Selectors, match: _Match) -> bool:
|
||||||
|
selector = {
|
||||||
|
"dataflow_pipeline": "pipeline_id",
|
||||||
|
"dataflow_pipeline_revision": "revision_id",
|
||||||
|
"dataflow_decision_set": "decision_set_id",
|
||||||
|
"dataflow_decision": "decision_id",
|
||||||
|
"dataflow_run": "run_id",
|
||||||
|
"dataflow_deployment": "deployment_id",
|
||||||
|
"dataflow_trigger": "trigger_id",
|
||||||
|
"dataflow_trigger_delivery": "delivery_id",
|
||||||
|
}[match.resource_type]
|
||||||
|
if _strip_prefix(selectors.direct.get(selector, "")) == str(match.row.id):
|
||||||
|
return True
|
||||||
|
pipeline_id = _strip_prefix(selectors.direct.get("pipeline_id", ""))
|
||||||
|
return bool(
|
||||||
|
pipeline_id
|
||||||
|
and (
|
||||||
|
(match.resource_type == "dataflow_pipeline" and match.row.id == pipeline_id)
|
||||||
|
or getattr(match.row, "pipeline_id", None) == pipeline_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _record(match: _Match) -> DsarRecordRef:
|
||||||
|
row = match.row
|
||||||
|
immutable = match.category == "dataflow_operator_attribution"
|
||||||
|
return DsarRecordRef(
|
||||||
|
provider_id="dataflow",
|
||||||
|
module_id="dataflow",
|
||||||
|
resource_type=match.resource_type,
|
||||||
|
resource_id=str(row.id),
|
||||||
|
category=match.category,
|
||||||
|
title=match.resource_type.removeprefix("dataflow_").replace("_", " ").title(),
|
||||||
|
data={
|
||||||
|
key: value
|
||||||
|
for key, value in _record_data(match.resource_type, row).items()
|
||||||
|
if value is not None
|
||||||
|
},
|
||||||
|
observed_at=_observed_at(row),
|
||||||
|
immutable_evidence=immutable,
|
||||||
|
retention_reason=(
|
||||||
|
"Institutional Dataflow authorship, decisions, runs, and deployment "
|
||||||
|
"activity remain attributable for governance and audit."
|
||||||
|
if immutable
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
source_path="/dataflow",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _record_data(resource_type: str, row: Any) -> dict[str, object]:
|
||||||
|
if resource_type == "dataflow_pipeline":
|
||||||
|
return {
|
||||||
|
"scope_type": row.scope_type,
|
||||||
|
"definition_kind": row.definition_kind,
|
||||||
|
"status": row.status,
|
||||||
|
"current_revision": row.current_revision,
|
||||||
|
"allow_run": row.allow_run,
|
||||||
|
"allow_reuse": row.allow_reuse,
|
||||||
|
"allow_automation": row.allow_automation,
|
||||||
|
"deleted_at": _iso(row.deleted_at),
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
"updated_at": _iso(row.updated_at),
|
||||||
|
}
|
||||||
|
if resource_type == "dataflow_pipeline_revision":
|
||||||
|
return {
|
||||||
|
"revision": row.revision,
|
||||||
|
"schema_version": row.schema_version,
|
||||||
|
"editor_mode": row.editor_mode,
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
}
|
||||||
|
if resource_type == "dataflow_decision_set":
|
||||||
|
return {
|
||||||
|
"resource_revision": row.resource_revision,
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
"updated_at": _iso(row.updated_at),
|
||||||
|
}
|
||||||
|
if resource_type == "dataflow_decision":
|
||||||
|
return {
|
||||||
|
"revision": row.revision,
|
||||||
|
"action": row.action,
|
||||||
|
"decided_at": _iso(row.decided_at),
|
||||||
|
}
|
||||||
|
if resource_type == "dataflow_run":
|
||||||
|
return {
|
||||||
|
"run_type": row.run_type,
|
||||||
|
"status": row.status,
|
||||||
|
"execution_backend": row.execution_backend,
|
||||||
|
"environment": row.environment,
|
||||||
|
"invocation_kind": row.invocation_kind,
|
||||||
|
"input_row_count": row.input_row_count,
|
||||||
|
"output_row_count": row.output_row_count,
|
||||||
|
"attempts": row.attempts,
|
||||||
|
"progress_percent": row.progress_percent,
|
||||||
|
"purged_at": _iso(row.purged_at),
|
||||||
|
"started_at": _iso(row.started_at),
|
||||||
|
"finished_at": _iso(row.finished_at),
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
}
|
||||||
|
if resource_type == "dataflow_deployment":
|
||||||
|
return {
|
||||||
|
"environment": row.environment,
|
||||||
|
"source_environment": row.source_environment,
|
||||||
|
"status": row.status,
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
"updated_at": _iso(row.updated_at),
|
||||||
|
}
|
||||||
|
if resource_type == "dataflow_trigger":
|
||||||
|
return {
|
||||||
|
"kind": row.kind,
|
||||||
|
"status": row.status,
|
||||||
|
"revision": row.revision,
|
||||||
|
"catch_up_policy": row.catch_up_policy,
|
||||||
|
"max_concurrent_runs": row.max_concurrent_runs,
|
||||||
|
"next_fire_at": _iso(row.next_fire_at),
|
||||||
|
"last_fire_at": _iso(row.last_fire_at),
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
"updated_at": _iso(row.updated_at),
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"invocation_kind": row.invocation_kind,
|
||||||
|
"status": row.status,
|
||||||
|
"attempts": row.attempts,
|
||||||
|
"scheduled_for": _iso(row.scheduled_for),
|
||||||
|
"finished_at": _iso(row.finished_at),
|
||||||
|
"created_at": _iso(row.created_at),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _planned_kind(record: DsarRecordRef) -> str:
|
||||||
|
if record.category in {"terminal_dataflow_run", "terminal_trigger_delivery"}:
|
||||||
|
return "anonymize"
|
||||||
|
if record.category == "dataflow_automation_authority":
|
||||||
|
return "revoke"
|
||||||
|
if record.category == "dataflow_operator_attribution":
|
||||||
|
return "retain"
|
||||||
|
return "manual_review"
|
||||||
|
|
||||||
|
|
||||||
|
def _rationale(record: DsarRecordRef, *, kind: str) -> str:
|
||||||
|
if kind == "anonymize":
|
||||||
|
return (
|
||||||
|
"Clear retained request, event, authorization, diagnostic, and error "
|
||||||
|
"detail while preserving minimal run or delivery evidence."
|
||||||
|
)
|
||||||
|
if kind == "revoke":
|
||||||
|
return (
|
||||||
|
"Disable automation and remove the subject-linked authorization "
|
||||||
|
"snapshot without deleting historical delivery evidence."
|
||||||
|
)
|
||||||
|
if kind == "retain":
|
||||||
|
return record.retention_reason or "Retain institutional attribution evidence."
|
||||||
|
return (
|
||||||
|
"A Dataflow owner must review definitions, reconciliation decisions, "
|
||||||
|
"active work, published outputs, and downstream dependencies."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _execute_action(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
resource_type: str,
|
||||||
|
row: Any,
|
||||||
|
kind: str,
|
||||||
|
) -> tuple[str, str]:
|
||||||
|
if resource_type == "dataflow_run" and kind == "anonymize":
|
||||||
|
if row.status not in {"succeeded", "failed", "cancelled", "outcome_unknown"}:
|
||||||
|
raise ValueError("Active Dataflow runs require manual review.")
|
||||||
|
authorization = dict(row.authorization_ or {})
|
||||||
|
authorization.pop("submitted_principal", None)
|
||||||
|
authorization["personal_data_purged"] = True
|
||||||
|
changed = _replace_fields(
|
||||||
|
row,
|
||||||
|
{
|
||||||
|
"request_": {},
|
||||||
|
"source_fingerprints": [],
|
||||||
|
"diagnostics": [],
|
||||||
|
"authorization_": authorization,
|
||||||
|
"correlation_id": None,
|
||||||
|
"causation_id": None,
|
||||||
|
"error": None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if row.purged_at is None:
|
||||||
|
row.purged_at = datetime.now(timezone.utc)
|
||||||
|
changed = True
|
||||||
|
elif resource_type == "dataflow_trigger_delivery" and kind == "anonymize":
|
||||||
|
if row.status not in {"succeeded", "failed", "skipped", "cancelled"}:
|
||||||
|
raise ValueError("Active Dataflow deliveries require manual review.")
|
||||||
|
changed = _replace_fields(
|
||||||
|
row,
|
||||||
|
{
|
||||||
|
"event_": None,
|
||||||
|
"authorization_provenance": {},
|
||||||
|
"error": None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
elif resource_type == "dataflow_trigger" and kind == "revoke":
|
||||||
|
changed = _replace_fields(
|
||||||
|
row,
|
||||||
|
{
|
||||||
|
"status": "disabled",
|
||||||
|
"authorization_account_id": "redacted",
|
||||||
|
"authorization_membership_id": "redacted",
|
||||||
|
"authorization_ref": f"redacted:{row.id}",
|
||||||
|
"grant_scopes": [],
|
||||||
|
"config_": {},
|
||||||
|
"publication_": None,
|
||||||
|
"last_error": None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError("Dataflow DSAR executable action is unsupported.")
|
||||||
|
if changed:
|
||||||
|
session.flush()
|
||||||
|
return "executed", "Personal Dataflow detail minimized."
|
||||||
|
return "unchanged", "Personal Dataflow detail was already minimized."
|
||||||
|
|
||||||
|
|
||||||
|
def _replace_fields(row: Any, values: dict[str, object]) -> bool:
|
||||||
|
changed = False
|
||||||
|
for field, value in values.items():
|
||||||
|
if getattr(row, field) != value:
|
||||||
|
setattr(row, field, value)
|
||||||
|
changed = True
|
||||||
|
return changed
|
||||||
|
|
||||||
|
|
||||||
|
def _already_revoked(resource_type: str, row: Any) -> bool:
|
||||||
|
return bool(
|
||||||
|
resource_type == "dataflow_trigger"
|
||||||
|
and row.status == "disabled"
|
||||||
|
and row.authorization_account_id == "redacted"
|
||||||
|
and row.authorization_membership_id == "redacted"
|
||||||
|
and row.authorization_ref == f"redacted:{row.id}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _selectors(subject: DsarSubjectRef) -> _Selectors | None:
|
||||||
|
refs = subject.external_references
|
||||||
|
canonical = (
|
||||||
|
_coalesce(
|
||||||
|
subject.account_id,
|
||||||
|
refs.get("dataflow.account"),
|
||||||
|
refs.get("access.account"),
|
||||||
|
),
|
||||||
|
_coalesce(
|
||||||
|
subject.identity_id,
|
||||||
|
refs.get("dataflow.identity"),
|
||||||
|
refs.get("identity.id"),
|
||||||
|
),
|
||||||
|
_coalesce(
|
||||||
|
subject.membership_id,
|
||||||
|
refs.get("dataflow.membership"),
|
||||||
|
refs.get("tenancy.membership"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if any(value is _CONFLICT for value in canonical):
|
||||||
|
return None
|
||||||
|
direct: dict[str, str] = {}
|
||||||
|
for selector, aliases in _DIRECT_ALIASES.items():
|
||||||
|
value = _coalesce(*(refs.get(alias) for alias in aliases))
|
||||||
|
if value is _CONFLICT:
|
||||||
|
return None
|
||||||
|
if value:
|
||||||
|
direct[selector] = str(value)
|
||||||
|
return _Selectors(
|
||||||
|
account_id=_optional(canonical[0]),
|
||||||
|
identity_id=_optional(canonical[1]),
|
||||||
|
membership_id=_optional(canonical[2]),
|
||||||
|
direct=direct,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _coalesce(*values: str | None) -> str | None | object:
|
||||||
|
normalized = {str(value).strip() for value in values if str(value or "").strip()}
|
||||||
|
if len(normalized) > 1:
|
||||||
|
return _CONFLICT
|
||||||
|
return next(iter(normalized), None)
|
||||||
|
|
||||||
|
|
||||||
|
def _optional(value: object) -> str | None:
|
||||||
|
return value if isinstance(value, str) and value else None
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_prefix(value: str) -> str:
|
||||||
|
return value.partition(":")[2] if ":" in value else value
|
||||||
|
|
||||||
|
|
||||||
|
def _observed_at(row: Any) -> datetime | None:
|
||||||
|
for field in ("decided_at", "finished_at", "updated_at", "created_at"):
|
||||||
|
value = getattr(row, field, None)
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
return _aware(value)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _iso(value: datetime | None) -> str | None:
|
||||||
|
aware = _aware(value)
|
||||||
|
return aware.isoformat() if aware else None
|
||||||
|
|
||||||
|
|
||||||
|
def _aware(value: datetime | None) -> datetime | None:
|
||||||
|
if value is None or value.tzinfo is not None:
|
||||||
|
return value
|
||||||
|
return value.replace(tzinfo=timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
|
def _session(value: object) -> Session:
|
||||||
|
if not isinstance(value, Session):
|
||||||
|
raise TypeError("Dataflow DSAR requires a SQLAlchemy Session.")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_record(record: DsarRecordRef) -> None:
|
||||||
|
if record.provider_id != "dataflow" or record.module_id != "dataflow":
|
||||||
|
raise ValueError("Dataflow DSAR cannot plan a foreign provider record.")
|
||||||
|
if record.resource_type not in _RESOURCE_MODELS or not record.resource_id:
|
||||||
|
raise ValueError("Dataflow DSAR record identity is invalid.")
|
||||||
|
|
||||||
|
|
||||||
|
def _validate_action(action: DsarErasureActionRef) -> None:
|
||||||
|
if action.provider_id != "dataflow" or action.module_id != "dataflow":
|
||||||
|
raise ValueError("Dataflow DSAR cannot execute a foreign provider action.")
|
||||||
|
if action.resource_type not in _RESOURCE_MODELS or not action.action_id.startswith(
|
||||||
|
"dataflow:"
|
||||||
|
):
|
||||||
|
raise ValueError("Dataflow DSAR action identity is invalid.")
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["DATAFLOW_DSAR_CAPABILITY", "DataflowDsarProvider"]
|
||||||
@@ -17,6 +17,7 @@ from govoplan_core.core.dataflows import (
|
|||||||
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER,
|
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER,
|
||||||
)
|
)
|
||||||
from govoplan_core.core.modules import (
|
from govoplan_core.core.modules import (
|
||||||
|
CapabilityDocumentation,
|
||||||
DocumentationTopic,
|
DocumentationTopic,
|
||||||
FrontendModule,
|
FrontendModule,
|
||||||
FrontendRoute,
|
FrontendRoute,
|
||||||
@@ -27,6 +28,7 @@ from govoplan_core.core.modules import (
|
|||||||
ModuleManifest,
|
ModuleManifest,
|
||||||
NavItem,
|
NavItem,
|
||||||
PermissionDefinition,
|
PermissionDefinition,
|
||||||
|
ProductAreaContribution,
|
||||||
RoleTemplate,
|
RoleTemplate,
|
||||||
)
|
)
|
||||||
from govoplan_core.core.provider_governance import declared_module_architecture
|
from govoplan_core.core.provider_governance import declared_module_architecture
|
||||||
@@ -46,11 +48,15 @@ from govoplan_core.core.search import SearchSourceProviderRegistration
|
|||||||
from govoplan_core.core.views import ViewSurface
|
from govoplan_core.core.views import ViewSurface
|
||||||
from govoplan_core.db.base import Base
|
from govoplan_core.db.base import Base
|
||||||
from govoplan_dataflow.backend.db import models as dataflow_models
|
from govoplan_dataflow.backend.db import models as dataflow_models
|
||||||
|
from govoplan_dataflow.backend.dsar_provider import (
|
||||||
|
DATAFLOW_DSAR_CAPABILITY,
|
||||||
|
DataflowDsarProvider,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
MODULE_ID = "dataflow"
|
MODULE_ID = "dataflow"
|
||||||
MODULE_NAME = "Dataflow"
|
MODULE_NAME = "Dataflow"
|
||||||
MODULE_VERSION = "0.1.15"
|
MODULE_VERSION = "0.1.19"
|
||||||
|
|
||||||
READ_SCOPE = "dataflow:pipeline:read"
|
READ_SCOPE = "dataflow:pipeline:read"
|
||||||
WRITE_SCOPE = "dataflow:pipeline:write"
|
WRITE_SCOPE = "dataflow:pipeline:write"
|
||||||
@@ -141,6 +147,27 @@ ROLE_TEMPLATES = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
DOCUMENTATION = (
|
DOCUMENTATION = (
|
||||||
|
DocumentationTopic(
|
||||||
|
id="dataflow.data-subject-requests",
|
||||||
|
title="Dataflow data-subject requests",
|
||||||
|
summary="Minimize retained transformation detail without treating derived flows as authoritative subject records.",
|
||||||
|
body=(
|
||||||
|
"Dataflow matches exact tenant-scoped pipeline, revision, reconciliation, run, deployment, trigger, and delivery identifiers plus minimized account, identity, and membership attribution. Results never copy graphs, SQL, request or event payloads, reconciliation corrections, authorization snapshots, provenance bodies, errors, source details, hashes, credentials, or output rows. Dataflow does not scan arbitrary transformation content for a person; the authoritative input module must locate and correct subject facts. "
|
||||||
|
"Explicitly identified terminal run and delivery detail can be minimized idempotently, and automation authority linked to the subject can be disabled and revoked. Definitions, reconciliation evidence, active work, deployments, broad pipeline packages, published Datasource outputs, and institutional attribution require authorized review or retention. Correct sources and refresh Datasource, Search, and Reporting derivatives after review."
|
||||||
|
),
|
||||||
|
layer="configured",
|
||||||
|
documentation_types=("admin", "user"),
|
||||||
|
audience=("user", "operator", "module_admin", "data_steward", "auditor"),
|
||||||
|
order=74,
|
||||||
|
related_modules=("core", "datasources", "reporting", "workflow_engine"),
|
||||||
|
metadata={
|
||||||
|
"help_contexts": [
|
||||||
|
"dataflow.data-subject-requests",
|
||||||
|
"dataflow.runs",
|
||||||
|
"dataflow.triggers",
|
||||||
|
],
|
||||||
|
},
|
||||||
|
),
|
||||||
DocumentationTopic(
|
DocumentationTopic(
|
||||||
id="dataflow.module-boundary",
|
id="dataflow.module-boundary",
|
||||||
title="Dataflow module boundary",
|
title="Dataflow module boundary",
|
||||||
@@ -200,7 +227,12 @@ DOCUMENTATION = (
|
|||||||
"Every graph node declares typed inputs, configuration, output schema, and validation rules. "
|
"Every graph node declares typed inputs, configuration, output schema, and validation rules. "
|
||||||
"Source nodes pin inline content or governed Datasource references; combine, filter, transform, "
|
"Source nodes pin inline content or governed Datasource references; combine, filter, transform, "
|
||||||
"quality, reconciliation, reusable-subflow, and output nodes remain explicit in the canonical graph. "
|
"quality, reconciliation, reusable-subflow, and output nodes remain explicit in the canonical graph. "
|
||||||
"Reconciliation rows expose stable key hashes, explicit before/after values, and input hashes. A separate decision-table input can annotate exact matches, invalidate changed inputs, and report orphaned decisions without silently rewriting business data. "
|
"Reconciliation rows expose stable key hashes, explicit before/after values, and input hashes. The "
|
||||||
|
"review dialog records accept, reject, correct, or defer decisions in tenant-owned immutable decision "
|
||||||
|
"sets. Their current projection is a fingerprinted Dataflow source; every superseded revision retains "
|
||||||
|
"the actor, reason, exact input hash, time, and optional correction. A reconcile.decisions node can "
|
||||||
|
"annotate exact matches, invalidate changed inputs, and report orphaned decisions without silently "
|
||||||
|
"rewriting business data. "
|
||||||
"Expressions use the typed Dataflow expression language and never execute arbitrary host or database "
|
"Expressions use the typed Dataflow expression language and never execute arbitrary host or database "
|
||||||
"code. Selecting a node may request a bounded intermediate preview; preview rows are transient, "
|
"code. Selecting a node may request a bounded intermediate preview; preview rows are transient, "
|
||||||
"privacy-filtered for the actor, and are not retained as run output. SQL editing compiles into the same "
|
"privacy-filtered for the actor, and are not retained as run output. SQL editing compiles into the same "
|
||||||
@@ -218,6 +250,7 @@ DOCUMENTATION = (
|
|||||||
"dataflow.field.expression",
|
"dataflow.field.expression",
|
||||||
"dataflow.field.schema",
|
"dataflow.field.schema",
|
||||||
"dataflow.action.preview-node",
|
"dataflow.action.preview-node",
|
||||||
|
"dataflow.action.review-decisions",
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -229,9 +262,16 @@ DOCUMENTATION = (
|
|||||||
"Scope determines ownership and Policy inheritance. Templates can be derived but not run; complete "
|
"Scope determines ownership and Policy inheritance. Templates can be derived but not run; complete "
|
||||||
"flows may be previewed, revisioned, automated, and executed when effective Policy allows it. Saving "
|
"flows may be previewed, revisioned, automated, and executed when effective Policy allows it. Saving "
|
||||||
"appends an immutable revision. A scoped copy pins its source revision and content hash. Triggers pin "
|
"appends an immutable revision. A scoped copy pins its source revision and content hash. Triggers pin "
|
||||||
"the revision and authorization grant, then re-evaluate authority for every delivery. Runs create "
|
"the revision and authorization grant, then re-evaluate authority for every delivery. Allow runs is "
|
||||||
|
"the definition-level admission boundary and does not grant a caller permission. A one-time run uses "
|
||||||
|
"the configured tenant-local date and time. The missed-run policy either coalesces elapsed interval "
|
||||||
|
"occurrences into one latest delivery or skips them; it never silently replays every missed occurrence. "
|
||||||
|
"The concurrency limit bounds active deliveries for that trigger and does not increase tenant worker "
|
||||||
|
"capacity. Runs create "
|
||||||
"durable command and recovery evidence. Publishing creates a governed Datasource materialization, and "
|
"durable command and recovery evidence. Publishing creates a governed Datasource materialization, and "
|
||||||
"environment promotion changes which immutable revision is eligible for staging or production runs. "
|
"environment promotion changes which immutable revision is eligible for staging or production runs. "
|
||||||
|
"Recording a reconciliation decision uses optimistic concurrency and appends an immutable revision; "
|
||||||
|
"it never mutates the reviewed business row. "
|
||||||
"Deletion prevents future use while retained run, deployment, lineage, audit, and recovery evidence "
|
"Deletion prevents future use while retained run, deployment, lineage, audit, and recovery evidence "
|
||||||
"continues under its retention policy."
|
"continues under its retention policy."
|
||||||
),
|
),
|
||||||
@@ -239,20 +279,32 @@ DOCUMENTATION = (
|
|||||||
documentation_types=("admin", "user"),
|
documentation_types=("admin", "user"),
|
||||||
audience=("operator", "module_admin", "power_user", "product_owner"),
|
audience=("operator", "module_admin", "power_user", "product_owner"),
|
||||||
order=77,
|
order=77,
|
||||||
related_modules=("datasources", "workflow_engine", "notifications", "policy", "audit"),
|
related_modules=(
|
||||||
|
"datasources",
|
||||||
|
"workflow_engine",
|
||||||
|
"notifications",
|
||||||
|
"policy",
|
||||||
|
"audit",
|
||||||
|
),
|
||||||
metadata={
|
metadata={
|
||||||
"help_contexts": [
|
"help_contexts": [
|
||||||
"dataflow.field.scope",
|
"dataflow.field.scope",
|
||||||
"dataflow.field.definition-kind",
|
"dataflow.field.definition-kind",
|
||||||
|
"dataflow.field.allow-runs",
|
||||||
|
"dataflow.field.trigger-run-at",
|
||||||
|
"dataflow.field.trigger-missed-runs",
|
||||||
|
"dataflow.field.trigger-concurrency",
|
||||||
"dataflow.action.save",
|
"dataflow.action.save",
|
||||||
"dataflow.action.derive",
|
"dataflow.action.derive",
|
||||||
"dataflow.action.trigger",
|
"dataflow.action.trigger",
|
||||||
|
"dataflow.action.record-decision",
|
||||||
"dataflow.action.delete",
|
"dataflow.action.delete",
|
||||||
],
|
],
|
||||||
"consequence_classes": {
|
"consequence_classes": {
|
||||||
"save_revision": "Appends an immutable pipeline definition revision.",
|
"save_revision": "Appends an immutable pipeline definition revision.",
|
||||||
"derive_copy": "Creates a separately governed copy pinned to the source revision and hash.",
|
"derive_copy": "Creates a separately governed copy pinned to the source revision and hash.",
|
||||||
"configure_trigger": "Creates or changes an automation command with revision and authorization evidence.",
|
"configure_trigger": "Creates or changes an automation command with revision and authorization evidence.",
|
||||||
|
"record_decision": "Appends an actor-attributed immutable decision revision against an exact input hash.",
|
||||||
"delete_pipeline": "Prevents future use while retained evidence remains governed.",
|
"delete_pipeline": "Prevents future use while retained evidence remains governed.",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -266,7 +318,11 @@ DOCUMENTATION = (
|
|||||||
"environment, progress, cancellation, output, and recovery state. Database-only runs commit atomically. "
|
"environment, progress, cancellation, output, and recovery state. Database-only runs commit atomically. "
|
||||||
"Publication to a governed Datasource uses forward recovery: an unknown provider outcome is reconciled "
|
"Publication to a governed Datasource uses forward recovery: an unknown provider outcome is reconciled "
|
||||||
"before retry so output is not duplicated. Staging and production promotion is explicit and does not "
|
"before retry so output is not duplicated. Staging and production promotion is explicit and does not "
|
||||||
"rewrite a revision. Cancellation is best effort once external work has started; the final evidence "
|
"rewrite a revision. Freezing a published state assigns a durable label to the exact immutable output; "
|
||||||
|
"it does not copy or detach the data from Datasources retention, hold, and access rules. Artifact-backed "
|
||||||
|
"outputs return the same stable publication, datasource, and materialization references as inline "
|
||||||
|
"outputs. Datasource warnings and review-required states remain visible to Workflow instead of being "
|
||||||
|
"collapsed into success. Cancellation is best effort once external work has started; the final evidence "
|
||||||
"states whether work stopped, completed, failed, or requires operator reconciliation. Scheduled, event, "
|
"states whether work stopped, completed, failed, or requires operator reconciliation. Scheduled, event, "
|
||||||
"and queued execution is partitioned by tenant module entitlement before a run is claimed. Disabling "
|
"and queued execution is partitioned by tenant module entitlement before a run is claimed. Disabling "
|
||||||
"Dataflow stops new admission and leaves accepted runs available for an explicit operator decision."
|
"Dataflow stops new admission and leaves accepted runs available for an explicit operator decision."
|
||||||
@@ -282,6 +338,7 @@ DOCUMENTATION = (
|
|||||||
"dataflow.runs",
|
"dataflow.runs",
|
||||||
"dataflow.action.queue-run",
|
"dataflow.action.queue-run",
|
||||||
"dataflow.action.publish",
|
"dataflow.action.publish",
|
||||||
|
"dataflow.field.freeze-publication",
|
||||||
"dataflow.action.promote-staging",
|
"dataflow.action.promote-staging",
|
||||||
"dataflow.action.promote-production",
|
"dataflow.action.promote-production",
|
||||||
"dataflow.state.recovery-attention",
|
"dataflow.state.recovery-attention",
|
||||||
@@ -312,6 +369,11 @@ def _run_provider(context: ModuleContext):
|
|||||||
return SqlDataflowRunLifecycleProvider(registry=context.registry)
|
return SqlDataflowRunLifecycleProvider(registry=context.registry)
|
||||||
|
|
||||||
|
|
||||||
|
def _dsar_provider(context: ModuleContext) -> DataflowDsarProvider:
|
||||||
|
del context
|
||||||
|
return DataflowDsarProvider()
|
||||||
|
|
||||||
|
|
||||||
def _run_worker(context: ModuleContext):
|
def _run_worker(context: ModuleContext):
|
||||||
from govoplan_dataflow.backend.run_worker import SqlDataflowRunWorker
|
from govoplan_dataflow.backend.run_worker import SqlDataflowRunWorker
|
||||||
|
|
||||||
@@ -356,9 +418,20 @@ def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
|
|||||||
),
|
),
|
||||||
"dataflow_trigger_deliveries": (
|
"dataflow_trigger_deliveries": (
|
||||||
session.query(dataflow_models.DataflowTriggerDelivery)
|
session.query(dataflow_models.DataflowTriggerDelivery)
|
||||||
|
.filter(dataflow_models.DataflowTriggerDelivery.tenant_id == tenant_id)
|
||||||
|
.count()
|
||||||
|
),
|
||||||
|
"dataflow_reconciliation_decision_sets": (
|
||||||
|
session.query(dataflow_models.DataflowReconciliationDecisionSet)
|
||||||
.filter(
|
.filter(
|
||||||
dataflow_models.DataflowTriggerDelivery.tenant_id
|
dataflow_models.DataflowReconciliationDecisionSet.tenant_id == tenant_id
|
||||||
== tenant_id
|
)
|
||||||
|
.count()
|
||||||
|
),
|
||||||
|
"dataflow_reconciliation_decisions": (
|
||||||
|
session.query(dataflow_models.DataflowReconciliationDecision)
|
||||||
|
.filter(
|
||||||
|
dataflow_models.DataflowReconciliationDecision.tenant_id == tenant_id
|
||||||
)
|
)
|
||||||
.count()
|
.count()
|
||||||
),
|
),
|
||||||
@@ -393,15 +466,22 @@ manifest = ModuleManifest(
|
|||||||
CAPABILITY_POLICY_DEFINITION_GOVERNANCE,
|
CAPABILITY_POLICY_DEFINITION_GOVERNANCE,
|
||||||
),
|
),
|
||||||
provides_interfaces=(
|
provides_interfaces=(
|
||||||
ModuleInterfaceProvider(name="dataflow.pipeline_catalog", version=MODULE_VERSION),
|
ModuleInterfaceProvider(
|
||||||
ModuleInterfaceProvider(name="dataflow.pipeline_preview", version=MODULE_VERSION),
|
name="dataflow.pipeline_catalog", version=MODULE_VERSION
|
||||||
|
),
|
||||||
|
ModuleInterfaceProvider(
|
||||||
|
name="dataflow.pipeline_preview", version=MODULE_VERSION
|
||||||
|
),
|
||||||
ModuleInterfaceProvider(name="dataflow.run_lifecycle", version=MODULE_VERSION),
|
ModuleInterfaceProvider(name="dataflow.run_lifecycle", version=MODULE_VERSION),
|
||||||
ModuleInterfaceProvider(name="dataflow.run_worker", version=MODULE_VERSION),
|
ModuleInterfaceProvider(name="dataflow.run_worker", version=MODULE_VERSION),
|
||||||
ModuleInterfaceProvider(name=CAPABILITY_DATAFLOW_DATASET_OUTPUT, version=MODULE_VERSION),
|
ModuleInterfaceProvider(
|
||||||
|
name=CAPABILITY_DATAFLOW_DATASET_OUTPUT, version=MODULE_VERSION
|
||||||
|
),
|
||||||
ModuleInterfaceProvider(
|
ModuleInterfaceProvider(
|
||||||
name="dataflow.trigger_dispatcher",
|
name="dataflow.trigger_dispatcher",
|
||||||
version=MODULE_VERSION,
|
version=MODULE_VERSION,
|
||||||
),
|
),
|
||||||
|
ModuleInterfaceProvider(name=DATAFLOW_DSAR_CAPABILITY, version="0.1.0"),
|
||||||
),
|
),
|
||||||
requires_interfaces=(
|
requires_interfaces=(
|
||||||
ModuleInterfaceRequirement(
|
ModuleInterfaceRequirement(
|
||||||
@@ -484,6 +564,17 @@ manifest = ModuleManifest(
|
|||||||
order=72,
|
order=72,
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
product_areas=(
|
||||||
|
ProductAreaContribution(
|
||||||
|
id="data-assurance",
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
label="i18n:govoplan-core.product_area.data_assurance",
|
||||||
|
icon="database-zap",
|
||||||
|
description="i18n:govoplan-core.product_area.data_assurance_description",
|
||||||
|
surface_ids=("dataflow.nav.dataflow", "dataflow.route.dataflow"),
|
||||||
|
order=60,
|
||||||
|
),
|
||||||
|
),
|
||||||
view_surfaces=(
|
view_surfaces=(
|
||||||
ViewSurface(
|
ViewSurface(
|
||||||
id="dataflow.page",
|
id="dataflow.page",
|
||||||
@@ -532,6 +623,14 @@ manifest = ModuleManifest(
|
|||||||
parent_id="dataflow.page",
|
parent_id="dataflow.page",
|
||||||
order=50,
|
order=50,
|
||||||
),
|
),
|
||||||
|
ViewSurface(
|
||||||
|
id="dataflow.decisions",
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
kind="action",
|
||||||
|
label="Reconciliation decisions",
|
||||||
|
parent_id="dataflow.results",
|
||||||
|
order=55,
|
||||||
|
),
|
||||||
ViewSurface(
|
ViewSurface(
|
||||||
id="dataflow.triggers",
|
id="dataflow.triggers",
|
||||||
module_id=MODULE_ID,
|
module_id=MODULE_ID,
|
||||||
@@ -566,6 +665,16 @@ manifest = ModuleManifest(
|
|||||||
CAPABILITY_DATAFLOW_RUN_LIFECYCLE: _run_provider,
|
CAPABILITY_DATAFLOW_RUN_LIFECYCLE: _run_provider,
|
||||||
CAPABILITY_DATAFLOW_RUN_WORKER: _run_worker,
|
CAPABILITY_DATAFLOW_RUN_WORKER: _run_worker,
|
||||||
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER: _trigger_provider,
|
CAPABILITY_DATAFLOW_TRIGGER_DISPATCHER: _trigger_provider,
|
||||||
|
DATAFLOW_DSAR_CAPABILITY: _dsar_provider,
|
||||||
|
},
|
||||||
|
capability_documentation={
|
||||||
|
DATAFLOW_DSAR_CAPABILITY: CapabilityDocumentation(
|
||||||
|
label="Dataflow data-subject request provider",
|
||||||
|
summary="Finds and minimizes subject-linked transformation state and automation authority.",
|
||||||
|
contract_version="0.1.0",
|
||||||
|
documentation_types=("admin", "user"),
|
||||||
|
audience=("privacy_officer", "data_steward", "user"),
|
||||||
|
),
|
||||||
},
|
},
|
||||||
search_sources=(
|
search_sources=(
|
||||||
SearchSourceProviderRegistration(
|
SearchSourceProviderRegistration(
|
||||||
@@ -580,6 +689,8 @@ manifest = ModuleManifest(
|
|||||||
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
||||||
retirement_supported=True,
|
retirement_supported=True,
|
||||||
retirement_provider=drop_table_retirement_provider(
|
retirement_provider=drop_table_retirement_provider(
|
||||||
|
dataflow_models.DataflowReconciliationDecision,
|
||||||
|
dataflow_models.DataflowReconciliationDecisionSet,
|
||||||
dataflow_models.DataflowTriggerDelivery,
|
dataflow_models.DataflowTriggerDelivery,
|
||||||
dataflow_models.DataflowTrigger,
|
dataflow_models.DataflowTrigger,
|
||||||
dataflow_models.DataflowRun,
|
dataflow_models.DataflowRun,
|
||||||
@@ -597,6 +708,8 @@ manifest = ModuleManifest(
|
|||||||
persistent_table_uninstall_guard(
|
persistent_table_uninstall_guard(
|
||||||
dataflow_models.DataflowPipeline,
|
dataflow_models.DataflowPipeline,
|
||||||
dataflow_models.DataflowPipelineRevision,
|
dataflow_models.DataflowPipelineRevision,
|
||||||
|
dataflow_models.DataflowReconciliationDecisionSet,
|
||||||
|
dataflow_models.DataflowReconciliationDecision,
|
||||||
dataflow_models.DataflowPipelineDeployment,
|
dataflow_models.DataflowPipelineDeployment,
|
||||||
dataflow_models.DataflowRun,
|
dataflow_models.DataflowRun,
|
||||||
dataflow_models.DataflowTrigger,
|
dataflow_models.DataflowTrigger,
|
||||||
@@ -614,8 +727,19 @@ manifest = ModuleManifest(
|
|||||||
known_limits=(
|
known_limits=(
|
||||||
"Execution adapters do not yet cover every declared node family.",
|
"Execution adapters do not yet cover every declared node family.",
|
||||||
),
|
),
|
||||||
owned_concepts=("dataflow definition", "dataflow revision", "dataflow run", "transformation graph"),
|
owned_concepts=(
|
||||||
non_owned_concepts=("datasource binding", "connector transport", "report presentation", "workflow task"),
|
"dataflow definition",
|
||||||
|
"dataflow revision",
|
||||||
|
"dataflow run",
|
||||||
|
"reconciliation decision set",
|
||||||
|
"transformation graph",
|
||||||
|
),
|
||||||
|
non_owned_concepts=(
|
||||||
|
"datasource binding",
|
||||||
|
"connector transport",
|
||||||
|
"report presentation",
|
||||||
|
"workflow task",
|
||||||
|
),
|
||||||
recovery_docs=("README.md", "docs/DURABLE_RUN_RECOVERY.md"),
|
recovery_docs=("README.md", "docs/DURABLE_RUN_RECOVERY.md"),
|
||||||
security_docs=("README.md",),
|
security_docs=("README.md",),
|
||||||
operations_docs=("README.md",),
|
operations_docs=("README.md",),
|
||||||
|
|||||||
+118
@@ -0,0 +1,118 @@
|
|||||||
|
"""Add durable Dataflow reconciliation decision sets.
|
||||||
|
|
||||||
|
Revision ID: a3d7f1c5e9b2
|
||||||
|
Revises: f6c2a9d4e7b1
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "a3d7f1c5e9b2"
|
||||||
|
down_revision = "f6c2a9d4e7b1"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"dataflow_reconciliation_decision_sets",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("pipeline_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=300), nullable=False),
|
||||||
|
sa.Column("node_id", sa.String(length=100), nullable=True),
|
||||||
|
sa.Column("resource_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("updated_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["pipeline_id"],
|
||||||
|
["dataflow_pipelines.id"],
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"pipeline_id",
|
||||||
|
"name",
|
||||||
|
name="uq_dataflow_decision_sets_pipeline_name",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_decision_sets_tenant_id",
|
||||||
|
"dataflow_reconciliation_decision_sets",
|
||||||
|
["tenant_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_decision_sets_pipeline_id",
|
||||||
|
"dataflow_reconciliation_decision_sets",
|
||||||
|
["pipeline_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_decision_sets_tenant_pipeline",
|
||||||
|
"dataflow_reconciliation_decision_sets",
|
||||||
|
["tenant_id", "pipeline_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("decision_set_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("key_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("input_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("action", sa.String(length=20), nullable=False),
|
||||||
|
sa.Column("reason", sa.Text(), nullable=False),
|
||||||
|
sa.Column("correction", sa.JSON(), nullable=True),
|
||||||
|
sa.Column("actor_ref", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("decided_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["decision_set_id"],
|
||||||
|
["dataflow_reconciliation_decision_sets.id"],
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"decision_set_id",
|
||||||
|
"revision",
|
||||||
|
name="uq_dataflow_reconciliation_decision_revision",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_reconciliation_decisions_tenant_id",
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
|
["tenant_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_reconciliation_decisions_decision_set_id",
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
|
["decision_set_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_reconciliation_decisions_key_hash",
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
|
["key_hash"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_reconciliation_decisions_current",
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
|
["tenant_id", "decision_set_id", "key_hash", "revision"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("dataflow_reconciliation_decisions")
|
||||||
|
op.drop_table("dataflow_reconciliation_decision_sets")
|
||||||
@@ -0,0 +1,380 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from sqlalchemy import and_, func, select
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
from sqlalchemy.orm import Session, joinedload, selectinload
|
||||||
|
|
||||||
|
from govoplan_core.core.concurrency import claim_revision
|
||||||
|
from govoplan_core.security.time import utc_now
|
||||||
|
from govoplan_dataflow.backend.db.models import (
|
||||||
|
DataflowPipeline,
|
||||||
|
DataflowReconciliationDecision,
|
||||||
|
DataflowReconciliationDecisionSet,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
DECISION_SET_REF_PREFIX = "dataflow-decision-set:"
|
||||||
|
DECISION_REF_PREFIX = "dataflow-decision:"
|
||||||
|
DECISION_ACTIONS = frozenset({"accept", "reject", "correct", "defer"})
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionError(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionNotFoundError(ReconciliationDecisionError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionConflictError(ReconciliationDecisionError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def list_decision_sets(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
pipeline_id: str | None = None,
|
||||||
|
include_decisions: bool = True,
|
||||||
|
) -> tuple[DataflowReconciliationDecisionSet, ...]:
|
||||||
|
options = [joinedload(DataflowReconciliationDecisionSet.pipeline)]
|
||||||
|
if include_decisions:
|
||||||
|
options.append(selectinload(DataflowReconciliationDecisionSet.decisions))
|
||||||
|
statement = (
|
||||||
|
select(DataflowReconciliationDecisionSet)
|
||||||
|
.options(*options)
|
||||||
|
.where(DataflowReconciliationDecisionSet.tenant_id == tenant_id)
|
||||||
|
.order_by(
|
||||||
|
DataflowReconciliationDecisionSet.name,
|
||||||
|
DataflowReconciliationDecisionSet.id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if pipeline_id:
|
||||||
|
statement = statement.where(
|
||||||
|
DataflowReconciliationDecisionSet.pipeline_id == pipeline_id
|
||||||
|
)
|
||||||
|
return tuple(session.scalars(statement).all())
|
||||||
|
|
||||||
|
|
||||||
|
def create_decision_set(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
pipeline_id: str,
|
||||||
|
name: str,
|
||||||
|
node_id: str | None,
|
||||||
|
actor_ref: str,
|
||||||
|
) -> DataflowReconciliationDecisionSet:
|
||||||
|
pipeline = session.get(DataflowPipeline, pipeline_id)
|
||||||
|
if (
|
||||||
|
pipeline is None
|
||||||
|
or pipeline.tenant_id not in {None, tenant_id}
|
||||||
|
or pipeline.deleted_at is not None
|
||||||
|
):
|
||||||
|
raise ReconciliationDecisionNotFoundError("Dataflow pipeline not found.")
|
||||||
|
item = DataflowReconciliationDecisionSet(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
pipeline_id=pipeline_id,
|
||||||
|
name=name.strip(),
|
||||||
|
node_id=node_id,
|
||||||
|
resource_revision=1,
|
||||||
|
created_by=actor_ref,
|
||||||
|
updated_by=actor_ref,
|
||||||
|
)
|
||||||
|
session.add(item)
|
||||||
|
try:
|
||||||
|
session.flush()
|
||||||
|
except IntegrityError as exc:
|
||||||
|
raise ReconciliationDecisionConflictError(
|
||||||
|
"A decision set with this name already exists for the pipeline."
|
||||||
|
) from exc
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def get_decision_set(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
decision_set_id: str,
|
||||||
|
include_decisions: bool = True,
|
||||||
|
) -> DataflowReconciliationDecisionSet:
|
||||||
|
options = [joinedload(DataflowReconciliationDecisionSet.pipeline)]
|
||||||
|
if include_decisions:
|
||||||
|
options.append(selectinload(DataflowReconciliationDecisionSet.decisions))
|
||||||
|
item = session.scalar(
|
||||||
|
select(DataflowReconciliationDecisionSet)
|
||||||
|
.options(*options)
|
||||||
|
.where(
|
||||||
|
DataflowReconciliationDecisionSet.id == decision_set_id,
|
||||||
|
DataflowReconciliationDecisionSet.tenant_id == tenant_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise ReconciliationDecisionNotFoundError(
|
||||||
|
"Reconciliation decision set not found."
|
||||||
|
)
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def record_decision(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
decision_set_id: str,
|
||||||
|
expected_revision: int,
|
||||||
|
key_hash: str,
|
||||||
|
input_hash: str,
|
||||||
|
action: str,
|
||||||
|
reason: str,
|
||||||
|
correction: dict[str, object] | None,
|
||||||
|
actor_ref: str,
|
||||||
|
) -> DataflowReconciliationDecisionSet:
|
||||||
|
item = get_decision_set(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
decision_set_id=decision_set_id,
|
||||||
|
)
|
||||||
|
if action not in DECISION_ACTIONS:
|
||||||
|
raise ReconciliationDecisionError("Unsupported reconciliation action.")
|
||||||
|
next_revision = claim_revision(
|
||||||
|
session,
|
||||||
|
model=DataflowReconciliationDecisionSet,
|
||||||
|
filters=(
|
||||||
|
DataflowReconciliationDecisionSet.id == decision_set_id,
|
||||||
|
DataflowReconciliationDecisionSet.tenant_id == tenant_id,
|
||||||
|
),
|
||||||
|
revision_attribute="resource_revision",
|
||||||
|
expected_revision=expected_revision,
|
||||||
|
resource_type="dataflow_reconciliation_decision_set",
|
||||||
|
resource_id=decision_set_id,
|
||||||
|
)
|
||||||
|
item.resource_revision = next_revision
|
||||||
|
item.updated_by = actor_ref
|
||||||
|
item.decisions.append(
|
||||||
|
DataflowReconciliationDecision(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
revision=next_revision,
|
||||||
|
key_hash=key_hash,
|
||||||
|
input_hash=input_hash,
|
||||||
|
action=action,
|
||||||
|
reason=reason.strip(),
|
||||||
|
correction=dict(correction) if correction else None,
|
||||||
|
actor_ref=actor_ref,
|
||||||
|
decided_at=utc_now(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.flush()
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def current_decisions(
|
||||||
|
item: DataflowReconciliationDecisionSet,
|
||||||
|
) -> tuple[DataflowReconciliationDecision, ...]:
|
||||||
|
current: dict[str, DataflowReconciliationDecision] = {}
|
||||||
|
for decision in sorted(item.decisions, key=lambda value: value.revision):
|
||||||
|
current[decision.key_hash] = decision
|
||||||
|
return tuple(
|
||||||
|
sorted(
|
||||||
|
current.values(),
|
||||||
|
key=lambda value: (value.key_hash, value.revision),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def decision_rows(
|
||||||
|
item: DataflowReconciliationDecisionSet,
|
||||||
|
) -> tuple[dict[str, object], ...]:
|
||||||
|
return tuple(_decision_row(value) for value in current_decisions(item))
|
||||||
|
|
||||||
|
|
||||||
|
def decision_set_fingerprint(
|
||||||
|
item: DataflowReconciliationDecisionSet,
|
||||||
|
) -> str:
|
||||||
|
payload = {
|
||||||
|
"decision_set_id": item.id,
|
||||||
|
"resource_revision": item.resource_revision,
|
||||||
|
}
|
||||||
|
return "sha256:" + hashlib.sha256(
|
||||||
|
json.dumps(
|
||||||
|
payload,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
ensure_ascii=True,
|
||||||
|
).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def decision_set_payload(
|
||||||
|
item: DataflowReconciliationDecisionSet,
|
||||||
|
*,
|
||||||
|
include_decisions: bool = True,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
current = current_decisions(item) if include_decisions else ()
|
||||||
|
history = (
|
||||||
|
sorted(item.decisions, key=lambda value: value.revision)
|
||||||
|
if include_decisions
|
||||||
|
else ()
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"ref": f"{DECISION_SET_REF_PREFIX}{item.id}",
|
||||||
|
"id": item.id,
|
||||||
|
"pipeline_id": item.pipeline_id,
|
||||||
|
"name": item.name,
|
||||||
|
"node_id": item.node_id,
|
||||||
|
"resource_revision": item.resource_revision,
|
||||||
|
"etag": item.strong_etag,
|
||||||
|
"fingerprint": decision_set_fingerprint(item),
|
||||||
|
"decisions_included": include_decisions,
|
||||||
|
"current_decisions": [_decision_payload(value) for value in current],
|
||||||
|
"history": [
|
||||||
|
_decision_payload(value)
|
||||||
|
for value in history
|
||||||
|
],
|
||||||
|
"created_by": item.created_by,
|
||||||
|
"updated_by": item.updated_by,
|
||||||
|
"created_at": item.created_at,
|
||||||
|
"updated_at": item.updated_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def decision_set_source_payload(
|
||||||
|
item: DataflowReconciliationDecisionSet,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"ref": f"{DECISION_SET_REF_PREFIX}{item.id}",
|
||||||
|
"provider": "dataflow.reconciliation_decisions",
|
||||||
|
"source_name": _source_name(item.name, item.id),
|
||||||
|
"name": item.name,
|
||||||
|
"description": "Current immutable reconciliation decisions; prior revisions remain in decision history.",
|
||||||
|
"mode": "static",
|
||||||
|
"columns": [
|
||||||
|
{"name": "key_hash", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "input_hash", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "decision_ref", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "action", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "actor_ref", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "decided_at", "data_type": "datetime", "nullable": False},
|
||||||
|
{"name": "reason", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "correction", "data_type": "object", "nullable": True},
|
||||||
|
],
|
||||||
|
"schema_version": "1",
|
||||||
|
"fingerprint": decision_set_fingerprint(item),
|
||||||
|
"row_count": None,
|
||||||
|
"byte_count": None,
|
||||||
|
"updated_at": item.updated_at,
|
||||||
|
"capabilities": ["preview", "read", "immutable_history"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def current_decision_rows(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
decision_set_id: str,
|
||||||
|
limit: int,
|
||||||
|
) -> tuple[tuple[dict[str, object], ...], int]:
|
||||||
|
bounded_limit = max(1, limit)
|
||||||
|
latest = (
|
||||||
|
select(
|
||||||
|
DataflowReconciliationDecision.key_hash.label("key_hash"),
|
||||||
|
func.max(DataflowReconciliationDecision.revision).label("revision"),
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
||||||
|
)
|
||||||
|
.group_by(DataflowReconciliationDecision.key_hash)
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
statement = (
|
||||||
|
select(DataflowReconciliationDecision)
|
||||||
|
.join(
|
||||||
|
latest,
|
||||||
|
and_(
|
||||||
|
DataflowReconciliationDecision.key_hash == latest.c.key_hash,
|
||||||
|
DataflowReconciliationDecision.revision == latest.c.revision,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
||||||
|
)
|
||||||
|
.order_by(DataflowReconciliationDecision.key_hash)
|
||||||
|
.limit(bounded_limit)
|
||||||
|
)
|
||||||
|
records = tuple(session.scalars(statement))
|
||||||
|
total = int(
|
||||||
|
session.scalar(
|
||||||
|
select(func.count(func.distinct(DataflowReconciliationDecision.key_hash))).where(
|
||||||
|
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
return tuple(_decision_row(value) for value in records), total
|
||||||
|
|
||||||
|
|
||||||
|
def decision_set_id_from_ref(value: str) -> str | None:
|
||||||
|
cleaned = str(value or "").strip()
|
||||||
|
if not cleaned.startswith(DECISION_SET_REF_PREFIX):
|
||||||
|
return None
|
||||||
|
identifier = cleaned[len(DECISION_SET_REF_PREFIX) :]
|
||||||
|
return identifier or None
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_row(
|
||||||
|
value: DataflowReconciliationDecision,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"key_hash": value.key_hash,
|
||||||
|
"input_hash": value.input_hash,
|
||||||
|
"decision_ref": f"{DECISION_REF_PREFIX}{value.id}",
|
||||||
|
"action": value.action,
|
||||||
|
"actor_ref": value.actor_ref,
|
||||||
|
"decided_at": value.decided_at.isoformat(),
|
||||||
|
"reason": value.reason,
|
||||||
|
"correction": dict(value.correction) if value.correction else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_payload(
|
||||||
|
value: DataflowReconciliationDecision,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"ref": f"{DECISION_REF_PREFIX}{value.id}",
|
||||||
|
"id": value.id,
|
||||||
|
"revision": value.revision,
|
||||||
|
"key_hash": value.key_hash,
|
||||||
|
"input_hash": value.input_hash,
|
||||||
|
"action": value.action,
|
||||||
|
"reason": value.reason,
|
||||||
|
"correction": dict(value.correction) if value.correction else None,
|
||||||
|
"actor_ref": value.actor_ref,
|
||||||
|
"decided_at": value.decided_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _source_name(name: str, identifier: str) -> str:
|
||||||
|
slug = re.sub(r"[^a-z0-9]+", "_", name.casefold()).strip("_")
|
||||||
|
return (f"decisions_{slug}" if slug else f"decisions_{identifier[:8]}")[:120]
|
||||||
|
|
||||||
|
|
||||||
|
__all__: Sequence[str] = (
|
||||||
|
"DECISION_SET_REF_PREFIX",
|
||||||
|
"ReconciliationDecisionConflictError",
|
||||||
|
"ReconciliationDecisionError",
|
||||||
|
"ReconciliationDecisionNotFoundError",
|
||||||
|
"create_decision_set",
|
||||||
|
"current_decision_rows",
|
||||||
|
"current_decisions",
|
||||||
|
"decision_rows",
|
||||||
|
"decision_set_fingerprint",
|
||||||
|
"decision_set_id_from_ref",
|
||||||
|
"decision_set_payload",
|
||||||
|
"decision_set_source_payload",
|
||||||
|
"get_decision_set",
|
||||||
|
"list_decision_sets",
|
||||||
|
"record_decision",
|
||||||
|
)
|
||||||
@@ -10,6 +10,7 @@ from govoplan_core.api.v1.schemas import (
|
|||||||
from govoplan_core.audit.logging import audit_event
|
from govoplan_core.audit.logging import audit_event
|
||||||
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
|
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
|
||||||
from govoplan_core.core.automation import AutomationInvocation
|
from govoplan_core.core.automation import AutomationInvocation
|
||||||
|
from govoplan_core.core.concurrency import RevisionConflictError
|
||||||
from govoplan_core.core.dataflows import (
|
from govoplan_core.core.dataflows import (
|
||||||
DataflowPublicationTarget,
|
DataflowPublicationTarget,
|
||||||
DataflowRunRequest,
|
DataflowRunRequest,
|
||||||
@@ -71,6 +72,10 @@ from govoplan_dataflow.backend.schemas import (
|
|||||||
PipelineSqlResponse,
|
PipelineSqlResponse,
|
||||||
PipelineUpdateRequest,
|
PipelineUpdateRequest,
|
||||||
PipelineValidationResponse,
|
PipelineValidationResponse,
|
||||||
|
ReconciliationDecisionSetCreateRequest,
|
||||||
|
ReconciliationDecisionSetListResponse,
|
||||||
|
ReconciliationDecisionSetResponse,
|
||||||
|
ReconciliationDecisionWriteRequest,
|
||||||
TabularSnapshotCreateRequest,
|
TabularSnapshotCreateRequest,
|
||||||
TabularSourceColumnResponse,
|
TabularSourceColumnResponse,
|
||||||
TabularSourceListResponse,
|
TabularSourceListResponse,
|
||||||
@@ -110,6 +115,17 @@ from govoplan_dataflow.backend.service import (
|
|||||||
validate_draft,
|
validate_draft,
|
||||||
)
|
)
|
||||||
from govoplan_dataflow.backend.run_worker import run_metrics
|
from govoplan_dataflow.backend.run_worker import run_metrics
|
||||||
|
from govoplan_dataflow.backend.reconciliation_decisions import (
|
||||||
|
ReconciliationDecisionConflictError,
|
||||||
|
ReconciliationDecisionError,
|
||||||
|
ReconciliationDecisionNotFoundError,
|
||||||
|
create_decision_set,
|
||||||
|
decision_set_payload,
|
||||||
|
decision_set_source_payload,
|
||||||
|
get_decision_set,
|
||||||
|
list_decision_sets,
|
||||||
|
record_decision,
|
||||||
|
)
|
||||||
from govoplan_dataflow.backend.recovery import dataflow_run_recovery_states
|
from govoplan_dataflow.backend.recovery import dataflow_run_recovery_states
|
||||||
from govoplan_dataflow.backend.triggers import (
|
from govoplan_dataflow.backend.triggers import (
|
||||||
create_trigger,
|
create_trigger,
|
||||||
@@ -145,6 +161,18 @@ def _actor_id(principal: ApiPrincipal) -> str | None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_actor_ref(principal: ApiPrincipal) -> str:
|
||||||
|
actor_id = _actor_id(principal)
|
||||||
|
if not actor_id:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN,
|
||||||
|
detail="A stable actor reference is required to record a decision.",
|
||||||
|
)
|
||||||
|
principal_ref = principal.to_platform_principal()
|
||||||
|
prefix = "service-account" if principal_ref.service_account_id else "account"
|
||||||
|
return f"{prefix}:{actor_id}"
|
||||||
|
|
||||||
|
|
||||||
def _http_error(exc: DataflowError) -> HTTPException:
|
def _http_error(exc: DataflowError) -> HTTPException:
|
||||||
if isinstance(exc, DataflowNotFoundError):
|
if isinstance(exc, DataflowNotFoundError):
|
||||||
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
||||||
@@ -161,6 +189,17 @@ def _http_error(exc: DataflowError) -> HTTPException:
|
|||||||
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_http_error(exc: ReconciliationDecisionError) -> HTTPException:
|
||||||
|
if isinstance(exc, ReconciliationDecisionNotFoundError):
|
||||||
|
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
||||||
|
if isinstance(exc, ReconciliationDecisionConflictError):
|
||||||
|
return HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc))
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail=str(exc),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _governance_http_error(exc: PermissionError | ValueError) -> HTTPException:
|
def _governance_http_error(exc: PermissionError | ValueError) -> HTTPException:
|
||||||
return HTTPException(
|
return HTTPException(
|
||||||
status_code=(
|
status_code=(
|
||||||
@@ -329,21 +368,42 @@ def api_list_sources(
|
|||||||
registry = get_registry()
|
registry = get_registry()
|
||||||
provider = datasource_catalogue(registry)
|
provider = datasource_catalogue(registry)
|
||||||
writer = datasource_lifecycle(registry)
|
writer = datasource_lifecycle(registry)
|
||||||
if provider is None:
|
sources = ()
|
||||||
return TabularSourceListResponse(available=False, writable=False, sources=[])
|
if provider is not None:
|
||||||
try:
|
try:
|
||||||
sources = provider.list_datasources(
|
sources = provider.list_datasources(
|
||||||
session,
|
session,
|
||||||
principal,
|
principal,
|
||||||
query=query,
|
query=query,
|
||||||
limit=100,
|
limit=100,
|
||||||
)
|
)
|
||||||
except DatasourceError as exc:
|
except DatasourceError as exc:
|
||||||
raise _source_http_error(exc) from exc
|
raise _source_http_error(exc) from exc
|
||||||
|
decision_sources = []
|
||||||
|
for item in list_decision_sets(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
include_decisions=False,
|
||||||
|
):
|
||||||
|
if query and query.casefold() not in item.name.casefold():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
require_definition_action(
|
||||||
|
item.pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=registry,
|
||||||
|
action="view",
|
||||||
|
)
|
||||||
|
except PermissionError:
|
||||||
|
continue
|
||||||
|
decision_sources.append(decision_set_source_payload(item))
|
||||||
return TabularSourceListResponse(
|
return TabularSourceListResponse(
|
||||||
available=True,
|
available=provider is not None or bool(decision_sources),
|
||||||
writable=writer is not None,
|
writable=writer is not None,
|
||||||
sources=[_source_response(source) for source in sources],
|
sources=[
|
||||||
|
*(_source_response(source) for source in sources),
|
||||||
|
*(TabularSourceResponse.model_validate(item) for item in decision_sources),
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -504,6 +564,212 @@ def api_create_pipeline(
|
|||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/pipelines/{pipeline_id}/decision-sets",
|
||||||
|
response_model=ReconciliationDecisionSetListResponse,
|
||||||
|
)
|
||||||
|
def api_list_reconciliation_decision_sets(
|
||||||
|
pipeline_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ReconciliationDecisionSetListResponse:
|
||||||
|
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
pipeline = get_pipeline(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
pipeline_id=pipeline_id,
|
||||||
|
)
|
||||||
|
require_definition_action(
|
||||||
|
pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=get_registry(),
|
||||||
|
action="view",
|
||||||
|
)
|
||||||
|
except PermissionError as exc:
|
||||||
|
raise _governance_http_error(exc) from exc
|
||||||
|
except DataflowError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return ReconciliationDecisionSetListResponse(
|
||||||
|
decision_sets=[
|
||||||
|
ReconciliationDecisionSetResponse.model_validate(
|
||||||
|
decision_set_payload(item, include_decisions=False)
|
||||||
|
)
|
||||||
|
for item in list_decision_sets(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
pipeline_id=pipeline_id,
|
||||||
|
include_decisions=False,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/pipelines/{pipeline_id}/decision-sets",
|
||||||
|
response_model=ReconciliationDecisionSetResponse,
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def api_create_reconciliation_decision_set(
|
||||||
|
pipeline_id: str,
|
||||||
|
payload: ReconciliationDecisionSetCreateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ReconciliationDecisionSetResponse:
|
||||||
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
pipeline = get_pipeline(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
pipeline_id=pipeline_id,
|
||||||
|
)
|
||||||
|
require_definition_action(
|
||||||
|
pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=get_registry(),
|
||||||
|
action="edit",
|
||||||
|
)
|
||||||
|
item = create_decision_set(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
pipeline_id=pipeline_id,
|
||||||
|
name=payload.name,
|
||||||
|
node_id=payload.node_id,
|
||||||
|
actor_ref=_decision_actor_ref(principal),
|
||||||
|
)
|
||||||
|
except PermissionError as exc:
|
||||||
|
raise _governance_http_error(exc) from exc
|
||||||
|
except DataflowError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
except ReconciliationDecisionError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _decision_http_error(exc) from exc
|
||||||
|
audit_event(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=getattr(principal.user, "id", None),
|
||||||
|
api_key_id=principal.api_key_id,
|
||||||
|
action="dataflow.reconciliation_decision_set.created",
|
||||||
|
object_type="dataflow_reconciliation_decision_set",
|
||||||
|
object_id=item.id,
|
||||||
|
details={"pipeline_id": pipeline_id, "node_id": payload.node_id},
|
||||||
|
)
|
||||||
|
response = ReconciliationDecisionSetResponse.model_validate(
|
||||||
|
decision_set_payload(item)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/decision-sets/{decision_set_id}",
|
||||||
|
response_model=ReconciliationDecisionSetResponse,
|
||||||
|
)
|
||||||
|
def api_get_reconciliation_decision_set(
|
||||||
|
decision_set_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ReconciliationDecisionSetResponse:
|
||||||
|
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
item = get_decision_set(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
decision_set_id=decision_set_id,
|
||||||
|
)
|
||||||
|
require_definition_action(
|
||||||
|
item.pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=get_registry(),
|
||||||
|
action="view",
|
||||||
|
)
|
||||||
|
except PermissionError as exc:
|
||||||
|
raise _governance_http_error(exc) from exc
|
||||||
|
except ReconciliationDecisionError as exc:
|
||||||
|
raise _decision_http_error(exc) from exc
|
||||||
|
return ReconciliationDecisionSetResponse.model_validate(
|
||||||
|
decision_set_payload(item)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/decision-sets/{decision_set_id}/decisions",
|
||||||
|
response_model=ReconciliationDecisionSetResponse,
|
||||||
|
)
|
||||||
|
def api_record_reconciliation_decision(
|
||||||
|
decision_set_id: str,
|
||||||
|
payload: ReconciliationDecisionWriteRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ReconciliationDecisionSetResponse:
|
||||||
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
item = get_decision_set(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
decision_set_id=decision_set_id,
|
||||||
|
)
|
||||||
|
pipeline = get_pipeline(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
pipeline_id=item.pipeline_id,
|
||||||
|
)
|
||||||
|
require_definition_action(
|
||||||
|
pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=get_registry(),
|
||||||
|
action="edit",
|
||||||
|
)
|
||||||
|
item = record_decision(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
decision_set_id=decision_set_id,
|
||||||
|
expected_revision=payload.base_revision,
|
||||||
|
key_hash=payload.key_hash,
|
||||||
|
input_hash=payload.input_hash,
|
||||||
|
action=payload.action,
|
||||||
|
reason=payload.reason,
|
||||||
|
correction=payload.correction,
|
||||||
|
actor_ref=_decision_actor_ref(principal),
|
||||||
|
)
|
||||||
|
except PermissionError as exc:
|
||||||
|
raise _governance_http_error(exc) from exc
|
||||||
|
except DataflowError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
except RevisionConflictError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_409_CONFLICT,
|
||||||
|
detail=exc.as_dict(),
|
||||||
|
) from exc
|
||||||
|
except ReconciliationDecisionError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _decision_http_error(exc) from exc
|
||||||
|
decision = item.decisions[-1]
|
||||||
|
audit_event(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=getattr(principal.user, "id", None),
|
||||||
|
api_key_id=principal.api_key_id,
|
||||||
|
action="dataflow.reconciliation_decision.recorded",
|
||||||
|
object_type="dataflow_reconciliation_decision",
|
||||||
|
object_id=decision.id,
|
||||||
|
details={
|
||||||
|
"decision_set_id": item.id,
|
||||||
|
"pipeline_id": item.pipeline_id,
|
||||||
|
"revision": decision.revision,
|
||||||
|
"key_hash": decision.key_hash,
|
||||||
|
"input_hash": decision.input_hash,
|
||||||
|
"decision_action": decision.action,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
response = ReconciliationDecisionSetResponse.model_validate(
|
||||||
|
decision_set_payload(item)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
@router.get("/pipelines/{pipeline_id}", response_model=PipelineResponse)
|
@router.get("/pipelines/{pipeline_id}", response_model=PipelineResponse)
|
||||||
def api_get_pipeline(
|
def api_get_pipeline(
|
||||||
pipeline_id: str,
|
pipeline_id: str,
|
||||||
|
|||||||
@@ -145,6 +145,68 @@ class PipelineListResponse(BaseModel):
|
|||||||
pipelines: list[PipelineResponse]
|
pipelines: list[PipelineResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionSetCreateRequest(BaseModel):
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
node_id: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
min_length=1,
|
||||||
|
max_length=100,
|
||||||
|
pattern=r"^[A-Za-z0-9_.:-]+$",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionWriteRequest(BaseModel):
|
||||||
|
base_revision: int = Field(ge=1)
|
||||||
|
key_hash: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||||
|
input_hash: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||||
|
action: Literal["accept", "reject", "correct", "defer"]
|
||||||
|
reason: str = Field(min_length=3, max_length=4_000)
|
||||||
|
correction: dict[str, Any] | None = None
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_correction(self) -> "ReconciliationDecisionWriteRequest":
|
||||||
|
if self.action == "correct" and not self.correction:
|
||||||
|
raise ValueError("Correct decisions require corrected field values.")
|
||||||
|
if self.action != "correct" and self.correction:
|
||||||
|
raise ValueError("Only correct decisions may include corrected field values.")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionResponse(BaseModel):
|
||||||
|
ref: str
|
||||||
|
id: str
|
||||||
|
revision: int
|
||||||
|
key_hash: str
|
||||||
|
input_hash: str
|
||||||
|
action: Literal["accept", "reject", "correct", "defer"]
|
||||||
|
reason: str
|
||||||
|
correction: dict[str, Any] | None
|
||||||
|
actor_ref: str
|
||||||
|
decided_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionSetResponse(BaseModel):
|
||||||
|
ref: str
|
||||||
|
id: str
|
||||||
|
pipeline_id: str
|
||||||
|
name: str
|
||||||
|
node_id: str | None
|
||||||
|
resource_revision: int = Field(ge=1)
|
||||||
|
etag: str
|
||||||
|
fingerprint: str
|
||||||
|
decisions_included: bool
|
||||||
|
current_decisions: list[ReconciliationDecisionResponse]
|
||||||
|
history: list[ReconciliationDecisionResponse]
|
||||||
|
created_by: str | None
|
||||||
|
updated_by: str | None
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionSetListResponse(BaseModel):
|
||||||
|
decision_sets: list[ReconciliationDecisionSetResponse]
|
||||||
|
|
||||||
|
|
||||||
class PipelineCreateRequest(BaseModel):
|
class PipelineCreateRequest(BaseModel):
|
||||||
name: str = Field(min_length=1, max_length=300)
|
name: str = Field(min_length=1, max_length=300)
|
||||||
description: str | None = Field(default=None, max_length=4000)
|
description: str | None = Field(default=None, max_length=4000)
|
||||||
|
|||||||
@@ -86,6 +86,12 @@ from govoplan_dataflow.backend.recovery import (
|
|||||||
begin_dataflow_run_recovery,
|
begin_dataflow_run_recovery,
|
||||||
dataflow_run_recovery_state,
|
dataflow_run_recovery_state,
|
||||||
)
|
)
|
||||||
|
from govoplan_dataflow.backend.reconciliation_decisions import (
|
||||||
|
current_decision_rows,
|
||||||
|
decision_set_fingerprint,
|
||||||
|
decision_set_id_from_ref,
|
||||||
|
get_decision_set,
|
||||||
|
)
|
||||||
from govoplan_dataflow.backend.sql_compiler import (
|
from govoplan_dataflow.backend.sql_compiler import (
|
||||||
SqlCompilationError,
|
SqlCompilationError,
|
||||||
compile_sql,
|
compile_sql,
|
||||||
@@ -1352,8 +1358,12 @@ def _run_authorization_payload(
|
|||||||
graph = PipelineGraph.model_validate(revision.graph)
|
graph = PipelineGraph.model_validate(revision.graph)
|
||||||
scopes = {RUN_SCOPE}
|
scopes = {RUN_SCOPE}
|
||||||
if any(
|
if any(
|
||||||
node.type == "source.reference"
|
(
|
||||||
or bool(node.config.get("source_ref"))
|
node.type == "source.reference"
|
||||||
|
or bool(node.config.get("source_ref"))
|
||||||
|
)
|
||||||
|
and decision_set_id_from_ref(str(node.config.get("source_ref") or ""))
|
||||||
|
is None
|
||||||
for node in graph.nodes
|
for node in graph.nodes
|
||||||
):
|
):
|
||||||
scopes.add(DATASOURCE_READ_SCOPE)
|
scopes.add(DATASOURCE_READ_SCOPE)
|
||||||
@@ -1574,6 +1584,26 @@ def _publish_pipeline_result(
|
|||||||
run.output_publication_ref = publication.ref
|
run.output_publication_ref = publication.ref
|
||||||
run.output_datasource_ref = publication.datasource.ref
|
run.output_datasource_ref = publication.datasource.ref
|
||||||
run.output_materialization_ref = publication.materialization.ref
|
run.output_materialization_ref = publication.materialization.ref
|
||||||
|
if publication.status in {"published_with_warnings", "review_required"}:
|
||||||
|
diagnostics = list(run.diagnostics)
|
||||||
|
diagnostics.append(
|
||||||
|
DataflowDiagnostic(
|
||||||
|
severity="warning",
|
||||||
|
code=(
|
||||||
|
"publication.review_required"
|
||||||
|
if publication.status == "review_required"
|
||||||
|
else "publication.warning"
|
||||||
|
),
|
||||||
|
message=(
|
||||||
|
"The output materialization requires review before it can "
|
||||||
|
"become the datasource's current state."
|
||||||
|
if publication.status == "review_required"
|
||||||
|
else "The output was published with datasource validation "
|
||||||
|
"warnings."
|
||||||
|
),
|
||||||
|
).model_dump(mode="json")
|
||||||
|
)
|
||||||
|
run.diagnostics = diagnostics
|
||||||
|
|
||||||
|
|
||||||
def _mark_pipeline_run_failed(
|
def _mark_pipeline_run_failed(
|
||||||
@@ -1880,6 +1910,49 @@ def _datasource_source_resolver(
|
|||||||
provider = datasource_catalogue(registry)
|
provider = datasource_catalogue(registry)
|
||||||
|
|
||||||
def resolve_source(node: GraphNode, limit: int) -> ResolvedSource:
|
def resolve_source(node: GraphNode, limit: int) -> ResolvedSource:
|
||||||
|
source_ref = str(node.config.get("source_ref") or "")
|
||||||
|
decision_set_id = decision_set_id_from_ref(source_ref)
|
||||||
|
if decision_set_id is not None:
|
||||||
|
try:
|
||||||
|
decision_set = get_decision_set(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
decision_set_id=decision_set_id,
|
||||||
|
include_decisions=False,
|
||||||
|
)
|
||||||
|
require_definition_action(
|
||||||
|
decision_set.pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=registry,
|
||||||
|
action="view",
|
||||||
|
)
|
||||||
|
except (PermissionError, ValueError) as exc:
|
||||||
|
raise PipelineExecutionError(
|
||||||
|
str(exc),
|
||||||
|
node_id=node.id,
|
||||||
|
) from exc
|
||||||
|
rows, total_rows = current_decision_rows(
|
||||||
|
session,
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
fingerprint = decision_set_fingerprint(decision_set)
|
||||||
|
expected_fingerprint = _clean_optional(
|
||||||
|
node.config.get("expected_fingerprint")
|
||||||
|
)
|
||||||
|
if expected_fingerprint and expected_fingerprint != fingerprint:
|
||||||
|
raise PipelineExecutionError(
|
||||||
|
"Reconciliation decisions changed; refresh the decision source before running it.",
|
||||||
|
node_id=node.id,
|
||||||
|
)
|
||||||
|
return ResolvedSource(
|
||||||
|
rows=rows,
|
||||||
|
source_ref=source_ref,
|
||||||
|
provider="dataflow.reconciliation_decisions",
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
total_rows=total_rows,
|
||||||
|
truncated=total_rows > len(rows),
|
||||||
|
)
|
||||||
if provider is None:
|
if provider is None:
|
||||||
raise PipelineExecutionError(
|
raise PipelineExecutionError(
|
||||||
"Datasource-backed execution requires the Datasources "
|
"Datasource-backed execution requires the Datasources "
|
||||||
|
|||||||
@@ -0,0 +1,546 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import unittest
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.dsar import (
|
||||||
|
DsarErasureActionRef,
|
||||||
|
DsarProvider,
|
||||||
|
DsarRecordRef,
|
||||||
|
DsarSubjectRef,
|
||||||
|
)
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.privacy.dsar_workflow import (
|
||||||
|
create_data_subject_request,
|
||||||
|
search_data_subject_request,
|
||||||
|
)
|
||||||
|
from govoplan_dataflow.backend.db.models import (
|
||||||
|
DataflowPipeline,
|
||||||
|
DataflowPipelineDeployment,
|
||||||
|
DataflowPipelineRevision,
|
||||||
|
DataflowReconciliationDecision,
|
||||||
|
DataflowReconciliationDecisionSet,
|
||||||
|
DataflowRun,
|
||||||
|
DataflowTrigger,
|
||||||
|
DataflowTriggerDelivery,
|
||||||
|
)
|
||||||
|
from govoplan_dataflow.backend.dsar_provider import (
|
||||||
|
DATAFLOW_DSAR_CAPABILITY,
|
||||||
|
DataflowDsarProvider,
|
||||||
|
)
|
||||||
|
from govoplan_dataflow.backend.manifest import manifest
|
||||||
|
|
||||||
|
|
||||||
|
NOW = datetime(2026, 8, 21, 20, 0, tzinfo=UTC)
|
||||||
|
SECRET = "private-dataflow-detail-do-not-export"
|
||||||
|
|
||||||
|
|
||||||
|
class _Registry:
|
||||||
|
def __init__(self, provider: DataflowDsarProvider, *, active: bool = True) -> None:
|
||||||
|
self.provider = provider
|
||||||
|
self.active = active
|
||||||
|
|
||||||
|
def capability_names(self):
|
||||||
|
return (DATAFLOW_DSAR_CAPABILITY,)
|
||||||
|
|
||||||
|
def capability_owner(self, name):
|
||||||
|
self._assert_capability(name)
|
||||||
|
return "dataflow"
|
||||||
|
|
||||||
|
def tenant_entitlement_resolver(self):
|
||||||
|
active = self.active
|
||||||
|
|
||||||
|
class _Resolver:
|
||||||
|
@staticmethod
|
||||||
|
def resolve(session, tenant_id):
|
||||||
|
del session, tenant_id
|
||||||
|
return type(
|
||||||
|
"State",
|
||||||
|
(),
|
||||||
|
{"effective_modules": ("dataflow",) if active else ()},
|
||||||
|
)()
|
||||||
|
|
||||||
|
return _Resolver()
|
||||||
|
|
||||||
|
def require_tenant_capability(self, name, session, **kwargs):
|
||||||
|
del session, kwargs
|
||||||
|
self._assert_capability(name)
|
||||||
|
return self.provider
|
||||||
|
|
||||||
|
def manifests(self):
|
||||||
|
return (type("Manifest", (), {"id": "dataflow"})(),)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _assert_capability(name: str) -> None:
|
||||||
|
if name != DATAFLOW_DSAR_CAPABILITY:
|
||||||
|
raise KeyError(name)
|
||||||
|
|
||||||
|
|
||||||
|
class DataflowDsarProviderTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(self.engine)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
self.provider = DataflowDsarProvider()
|
||||||
|
self.assertIsInstance(self.provider, DsarProvider)
|
||||||
|
self._seed()
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def _seed(self) -> None:
|
||||||
|
pipeline = DataflowPipeline(
|
||||||
|
id="pipeline-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scope_type="tenant",
|
||||||
|
definition_kind="flow",
|
||||||
|
name=SECRET,
|
||||||
|
description=SECRET,
|
||||||
|
status="active",
|
||||||
|
current_revision=1,
|
||||||
|
created_by="account-1",
|
||||||
|
updated_by="account-1",
|
||||||
|
metadata_={"secret": SECRET},
|
||||||
|
derivation_provenance={"secret": SECRET},
|
||||||
|
)
|
||||||
|
other = DataflowPipeline(
|
||||||
|
id="pipeline-other",
|
||||||
|
tenant_id="tenant-2",
|
||||||
|
scope_type="tenant",
|
||||||
|
definition_kind="flow",
|
||||||
|
name=SECRET,
|
||||||
|
status="active",
|
||||||
|
current_revision=1,
|
||||||
|
created_by="account-1",
|
||||||
|
updated_by="account-1",
|
||||||
|
)
|
||||||
|
self.session.add_all((pipeline, other))
|
||||||
|
self.session.flush()
|
||||||
|
revision = DataflowPipelineRevision(
|
||||||
|
id="revision-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=pipeline.id,
|
||||||
|
revision=1,
|
||||||
|
schema_version=1,
|
||||||
|
graph={"secret": SECRET},
|
||||||
|
sql_text=SECRET,
|
||||||
|
editor_mode="graph",
|
||||||
|
content_hash="a" * 64,
|
||||||
|
created_by="account-1",
|
||||||
|
)
|
||||||
|
self.session.add(revision)
|
||||||
|
self.session.flush()
|
||||||
|
decision_set = DataflowReconciliationDecisionSet(
|
||||||
|
id="decision-set-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=pipeline.id,
|
||||||
|
name=SECRET,
|
||||||
|
node_id="reconcile",
|
||||||
|
created_by="account-1",
|
||||||
|
updated_by="account-1",
|
||||||
|
)
|
||||||
|
self.session.add(decision_set)
|
||||||
|
self.session.flush()
|
||||||
|
run = DataflowRun(
|
||||||
|
id="run-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=pipeline.id,
|
||||||
|
pipeline_revision_id=revision.id,
|
||||||
|
run_type="execute",
|
||||||
|
status="succeeded",
|
||||||
|
execution_backend="python",
|
||||||
|
environment="production",
|
||||||
|
executor_version="dataflow-v1",
|
||||||
|
definition_hash="b" * 64,
|
||||||
|
idempotency_key=SECRET,
|
||||||
|
request_hash="c" * 64,
|
||||||
|
request_={"secret": SECRET},
|
||||||
|
invocation_kind="manual",
|
||||||
|
correlation_id=SECRET,
|
||||||
|
causation_id=SECRET,
|
||||||
|
source_fingerprints=[{"secret": SECRET}],
|
||||||
|
result_schema=[{"secret": SECRET}],
|
||||||
|
diagnostics=[{"secret": SECRET}],
|
||||||
|
input_row_count=1,
|
||||||
|
output_row_count=1,
|
||||||
|
output_publication_ref=SECRET,
|
||||||
|
output_datasource_ref=SECRET,
|
||||||
|
output_materialization_ref=SECRET,
|
||||||
|
attempts=1,
|
||||||
|
progress_percent=100,
|
||||||
|
progress_phase="complete",
|
||||||
|
retention_until=NOW + timedelta(days=30),
|
||||||
|
authorization_={"submitted_principal": {"secret": SECRET}},
|
||||||
|
resource_budget={"secret": SECRET},
|
||||||
|
started_at=NOW,
|
||||||
|
finished_at=NOW,
|
||||||
|
error=SECRET,
|
||||||
|
created_by="account-1",
|
||||||
|
)
|
||||||
|
trigger = DataflowTrigger(
|
||||||
|
id="trigger-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=pipeline.id,
|
||||||
|
pipeline_revision_id=revision.id,
|
||||||
|
name=SECRET,
|
||||||
|
kind="event",
|
||||||
|
status="active",
|
||||||
|
revision=1,
|
||||||
|
config_={"secret": SECRET},
|
||||||
|
publication_={"secret": SECRET},
|
||||||
|
row_limit=100,
|
||||||
|
catch_up_policy="coalesce",
|
||||||
|
max_concurrent_runs=1,
|
||||||
|
next_fire_at=NOW + timedelta(hours=1),
|
||||||
|
last_error=SECRET,
|
||||||
|
authorization_account_id="account-1",
|
||||||
|
authorization_membership_id="membership-1",
|
||||||
|
authorization_ref=SECRET,
|
||||||
|
grant_scopes=[SECRET],
|
||||||
|
created_by="account-1",
|
||||||
|
updated_by="account-1",
|
||||||
|
)
|
||||||
|
self.session.add_all(
|
||||||
|
(
|
||||||
|
DataflowReconciliationDecision(
|
||||||
|
id="decision-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
revision=1,
|
||||||
|
key_hash="d" * 64,
|
||||||
|
input_hash="e" * 64,
|
||||||
|
action="correct",
|
||||||
|
reason=SECRET,
|
||||||
|
correction={"secret": SECRET},
|
||||||
|
actor_ref="account-1",
|
||||||
|
decided_at=NOW,
|
||||||
|
),
|
||||||
|
run,
|
||||||
|
DataflowPipelineDeployment(
|
||||||
|
id="deployment-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=pipeline.id,
|
||||||
|
pipeline_revision_id=revision.id,
|
||||||
|
environment="production",
|
||||||
|
source_environment="staging",
|
||||||
|
status="active",
|
||||||
|
provenance={"secret": SECRET},
|
||||||
|
promoted_by="account-1",
|
||||||
|
),
|
||||||
|
trigger,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.session.flush()
|
||||||
|
self.session.add(
|
||||||
|
DataflowTriggerDelivery(
|
||||||
|
id="delivery-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
trigger_id=trigger.id,
|
||||||
|
pipeline_id=pipeline.id,
|
||||||
|
pipeline_revision_id=revision.id,
|
||||||
|
source_key=SECRET,
|
||||||
|
invocation_kind="event",
|
||||||
|
status="succeeded",
|
||||||
|
scheduled_for=NOW,
|
||||||
|
event_={"secret": SECRET},
|
||||||
|
run_id=run.id,
|
||||||
|
attempts=1,
|
||||||
|
authorization_provenance={"secret": SECRET},
|
||||||
|
error=SECRET,
|
||||||
|
finished_at=NOW,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_canonical_selector_is_minimized_and_marks_automation_authority(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(7, len(records))
|
||||||
|
trigger = next(
|
||||||
|
item for item in records if item.resource_type == "dataflow_trigger"
|
||||||
|
)
|
||||||
|
self.assertEqual("dataflow_automation_authority", trigger.category)
|
||||||
|
exported = json.dumps([record.to_dict() for record in records])
|
||||||
|
self.assertNotIn(SECRET, exported)
|
||||||
|
self.assertNotIn("account-1", exported)
|
||||||
|
self.assertNotIn("membership-1", exported)
|
||||||
|
self.assertNotIn("pipeline-other", exported)
|
||||||
|
|
||||||
|
def test_exact_pipeline_package_is_review_only_and_minimized(self) -> None:
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
account_id="account-1",
|
||||||
|
external_references={"dataflow.pipeline": "pipeline:pipeline-1"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(8, len(records))
|
||||||
|
exported = json.dumps([record.to_dict() for record in records])
|
||||||
|
self.assertNotIn(SECRET, exported)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
external_references={"dataflow.pipeline": "pipeline-1"}
|
||||||
|
),
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
self.assertEqual({"manual_review"}, {action.kind for action in actions})
|
||||||
|
|
||||||
|
def test_every_exact_reference_and_fail_closed_correlation(self) -> None:
|
||||||
|
references = {
|
||||||
|
"dataflow.pipeline_revision": "revision-1",
|
||||||
|
"dataflow.decision_set": "decision-set-1",
|
||||||
|
"dataflow.decision": "decision-1",
|
||||||
|
"dataflow.run": "dataflow-run:run-1",
|
||||||
|
"dataflow.deployment": "deployment-1",
|
||||||
|
"dataflow.trigger": "trigger-1",
|
||||||
|
"dataflow.trigger_delivery": "delivery-1",
|
||||||
|
}
|
||||||
|
for key, value in references.items():
|
||||||
|
with self.subTest(key=key):
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(external_references={key: value}),
|
||||||
|
)
|
||||||
|
self.assertEqual(1, len(records))
|
||||||
|
|
||||||
|
mismatch = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
account_id="account-2",
|
||||||
|
external_references={"dataflow.run": "run-1"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
wrong_tenant = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-2",
|
||||||
|
subject=DsarSubjectRef(external_references={"dataflow.run": "run-1"}),
|
||||||
|
)
|
||||||
|
conflict = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=DsarSubjectRef(
|
||||||
|
external_references={
|
||||||
|
"dataflow.pipeline_revision": "revision-1",
|
||||||
|
"dataflow.revision": "different",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertEqual((), mismatch)
|
||||||
|
self.assertEqual((), wrong_tenant)
|
||||||
|
self.assertEqual((), conflict)
|
||||||
|
|
||||||
|
def test_terminal_run_and_delivery_minimization_is_idempotent(self) -> None:
|
||||||
|
subject = DsarSubjectRef(
|
||||||
|
external_references={
|
||||||
|
"dataflow.run": "run-1",
|
||||||
|
"dataflow.delivery": "delivery-1",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=subject,
|
||||||
|
)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=subject,
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
self.assertEqual({"anonymize"}, {action.kind for action in actions})
|
||||||
|
first = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=subject,
|
||||||
|
actions=actions,
|
||||||
|
request_id="dsar-1",
|
||||||
|
)
|
||||||
|
second = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=subject,
|
||||||
|
actions=actions,
|
||||||
|
request_id="dsar-1-retry",
|
||||||
|
)
|
||||||
|
self.assertTrue(all(result.status == "executed" for result in first))
|
||||||
|
self.assertTrue(all(result.status == "unchanged" for result in second))
|
||||||
|
run = self.session.get(DataflowRun, "run-1")
|
||||||
|
delivery = self.session.get(DataflowTriggerDelivery, "delivery-1")
|
||||||
|
self.assertEqual({}, run.request_)
|
||||||
|
self.assertEqual([], run.diagnostics)
|
||||||
|
self.assertIsNotNone(run.purged_at)
|
||||||
|
self.assertIsNone(delivery.event_)
|
||||||
|
self.assertEqual({}, delivery.authorization_provenance)
|
||||||
|
self.assertEqual(SECRET, delivery.source_key)
|
||||||
|
|
||||||
|
def test_automation_authority_is_revoked_with_retry_support(self) -> None:
|
||||||
|
subject = DsarSubjectRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
)
|
||||||
|
records = self.provider.search_subject(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=subject,
|
||||||
|
)
|
||||||
|
actions = self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=subject,
|
||||||
|
records=records,
|
||||||
|
)
|
||||||
|
trigger_action = next(
|
||||||
|
action for action in actions if action.resource_type == "dataflow_trigger"
|
||||||
|
)
|
||||||
|
self.assertEqual("revoke", trigger_action.kind)
|
||||||
|
self.assertTrue(trigger_action.executable)
|
||||||
|
first = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=subject,
|
||||||
|
actions=(trigger_action,),
|
||||||
|
request_id="dsar-2",
|
||||||
|
)
|
||||||
|
second = self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=subject,
|
||||||
|
actions=(trigger_action,),
|
||||||
|
request_id="dsar-2-retry",
|
||||||
|
)
|
||||||
|
self.assertEqual("executed", first[0].status)
|
||||||
|
self.assertEqual("unchanged", second[0].status)
|
||||||
|
trigger = self.session.get(DataflowTrigger, "trigger-1")
|
||||||
|
self.assertEqual("disabled", trigger.status)
|
||||||
|
self.assertEqual("redacted", trigger.authorization_account_id)
|
||||||
|
self.assertEqual("redacted", trigger.authorization_membership_id)
|
||||||
|
self.assertEqual([], trigger.grant_scopes)
|
||||||
|
self.assertEqual({}, trigger.config_)
|
||||||
|
|
||||||
|
def test_foreign_records_and_actions_are_rejected(self) -> None:
|
||||||
|
subject = DsarSubjectRef(account_id="account-1")
|
||||||
|
with self.assertRaisesRegex(ValueError, "foreign provider record"):
|
||||||
|
self.provider.plan_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=subject,
|
||||||
|
records=(
|
||||||
|
DsarRecordRef(
|
||||||
|
provider_id="cases",
|
||||||
|
module_id="cases",
|
||||||
|
resource_type="case",
|
||||||
|
resource_id="case-1",
|
||||||
|
category="case",
|
||||||
|
title="Case",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
with self.assertRaisesRegex(ValueError, "foreign provider action"):
|
||||||
|
self.provider.execute_erasure(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
subject=subject,
|
||||||
|
actions=(
|
||||||
|
DsarErasureActionRef(
|
||||||
|
action_id="cases:delete:case:case-1",
|
||||||
|
provider_id="cases",
|
||||||
|
module_id="cases",
|
||||||
|
kind="delete",
|
||||||
|
resource_type="case",
|
||||||
|
resource_id="case-1",
|
||||||
|
title="Delete case",
|
||||||
|
rationale="Foreign",
|
||||||
|
executable=True,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
request_id="dsar-3",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_core_workflow_reports_active_and_inactive_provider(self) -> None:
|
||||||
|
row = create_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
reference="DSAR-DATAFLOW-1",
|
||||||
|
request_kind="access_and_erasure",
|
||||||
|
subject=DsarSubjectRef(account_id="account-1"),
|
||||||
|
purpose="Respond to a verified request.",
|
||||||
|
legal_basis="Article 15 and 17 GDPR",
|
||||||
|
due_at=None,
|
||||||
|
requested_by_account_id="privacy-officer",
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
search_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
registry=_Registry(self.provider),
|
||||||
|
row=row,
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
[DATAFLOW_DSAR_CAPABILITY],
|
||||||
|
row.coverage["provider_capabilities"],
|
||||||
|
)
|
||||||
|
self.assertEqual(7, row.search_result["record_count"])
|
||||||
|
|
||||||
|
inactive = create_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
reference="DSAR-DATAFLOW-2",
|
||||||
|
request_kind="access",
|
||||||
|
subject=DsarSubjectRef(account_id="account-1"),
|
||||||
|
purpose="Respond to a verified request.",
|
||||||
|
legal_basis="Article 15 GDPR",
|
||||||
|
due_at=None,
|
||||||
|
requested_by_account_id="privacy-officer",
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
search_data_subject_request(
|
||||||
|
self.session,
|
||||||
|
registry=_Registry(self.provider, active=False),
|
||||||
|
row=inactive,
|
||||||
|
expected_revision=1,
|
||||||
|
)
|
||||||
|
self.assertEqual([], inactive.coverage["provider_capabilities"])
|
||||||
|
self.assertEqual(
|
||||||
|
[DATAFLOW_DSAR_CAPABILITY],
|
||||||
|
inactive.coverage["inactive_provider_capabilities"],
|
||||||
|
)
|
||||||
|
self.assertEqual(0, inactive.search_result["record_count"])
|
||||||
|
|
||||||
|
def test_manifest_registers_and_documents_capability(self) -> None:
|
||||||
|
self.assertIn(DATAFLOW_DSAR_CAPABILITY, manifest.capability_factories)
|
||||||
|
self.assertIn(DATAFLOW_DSAR_CAPABILITY, manifest.capability_documentation)
|
||||||
|
self.assertIn(
|
||||||
|
DATAFLOW_DSAR_CAPABILITY,
|
||||||
|
{item.name for item in manifest.provides_interfaces},
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
any(
|
||||||
|
topic.id == "dataflow.data-subject-requests"
|
||||||
|
and {"admin", "user"}.issubset(topic.documentation_types)
|
||||||
|
for topic in manifest.documentation
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -22,6 +22,7 @@ class DataflowInterfaceDocumentationContractTests(unittest.TestCase):
|
|||||||
"dataflow.sql",
|
"dataflow.sql",
|
||||||
"dataflow.inspector",
|
"dataflow.inspector",
|
||||||
"dataflow.results",
|
"dataflow.results",
|
||||||
|
"dataflow.decisions",
|
||||||
"dataflow.triggers",
|
"dataflow.triggers",
|
||||||
"dataflow.runs",
|
"dataflow.runs",
|
||||||
"dataflow.widget.pipelines",
|
"dataflow.widget.pipelines",
|
||||||
@@ -38,6 +39,7 @@ class DataflowInterfaceDocumentationContractTests(unittest.TestCase):
|
|||||||
"dataflow.runs",
|
"dataflow.runs",
|
||||||
):
|
):
|
||||||
self.assertEqual("dataflow.page", surfaces[surface_id].parent_id)
|
self.assertEqual("dataflow.page", surfaces[surface_id].parent_id)
|
||||||
|
self.assertEqual("dataflow.results", surfaces["dataflow.decisions"].parent_id)
|
||||||
|
|
||||||
def test_help_and_consequence_metadata_remain_published(self) -> None:
|
def test_help_and_consequence_metadata_remain_published(self) -> None:
|
||||||
topics = {topic.id: topic for topic in get_manifest().documentation}
|
topics = {topic.id: topic for topic in get_manifest().documentation}
|
||||||
@@ -48,8 +50,10 @@ class DataflowInterfaceDocumentationContractTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertIn("dataflow.state.read-only", boundary.metadata["help_contexts"])
|
self.assertIn("dataflow.state.read-only", boundary.metadata["help_contexts"])
|
||||||
self.assertIn("dataflow.field.expression", nodes.metadata["help_contexts"])
|
self.assertIn("dataflow.field.expression", nodes.metadata["help_contexts"])
|
||||||
|
self.assertIn("dataflow.action.review-decisions", nodes.metadata["help_contexts"])
|
||||||
self.assertIn("save_revision", fields.metadata["consequence_classes"])
|
self.assertIn("save_revision", fields.metadata["consequence_classes"])
|
||||||
self.assertIn("delete_pipeline", fields.metadata["consequence_classes"])
|
self.assertIn("delete_pipeline", fields.metadata["consequence_classes"])
|
||||||
|
self.assertIn("record_decision", fields.metadata["consequence_classes"])
|
||||||
self.assertIn("publish_output", execution.metadata["consequence_classes"])
|
self.assertIn("publish_output", execution.metadata["consequence_classes"])
|
||||||
self.assertIn("promote_revision", execution.metadata["consequence_classes"])
|
self.assertIn("promote_revision", execution.metadata["consequence_classes"])
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,7 @@ class DataflowManifestTests(unittest.TestCase):
|
|||||||
"dataflow.run_worker",
|
"dataflow.run_worker",
|
||||||
"dataflow.dataset_output",
|
"dataflow.dataset_output",
|
||||||
"dataflow.trigger_dispatcher",
|
"dataflow.trigger_dispatcher",
|
||||||
|
"privacy.dsar.dataflow",
|
||||||
},
|
},
|
||||||
{item.name for item in manifest.provides_interfaces},
|
{item.name for item in manifest.provides_interfaces},
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ class DataflowMigrationTests(unittest.TestCase):
|
|||||||
try:
|
try:
|
||||||
with engine.connect() as connection:
|
with engine.connect() as connection:
|
||||||
self.assertIn(
|
self.assertIn(
|
||||||
"f6c2a9d4e7b1",
|
"a3d7f1c5e9b2",
|
||||||
set(MigrationContext.configure(connection).get_current_heads()),
|
set(MigrationContext.configure(connection).get_current_heads()),
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@@ -32,6 +32,8 @@ class DataflowMigrationTests(unittest.TestCase):
|
|||||||
"dataflow_pipelines",
|
"dataflow_pipelines",
|
||||||
"dataflow_pipeline_revisions",
|
"dataflow_pipeline_revisions",
|
||||||
"dataflow_pipeline_deployments",
|
"dataflow_pipeline_deployments",
|
||||||
|
"dataflow_reconciliation_decision_sets",
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
"dataflow_runs",
|
"dataflow_runs",
|
||||||
"dataflow_triggers",
|
"dataflow_triggers",
|
||||||
"dataflow_trigger_deliveries",
|
"dataflow_trigger_deliveries",
|
||||||
|
|||||||
@@ -0,0 +1,247 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.concurrency import RevisionConflictError
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_dataflow.backend.db.models import (
|
||||||
|
DataflowPipeline,
|
||||||
|
DataflowReconciliationDecision,
|
||||||
|
DataflowReconciliationDecisionSet,
|
||||||
|
)
|
||||||
|
from govoplan_dataflow.backend.executor import PipelineExecutionError
|
||||||
|
from govoplan_dataflow.backend.reconciliation_decisions import (
|
||||||
|
create_decision_set,
|
||||||
|
current_decision_rows,
|
||||||
|
current_decisions,
|
||||||
|
decision_set_fingerprint,
|
||||||
|
decision_set_payload,
|
||||||
|
decision_set_source_payload,
|
||||||
|
record_decision,
|
||||||
|
)
|
||||||
|
from govoplan_dataflow.backend.schemas import GraphNode, GraphPosition
|
||||||
|
from govoplan_dataflow.backend.service import _datasource_source_resolver
|
||||||
|
|
||||||
|
|
||||||
|
TABLES = (
|
||||||
|
DataflowPipeline.__table__,
|
||||||
|
DataflowReconciliationDecisionSet.__table__,
|
||||||
|
DataflowReconciliationDecision.__table__,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(self.engine, tables=TABLES)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
self.pipeline = DataflowPipeline(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scope_type="tenant",
|
||||||
|
scope_id="tenant-1",
|
||||||
|
name="Monthly reconciliation",
|
||||||
|
status="active",
|
||||||
|
created_by="account-1",
|
||||||
|
updated_by="account-1",
|
||||||
|
)
|
||||||
|
self.session.add(self.pipeline)
|
||||||
|
self.session.flush()
|
||||||
|
self.principal = ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset(
|
||||||
|
{
|
||||||
|
"dataflow:pipeline:read",
|
||||||
|
"dataflow:pipeline:write",
|
||||||
|
"dataflow:pipeline:run",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
),
|
||||||
|
account=SimpleNamespace(id="account-1"),
|
||||||
|
user=SimpleNamespace(id="membership-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_decisions_are_immutable_current_rows_and_occ_protected(self) -> None:
|
||||||
|
decision_set = create_decision_set(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=self.pipeline.id,
|
||||||
|
name="July review",
|
||||||
|
node_id="reconcile",
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
first = record_decision(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
expected_revision=1,
|
||||||
|
key_hash="a" * 64,
|
||||||
|
input_hash="b" * 64,
|
||||||
|
action="accept",
|
||||||
|
reason="Checked against the source record.",
|
||||||
|
correction=None,
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
first_fingerprint = decision_set_fingerprint(first)
|
||||||
|
second = record_decision(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
expected_revision=2,
|
||||||
|
key_hash="a" * 64,
|
||||||
|
input_hash="c" * 64,
|
||||||
|
action="correct",
|
||||||
|
reason="The monthly source changed after review.",
|
||||||
|
correction={"amount": 25},
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(2, len(second.decisions))
|
||||||
|
self.assertEqual(1, len(current_decisions(second)))
|
||||||
|
self.assertEqual("c" * 64, current_decisions(second)[0].input_hash)
|
||||||
|
self.assertNotEqual(first_fingerprint, decision_set_fingerprint(second))
|
||||||
|
payload = decision_set_payload(second)
|
||||||
|
self.assertEqual(2, len(payload["history"]))
|
||||||
|
self.assertEqual(1, len(payload["current_decisions"]))
|
||||||
|
with self.assertRaises(RevisionConflictError):
|
||||||
|
record_decision(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
expected_revision=2,
|
||||||
|
key_hash="d" * 64,
|
||||||
|
input_hash="e" * 64,
|
||||||
|
action="defer",
|
||||||
|
reason="Needs another source.",
|
||||||
|
correction=None,
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_decision_set_is_a_fingerprinted_reference_source(self) -> None:
|
||||||
|
decision_set = create_decision_set(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=self.pipeline.id,
|
||||||
|
name="July review",
|
||||||
|
node_id="reconcile",
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
record_decision(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
expected_revision=1,
|
||||||
|
key_hash="a" * 64,
|
||||||
|
input_hash="b" * 64,
|
||||||
|
action="reject",
|
||||||
|
reason="The observed record belongs to another case.",
|
||||||
|
correction=None,
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
source = decision_set_source_payload(decision_set)
|
||||||
|
resolver = _datasource_source_resolver(
|
||||||
|
session=self.session,
|
||||||
|
principal=self.principal,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
node = GraphNode(
|
||||||
|
id="decisions",
|
||||||
|
type="source.reference",
|
||||||
|
label="Review decisions",
|
||||||
|
position=GraphPosition(x=0, y=0),
|
||||||
|
config={
|
||||||
|
"source_ref": source["ref"],
|
||||||
|
"expected_fingerprint": source["fingerprint"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
resolved = resolver(node, 100)
|
||||||
|
|
||||||
|
self.assertEqual("dataflow.reconciliation_decisions", resolved.provider)
|
||||||
|
self.assertEqual(1, resolved.total_rows)
|
||||||
|
self.assertEqual("reject", resolved.rows[0]["action"])
|
||||||
|
stale = node.model_copy(deep=True)
|
||||||
|
stale.config["expected_fingerprint"] = "sha256:" + "0" * 64
|
||||||
|
with self.assertRaisesRegex(PipelineExecutionError, "changed"):
|
||||||
|
resolver(stale, 100)
|
||||||
|
|
||||||
|
def test_tenant_can_keep_decisions_for_visible_system_pipeline(self) -> None:
|
||||||
|
system_pipeline = DataflowPipeline(
|
||||||
|
tenant_id=None,
|
||||||
|
scope_type="system",
|
||||||
|
scope_id=None,
|
||||||
|
name="Governed monthly reconciliation",
|
||||||
|
status="active",
|
||||||
|
created_by="system-admin",
|
||||||
|
updated_by="system-admin",
|
||||||
|
)
|
||||||
|
self.session.add(system_pipeline)
|
||||||
|
self.session.flush()
|
||||||
|
|
||||||
|
decision_set = create_decision_set(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=system_pipeline.id,
|
||||||
|
name="Tenant July review",
|
||||||
|
node_id=None,
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("tenant-1", decision_set.tenant_id)
|
||||||
|
self.assertEqual(system_pipeline.id, decision_set.pipeline_id)
|
||||||
|
|
||||||
|
def test_summary_and_current_projection_do_not_require_full_history(self) -> None:
|
||||||
|
decision_set = create_decision_set(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=self.pipeline.id,
|
||||||
|
name="Bounded review",
|
||||||
|
node_id="reconcile",
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
for expected_revision, key_hash, input_hash in (
|
||||||
|
(1, "a" * 64, "b" * 64),
|
||||||
|
(2, "c" * 64, "d" * 64),
|
||||||
|
(3, "a" * 64, "e" * 64),
|
||||||
|
):
|
||||||
|
record_decision(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
expected_revision=expected_revision,
|
||||||
|
key_hash=key_hash,
|
||||||
|
input_hash=input_hash,
|
||||||
|
action="accept",
|
||||||
|
reason="Reviewed against the current input.",
|
||||||
|
correction=None,
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
summary = decision_set_payload(decision_set, include_decisions=False)
|
||||||
|
rows, total = current_decision_rows(
|
||||||
|
self.session,
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
limit=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertFalse(summary["decisions_included"])
|
||||||
|
self.assertEqual([], summary["current_decisions"])
|
||||||
|
self.assertEqual([], summary["history"])
|
||||||
|
self.assertEqual(2, total)
|
||||||
|
self.assertEqual(1, len(rows))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
+33
-2
@@ -123,8 +123,9 @@ def runtime_identity() -> RuntimeIdentity:
|
|||||||
|
|
||||||
|
|
||||||
class FakePublicationProvider:
|
class FakePublicationProvider:
|
||||||
def __init__(self) -> None:
|
def __init__(self, status: str = "published") -> None:
|
||||||
self.requests = []
|
self.requests = []
|
||||||
|
self.status = status
|
||||||
|
|
||||||
def publish_rows(self, _session, _principal, *, request):
|
def publish_rows(self, _session, _principal, *, request):
|
||||||
self.requests.append(request)
|
self.requests.append(request)
|
||||||
@@ -139,7 +140,7 @@ class FakePublicationProvider:
|
|||||||
)
|
)
|
||||||
return DatasourcePublicationResult(
|
return DatasourcePublicationResult(
|
||||||
ref="publication:publication-1",
|
ref="publication:publication-1",
|
||||||
status="published",
|
status=self.status,
|
||||||
datasource=descriptor,
|
datasource=descriptor,
|
||||||
materialization=DatasourceMaterialization(
|
materialization=DatasourceMaterialization(
|
||||||
ref="materialization:materialization-1",
|
ref="materialization:materialization-1",
|
||||||
@@ -539,6 +540,36 @@ class DataflowServiceTests(unittest.TestCase):
|
|||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_publication_review_state_is_exposed_to_workflow_as_a_warning(
|
||||||
|
self,
|
||||||
|
) -> None:
|
||||||
|
pipeline = self._create()
|
||||||
|
provider = FakePublicationProvider("review_required")
|
||||||
|
|
||||||
|
run, _ = start_pipeline_run(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
actor_id="user-1",
|
||||||
|
principal=principal(),
|
||||||
|
registry=FakeRegistry(provider),
|
||||||
|
request=DataflowRunRequest(
|
||||||
|
pipeline_ref=f"pipeline:{pipeline.id}",
|
||||||
|
revision=1,
|
||||||
|
idempotency_key="review-publication",
|
||||||
|
publication=DataflowPublicationTarget(
|
||||||
|
name="Review output",
|
||||||
|
source_name="review_output",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("succeeded", run.status)
|
||||||
|
self.assertEqual(
|
||||||
|
"publication.review_required",
|
||||||
|
run.diagnostics[-1]["code"],
|
||||||
|
)
|
||||||
|
self.assertEqual("warning", run.diagnostics[-1]["severity"])
|
||||||
|
|
||||||
def test_publication_without_datasources_finishes_as_failed_run(self) -> None:
|
def test_publication_without_datasources_finishes_as_failed_run(self) -> None:
|
||||||
pipeline = self._create()
|
pipeline = self._create()
|
||||||
run, _ = start_pipeline_run(
|
run, _ = start_pipeline_run(
|
||||||
|
|||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@govoplan/dataflow-webui",
|
"name": "@govoplan/dataflow-webui",
|
||||||
"version": "0.1.15",
|
"version": "0.1.19",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "src/index.ts",
|
"main": "src/index.ts",
|
||||||
@@ -18,7 +18,7 @@
|
|||||||
"test:structure": "node scripts/test-dataflow-page-structure.mjs"
|
"test:structure": "node scripts/test-dataflow-page-structure.mjs"
|
||||||
},
|
},
|
||||||
"peerDependencies": {
|
"peerDependencies": {
|
||||||
"@govoplan/core-webui": "^0.1.15",
|
"@govoplan/core-webui": "^0.1.18",
|
||||||
"@xyflow/react": "^12.11.2",
|
"@xyflow/react": "^12.11.2",
|
||||||
"lucide-react": "^1.23.0",
|
"lucide-react": "^1.23.0",
|
||||||
"react": ">=19.2.7 <20",
|
"react": ">=19.2.7 <20",
|
||||||
|
|||||||
@@ -60,11 +60,14 @@ const checks = [
|
|||||||
moduleEntry.indexOf("./styles/dataflow.css"),
|
moduleEntry.indexOf("./styles/dataflow.css"),
|
||||||
"XyFlow base styles before GovOPlaN overrides"
|
"XyFlow base styles before GovOPlaN overrides"
|
||||||
],
|
],
|
||||||
[css.includes("height: calc(100vh - 115px)"), "full-height workspace"],
|
[
|
||||||
|
page.includes('<WorkspaceFrame as="main" height="viewport"'),
|
||||||
|
"central full-height workspace"
|
||||||
|
],
|
||||||
[css.includes(".dataflow-preview-table-wrap"), "bounded preview scrolling"],
|
[css.includes(".dataflow-preview-table-wrap"), "bounded preview scrolling"],
|
||||||
[
|
[
|
||||||
/\.dataflow-palette-items\s*\{[^}]*grid-auto-rows:\s*max-content;[^}]*overflow-y:\s*auto;/s.test(css),
|
page.includes("<DefinitionPalette") && !css.includes(".dataflow-palette-items"),
|
||||||
"bounded palette scrolling"
|
"centralized bounded palette scrolling"
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
/\.dataflow-inspector-fields\s*\{[^}]*grid-auto-rows:\s*max-content;[^}]*overflow-y:\s*auto;/s.test(css),
|
/\.dataflow-inspector-fields\s*\{[^}]*grid-auto-rows:\s*max-content;[^}]*overflow-y:\s*auto;/s.test(css),
|
||||||
|
|||||||
@@ -205,6 +205,7 @@ export type TabularSource = {
|
|||||||
columns: TabularSourceColumn[];
|
columns: TabularSourceColumn[];
|
||||||
schema_version: string;
|
schema_version: string;
|
||||||
fingerprint: string;
|
fingerprint: string;
|
||||||
|
decisions_included: boolean;
|
||||||
row_count?: number | null;
|
row_count?: number | null;
|
||||||
byte_count?: number | null;
|
byte_count?: number | null;
|
||||||
updated_at?: string | null;
|
updated_at?: string | null;
|
||||||
@@ -217,6 +218,38 @@ export type TabularSourceCatalogue = {
|
|||||||
sources: TabularSource[];
|
sources: TabularSource[];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type ReconciliationDecisionAction = "accept" | "reject" | "correct" | "defer";
|
||||||
|
|
||||||
|
export type ReconciliationDecision = {
|
||||||
|
ref: string;
|
||||||
|
id: string;
|
||||||
|
revision: number;
|
||||||
|
key_hash: string;
|
||||||
|
input_hash: string;
|
||||||
|
action: ReconciliationDecisionAction;
|
||||||
|
reason: string;
|
||||||
|
correction?: Record<string, unknown> | null;
|
||||||
|
actor_ref: string;
|
||||||
|
decided_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ReconciliationDecisionSet = {
|
||||||
|
ref: string;
|
||||||
|
id: string;
|
||||||
|
pipeline_id: string;
|
||||||
|
name: string;
|
||||||
|
node_id?: string | null;
|
||||||
|
resource_revision: number;
|
||||||
|
etag: string;
|
||||||
|
fingerprint: string;
|
||||||
|
current_decisions: ReconciliationDecision[];
|
||||||
|
history: ReconciliationDecision[];
|
||||||
|
created_by?: string | null;
|
||||||
|
updated_by?: string | null;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
export type PipelineRun = {
|
export type PipelineRun = {
|
||||||
ref: string;
|
ref: string;
|
||||||
pipeline_id: string;
|
pipeline_id: string;
|
||||||
@@ -363,6 +396,58 @@ export function createDataflowSourceSnapshot(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function listDataflowDecisionSets(
|
||||||
|
settings: ApiSettings,
|
||||||
|
pipelineId: string
|
||||||
|
): Promise<ReconciliationDecisionSet[]> {
|
||||||
|
const response = await apiFetch<{ decision_sets: ReconciliationDecisionSet[] }>(
|
||||||
|
settings,
|
||||||
|
`/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/decision-sets`
|
||||||
|
);
|
||||||
|
return response.decision_sets;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createDataflowDecisionSet(
|
||||||
|
settings: ApiSettings,
|
||||||
|
pipelineId: string,
|
||||||
|
payload: { name: string; node_id?: string | null }
|
||||||
|
): Promise<ReconciliationDecisionSet> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/decision-sets`,
|
||||||
|
{ method: "POST", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getDataflowDecisionSet(
|
||||||
|
settings: ApiSettings,
|
||||||
|
decisionSetId: string
|
||||||
|
): Promise<ReconciliationDecisionSet> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`/api/v1/dataflow/decision-sets/${encodeURIComponent(decisionSetId)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function recordDataflowDecision(
|
||||||
|
settings: ApiSettings,
|
||||||
|
decisionSetId: string,
|
||||||
|
payload: {
|
||||||
|
base_revision: number;
|
||||||
|
key_hash: string;
|
||||||
|
input_hash: string;
|
||||||
|
action: ReconciliationDecisionAction;
|
||||||
|
reason: string;
|
||||||
|
correction?: Record<string, unknown> | null;
|
||||||
|
}
|
||||||
|
): Promise<ReconciliationDecisionSet> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`/api/v1/dataflow/decision-sets/${encodeURIComponent(decisionSetId)}/decisions`,
|
||||||
|
{ method: "POST", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export async function listDataflowPipelines(settings: ApiSettings): Promise<Pipeline[]> {
|
export async function listDataflowPipelines(settings: ApiSettings): Promise<Pipeline[]> {
|
||||||
const response = await apiFetch<{ pipelines: Pipeline[] }>(settings, "/api/v1/dataflow/pipelines");
|
const response = await apiFetch<{ pipelines: Pipeline[] }>(settings, "/api/v1/dataflow/pipelines");
|
||||||
return response.pipelines;
|
return response.pipelines;
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import {
|
|||||||
type Edge,
|
type Edge,
|
||||||
type ReactFlowInstance
|
type ReactFlowInstance
|
||||||
} from "@xyflow/react";
|
} from "@xyflow/react";
|
||||||
|
import { StatePanel } from "@govoplan/core-webui";
|
||||||
import { definitionConnectionError } from "@govoplan/core-webui/definition-graph";
|
import { definitionConnectionError } from "@govoplan/core-webui/definition-graph";
|
||||||
import type {
|
import type {
|
||||||
DataflowDiagnostic,
|
DataflowDiagnostic,
|
||||||
@@ -226,7 +227,7 @@ export default function DataflowCanvas({
|
|||||||
return (
|
return (
|
||||||
<div
|
<div
|
||||||
ref={canvasRef}
|
ref={canvasRef}
|
||||||
className="dataflow-canvas"
|
className="definition-graph-canvas dataflow-canvas"
|
||||||
tabIndex={0}
|
tabIndex={0}
|
||||||
aria-label="Dataflow graph canvas"
|
aria-label="Dataflow graph canvas"
|
||||||
onDragOver={(event) => {
|
onDragOver={(event) => {
|
||||||
@@ -360,7 +361,7 @@ export default function DataflowCanvas({
|
|||||||
<Controls showInteractive={false} />
|
<Controls showInteractive={false} />
|
||||||
</ReactFlow>
|
</ReactFlow>
|
||||||
{!graph.nodes.length ? (
|
{!graph.nodes.length ? (
|
||||||
<div className="dataflow-canvas-empty">Drop a source here</div>
|
<StatePanel className="definition-graph-canvas-empty" size="fill" description="Drop a source here" />
|
||||||
) : null}
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { Handle, Position, type Node, type NodeProps } from "@xyflow/react";
|
import { Handle, Position, type Node, type NodeProps } from "@xyflow/react";
|
||||||
|
import { CountBadge, DefinitionNodeIcon } from "@govoplan/core-webui";
|
||||||
import type { NodeTypeDefinition } from "../../api/dataflow";
|
import type { NodeTypeDefinition } from "../../api/dataflow";
|
||||||
import { dataflowNodeIcon } from "./nodeIcons";
|
import { dataflowNodeIcon } from "./nodeIcons";
|
||||||
|
|
||||||
@@ -31,7 +32,7 @@ export default function DataflowNode({ data, selected }: NodeProps<DataflowFlowN
|
|||||||
id={port.id}
|
id={port.id}
|
||||||
type="target"
|
type="target"
|
||||||
position={Position.Left}
|
position={Position.Left}
|
||||||
className="dataflow-node-handle dataflow-node-handle-input"
|
className="definition-node-handle dataflow-node-handle-input"
|
||||||
style={{ top: portPosition(index, data.definition.input_ports.length) }}
|
style={{ top: portPosition(index, data.definition.input_ports.length) }}
|
||||||
/>
|
/>
|
||||||
))}
|
))}
|
||||||
@@ -46,15 +47,15 @@ export default function DataflowNode({ data, selected }: NodeProps<DataflowFlowN
|
|||||||
</span>
|
</span>
|
||||||
))
|
))
|
||||||
: null}
|
: null}
|
||||||
<span className="dataflow-node-icon" aria-hidden="true">
|
<DefinitionNodeIcon aria-hidden="true">
|
||||||
<Icon size={17} strokeWidth={1.8} />
|
<Icon size={17} strokeWidth={1.8} />
|
||||||
</span>
|
</DefinitionNodeIcon>
|
||||||
<span className="dataflow-node-copy">
|
<span className="dataflow-node-copy">
|
||||||
<strong>{data.label}</strong>
|
<strong>{data.label}</strong>
|
||||||
<small>{data.definition.label}</small>
|
<small>{data.definition.label}</small>
|
||||||
</span>
|
</span>
|
||||||
{typeof data.outputRows === "number" ? (
|
{typeof data.outputRows === "number" ? (
|
||||||
<span className="dataflow-node-count">{data.outputRows}</span>
|
<CountBadge tone="neutral" size="compact">{data.outputRows}</CountBadge>
|
||||||
) : null}
|
) : null}
|
||||||
{data.definition.output_ports.map((port, index) => (
|
{data.definition.output_ports.map((port, index) => (
|
||||||
<Handle
|
<Handle
|
||||||
@@ -62,7 +63,7 @@ export default function DataflowNode({ data, selected }: NodeProps<DataflowFlowN
|
|||||||
id={port.id}
|
id={port.id}
|
||||||
type="source"
|
type="source"
|
||||||
position={Position.Right}
|
position={Position.Right}
|
||||||
className="dataflow-node-handle dataflow-node-handle-output"
|
className="definition-node-handle dataflow-node-handle-output"
|
||||||
style={{ top: portPosition(index, data.definition.output_ports.length) }}
|
style={{ top: portPosition(index, data.definition.output_ports.length) }}
|
||||||
/>
|
/>
|
||||||
))}
|
))}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import {
|
|||||||
Code2,
|
Code2,
|
||||||
CopyPlus,
|
CopyPlus,
|
||||||
DatabaseZap,
|
DatabaseZap,
|
||||||
|
ListChecks,
|
||||||
Network,
|
Network,
|
||||||
Play,
|
Play,
|
||||||
Plus,
|
Plus,
|
||||||
@@ -25,20 +26,34 @@ import {
|
|||||||
TriangleAlert,
|
TriangleAlert,
|
||||||
Upload
|
Upload
|
||||||
} from "lucide-react";
|
} from "lucide-react";
|
||||||
import {
|
import { DialogSection, ActionToolbar,
|
||||||
ActionBlockerHint,
|
ActionBlockerHint,
|
||||||
Button,
|
Button,
|
||||||
ConfirmDialog,
|
ConfirmDialog,
|
||||||
|
ContentGrid,
|
||||||
|
ContentSection,
|
||||||
Dialog,
|
Dialog,
|
||||||
DismissibleAlert,
|
DismissibleAlert,
|
||||||
|
DefinitionPalette,
|
||||||
|
DefinitionPaletteGroup,
|
||||||
|
DefinitionPaletteItem,
|
||||||
DocumentationHelpLink,
|
DocumentationHelpLink,
|
||||||
|
FilterBar,
|
||||||
|
FloatingStatus,
|
||||||
FormField,
|
FormField,
|
||||||
IconButton,
|
IconButton,
|
||||||
LoadingFrame,
|
LoadingFrame,
|
||||||
ReferenceSelect,
|
ReferenceSelect,
|
||||||
SegmentedControl,
|
SegmentedControl,
|
||||||
|
SelectionList,
|
||||||
|
SelectionListItem,
|
||||||
|
SelectionListItemContent,
|
||||||
|
StatePanel,
|
||||||
StatusBadge,
|
StatusBadge,
|
||||||
ToggleSwitch,
|
ToggleSwitch,
|
||||||
|
WorkspaceActionBar,
|
||||||
|
WorkspaceFrame,
|
||||||
|
WorkspaceLayout,
|
||||||
hasScope,
|
hasScope,
|
||||||
isApiError,
|
isApiError,
|
||||||
useUnsavedChanges,
|
useUnsavedChanges,
|
||||||
@@ -51,6 +66,7 @@ import { useLocation } from "react-router";
|
|||||||
import {
|
import {
|
||||||
compileDataflowSql,
|
compileDataflowSql,
|
||||||
cancelDataflowPipelineRun,
|
cancelDataflowPipelineRun,
|
||||||
|
createDataflowDecisionSet,
|
||||||
createDataflowTrigger,
|
createDataflowTrigger,
|
||||||
createDataflowPipeline,
|
createDataflowPipeline,
|
||||||
createDataflowSourceSnapshot,
|
createDataflowSourceSnapshot,
|
||||||
@@ -58,7 +74,9 @@ import {
|
|||||||
deleteDataflowTrigger,
|
deleteDataflowTrigger,
|
||||||
dataflowScopeReferenceProvider,
|
dataflowScopeReferenceProvider,
|
||||||
deriveDataflowPipeline,
|
deriveDataflowPipeline,
|
||||||
|
getDataflowDecisionSet,
|
||||||
listDataflowNodeTypes,
|
listDataflowNodeTypes,
|
||||||
|
listDataflowDecisionSets,
|
||||||
listDataflowPipelineRuns,
|
listDataflowPipelineRuns,
|
||||||
listDataflowPipelineDeployments,
|
listDataflowPipelineDeployments,
|
||||||
listDataflowPipelines,
|
listDataflowPipelines,
|
||||||
@@ -66,6 +84,7 @@ import {
|
|||||||
listDataflowTriggers,
|
listDataflowTriggers,
|
||||||
previewDataflowPipeline,
|
previewDataflowPipeline,
|
||||||
promoteDataflowPipeline,
|
promoteDataflowPipeline,
|
||||||
|
recordDataflowDecision,
|
||||||
runDataflowPipeline,
|
runDataflowPipeline,
|
||||||
renderDataflowSql,
|
renderDataflowSql,
|
||||||
updateDataflowPipeline,
|
updateDataflowPipeline,
|
||||||
@@ -84,6 +103,8 @@ import {
|
|||||||
type PipelinePreview,
|
type PipelinePreview,
|
||||||
type PipelineDeployment,
|
type PipelineDeployment,
|
||||||
type PipelineRun,
|
type PipelineRun,
|
||||||
|
type ReconciliationDecisionAction,
|
||||||
|
type ReconciliationDecisionSet,
|
||||||
type TabularSource
|
type TabularSource
|
||||||
} from "../../api/dataflow";
|
} from "../../api/dataflow";
|
||||||
import DataflowCanvas, { updateGraphNode } from "./DataflowCanvas";
|
import DataflowCanvas, { updateGraphNode } from "./DataflowCanvas";
|
||||||
@@ -141,6 +162,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
const [definitionSettingsOpen, setDefinitionSettingsOpen] = useState(false);
|
const [definitionSettingsOpen, setDefinitionSettingsOpen] = useState(false);
|
||||||
const [deriveOpen, setDeriveOpen] = useState(false);
|
const [deriveOpen, setDeriveOpen] = useState(false);
|
||||||
const [triggersOpen, setTriggersOpen] = useState(false);
|
const [triggersOpen, setTriggersOpen] = useState(false);
|
||||||
|
const [decisionReviewOpen, setDecisionReviewOpen] = useState(false);
|
||||||
const [nodeLibrary, setNodeLibrary] = useState<NodeTypeDefinition[]>(FALLBACK_NODE_LIBRARY);
|
const [nodeLibrary, setNodeLibrary] = useState<NodeTypeDefinition[]>(FALLBACK_NODE_LIBRARY);
|
||||||
const [sources, setSources] = useState<TabularSource[]>([]);
|
const [sources, setSources] = useState<TabularSource[]>([]);
|
||||||
const [sourceCatalogueAvailable, setSourceCatalogueAvailable] = useState(false);
|
const [sourceCatalogueAvailable, setSourceCatalogueAvailable] = useState(false);
|
||||||
@@ -576,31 +598,33 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<main className="dataflow-page">
|
<WorkspaceFrame as="main" height="viewport" surface="plain" className="dataflow-page" label="Dataflow workspace">
|
||||||
<div className="dataflow-shell">
|
<WorkspaceLayout
|
||||||
<aside className="dataflow-pipeline-panel" aria-label="Data pipelines">
|
variant="split"
|
||||||
<div className="dataflow-panel-toolbar">
|
primarySize="compact"
|
||||||
<strong>Pipelines</strong>
|
surface="contained"
|
||||||
<span className="dataflow-toolbar-actions">
|
primaryScrollable={false}
|
||||||
<IconButton
|
contentScrollable={false}
|
||||||
label="Refresh pipelines"
|
primaryLabel="Data pipelines"
|
||||||
icon={<RefreshCw size={16} />}
|
contentLabel="Pipeline editor"
|
||||||
variant="ghost"
|
contentClassName="dataflow-workspace"
|
||||||
onClick={() => requestNavigation(() => void loadPipelines(draft?.id))}
|
primary={<>
|
||||||
disabled={loading}
|
<WorkspaceActionBar
|
||||||
disabledReason={loading ? DATAFLOW_I18N.loading : undefined}
|
scope="collection-pane"
|
||||||
/>
|
variant="collection"
|
||||||
<IconButton
|
refreshable
|
||||||
label="New pipeline"
|
reloadAction={{ onReload: () => void loadPipelines(draft?.id), loading, label: "Refresh pipelines" }}
|
||||||
icon={<Plus size={17} />}
|
contextActions={<strong>Pipelines</strong>}
|
||||||
variant="primary"
|
createAction={<IconButton
|
||||||
onClick={createNew}
|
label="New pipeline"
|
||||||
disabled={!canWrite}
|
icon={<Plus size={17} />}
|
||||||
disabledReason={!canWrite ? DATAFLOW_I18N.writeReason : undefined}
|
variant="primary"
|
||||||
/>
|
onClick={createNew}
|
||||||
</span>
|
disabled={!canWrite}
|
||||||
</div>
|
disabledReason={!canWrite ? DATAFLOW_I18N.writeReason : undefined}
|
||||||
<div className="dataflow-pipeline-search">
|
/>}
|
||||||
|
/>
|
||||||
|
<FilterBar surface="panel">
|
||||||
<input
|
<input
|
||||||
type="search"
|
type="search"
|
||||||
value={search}
|
value={search}
|
||||||
@@ -608,40 +632,37 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
placeholder="Search pipelines"
|
placeholder="Search pipelines"
|
||||||
aria-label="Search pipelines"
|
aria-label="Search pipelines"
|
||||||
/>
|
/>
|
||||||
</div>
|
</FilterBar>
|
||||||
<LoadingFrame loading={loading} label="Loading pipelines" className="dataflow-pipeline-list-frame">
|
<LoadingFrame loading={loading} label="Loading pipelines" className="dataflow-pipeline-list-frame">
|
||||||
<div className="dataflow-pipeline-list">
|
<SelectionList variant="navigation" label="Data pipelines">
|
||||||
{filteredPipelines.map((pipeline) => (
|
{filteredPipelines.map((pipeline) => (
|
||||||
<button
|
<SelectionListItem
|
||||||
key={pipeline.id}
|
key={pipeline.id}
|
||||||
type="button"
|
selected={pipeline.id === draft?.id}
|
||||||
className={pipeline.id === draft?.id ? "is-selected" : ""}
|
|
||||||
onClick={() => selectPipeline(pipeline)}
|
onClick={() => selectPipeline(pipeline)}
|
||||||
>
|
>
|
||||||
<span>
|
<SelectionListItemContent
|
||||||
<strong>{pipeline.name}</strong>
|
title={pipeline.name}
|
||||||
<small>Revision {pipeline.current_revision}</small>
|
description={`Revision ${pipeline.current_revision} · ${pipeline.governance.scope_type} · ${pipeline.governance.definition_kind}`}
|
||||||
<small>
|
/>
|
||||||
{pipeline.governance.scope_type} · {pipeline.governance.definition_kind}
|
|
||||||
</small>
|
|
||||||
</span>
|
|
||||||
<StatusBadge status={pipeline.status} />
|
<StatusBadge status={pipeline.status} />
|
||||||
</button>
|
</SelectionListItem>
|
||||||
))}
|
))}
|
||||||
{!loading && !filteredPipelines.length ? (
|
{!loading && !filteredPipelines.length ? (
|
||||||
<div className="dataflow-pipeline-empty">
|
<StatePanel size="compact" description={search ? "No matching pipelines" : "No pipelines yet"} />
|
||||||
{search ? "No matching pipelines" : "No pipelines yet"}
|
|
||||||
</div>
|
|
||||||
) : null}
|
) : null}
|
||||||
</div>
|
</SelectionList>
|
||||||
</LoadingFrame>
|
</LoadingFrame>
|
||||||
</aside>
|
</>}
|
||||||
|
>
|
||||||
<section className="dataflow-workspace">
|
|
||||||
{draft ? (
|
{draft ? (
|
||||||
<>
|
<>
|
||||||
<div className="dataflow-workspace-toolbar">
|
<WorkspaceActionBar
|
||||||
<div className="dataflow-identity-fields">
|
scope="editor-pane"
|
||||||
|
variant="editor"
|
||||||
|
state={saving ? "saving" : dirty && !draft.name.trim() ? "invalid" : dirty ? "dirty" : "clean"}
|
||||||
|
className="dataflow-workspace-toolbar"
|
||||||
|
contextActions={<div className="dataflow-identity-fields">
|
||||||
<input
|
<input
|
||||||
className="dataflow-name-input"
|
className="dataflow-name-input"
|
||||||
value={draft.name}
|
value={draft.name}
|
||||||
@@ -669,9 +690,9 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
</option>
|
</option>
|
||||||
<option value="archived">Archived</option>
|
<option value="archived">Archived</option>
|
||||||
</select>
|
</select>
|
||||||
</div>
|
</div>}
|
||||||
<div className="dataflow-command-bar">
|
helpAction={<DocumentationHelpLink reference={DATAFLOW_DOCUMENTATION} />}
|
||||||
<DocumentationHelpLink reference={DATAFLOW_DOCUMENTATION} />
|
primaryActions={<div className="dataflow-command-bar">
|
||||||
<SegmentedControl<EditorMode>
|
<SegmentedControl<EditorMode>
|
||||||
ariaLabel="Pipeline editor mode"
|
ariaLabel="Pipeline editor mode"
|
||||||
options={[
|
options={[
|
||||||
@@ -755,50 +776,28 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
disabledReason={!canViewTriggers ? DATAFLOW_I18N.writeReason : undefined}
|
disabledReason={!canViewTriggers ? DATAFLOW_I18N.writeReason : undefined}
|
||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
|
</div>}
|
||||||
|
destructiveActions={draft.id ? (
|
||||||
<IconButton
|
<IconButton
|
||||||
label="Discard changes"
|
label="Delete pipeline"
|
||||||
icon={<RotateCcw size={16} />}
|
icon={<Trash2 size={16} />}
|
||||||
variant="ghost"
|
variant="danger"
|
||||||
onClick={() => requestDiscard(() => undefined)}
|
onClick={() => setDeleteOpen(true)}
|
||||||
disabled={!dirty || saving}
|
disabled={!canEdit || saving}
|
||||||
disabledReason={
|
disabledReason={saving ? DATAFLOW_I18N.working : !canEdit ? editBlockedReason : undefined}
|
||||||
saving
|
|
||||||
? DATAFLOW_I18N.working
|
|
||||||
: !dirty
|
|
||||||
? DATAFLOW_I18N.noChanges
|
|
||||||
: undefined
|
|
||||||
}
|
|
||||||
/>
|
/>
|
||||||
{draft.id ? (
|
) : undefined}
|
||||||
<IconButton
|
discardAction={{
|
||||||
label="Delete pipeline"
|
label: <><RotateCcw size={16} /> Discard</>,
|
||||||
icon={<Trash2 size={16} />}
|
onClick: () => requestDiscard(() => undefined)
|
||||||
variant="danger"
|
}}
|
||||||
onClick={() => setDeleteOpen(true)}
|
saveAction={{
|
||||||
disabled={!canEdit || saving}
|
label: <><Save size={16} /> {saving ? "Saving..." : "Save"}</>,
|
||||||
disabledReason={
|
onClick: () => void saveDraft(),
|
||||||
saving ? DATAFLOW_I18N.working : !canEdit ? editBlockedReason : undefined
|
disabled: working || !canEdit,
|
||||||
}
|
disabledReason: working ? DATAFLOW_I18N.working : !canEdit ? editBlockedReason : undefined
|
||||||
/>
|
}}
|
||||||
) : null}
|
/>
|
||||||
<Button
|
|
||||||
variant="primary"
|
|
||||||
onClick={() => void saveDraft()}
|
|
||||||
disabled={saving || working || !dirty || !canEdit}
|
|
||||||
disabledReason={
|
|
||||||
saving || working
|
|
||||||
? DATAFLOW_I18N.working
|
|
||||||
: !canEdit
|
|
||||||
? editBlockedReason
|
|
||||||
: !dirty
|
|
||||||
? DATAFLOW_I18N.noChanges
|
|
||||||
: undefined
|
|
||||||
}
|
|
||||||
>
|
|
||||||
<Save size={16} /> {saving ? "Saving..." : "Save"}
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{error ? (
|
{error ? (
|
||||||
<DismissibleAlert tone="danger" resetKey={error} floating>{error}</DismissibleAlert>
|
<DismissibleAlert tone="danger" resetKey={error} floating>{error}</DismissibleAlert>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -827,10 +826,10 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
) : null}
|
) : null}
|
||||||
<div className={`dataflow-editor ${draft.editorMode === "sql" ? "is-sql" : ""}`}>
|
<div className={`dataflow-editor ${draft.editorMode === "sql" ? "is-sql" : ""}`}>
|
||||||
{draft.editorMode === "graph" ? (
|
{draft.editorMode === "graph" ? (
|
||||||
<aside className="dataflow-palette" aria-label="Transform palette">
|
<DefinitionPalette
|
||||||
<div className="dataflow-panel-heading">
|
label="Nodes"
|
||||||
<strong>Nodes</strong>
|
aria-label="Transform palette"
|
||||||
{canImportSources ? (
|
actions={canImportSources ? (
|
||||||
<IconButton
|
<IconButton
|
||||||
label="Stage datasource"
|
label="Stage datasource"
|
||||||
icon={<Upload size={15} />}
|
icon={<Upload size={15} />}
|
||||||
@@ -838,36 +837,30 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
onClick={() => setSnapshotOpen(true)}
|
onClick={() => setSnapshotOpen(true)}
|
||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
</div>
|
>
|
||||||
<div className="dataflow-palette-items">
|
|
||||||
{paletteGroups.map((group) => (
|
{paletteGroups.map((group) => (
|
||||||
<section key={group.category} className="dataflow-palette-group">
|
<DefinitionPaletteGroup key={group.category} label={group.label}>
|
||||||
<h3>{group.label}</h3>
|
|
||||||
{group.nodes.map((definition) => {
|
{group.nodes.map((definition) => {
|
||||||
const Icon = dataflowNodeIcon(definition.icon);
|
const Icon = dataflowNodeIcon(definition.icon);
|
||||||
const disabled = !canEdit || uniqueNodeExists(draft, definition.type);
|
const disabled = !canEdit || uniqueNodeExists(draft, definition.type);
|
||||||
return (
|
return (
|
||||||
<button
|
<DefinitionPaletteItem
|
||||||
key={definition.type}
|
key={definition.type}
|
||||||
type="button"
|
|
||||||
title={definition.description}
|
title={definition.description}
|
||||||
|
icon={<Icon size={16} />}
|
||||||
|
label={definition.label}
|
||||||
draggable={!disabled}
|
draggable={!disabled}
|
||||||
disabled={disabled}
|
disabled={disabled}
|
||||||
onDragStart={(event) => startPaletteDrag(event, definition.type)}
|
onDragStart={(event) => startPaletteDrag(event, definition.type)}
|
||||||
onClick={() => addNode(definition.type)}
|
onClick={() => addNode(definition.type)}
|
||||||
>
|
/>
|
||||||
<Icon size={16} />
|
|
||||||
<span>{definition.label}</span>
|
|
||||||
<Plus size={14} className="dataflow-palette-add" />
|
|
||||||
</button>
|
|
||||||
);
|
);
|
||||||
})}
|
})}
|
||||||
</section>
|
</DefinitionPaletteGroup>
|
||||||
))}
|
))}
|
||||||
</div>
|
</DefinitionPalette>
|
||||||
</aside>
|
|
||||||
) : null}
|
) : null}
|
||||||
<section className="dataflow-editor-surface">
|
<section className="definition-editor-surface dataflow-editor-surface">
|
||||||
{draft.editorMode === "graph" ? (
|
{draft.editorMode === "graph" ? (
|
||||||
<ReactFlowProvider>
|
<ReactFlowProvider>
|
||||||
<DataflowCanvas
|
<DataflowCanvas
|
||||||
@@ -883,7 +876,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
</ReactFlowProvider>
|
</ReactFlowProvider>
|
||||||
) : (
|
) : (
|
||||||
<div className="dataflow-sql-workbench">
|
<div className="dataflow-sql-workbench">
|
||||||
<div className="dataflow-sql-toolbar">
|
<ActionToolbar surface="section-header" className="dataflow-sql-toolbar">
|
||||||
<span>Constrained Dataflow SQL</span>
|
<span>Constrained Dataflow SQL</span>
|
||||||
<Button
|
<Button
|
||||||
variant="primary"
|
variant="primary"
|
||||||
@@ -892,7 +885,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
>
|
>
|
||||||
<Code2 size={16} /> Apply SQL
|
<Code2 size={16} /> Apply SQL
|
||||||
</Button>
|
</Button>
|
||||||
</div>
|
</ActionToolbar>
|
||||||
<textarea
|
<textarea
|
||||||
value={draft.sqlText}
|
value={draft.sqlText}
|
||||||
onChange={(event) => updateDraft({ sqlText: event.target.value })}
|
onChange={(event) => updateDraft({ sqlText: event.target.value })}
|
||||||
@@ -932,27 +925,37 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
diagnostics={diagnostics}
|
diagnostics={diagnostics}
|
||||||
nodeDiagnostics={nodeDiagnostics}
|
nodeDiagnostics={nodeDiagnostics}
|
||||||
selectedNodeId={selectedNodeId}
|
selectedNodeId={selectedNodeId}
|
||||||
|
decisionReviewDisabledReason={
|
||||||
|
!draft.id
|
||||||
|
? "Save the pipeline before recording review decisions."
|
||||||
|
: dirty
|
||||||
|
? "Save the current pipeline revision before recording review decisions."
|
||||||
|
: !canEdit
|
||||||
|
? editBlockedReason
|
||||||
|
: undefined
|
||||||
|
}
|
||||||
|
onReviewDecisions={() => setDecisionReviewOpen(true)}
|
||||||
onClose={() => setResultOpen(false)}
|
onClose={() => setResultOpen(false)}
|
||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
) : (
|
) : (
|
||||||
<div className="dataflow-workspace-empty">
|
<StatePanel
|
||||||
<Network size={30} />
|
size="fill"
|
||||||
<strong>No pipeline selected</strong>
|
icon={<Network size={30} />}
|
||||||
<Button
|
title="No pipeline selected"
|
||||||
|
actions={<Button
|
||||||
variant="primary"
|
variant="primary"
|
||||||
onClick={createNew}
|
onClick={createNew}
|
||||||
disabled={!canWrite}
|
disabled={!canWrite}
|
||||||
disabledReason={!canWrite ? DATAFLOW_I18N.writeReason : undefined}
|
disabledReason={!canWrite ? DATAFLOW_I18N.writeReason : undefined}
|
||||||
>
|
>
|
||||||
<Plus size={16} /> New pipeline
|
<Plus size={16} /> New pipeline
|
||||||
</Button>
|
</Button>}
|
||||||
</div>
|
/>
|
||||||
)}
|
)}
|
||||||
{working ? <div className="dataflow-working-indicator" role="status">Working...</div> : null}
|
{working ? <FloatingStatus>Working...</FloatingStatus> : null}
|
||||||
</section>
|
</WorkspaceLayout>
|
||||||
</div>
|
|
||||||
<ConfirmDialog
|
<ConfirmDialog
|
||||||
open={deleteOpen}
|
open={deleteOpen}
|
||||||
title="Delete pipeline"
|
title="Delete pipeline"
|
||||||
@@ -1068,7 +1071,23 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
editable={canManageTriggers}
|
editable={canManageTriggers}
|
||||||
onClose={() => setTriggersOpen(false)}
|
onClose={() => setTriggersOpen(false)}
|
||||||
/>
|
/>
|
||||||
</main>
|
<ReconciliationDecisionDialog
|
||||||
|
open={decisionReviewOpen}
|
||||||
|
settings={settings}
|
||||||
|
pipeline={draft?.id ? { id: draft.id, name: draft.name } : null}
|
||||||
|
node={selectedPreviewNode(draft?.graph.nodes ?? [], preview, selectedNodeId)}
|
||||||
|
preview={preview}
|
||||||
|
editable={canEdit && !dirty}
|
||||||
|
onClose={() => setDecisionReviewOpen(false)}
|
||||||
|
onChanged={() => {
|
||||||
|
void listDataflowSources(settings).then((catalogue) => {
|
||||||
|
setSources(catalogue.sources);
|
||||||
|
setSourceCatalogueAvailable(catalogue.available);
|
||||||
|
setSourceCatalogueWritable(catalogue.writable);
|
||||||
|
}).catch((sourceError) => setError(apiErrorMessage(sourceError)));
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</WorkspaceFrame>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1173,7 +1192,7 @@ function DefinitionSettingsDialog({
|
|||||||
<option value="template">Template</option>
|
<option value="template">Template</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</FormField>
|
||||||
<div className="dataflow-definition-toggles">
|
<ContentGrid columns={2} gap="default" collapseAt="narrow">
|
||||||
<ToggleSwitch
|
<ToggleSwitch
|
||||||
label="Visible to lower scopes"
|
label="Visible to lower scopes"
|
||||||
checked={draft.inheritToLowerScopes}
|
checked={draft.inheritToLowerScopes}
|
||||||
@@ -1182,6 +1201,10 @@ function DefinitionSettingsDialog({
|
|||||||
/>
|
/>
|
||||||
<ToggleSwitch
|
<ToggleSwitch
|
||||||
label="Allow runs"
|
label="Allow runs"
|
||||||
|
interfaceId="dataflow.field.allow-runs"
|
||||||
|
helpContextId="dataflow.field.allow-runs"
|
||||||
|
helpModuleId="dataflow"
|
||||||
|
helpTopicId="dataflow.reference.fields-and-consequences"
|
||||||
checked={draft.allowRun}
|
checked={draft.allowRun}
|
||||||
disabled={!editable}
|
disabled={!editable}
|
||||||
onChange={(value) => onChange({ allowRun: value })}
|
onChange={(value) => onChange({ allowRun: value })}
|
||||||
@@ -1198,9 +1221,9 @@ function DefinitionSettingsDialog({
|
|||||||
disabled={!editable}
|
disabled={!editable}
|
||||||
onChange={(value) => onChange({ allowAutomation: value })}
|
onChange={(value) => onChange({ allowAutomation: value })}
|
||||||
/>
|
/>
|
||||||
</div>
|
</ContentGrid>
|
||||||
{draft.governance?.derived_from_pipeline_id ? (
|
{draft.governance?.derived_from_pipeline_id ? (
|
||||||
<section className="dataflow-provenance">
|
<ContentSection spacing="none" surface="subtle" density="compact" layout="stack" className="dataflow-provenance">
|
||||||
<strong>Derived from</strong>
|
<strong>Derived from</strong>
|
||||||
<span>
|
<span>
|
||||||
{draft.governance.derived_from_pipeline_id}
|
{draft.governance.derived_from_pipeline_id}
|
||||||
@@ -1208,10 +1231,10 @@ function DefinitionSettingsDialog({
|
|||||||
{draft.governance.derived_from_revision}
|
{draft.governance.derived_from_revision}
|
||||||
</span>
|
</span>
|
||||||
<code>{draft.governance.derived_from_hash}</code>
|
<code>{draft.governance.derived_from_hash}</code>
|
||||||
</section>
|
</ContentSection>
|
||||||
) : null}
|
) : null}
|
||||||
{provenance.length ? (
|
{provenance.length ? (
|
||||||
<section className="dataflow-provenance">
|
<ContentSection spacing="none" surface="subtle" density="compact" layout="stack" className="dataflow-provenance">
|
||||||
<strong>Effective Policy path</strong>
|
<strong>Effective Policy path</strong>
|
||||||
{provenance.map((item, index) => (
|
{provenance.map((item, index) => (
|
||||||
<span key={`${String(item.path ?? item.scope_type)}-${index}`}>
|
<span key={`${String(item.path ?? item.scope_type)}-${index}`}>
|
||||||
@@ -1221,7 +1244,7 @@ function DefinitionSettingsDialog({
|
|||||||
{!editable && draft.governance?.actions.edit?.reason ? (
|
{!editable && draft.governance?.actions.edit?.reason ? (
|
||||||
<small>{draft.governance.actions.edit.reason}</small>
|
<small>{draft.governance.actions.edit.reason}</small>
|
||||||
) : null}
|
) : null}
|
||||||
</section>
|
</ContentSection>
|
||||||
) : null}
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -1718,7 +1741,7 @@ function DataflowTriggersDialog({
|
|||||||
</select>
|
</select>
|
||||||
</FormField>
|
</FormField>
|
||||||
{kind === "once" ? (
|
{kind === "once" ? (
|
||||||
<FormField label="Run at" documentation={DATAFLOW_FIELDS_DOCUMENTATION}>
|
<FormField label="Run at" interfaceId="dataflow.field.trigger-run-at" helpContextId="dataflow.field.trigger-run-at" helpModuleId="dataflow" documentation={DATAFLOW_FIELDS_DOCUMENTATION}>
|
||||||
<input disabled={!editable} type="datetime-local" value={runAt} onChange={(event) => setRunAt(event.target.value)} />
|
<input disabled={!editable} type="datetime-local" value={runAt} onChange={(event) => setRunAt(event.target.value)} />
|
||||||
</FormField>
|
</FormField>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -1733,7 +1756,7 @@ function DataflowTriggersDialog({
|
|||||||
onChange={(event) => setIntervalMinutes(Math.max(1, Number(event.target.value)))}
|
onChange={(event) => setIntervalMinutes(Math.max(1, Number(event.target.value)))}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</FormField>
|
||||||
<FormField label="Missed runs" documentation={DATAFLOW_FIELDS_DOCUMENTATION}>
|
<FormField label="Missed runs" interfaceId="dataflow.field.trigger-missed-runs" helpContextId="dataflow.field.trigger-missed-runs" helpModuleId="dataflow" documentation={DATAFLOW_FIELDS_DOCUMENTATION}>
|
||||||
<select
|
<select
|
||||||
disabled={!editable}
|
disabled={!editable}
|
||||||
value={catchUpPolicy}
|
value={catchUpPolicy}
|
||||||
@@ -1768,7 +1791,7 @@ function DataflowTriggersDialog({
|
|||||||
disabled={busy || !editable}
|
disabled={busy || !editable}
|
||||||
onChange={setEnabled}
|
onChange={setEnabled}
|
||||||
/>
|
/>
|
||||||
<FormField label="Concurrent runs" documentation={DATAFLOW_FIELDS_DOCUMENTATION}>
|
<FormField label="Concurrent runs" interfaceId="dataflow.field.trigger-concurrency" helpContextId="dataflow.field.trigger-concurrency" helpModuleId="dataflow" documentation={DATAFLOW_FIELDS_DOCUMENTATION}>
|
||||||
<input
|
<input
|
||||||
type="number"
|
type="number"
|
||||||
min={1}
|
min={1}
|
||||||
@@ -2036,7 +2059,7 @@ function RunPipelineDialog({
|
|||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
<div className="dataflow-run-dialog-content">
|
<DialogSection className="dataflow-run-dialog-content">
|
||||||
{error ? (
|
{error ? (
|
||||||
<DismissibleAlert tone="danger" resetKey={error}>
|
<DismissibleAlert tone="danger" resetKey={error}>
|
||||||
{error}
|
{error}
|
||||||
@@ -2136,6 +2159,10 @@ function RunPipelineDialog({
|
|||||||
<div className="dataflow-run-freeze">
|
<div className="dataflow-run-freeze">
|
||||||
<ToggleSwitch
|
<ToggleSwitch
|
||||||
label="Freeze published state"
|
label="Freeze published state"
|
||||||
|
interfaceId="dataflow.field.freeze-publication"
|
||||||
|
helpContextId="dataflow.field.freeze-publication"
|
||||||
|
helpModuleId="dataflow"
|
||||||
|
helpTopicId="dataflow.execution-and-recovery"
|
||||||
checked={freeze}
|
checked={freeze}
|
||||||
onChange={setFreeze}
|
onChange={setFreeze}
|
||||||
/>
|
/>
|
||||||
@@ -2223,7 +2250,7 @@ function RunPipelineDialog({
|
|||||||
</table>
|
</table>
|
||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
</div>
|
</DialogSection>
|
||||||
</Dialog>
|
</Dialog>
|
||||||
<ConfirmDialog
|
<ConfirmDialog
|
||||||
open={confirmation !== null}
|
open={confirmation !== null}
|
||||||
@@ -2377,7 +2404,7 @@ function SourceSnapshotDialog({
|
|||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
>
|
>
|
||||||
<div className="dataflow-source-dialog-fields">
|
<DialogSection className="dataflow-source-dialog-fields">
|
||||||
{error ? (
|
{error ? (
|
||||||
<DismissibleAlert tone="danger" resetKey={error}>{error}</DismissibleAlert>
|
<DismissibleAlert tone="danger" resetKey={error}>{error}</DismissibleAlert>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -2443,6 +2470,376 @@ function SourceSnapshotDialog({
|
|||||||
/>
|
/>
|
||||||
</FormField>
|
</FormField>
|
||||||
)}
|
)}
|
||||||
|
</DialogSection>
|
||||||
|
</Dialog>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
type ReviewableReconciliationRow = {
|
||||||
|
keyHash: string;
|
||||||
|
inputHash: string;
|
||||||
|
row: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
function ReconciliationDecisionDialog({
|
||||||
|
open,
|
||||||
|
settings,
|
||||||
|
pipeline,
|
||||||
|
node,
|
||||||
|
preview,
|
||||||
|
editable,
|
||||||
|
onClose,
|
||||||
|
onChanged
|
||||||
|
}: {
|
||||||
|
open: boolean;
|
||||||
|
settings: ApiSettings;
|
||||||
|
pipeline: { id: string; name: string } | null;
|
||||||
|
node: PipelineGraphNode | null;
|
||||||
|
preview: PipelinePreview | null;
|
||||||
|
editable: boolean;
|
||||||
|
onClose: () => void;
|
||||||
|
onChanged: () => void;
|
||||||
|
}) {
|
||||||
|
const [decisionSets, setDecisionSets] = useState<ReconciliationDecisionSet[]>([]);
|
||||||
|
const [selectedSetId, setSelectedSetId] = useState("");
|
||||||
|
const [selectedKeyHash, setSelectedKeyHash] = useState("");
|
||||||
|
const [newSetName, setNewSetName] = useState("");
|
||||||
|
const [action, setAction] = useState<ReconciliationDecisionAction>("accept");
|
||||||
|
const [reason, setReason] = useState("");
|
||||||
|
const [correctionText, setCorrectionText] = useState("{}");
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [error, setError] = useState("");
|
||||||
|
const [success, setSuccess] = useState("");
|
||||||
|
|
||||||
|
const rows = useMemo<ReviewableReconciliationRow[]>(() => {
|
||||||
|
const result = preview?.node_preview ?? preview;
|
||||||
|
if (!result || node?.type !== "reconcile.compare") return [];
|
||||||
|
return result.rows.flatMap((row) => {
|
||||||
|
const keyHash = row._reconciliation_key_hash;
|
||||||
|
const inputHash = row._reconciliation_input_hash;
|
||||||
|
if (!isSha256Hex(keyHash) || !isSha256Hex(inputHash)) return [];
|
||||||
|
return [{ keyHash, inputHash, row }];
|
||||||
|
});
|
||||||
|
}, [node?.type, preview]);
|
||||||
|
|
||||||
|
const selectedSet = decisionSets.find((item) => item.id === selectedSetId) ?? null;
|
||||||
|
const selectedRow = rows.find((item) => item.keyHash === selectedKeyHash) ?? null;
|
||||||
|
const currentDecision = selectedSet?.current_decisions.find(
|
||||||
|
(item) => item.key_hash === selectedRow?.keyHash
|
||||||
|
) ?? null;
|
||||||
|
const exactDecision = currentDecision?.input_hash === selectedRow?.inputHash
|
||||||
|
? currentDecision
|
||||||
|
: null;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open) return;
|
||||||
|
setSelectedKeyHash((current) => rows.some((item) => item.keyHash === current)
|
||||||
|
? current
|
||||||
|
: rows[0]?.keyHash ?? "");
|
||||||
|
}, [open, rows]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open || !pipeline) {
|
||||||
|
setDecisionSets([]);
|
||||||
|
setSelectedSetId("");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let cancelled = false;
|
||||||
|
setLoading(true);
|
||||||
|
setError("");
|
||||||
|
void listDataflowDecisionSets(settings, pipeline.id)
|
||||||
|
.then((items) => {
|
||||||
|
if (cancelled) return;
|
||||||
|
const relevant = items.filter((item) => !item.node_id || item.node_id === node?.id);
|
||||||
|
setDecisionSets(relevant);
|
||||||
|
setSelectedSetId((current) => relevant.some((item) => item.id === current)
|
||||||
|
? current
|
||||||
|
: relevant[0]?.id ?? "");
|
||||||
|
})
|
||||||
|
.catch((loadError) => {
|
||||||
|
if (!cancelled) setError(apiErrorMessage(loadError));
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (!cancelled) setLoading(false);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [node?.id, open, pipeline?.id, settings]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open || !selectedSetId) return;
|
||||||
|
let cancelled = false;
|
||||||
|
setLoading(true);
|
||||||
|
void getDataflowDecisionSet(settings, selectedSetId)
|
||||||
|
.then((item) => {
|
||||||
|
if (cancelled) return;
|
||||||
|
setDecisionSets((current) => current.map(
|
||||||
|
(candidate) => candidate.id === item.id ? item : candidate
|
||||||
|
));
|
||||||
|
})
|
||||||
|
.catch((loadError) => {
|
||||||
|
if (!cancelled) setError(apiErrorMessage(loadError));
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (!cancelled) setLoading(false);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [open, selectedSetId, settings]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setAction(exactDecision?.action ?? "accept");
|
||||||
|
setReason(exactDecision?.reason ?? "");
|
||||||
|
setCorrectionText(JSON.stringify(exactDecision?.correction ?? {}, null, 2));
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
}, [exactDecision?.id, selectedKeyHash, selectedSetId]);
|
||||||
|
|
||||||
|
const createSet = async () => {
|
||||||
|
if (!pipeline || !node || !newSetName.trim() || !editable) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
try {
|
||||||
|
const created = await createDataflowDecisionSet(settings, pipeline.id, {
|
||||||
|
name: newSetName.trim(),
|
||||||
|
node_id: node.id
|
||||||
|
});
|
||||||
|
setDecisionSets((current) => [created, ...current.filter((item) => item.id !== created.id)]);
|
||||||
|
setSelectedSetId(created.id);
|
||||||
|
setNewSetName("");
|
||||||
|
setSuccess("Decision set created. It is now available as a governed Dataflow source.");
|
||||||
|
onChanged();
|
||||||
|
} catch (createError) {
|
||||||
|
setError(apiErrorMessage(createError));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const saveDecision = async () => {
|
||||||
|
if (!selectedSet || !selectedRow || !editable) return;
|
||||||
|
if (reason.trim().length < 3) {
|
||||||
|
setError("Record a reason of at least three characters.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let correction: Record<string, unknown> | null = null;
|
||||||
|
if (action === "correct") {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(correctionText) as unknown;
|
||||||
|
if (!isRecord(parsed) || !Object.keys(parsed).length) {
|
||||||
|
setError("Correct decisions require a JSON object with at least one corrected field.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
correction = parsed;
|
||||||
|
} catch {
|
||||||
|
setError("Correction must be a valid JSON object.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
try {
|
||||||
|
const updated = await recordDataflowDecision(settings, selectedSet.id, {
|
||||||
|
base_revision: selectedSet.resource_revision,
|
||||||
|
key_hash: selectedRow.keyHash,
|
||||||
|
input_hash: selectedRow.inputHash,
|
||||||
|
action,
|
||||||
|
reason: reason.trim(),
|
||||||
|
correction
|
||||||
|
});
|
||||||
|
setDecisionSets((current) => current.map((item) => item.id === updated.id ? updated : item));
|
||||||
|
setSuccess(`Recorded decision revision ${updated.resource_revision}.`);
|
||||||
|
onChanged();
|
||||||
|
} catch (saveError) {
|
||||||
|
setError(apiErrorMessage(saveError));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const reviewedCount = selectedSet
|
||||||
|
? rows.filter((row) => selectedSet.current_decisions.some(
|
||||||
|
(decision) => decision.key_hash === row.keyHash && decision.input_hash === row.inputHash
|
||||||
|
)).length
|
||||||
|
: 0;
|
||||||
|
const staleCount = selectedSet
|
||||||
|
? rows.filter((row) => selectedSet.current_decisions.some(
|
||||||
|
(decision) => decision.key_hash === row.keyHash && decision.input_hash !== row.inputHash
|
||||||
|
)).length
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Dialog
|
||||||
|
open={open}
|
||||||
|
title={`Review reconciliation${node ? `: ${node.label}` : ""}`}
|
||||||
|
className="dataflow-decision-dialog"
|
||||||
|
bodyClassName="dataflow-decision-dialog-body"
|
||||||
|
closeDisabled={busy}
|
||||||
|
onClose={onClose}
|
||||||
|
footer={(
|
||||||
|
<>
|
||||||
|
<span className="dataflow-decision-history-summary">
|
||||||
|
{selectedSet ? `${selectedSet.history.length} immutable decision revision(s)` : "No decision set selected"}
|
||||||
|
</span>
|
||||||
|
<Button onClick={onClose} disabled={busy}>Close</Button>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
<div className="dataflow-decision-shell">
|
||||||
|
{error ? <DismissibleAlert tone="danger" resetKey={error}>{error}</DismissibleAlert> : null}
|
||||||
|
{success ? <DismissibleAlert tone="success" resetKey={success}>{success}</DismissibleAlert> : null}
|
||||||
|
<ActionToolbar className="dataflow-decision-set-toolbar">
|
||||||
|
<FormField
|
||||||
|
label="Decision set"
|
||||||
|
help="The current projection is consumable as a source; every prior decision remains in immutable history."
|
||||||
|
documentation={DATAFLOW_FIELDS_DOCUMENTATION}
|
||||||
|
>
|
||||||
|
<select
|
||||||
|
value={selectedSetId}
|
||||||
|
onChange={(event) => setSelectedSetId(event.target.value)}
|
||||||
|
disabled={loading || !decisionSets.length}
|
||||||
|
>
|
||||||
|
{!decisionSets.length ? <option value="">No decision sets</option> : null}
|
||||||
|
{decisionSets.map((item) => (
|
||||||
|
<option key={item.id} value={item.id}>{item.name}</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField
|
||||||
|
label="New decision set"
|
||||||
|
help="Create one set per review purpose or review round."
|
||||||
|
documentation={DATAFLOW_FIELDS_DOCUMENTATION}
|
||||||
|
>
|
||||||
|
<div className="dataflow-decision-create">
|
||||||
|
<input
|
||||||
|
value={newSetName}
|
||||||
|
onChange={(event) => setNewSetName(event.target.value)}
|
||||||
|
placeholder={`${pipeline?.name ?? "Pipeline"} review`}
|
||||||
|
disabled={!editable || busy}
|
||||||
|
/>
|
||||||
|
<Button
|
||||||
|
variant="primary"
|
||||||
|
onClick={() => void createSet()}
|
||||||
|
disabled={!editable || busy || !newSetName.trim()}
|
||||||
|
>
|
||||||
|
<Plus size={16} /> Create
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</FormField>
|
||||||
|
<div className="dataflow-decision-counts" aria-label="Review progress">
|
||||||
|
<span><strong>{reviewedCount}</strong> reviewed</span>
|
||||||
|
<span className={staleCount ? "is-warning" : ""}><strong>{staleCount}</strong> stale</span>
|
||||||
|
<span><strong>{Math.max(0, rows.length - reviewedCount - staleCount)}</strong> open</span>
|
||||||
|
</div>
|
||||||
|
</ActionToolbar>
|
||||||
|
{!rows.length ? (
|
||||||
|
<StatePanel size="compact" description="Run a preview of a reconciliation comparison that emits stable key and input hashes." />
|
||||||
|
) : (
|
||||||
|
<div className="dataflow-decision-layout">
|
||||||
|
<div className="dataflow-decision-rows" role="listbox" aria-label="Reconciliation rows">
|
||||||
|
<div className="dataflow-decision-row-heading">
|
||||||
|
<span>Record</span><span>Comparison</span><span>Decision</span>
|
||||||
|
</div>
|
||||||
|
{rows.map((item) => {
|
||||||
|
const decision = selectedSet?.current_decisions.find(
|
||||||
|
(candidate) => candidate.key_hash === item.keyHash
|
||||||
|
);
|
||||||
|
const stale = Boolean(decision && decision.input_hash !== item.inputHash);
|
||||||
|
const statusLabel = stale ? "stale" : decision?.action ?? "open";
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
key={item.keyHash}
|
||||||
|
type="button"
|
||||||
|
role="option"
|
||||||
|
aria-selected={item.keyHash === selectedKeyHash}
|
||||||
|
className={item.keyHash === selectedKeyHash ? "is-selected" : ""}
|
||||||
|
onClick={() => setSelectedKeyHash(item.keyHash)}
|
||||||
|
>
|
||||||
|
<span title={item.keyHash}>{reviewRowLabel(item.row)}</span>
|
||||||
|
<span>{formatCell(item.row._reconciliation_status)}</span>
|
||||||
|
<span className={`dataflow-decision-state is-${statusLabel}`}>{statusLabel}</span>
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
<div className="dataflow-decision-editor">
|
||||||
|
{selectedRow && selectedSet ? (
|
||||||
|
<>
|
||||||
|
<div className="dataflow-decision-editor-heading">
|
||||||
|
<div>
|
||||||
|
<strong>{reviewRowLabel(selectedRow.row)}</strong>
|
||||||
|
<small title={selectedRow.keyHash}>{selectedRow.keyHash}</small>
|
||||||
|
</div>
|
||||||
|
{currentDecision && !exactDecision ? (
|
||||||
|
<StatusBadge status="warning" label="Prior decision is stale" />
|
||||||
|
) : exactDecision ? (
|
||||||
|
<StatusBadge status="success" label={`Current: ${exactDecision.action}`} />
|
||||||
|
) : (
|
||||||
|
<StatusBadge status="inactive" label="Unreviewed" />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<FormField label="Decision" documentation={DATAFLOW_FIELDS_DOCUMENTATION}>
|
||||||
|
<SegmentedControl<ReconciliationDecisionAction>
|
||||||
|
ariaLabel="Reconciliation decision"
|
||||||
|
options={[
|
||||||
|
{ id: "accept", label: "Accept" },
|
||||||
|
{ id: "reject", label: "Reject" },
|
||||||
|
{ id: "correct", label: "Correct" },
|
||||||
|
{ id: "defer", label: "Defer" }
|
||||||
|
]}
|
||||||
|
value={action}
|
||||||
|
onChange={setAction}
|
||||||
|
disabled={!editable || busy}
|
||||||
|
/>
|
||||||
|
</FormField>
|
||||||
|
<FormField
|
||||||
|
label="Reason"
|
||||||
|
help="The reason is retained with the actor, exact input hash, and immutable revision."
|
||||||
|
documentation={DATAFLOW_FIELDS_DOCUMENTATION}
|
||||||
|
>
|
||||||
|
<textarea
|
||||||
|
rows={4}
|
||||||
|
value={reason}
|
||||||
|
onChange={(event) => setReason(event.target.value)}
|
||||||
|
disabled={!editable || busy}
|
||||||
|
/>
|
||||||
|
</FormField>
|
||||||
|
{action === "correct" ? (
|
||||||
|
<FormField
|
||||||
|
label="Corrected fields (JSON)"
|
||||||
|
help="Corrections are annotations. A downstream governed transform decides whether to apply them."
|
||||||
|
documentation={DATAFLOW_FIELDS_DOCUMENTATION}
|
||||||
|
>
|
||||||
|
<textarea
|
||||||
|
className="dataflow-json-editor"
|
||||||
|
value={correctionText}
|
||||||
|
onChange={(event) => setCorrectionText(event.target.value)}
|
||||||
|
spellCheck={false}
|
||||||
|
disabled={!editable || busy}
|
||||||
|
/>
|
||||||
|
</FormField>
|
||||||
|
) : null}
|
||||||
|
<div className="dataflow-decision-editor-actions">
|
||||||
|
<Button
|
||||||
|
variant="primary"
|
||||||
|
onClick={() => void saveDecision()}
|
||||||
|
disabled={!editable || busy || reason.trim().length < 3}
|
||||||
|
>
|
||||||
|
<Save size={16} /> {busy ? "Recording..." : "Record revision"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<StatePanel size="compact" description={loading ? "Loading decision sets..." : "Create or select a decision set to review this row."} />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
</Dialog>
|
</Dialog>
|
||||||
);
|
);
|
||||||
@@ -2456,6 +2853,8 @@ function ResultPanel({
|
|||||||
diagnostics,
|
diagnostics,
|
||||||
nodeDiagnostics,
|
nodeDiagnostics,
|
||||||
selectedNodeId,
|
selectedNodeId,
|
||||||
|
decisionReviewDisabledReason,
|
||||||
|
onReviewDecisions,
|
||||||
onClose
|
onClose
|
||||||
}: {
|
}: {
|
||||||
tab: ResultTab;
|
tab: ResultTab;
|
||||||
@@ -2465,6 +2864,8 @@ function ResultPanel({
|
|||||||
diagnostics: DataflowDiagnostic[];
|
diagnostics: DataflowDiagnostic[];
|
||||||
nodeDiagnostics: NodePreviewDiagnostic[];
|
nodeDiagnostics: NodePreviewDiagnostic[];
|
||||||
selectedNodeId: string | null;
|
selectedNodeId: string | null;
|
||||||
|
decisionReviewDisabledReason?: string;
|
||||||
|
onReviewDecisions: () => void;
|
||||||
onClose: () => void;
|
onClose: () => void;
|
||||||
}) {
|
}) {
|
||||||
const outputNodeId = nodes.find((node) => node.type === "output")?.id ?? "";
|
const outputNodeId = nodes.find((node) => node.type === "output")?.id ?? "";
|
||||||
@@ -2473,7 +2874,7 @@ function ResultPanel({
|
|||||||
const previewRows = preview?.node_preview?.total_rows ?? preview?.total_rows;
|
const previewRows = preview?.node_preview?.total_rows ?? preview?.total_rows;
|
||||||
return (
|
return (
|
||||||
<section className="dataflow-results" aria-label="Pipeline results">
|
<section className="dataflow-results" aria-label="Pipeline results">
|
||||||
<div className="dataflow-results-toolbar">
|
<ActionToolbar surface="section-header" className="dataflow-results-toolbar">
|
||||||
<div className="dataflow-results-view-controls">
|
<div className="dataflow-results-view-controls">
|
||||||
<SegmentedControl<ResultTab>
|
<SegmentedControl<ResultTab>
|
||||||
ariaLabel="Result view"
|
ariaLabel="Result view"
|
||||||
@@ -2491,8 +2892,19 @@ function ResultPanel({
|
|||||||
</span>
|
</span>
|
||||||
) : null}
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
<Button variant="ghost" onClick={onClose}>Close</Button>
|
<div className="dataflow-results-actions">
|
||||||
</div>
|
{tab === "preview" && previewNode?.type === "reconcile.compare" ? (
|
||||||
|
<Button
|
||||||
|
onClick={onReviewDecisions}
|
||||||
|
disabled={!preview || Boolean(decisionReviewDisabledReason)}
|
||||||
|
disabledReason={decisionReviewDisabledReason}
|
||||||
|
>
|
||||||
|
<ListChecks size={16} /> Review decisions
|
||||||
|
</Button>
|
||||||
|
) : null}
|
||||||
|
<Button variant="ghost" onClick={onClose}>Close</Button>
|
||||||
|
</div>
|
||||||
|
</ActionToolbar>
|
||||||
{tab === "preview" ? (
|
{tab === "preview" ? (
|
||||||
<PreviewTable preview={preview} />
|
<PreviewTable preview={preview} />
|
||||||
) : (
|
) : (
|
||||||
@@ -2503,9 +2915,9 @@ function ResultPanel({
|
|||||||
}
|
}
|
||||||
|
|
||||||
function PreviewTable({ preview }: { preview: PipelinePreview | null }) {
|
function PreviewTable({ preview }: { preview: PipelinePreview | null }) {
|
||||||
if (!preview) return <div className="dataflow-results-empty">No preview has been run.</div>;
|
if (!preview) return <StatePanel size="compact" description="No preview has been run." />;
|
||||||
if (preview.status === "failed" && !preview.node_preview) {
|
if (preview.status === "failed" && !preview.node_preview) {
|
||||||
return <div className="dataflow-results-empty">Preview failed.</div>;
|
return <StatePanel size="compact" tone="danger" description="Preview failed." />;
|
||||||
}
|
}
|
||||||
const result = preview.node_preview ?? preview;
|
const result = preview.node_preview ?? preview;
|
||||||
return (
|
return (
|
||||||
@@ -2555,7 +2967,7 @@ function DiagnosticsPanel({
|
|||||||
nodeDiagnostics: NodePreviewDiagnostic[];
|
nodeDiagnostics: NodePreviewDiagnostic[];
|
||||||
}) {
|
}) {
|
||||||
if (!diagnostics.length && !nodeDiagnostics.length) {
|
if (!diagnostics.length && !nodeDiagnostics.length) {
|
||||||
return <div className="dataflow-results-empty">No diagnostics.</div>;
|
return <StatePanel size="compact" description="No diagnostics." />;
|
||||||
}
|
}
|
||||||
return (
|
return (
|
||||||
<div className="dataflow-diagnostics-list">
|
<div className="dataflow-diagnostics-list">
|
||||||
@@ -2625,6 +3037,27 @@ function formatCell(value: unknown): string {
|
|||||||
return String(value);
|
return String(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function selectedPreviewNode(
|
||||||
|
nodes: PipelineGraphNode[],
|
||||||
|
preview: PipelinePreview | null,
|
||||||
|
selectedNodeId: string | null
|
||||||
|
): PipelineGraphNode | null {
|
||||||
|
const nodeId = preview?.node_preview?.node_id ?? selectedNodeId;
|
||||||
|
return nodes.find((node) => node.id === nodeId) ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSha256Hex(value: unknown): value is string {
|
||||||
|
return typeof value === "string" && /^[0-9a-f]{64}$/.test(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function reviewRowLabel(row: Record<string, unknown>): string {
|
||||||
|
const values = Object.entries(row)
|
||||||
|
.filter(([key, value]) => !key.startsWith("_") && value !== null && value !== undefined)
|
||||||
|
.slice(0, 3)
|
||||||
|
.map(([key, value]) => `${key}: ${formatCell(value)}`);
|
||||||
|
return values.join(" · ") || "Reconciliation row";
|
||||||
|
}
|
||||||
|
|
||||||
function sourceNameFromFile(value: string): string {
|
function sourceNameFromFile(value: string): string {
|
||||||
const normalized = value
|
const normalized = value
|
||||||
.normalize("NFKD")
|
.normalize("NFKD")
|
||||||
|
|||||||
@@ -1,9 +1,11 @@
|
|||||||
import { useEffect, useState, type ComponentProps } from "react";
|
import { useEffect, useState, type ComponentProps } from "react";
|
||||||
import { Play, Trash2 } from "lucide-react";
|
import { Play, Trash2 } from "lucide-react";
|
||||||
import {
|
import {
|
||||||
|
ActionToolbar,
|
||||||
DismissibleAlert,
|
DismissibleAlert,
|
||||||
FormField as CoreFormField,
|
FormField as CoreFormField,
|
||||||
IconButton
|
IconButton,
|
||||||
|
StatePanel
|
||||||
} from "@govoplan/core-webui";
|
} from "@govoplan/core-webui";
|
||||||
import type {
|
import type {
|
||||||
NodeTypeDefinition,
|
NodeTypeDefinition,
|
||||||
@@ -14,7 +16,7 @@ import { DATAFLOW_NODE_DOCUMENTATION } from "./interfacePatterns";
|
|||||||
|
|
||||||
type NodeFormFieldProps = ComponentProps<typeof CoreFormField>;
|
type NodeFormFieldProps = ComponentProps<typeof CoreFormField>;
|
||||||
|
|
||||||
function FormField({ documentation, ...props }: NodeFormFieldProps) {
|
function NodeFormField({ documentation, ...props }: NodeFormFieldProps) {
|
||||||
return (
|
return (
|
||||||
<CoreFormField
|
<CoreFormField
|
||||||
{...props}
|
{...props}
|
||||||
@@ -73,10 +75,10 @@ export default function NodeInspector({
|
|||||||
if (!node) {
|
if (!node) {
|
||||||
return (
|
return (
|
||||||
<aside className="dataflow-inspector" aria-label="Node inspector">
|
<aside className="dataflow-inspector" aria-label="Node inspector">
|
||||||
<div className="dataflow-panel-heading">
|
<ActionToolbar surface="section-header" className="dataflow-panel-heading">
|
||||||
<strong>Inspector</strong>
|
<strong>Inspector</strong>
|
||||||
</div>
|
</ActionToolbar>
|
||||||
<div className="dataflow-inspector-empty">No node selected</div>
|
<StatePanel size="compact" description="No node selected" />
|
||||||
</aside>
|
</aside>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -161,7 +163,7 @@ export default function NodeInspector({
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<aside className="dataflow-inspector" aria-label="Node inspector">
|
<aside className="dataflow-inspector" aria-label="Node inspector">
|
||||||
<div className="dataflow-panel-heading">
|
<ActionToolbar surface="section-header" className="dataflow-panel-heading">
|
||||||
<span>
|
<span>
|
||||||
<strong>Inspector</strong>
|
<strong>Inspector</strong>
|
||||||
<small>{definition?.label ?? node.type}</small>
|
<small>{definition?.label ?? node.type}</small>
|
||||||
@@ -182,32 +184,32 @@ export default function NodeInspector({
|
|||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</ActionToolbar>
|
||||||
<div className="dataflow-inspector-fields">
|
<div className="dataflow-inspector-fields">
|
||||||
{localError ? (
|
{localError ? (
|
||||||
<DismissibleAlert tone="danger" resetKey={localError}>
|
<DismissibleAlert tone="danger" resetKey={localError}>
|
||||||
{localError}
|
{localError}
|
||||||
</DismissibleAlert>
|
</DismissibleAlert>
|
||||||
) : null}
|
) : null}
|
||||||
<FormField label="Name">
|
<NodeFormField label="Name">
|
||||||
<input
|
<input
|
||||||
value={node.label}
|
value={node.label}
|
||||||
onChange={(event) => onChange({ ...node, label: event.target.value })}
|
onChange={(event) => onChange({ ...node, label: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
{node.type.startsWith("source.") ? (
|
{node.type.startsWith("source.") ? (
|
||||||
<FormField label="Logical source name">
|
<NodeFormField label="Logical source name">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.source_name)}
|
value={textValue(node.config.source_name)}
|
||||||
onChange={(event) => updateConfig({ source_name: event.target.value })}
|
onChange={(event) => updateConfig({ source_name: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "source.reference" ? (
|
{node.type === "source.reference" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Source">
|
<NodeFormField label="Source">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.source_ref)}
|
value={textValue(node.config.source_ref)}
|
||||||
onChange={(event) => {
|
onChange={(event) => {
|
||||||
@@ -237,8 +239,8 @@ export default function NodeInspector({
|
|||||||
</option>
|
</option>
|
||||||
))}
|
))}
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="State">
|
<NodeFormField label="State">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.consistency) || "current"}
|
value={textValue(node.config.consistency) || "current"}
|
||||||
onChange={(event) => updateConfig({ consistency: event.target.value })}
|
onChange={(event) => updateConfig({ consistency: event.target.value })}
|
||||||
@@ -248,20 +250,20 @@ export default function NodeInspector({
|
|||||||
<option value="live">Live</option>
|
<option value="live">Live</option>
|
||||||
<option value="frozen">Latest frozen</option>
|
<option value="frozen">Latest frozen</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
{textValue(node.config.expected_fingerprint) ? (
|
{textValue(node.config.expected_fingerprint) ? (
|
||||||
<FormField label="Pinned fingerprint">
|
<NodeFormField label="Pinned fingerprint">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.expected_fingerprint)}
|
value={textValue(node.config.expected_fingerprint)}
|
||||||
readOnly
|
readOnly
|
||||||
title={textValue(node.config.expected_fingerprint)}
|
title={textValue(node.config.expected_fingerprint)}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "source.inline" ? (
|
{node.type === "source.inline" ? (
|
||||||
<FormField label="Rows">
|
<NodeFormField label="Rows">
|
||||||
<textarea
|
<textarea
|
||||||
className="dataflow-json-editor"
|
className="dataflow-json-editor"
|
||||||
value={rowsText}
|
value={rowsText}
|
||||||
@@ -270,18 +272,18 @@ export default function NodeInspector({
|
|||||||
spellCheck={false}
|
spellCheck={false}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "filter" ? (
|
{node.type === "filter" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Column">
|
<NodeFormField label="Column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.column)}
|
value={textValue(node.config.column)}
|
||||||
onChange={(event) => updateConfig({ column: event.target.value })}
|
onChange={(event) => updateConfig({ column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Operator">
|
<NodeFormField label="Operator">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.operator) || "eq"}
|
value={textValue(node.config.operator) || "eq"}
|
||||||
onChange={(event) => updateConfig({ operator: event.target.value })}
|
onChange={(event) => updateConfig({ operator: event.target.value })}
|
||||||
@@ -297,20 +299,20 @@ export default function NodeInspector({
|
|||||||
<option value="is_null">is null</option>
|
<option value="is_null">is null</option>
|
||||||
<option value="not_null">is not null</option>
|
<option value="not_null">is not null</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
{!["is_null", "not_null"].includes(textValue(node.config.operator)) ? (
|
{!["is_null", "not_null"].includes(textValue(node.config.operator)) ? (
|
||||||
<FormField label="Value">
|
<NodeFormField label="Value">
|
||||||
<input
|
<input
|
||||||
value={displayScalar(node.config.value)}
|
value={displayScalar(node.config.value)}
|
||||||
onChange={(event) => updateConfig({ value: parseScalar(event.target.value) })}
|
onChange={(event) => updateConfig({ value: parseScalar(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "filter.expression" ? (
|
{node.type === "filter.expression" ? (
|
||||||
<FormField label="Expression">
|
<NodeFormField label="Expression">
|
||||||
<textarea
|
<textarea
|
||||||
className="dataflow-expression-editor"
|
className="dataflow-expression-editor"
|
||||||
value={textValue(node.config.expression)}
|
value={textValue(node.config.expression)}
|
||||||
@@ -318,20 +320,20 @@ export default function NodeInspector({
|
|||||||
spellCheck={false}
|
spellCheck={false}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "distinct" ? (
|
{node.type === "distinct" ? (
|
||||||
<FormField label="Key columns">
|
<NodeFormField label="Key columns">
|
||||||
<input
|
<input
|
||||||
value={stringList(node.config.columns).join(", ")}
|
value={stringList(node.config.columns).join(", ")}
|
||||||
onChange={(event) => updateConfig({ columns: commaList(event.target.value) })}
|
onChange={(event) => updateConfig({ columns: commaList(event.target.value) })}
|
||||||
placeholder="All columns"
|
placeholder="All columns"
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "combine.union" ? (
|
{node.type === "combine.union" ? (
|
||||||
<FormField label="Duplicates">
|
<NodeFormField label="Duplicates">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.mode) || "all"}
|
value={textValue(node.config.mode) || "all"}
|
||||||
onChange={(event) => updateConfig({ mode: event.target.value })}
|
onChange={(event) => updateConfig({ mode: event.target.value })}
|
||||||
@@ -340,11 +342,11 @@ export default function NodeInspector({
|
|||||||
<option value="all">Keep all rows</option>
|
<option value="all">Keep all rows</option>
|
||||||
<option value="distinct">Remove duplicates</option>
|
<option value="distinct">Remove duplicates</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "combine.join" ? (
|
{node.type === "combine.join" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Join type">
|
<NodeFormField label="Join type">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.join_type) || "inner"}
|
value={textValue(node.config.join_type) || "inner"}
|
||||||
onChange={(event) => updateConfig({ join_type: event.target.value })}
|
onChange={(event) => updateConfig({ join_type: event.target.value })}
|
||||||
@@ -357,51 +359,51 @@ export default function NodeInspector({
|
|||||||
<option value="semi">Left rows with a match</option>
|
<option value="semi">Left rows with a match</option>
|
||||||
<option value="anti">Left rows without a match</option>
|
<option value="anti">Left rows without a match</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Left keys">
|
<NodeFormField label="Left keys">
|
||||||
<input
|
<input
|
||||||
value={stringList(node.config.left_keys).join(", ")}
|
value={stringList(node.config.left_keys).join(", ")}
|
||||||
onChange={(event) => updateConfig({ left_keys: commaList(event.target.value) })}
|
onChange={(event) => updateConfig({ left_keys: commaList(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Right keys">
|
<NodeFormField label="Right keys">
|
||||||
<input
|
<input
|
||||||
value={stringList(node.config.right_keys).join(", ")}
|
value={stringList(node.config.right_keys).join(", ")}
|
||||||
onChange={(event) => updateConfig({ right_keys: commaList(event.target.value) })}
|
onChange={(event) => updateConfig({ right_keys: commaList(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
{!["semi", "anti"].includes(textValue(node.config.join_type)) ? (
|
{!["semi", "anti"].includes(textValue(node.config.join_type)) ? (
|
||||||
<FormField label="Right-column prefix">
|
<NodeFormField label="Right-column prefix">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.right_prefix)}
|
value={textValue(node.config.right_prefix)}
|
||||||
onChange={(event) => updateConfig({ right_prefix: event.target.value })}
|
onChange={(event) => updateConfig({ right_prefix: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "select" ? (
|
{node.type === "select" ? (
|
||||||
<FormField label="Columns">
|
<NodeFormField label="Columns">
|
||||||
<input
|
<input
|
||||||
value={selectFieldsToText(node.config.fields)}
|
value={selectFieldsToText(node.config.fields)}
|
||||||
onChange={(event) => updateConfig({ fields: selectFieldsFromText(event.target.value) })}
|
onChange={(event) => updateConfig({ fields: selectFieldsFromText(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "aggregate" ? (
|
{node.type === "aggregate" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Group by">
|
<NodeFormField label="Group by">
|
||||||
<input
|
<input
|
||||||
value={stringList(node.config.group_by).join(", ")}
|
value={stringList(node.config.group_by).join(", ")}
|
||||||
onChange={(event) => updateConfig({ group_by: commaList(event.target.value) })}
|
onChange={(event) => updateConfig({ group_by: commaList(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Aggregates">
|
<NodeFormField label="Aggregates">
|
||||||
<textarea
|
<textarea
|
||||||
className="dataflow-expression-editor"
|
className="dataflow-expression-editor"
|
||||||
value={aggregateText}
|
value={aggregateText}
|
||||||
@@ -410,19 +412,19 @@ export default function NodeInspector({
|
|||||||
spellCheck={false}
|
spellCheck={false}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "derive" ? (
|
{node.type === "derive" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Output column">
|
<NodeFormField label="Output column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.target_column)}
|
value={textValue(node.config.target_column)}
|
||||||
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Operation">
|
<NodeFormField label="Operation">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.operation) || "copy"}
|
value={textValue(node.config.operation) || "copy"}
|
||||||
onChange={(event) => updateConfig({ operation: event.target.value })}
|
onChange={(event) => updateConfig({ operation: event.target.value })}
|
||||||
@@ -439,35 +441,35 @@ export default function NodeInspector({
|
|||||||
<option value="multiply">Multiply</option>
|
<option value="multiply">Multiply</option>
|
||||||
<option value="divide">Divide</option>
|
<option value="divide">Divide</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Source columns">
|
<NodeFormField label="Source columns">
|
||||||
<input
|
<input
|
||||||
value={stringList(node.config.source_columns).join(", ")}
|
value={stringList(node.config.source_columns).join(", ")}
|
||||||
onChange={(event) => updateConfig({ source_columns: commaList(event.target.value) })}
|
onChange={(event) => updateConfig({ source_columns: commaList(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
{textValue(node.config.operation) === "concat" ? (
|
{textValue(node.config.operation) === "concat" ? (
|
||||||
<FormField label="Separator">
|
<NodeFormField label="Separator">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.separator)}
|
value={textValue(node.config.separator)}
|
||||||
onChange={(event) => updateConfig({ separator: event.target.value })}
|
onChange={(event) => updateConfig({ separator: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "expression" ? (
|
{node.type === "expression" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Output column">
|
<NodeFormField label="Output column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.target_column)}
|
value={textValue(node.config.target_column)}
|
||||||
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Expression">
|
<NodeFormField label="Expression">
|
||||||
<textarea
|
<textarea
|
||||||
className="dataflow-expression-editor"
|
className="dataflow-expression-editor"
|
||||||
value={textValue(node.config.expression)}
|
value={textValue(node.config.expression)}
|
||||||
@@ -475,8 +477,8 @@ export default function NodeInspector({
|
|||||||
spellCheck={false}
|
spellCheck={false}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Expected type">
|
<NodeFormField label="Expected type">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.result_type) || "unknown"}
|
value={textValue(node.config.result_type) || "unknown"}
|
||||||
onChange={(event) => updateConfig({ result_type: event.target.value })}
|
onChange={(event) => updateConfig({ result_type: event.target.value })}
|
||||||
@@ -490,11 +492,11 @@ export default function NodeInspector({
|
|||||||
<option value="date">Date</option>
|
<option value="date">Date</option>
|
||||||
<option value="datetime">Date and time</option>
|
<option value="datetime">Date and time</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "calculate" ? (
|
{node.type === "calculate" ? (
|
||||||
<FormField label="Calculated columns">
|
<NodeFormField label="Calculated columns">
|
||||||
<textarea
|
<textarea
|
||||||
className="dataflow-expression-editor"
|
className="dataflow-expression-editor"
|
||||||
value={calculationText}
|
value={calculationText}
|
||||||
@@ -503,25 +505,25 @@ export default function NodeInspector({
|
|||||||
spellCheck={false}
|
spellCheck={false}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "convert" ? (
|
{node.type === "convert" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Source column">
|
<NodeFormField label="Source column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.source_column)}
|
value={textValue(node.config.source_column)}
|
||||||
onChange={(event) => updateConfig({ source_column: event.target.value })}
|
onChange={(event) => updateConfig({ source_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Output column">
|
<NodeFormField label="Output column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.target_column)}
|
value={textValue(node.config.target_column)}
|
||||||
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Data type">
|
<NodeFormField label="Data type">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.target_type) || "string"}
|
value={textValue(node.config.target_type) || "string"}
|
||||||
onChange={(event) => updateConfig({ target_type: event.target.value })}
|
onChange={(event) => updateConfig({ target_type: event.target.value })}
|
||||||
@@ -534,8 +536,8 @@ export default function NodeInspector({
|
|||||||
<option value="date">Date</option>
|
<option value="date">Date</option>
|
||||||
<option value="datetime">Date and time</option>
|
<option value="datetime">Date and time</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Conversion error">
|
<NodeFormField label="Conversion error">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.on_error) || "fail"}
|
value={textValue(node.config.on_error) || "fail"}
|
||||||
onChange={(event) => updateConfig({ on_error: event.target.value })}
|
onChange={(event) => updateConfig({ on_error: event.target.value })}
|
||||||
@@ -545,26 +547,26 @@ export default function NodeInspector({
|
|||||||
<option value="null">Use null</option>
|
<option value="null">Use null</option>
|
||||||
<option value="keep">Keep original</option>
|
<option value="keep">Keep original</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "replace" ? (
|
{node.type === "replace" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Source column">
|
<NodeFormField label="Source column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.source_column)}
|
value={textValue(node.config.source_column)}
|
||||||
onChange={(event) => updateConfig({ source_column: event.target.value })}
|
onChange={(event) => updateConfig({ source_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Output column">
|
<NodeFormField label="Output column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.target_column)}
|
value={textValue(node.config.target_column)}
|
||||||
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Mode">
|
<NodeFormField label="Mode">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.mode) || "exact"}
|
value={textValue(node.config.mode) || "exact"}
|
||||||
onChange={(event) => updateConfig({ mode: event.target.value })}
|
onChange={(event) => updateConfig({ mode: event.target.value })}
|
||||||
@@ -573,25 +575,25 @@ export default function NodeInspector({
|
|||||||
<option value="exact">Exact value</option>
|
<option value="exact">Exact value</option>
|
||||||
<option value="text">Text fragment</option>
|
<option value="text">Text fragment</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Find">
|
<NodeFormField label="Find">
|
||||||
<input
|
<input
|
||||||
value={displayScalar(node.config.find)}
|
value={displayScalar(node.config.find)}
|
||||||
onChange={(event) => updateConfig({ find: parseScalar(event.target.value) })}
|
onChange={(event) => updateConfig({ find: parseScalar(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Replacement">
|
<NodeFormField label="Replacement">
|
||||||
<input
|
<input
|
||||||
value={displayScalar(node.config.replacement)}
|
value={displayScalar(node.config.replacement)}
|
||||||
onChange={(event) => updateConfig({ replacement: parseScalar(event.target.value) })}
|
onChange={(event) => updateConfig({ replacement: parseScalar(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "sort" ? (
|
{node.type === "sort" ? (
|
||||||
<FormField label="Sort fields">
|
<NodeFormField label="Sort fields">
|
||||||
<textarea
|
<textarea
|
||||||
className="dataflow-expression-editor"
|
className="dataflow-expression-editor"
|
||||||
value={sortText}
|
value={sortText}
|
||||||
@@ -600,11 +602,11 @@ export default function NodeInspector({
|
|||||||
spellCheck={false}
|
spellCheck={false}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "window.rank" ? (
|
{node.type === "window.rank" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Method">
|
<NodeFormField label="Method">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.method) || "row_number"}
|
value={textValue(node.config.method) || "row_number"}
|
||||||
onChange={(event) => updateConfig({ method: event.target.value })}
|
onChange={(event) => updateConfig({ method: event.target.value })}
|
||||||
@@ -614,22 +616,22 @@ export default function NodeInspector({
|
|||||||
<option value="rank">Rank with gaps</option>
|
<option value="rank">Rank with gaps</option>
|
||||||
<option value="dense_rank">Dense rank</option>
|
<option value="dense_rank">Dense rank</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Output column">
|
<NodeFormField label="Output column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.target_column)}
|
value={textValue(node.config.target_column)}
|
||||||
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
onChange={(event) => updateConfig({ target_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Partition by">
|
<NodeFormField label="Partition by">
|
||||||
<input
|
<input
|
||||||
value={stringList(node.config.partition_by).join(", ")}
|
value={stringList(node.config.partition_by).join(", ")}
|
||||||
onChange={(event) => updateConfig({ partition_by: commaList(event.target.value) })}
|
onChange={(event) => updateConfig({ partition_by: commaList(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Order by">
|
<NodeFormField label="Order by">
|
||||||
<textarea
|
<textarea
|
||||||
className="dataflow-expression-editor"
|
className="dataflow-expression-editor"
|
||||||
value={rankSortText}
|
value={rankSortText}
|
||||||
@@ -638,11 +640,11 @@ export default function NodeInspector({
|
|||||||
spellCheck={false}
|
spellCheck={false}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "limit" ? (
|
{node.type === "limit" ? (
|
||||||
<FormField label="Maximum rows">
|
<NodeFormField label="Maximum rows">
|
||||||
<input
|
<input
|
||||||
type="number"
|
type="number"
|
||||||
min={1}
|
min={1}
|
||||||
@@ -651,11 +653,11 @@ export default function NodeInspector({
|
|||||||
onChange={(event) => updateConfig({ count: Number(event.target.value) })}
|
onChange={(event) => updateConfig({ count: Number(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "quality.rules" ? (
|
{node.type === "quality.rules" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Rules">
|
<NodeFormField label="Rules">
|
||||||
<textarea
|
<textarea
|
||||||
className="dataflow-json-editor"
|
className="dataflow-json-editor"
|
||||||
value={rulesText}
|
value={rulesText}
|
||||||
@@ -664,8 +666,8 @@ export default function NodeInspector({
|
|||||||
spellCheck={false}
|
spellCheck={false}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Invalid rows">
|
<NodeFormField label="Invalid rows">
|
||||||
<select
|
<select
|
||||||
value={textValue(node.config.action) || "annotate"}
|
value={textValue(node.config.action) || "annotate"}
|
||||||
onChange={(event) => updateConfig({ action: event.target.value })}
|
onChange={(event) => updateConfig({ action: event.target.value })}
|
||||||
@@ -675,26 +677,26 @@ export default function NodeInspector({
|
|||||||
<option value="drop">Drop</option>
|
<option value="drop">Drop</option>
|
||||||
<option value="fail">Stop</option>
|
<option value="fail">Stop</option>
|
||||||
</select>
|
</select>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "reconcile.compare" ? (
|
{node.type === "reconcile.compare" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Expected keys">
|
<NodeFormField label="Expected keys">
|
||||||
<input
|
<input
|
||||||
value={stringList(node.config.left_keys).join(", ")}
|
value={stringList(node.config.left_keys).join(", ")}
|
||||||
onChange={(event) => updateConfig({ left_keys: commaList(event.target.value) })}
|
onChange={(event) => updateConfig({ left_keys: commaList(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Observed keys">
|
<NodeFormField label="Observed keys">
|
||||||
<input
|
<input
|
||||||
value={stringList(node.config.right_keys).join(", ")}
|
value={stringList(node.config.right_keys).join(", ")}
|
||||||
onChange={(event) => updateConfig({ right_keys: commaList(event.target.value) })}
|
onChange={(event) => updateConfig({ right_keys: commaList(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Compared columns">
|
<NodeFormField label="Compared columns">
|
||||||
<input
|
<input
|
||||||
value={comparisonFieldsToText(node.config.compare_columns)}
|
value={comparisonFieldsToText(node.config.compare_columns)}
|
||||||
onChange={(event) => updateConfig({
|
onChange={(event) => updateConfig({
|
||||||
@@ -702,100 +704,100 @@ export default function NodeInspector({
|
|||||||
})}
|
})}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Observed prefix">
|
<NodeFormField label="Observed prefix">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.right_prefix)}
|
value={textValue(node.config.right_prefix)}
|
||||||
onChange={(event) => updateConfig({ right_prefix: event.target.value })}
|
onChange={(event) => updateConfig({ right_prefix: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "reconcile.decisions" ? (
|
{node.type === "reconcile.decisions" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Decision key hash column">
|
<NodeFormField label="Decision key hash column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.decision_key_column)}
|
value={textValue(node.config.decision_key_column)}
|
||||||
onChange={(event) => updateConfig({ decision_key_column: event.target.value })}
|
onChange={(event) => updateConfig({ decision_key_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Decision input hash column">
|
<NodeFormField label="Decision input hash column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.decision_input_column)}
|
value={textValue(node.config.decision_input_column)}
|
||||||
onChange={(event) => updateConfig({ decision_input_column: event.target.value })}
|
onChange={(event) => updateConfig({ decision_input_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Decision reference column">
|
<NodeFormField label="Decision reference column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.decision_ref_column)}
|
value={textValue(node.config.decision_ref_column)}
|
||||||
onChange={(event) => updateConfig({ decision_ref_column: event.target.value })}
|
onChange={(event) => updateConfig({ decision_ref_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Action column">
|
<NodeFormField label="Action column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.action_column)}
|
value={textValue(node.config.action_column)}
|
||||||
onChange={(event) => updateConfig({ action_column: event.target.value })}
|
onChange={(event) => updateConfig({ action_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Actor reference column">
|
<NodeFormField label="Actor reference column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.actor_column)}
|
value={textValue(node.config.actor_column)}
|
||||||
onChange={(event) => updateConfig({ actor_column: event.target.value })}
|
onChange={(event) => updateConfig({ actor_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Decision time column">
|
<NodeFormField label="Decision time column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.decided_at_column)}
|
value={textValue(node.config.decided_at_column)}
|
||||||
onChange={(event) => updateConfig({ decided_at_column: event.target.value })}
|
onChange={(event) => updateConfig({ decided_at_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Reason column">
|
<NodeFormField label="Reason column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.reason_column)}
|
value={textValue(node.config.reason_column)}
|
||||||
onChange={(event) => updateConfig({ reason_column: event.target.value })}
|
onChange={(event) => updateConfig({ reason_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Correction column">
|
<NodeFormField label="Correction column">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.correction_column)}
|
value={textValue(node.config.correction_column)}
|
||||||
onChange={(event) => updateConfig({ correction_column: event.target.value })}
|
onChange={(event) => updateConfig({ correction_column: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Allowed actions">
|
<NodeFormField label="Allowed actions">
|
||||||
<input
|
<input
|
||||||
value={stringList(node.config.allowed_actions).join(", ")}
|
value={stringList(node.config.allowed_actions).join(", ")}
|
||||||
onChange={(event) => updateConfig({ allowed_actions: commaList(event.target.value) })}
|
onChange={(event) => updateConfig({ allowed_actions: commaList(event.target.value) })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
{node.type === "subflow" ? (
|
{node.type === "subflow" ? (
|
||||||
<>
|
<>
|
||||||
<FormField label="Template reference">
|
<NodeFormField label="Template reference">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.template_ref)}
|
value={textValue(node.config.template_ref)}
|
||||||
onChange={(event) => updateConfig({ template_ref: event.target.value })}
|
onChange={(event) => updateConfig({ template_ref: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Template version">
|
<NodeFormField label="Template version">
|
||||||
<input
|
<input
|
||||||
value={textValue(node.config.template_version)}
|
value={textValue(node.config.template_version)}
|
||||||
onChange={(event) => updateConfig({ template_version: event.target.value })}
|
onChange={(event) => updateConfig({ template_version: event.target.value })}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Parameters">
|
<NodeFormField label="Parameters">
|
||||||
<textarea
|
<textarea
|
||||||
className="dataflow-json-editor"
|
className="dataflow-json-editor"
|
||||||
value={parametersText}
|
value={parametersText}
|
||||||
@@ -804,8 +806,8 @@ export default function NodeInspector({
|
|||||||
spellCheck={false}
|
spellCheck={false}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
<FormField label="Pinned graph">
|
<NodeFormField label="Pinned graph">
|
||||||
<textarea
|
<textarea
|
||||||
className="dataflow-json-editor"
|
className="dataflow-json-editor"
|
||||||
value={subflowGraphText}
|
value={subflowGraphText}
|
||||||
@@ -814,7 +816,7 @@ export default function NodeInspector({
|
|||||||
spellCheck={false}
|
spellCheck={false}
|
||||||
disabled={readOnly}
|
disabled={readOnly}
|
||||||
/>
|
/>
|
||||||
</FormField>
|
</NodeFormField>
|
||||||
</>
|
</>
|
||||||
) : null}
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
+217
-439
@@ -1,69 +1,10 @@
|
|||||||
.dataflow-page {
|
|
||||||
position: relative;
|
|
||||||
height: calc(100vh - 115px);
|
|
||||||
min-width: 0;
|
|
||||||
min-height: 0;
|
|
||||||
padding: 0;
|
|
||||||
overflow: hidden;
|
|
||||||
color: var(--text);
|
|
||||||
background: var(--bg);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-page *,
|
|
||||||
.dataflow-page *::before,
|
|
||||||
.dataflow-page *::after {
|
|
||||||
box-sizing: border-box;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-shell {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: minmax(250px, 300px) minmax(0, 1fr);
|
|
||||||
width: 100%;
|
|
||||||
height: 100%;
|
|
||||||
min-width: 0;
|
|
||||||
min-height: 0;
|
|
||||||
overflow: hidden;
|
|
||||||
border: var(--border-line);
|
|
||||||
background: var(--panel);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-panel,
|
|
||||||
.dataflow-workspace,
|
|
||||||
.dataflow-editor,
|
.dataflow-editor,
|
||||||
.dataflow-editor-surface,
|
|
||||||
.dataflow-canvas,
|
.dataflow-canvas,
|
||||||
.dataflow-pipeline-list-frame {
|
.dataflow-pipeline-list-frame {
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
min-height: 0;
|
min-height: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-pipeline-panel {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
overflow: hidden;
|
|
||||||
border-right: var(--border-line);
|
|
||||||
background: var(--panel-soft);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-panel-toolbar,
|
|
||||||
.dataflow-workspace-toolbar,
|
|
||||||
.dataflow-results-toolbar,
|
|
||||||
.dataflow-sql-toolbar,
|
|
||||||
.dataflow-panel-heading {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: space-between;
|
|
||||||
gap: 10px;
|
|
||||||
flex: 0 0 auto;
|
|
||||||
border-bottom: var(--border-line);
|
|
||||||
background: var(--panel-header);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-panel-toolbar {
|
|
||||||
min-height: 52px;
|
|
||||||
padding: 8px 10px 8px 14px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-toolbar-actions,
|
.dataflow-toolbar-actions,
|
||||||
.dataflow-command-bar,
|
.dataflow-command-bar,
|
||||||
.dataflow-identity-fields {
|
.dataflow-identity-fields {
|
||||||
@@ -72,66 +13,11 @@
|
|||||||
gap: 7px;
|
gap: 7px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-pipeline-search {
|
|
||||||
padding: 10px;
|
|
||||||
border-bottom: var(--border-line);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-search input {
|
|
||||||
min-height: 34px;
|
|
||||||
padding: 7px 9px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-list-frame {
|
.dataflow-pipeline-list-frame {
|
||||||
flex: 1 1 auto;
|
flex: 1 1 auto;
|
||||||
overflow: hidden;
|
overflow: hidden;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-pipeline-list {
|
|
||||||
height: 100%;
|
|
||||||
overflow: auto;
|
|
||||||
padding: 6px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-list > button {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: minmax(0, 1fr) auto;
|
|
||||||
align-items: center;
|
|
||||||
gap: 8px;
|
|
||||||
width: 100%;
|
|
||||||
min-height: 56px;
|
|
||||||
border: 0;
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
background: transparent;
|
|
||||||
color: var(--text);
|
|
||||||
cursor: pointer;
|
|
||||||
padding: 8px 9px;
|
|
||||||
text-align: left;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-list > button:hover,
|
|
||||||
.dataflow-pipeline-list > button:focus-visible {
|
|
||||||
background: var(--primary-soft);
|
|
||||||
outline: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-list > button.is-selected {
|
|
||||||
background: var(--primary-soft-strong);
|
|
||||||
box-shadow: inset 3px 0 0 var(--accent);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-list > button > span:first-child {
|
|
||||||
min-width: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-list strong,
|
|
||||||
.dataflow-pipeline-list small {
|
|
||||||
display: block;
|
|
||||||
overflow: hidden;
|
|
||||||
text-overflow: ellipsis;
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-definition-dialog {
|
.dataflow-definition-dialog {
|
||||||
width: min(620px, calc(100vw - 32px));
|
width: min(620px, calc(100vw - 32px));
|
||||||
}
|
}
|
||||||
@@ -150,13 +36,6 @@
|
|||||||
width: 100%;
|
width: 100%;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-definition-toggles {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
|
||||||
gap: 10px 18px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-provenance,
|
|
||||||
.dataflow-trigger-security {
|
.dataflow-trigger-security {
|
||||||
display: grid;
|
display: grid;
|
||||||
gap: 6px;
|
gap: 6px;
|
||||||
@@ -241,8 +120,7 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 760px) {
|
@media (max-width: 760px) {
|
||||||
.dataflow-triggers-layout,
|
.dataflow-triggers-layout {
|
||||||
.dataflow-definition-toggles {
|
|
||||||
grid-template-columns: 1fr;
|
grid-template-columns: 1fr;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -254,49 +132,12 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-pipeline-list strong {
|
|
||||||
color: var(--text-strong);
|
|
||||||
font-size: 13px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-list small {
|
|
||||||
margin-top: 4px;
|
|
||||||
color: var(--muted);
|
|
||||||
font-size: 11px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-list .status-badge {
|
|
||||||
max-width: 74px;
|
|
||||||
overflow: hidden;
|
|
||||||
text-overflow: ellipsis;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-empty,
|
|
||||||
.dataflow-inspector-empty,
|
|
||||||
.dataflow-results-empty {
|
|
||||||
display: grid;
|
|
||||||
place-items: center;
|
|
||||||
min-height: 100px;
|
|
||||||
color: var(--muted);
|
|
||||||
font-size: 13px;
|
|
||||||
text-align: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-workspace {
|
|
||||||
position: relative;
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
overflow: hidden;
|
|
||||||
background: var(--bg);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-action-guidance {
|
.dataflow-action-guidance {
|
||||||
padding: 0 10px 8px;
|
padding: 0 10px 8px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-workspace-toolbar {
|
.dataflow-workspace-toolbar {
|
||||||
min-height: 58px;
|
min-width: 0;
|
||||||
padding: 8px 10px;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-identity-fields {
|
.dataflow-identity-fields {
|
||||||
@@ -368,7 +209,6 @@
|
|||||||
outline-offset: -2px;
|
outline-offset: -2px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-palette,
|
|
||||||
.dataflow-inspector {
|
.dataflow-inspector {
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
min-height: 0;
|
min-height: 0;
|
||||||
@@ -376,19 +216,10 @@
|
|||||||
background: var(--panel-soft);
|
background: var(--panel-soft);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-palette {
|
|
||||||
border-right: var(--border-line);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-inspector {
|
.dataflow-inspector {
|
||||||
border-left: var(--border-line);
|
border-left: var(--border-line);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-panel-heading {
|
|
||||||
min-height: 44px;
|
|
||||||
padding: 8px 10px 8px 12px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-panel-heading > span {
|
.dataflow-panel-heading > span {
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
}
|
}
|
||||||
@@ -413,127 +244,6 @@
|
|||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-palette-items {
|
|
||||||
display: grid;
|
|
||||||
grid-auto-rows: max-content;
|
|
||||||
align-content: start;
|
|
||||||
gap: 4px;
|
|
||||||
height: calc(100% - 44px);
|
|
||||||
padding: 8px;
|
|
||||||
overflow-x: hidden;
|
|
||||||
overflow-y: auto;
|
|
||||||
scrollbar-gutter: stable;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-group {
|
|
||||||
display: grid;
|
|
||||||
grid-auto-rows: max-content;
|
|
||||||
align-content: start;
|
|
||||||
gap: 2px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-group + .dataflow-palette-group {
|
|
||||||
margin-top: 6px;
|
|
||||||
padding-top: 8px;
|
|
||||||
border-top: var(--border-line);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-group h3 {
|
|
||||||
margin: 0;
|
|
||||||
color: var(--muted);
|
|
||||||
font-size: 10px;
|
|
||||||
font-weight: 700;
|
|
||||||
letter-spacing: 0;
|
|
||||||
padding: 3px 8px;
|
|
||||||
text-transform: uppercase;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-items button {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: 18px minmax(0, 1fr) 14px;
|
|
||||||
align-items: center;
|
|
||||||
gap: 7px;
|
|
||||||
min-height: 38px;
|
|
||||||
border: 0;
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
background: transparent;
|
|
||||||
color: var(--text);
|
|
||||||
cursor: grab;
|
|
||||||
font: inherit;
|
|
||||||
font-size: 12px;
|
|
||||||
padding: 7px 8px;
|
|
||||||
text-align: left;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-items button:hover:not(:disabled),
|
|
||||||
.dataflow-palette-items button:focus-visible:not(:disabled) {
|
|
||||||
background: var(--primary-soft);
|
|
||||||
color: var(--text-strong);
|
|
||||||
outline: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-items button:active:not(:disabled) {
|
|
||||||
cursor: grabbing;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-items button:disabled {
|
|
||||||
cursor: default;
|
|
||||||
opacity: .42;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-add {
|
|
||||||
color: var(--accent);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-editor-surface {
|
|
||||||
position: relative;
|
|
||||||
overflow: hidden;
|
|
||||||
background: var(--bg);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-canvas {
|
|
||||||
position: absolute;
|
|
||||||
inset: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-canvas .react-flow {
|
|
||||||
background: var(--bg);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-canvas .react-flow__background {
|
|
||||||
color: var(--line-dark);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-canvas .react-flow__controls,
|
|
||||||
.dataflow-canvas .react-flow__minimap {
|
|
||||||
overflow: hidden;
|
|
||||||
border: var(--border-line);
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
background: var(--panel);
|
|
||||||
box-shadow: var(--shadow-xs);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-canvas .react-flow__controls-button {
|
|
||||||
border-bottom: var(--border-line);
|
|
||||||
background: var(--panel);
|
|
||||||
color: var(--text);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-canvas .react-flow__minimap-mask {
|
|
||||||
fill: color-mix(in srgb, var(--bg) 76%, transparent);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-canvas .react-flow__edge-path {
|
|
||||||
stroke: var(--line-dark);
|
|
||||||
stroke-width: 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-canvas .react-flow__edge.selected .react-flow__edge-path,
|
|
||||||
.dataflow-canvas .react-flow__edge:hover .react-flow__edge-path {
|
|
||||||
stroke: var(--accent);
|
|
||||||
stroke-width: 3;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-canvas .react-flow__edge.dataflow-edge-proximity .react-flow__edge-path {
|
.dataflow-canvas .react-flow__edge.dataflow-edge-proximity .react-flow__edge-path {
|
||||||
stroke: var(--accent);
|
stroke: var(--accent);
|
||||||
stroke-width: 3;
|
stroke-width: 3;
|
||||||
@@ -545,15 +255,6 @@
|
|||||||
stroke-width: 3;
|
stroke-width: 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-canvas-empty {
|
|
||||||
position: absolute;
|
|
||||||
inset: 0;
|
|
||||||
display: grid;
|
|
||||||
place-items: center;
|
|
||||||
color: var(--muted);
|
|
||||||
pointer-events: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-node {
|
.dataflow-node {
|
||||||
position: relative;
|
position: relative;
|
||||||
display: flex;
|
display: flex;
|
||||||
@@ -562,8 +263,8 @@
|
|||||||
width: 180px;
|
width: 180px;
|
||||||
min-height: 54px;
|
min-height: 54px;
|
||||||
border: 1px solid var(--line-dark);
|
border: 1px solid var(--line-dark);
|
||||||
border-left: 4px solid #3d6f9e;
|
border-left: 4px solid var(--data-category-blue);
|
||||||
border-radius: 6px;
|
border-radius: var(--radius-compact);
|
||||||
background: var(--panel);
|
background: var(--panel);
|
||||||
box-shadow: var(--shadow-xs);
|
box-shadow: var(--shadow-xs);
|
||||||
padding: 8px 10px;
|
padding: 8px 10px;
|
||||||
@@ -571,34 +272,34 @@
|
|||||||
|
|
||||||
.dataflow-node-source-inline,
|
.dataflow-node-source-inline,
|
||||||
.dataflow-node-source-reference {
|
.dataflow-node-source-reference {
|
||||||
border-left-color: #2f7d6d;
|
border-left-color: var(--data-category-green);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-node-filter {
|
.dataflow-node-filter {
|
||||||
border-left-color: #b7791f;
|
border-left-color: var(--data-category-amber);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-node-combine-union,
|
.dataflow-node-combine-union,
|
||||||
.dataflow-node-combine-join {
|
.dataflow-node-combine-join {
|
||||||
min-height: 64px;
|
min-height: 64px;
|
||||||
border-left-color: #2f7d6d;
|
border-left-color: var(--data-category-green);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-node-distinct {
|
.dataflow-node-distinct {
|
||||||
border-left-color: #b7791f;
|
border-left-color: var(--data-category-amber);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-node-derive,
|
.dataflow-node-derive,
|
||||||
.dataflow-node-aggregate {
|
.dataflow-node-aggregate {
|
||||||
border-left-color: #76569b;
|
border-left-color: var(--data-category-purple);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-node-sort {
|
.dataflow-node-sort {
|
||||||
border-left-color: #3d6f9e;
|
border-left-color: var(--data-category-blue);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-node-output {
|
.dataflow-node-output {
|
||||||
border-left-color: #9d4e63;
|
border-left-color: var(--data-category-rose);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-node.is-selected {
|
.dataflow-node.is-selected {
|
||||||
@@ -610,17 +311,6 @@
|
|||||||
border-color: var(--danger-text);
|
border-color: var(--danger-text);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-node-icon {
|
|
||||||
display: grid;
|
|
||||||
width: 28px;
|
|
||||||
height: 28px;
|
|
||||||
flex: 0 0 28px;
|
|
||||||
place-items: center;
|
|
||||||
border-radius: 5px;
|
|
||||||
background: var(--panel-soft);
|
|
||||||
color: var(--text-strong);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-node-copy {
|
.dataflow-node-copy {
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
flex: 1 1 auto;
|
flex: 1 1 auto;
|
||||||
@@ -645,32 +335,6 @@
|
|||||||
font-size: 10px;
|
font-size: 10px;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-node-count {
|
|
||||||
min-width: 22px;
|
|
||||||
border-radius: 999px;
|
|
||||||
background: var(--primary-soft);
|
|
||||||
color: var(--text-strong);
|
|
||||||
font-size: 10px;
|
|
||||||
font-variant-numeric: tabular-nums;
|
|
||||||
padding: 3px 5px;
|
|
||||||
text-align: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-node-handle {
|
|
||||||
width: 13px;
|
|
||||||
height: 13px;
|
|
||||||
border: 3px solid var(--panel);
|
|
||||||
background: var(--line-dark);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-node-handle:hover,
|
|
||||||
.dataflow-node-handle.connectingto,
|
|
||||||
.dataflow-node-handle.valid {
|
|
||||||
width: 17px;
|
|
||||||
height: 17px;
|
|
||||||
background: var(--accent);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-node-port-label {
|
.dataflow-node-port-label {
|
||||||
position: absolute;
|
position: absolute;
|
||||||
left: 8px;
|
left: 8px;
|
||||||
@@ -685,7 +349,7 @@
|
|||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-node-combine-join .dataflow-node-icon {
|
.dataflow-node-combine-join .definition-node-icon {
|
||||||
margin-left: 31px;
|
margin-left: 31px;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -736,6 +400,210 @@
|
|||||||
width: min(720px, calc(100vw - 32px));
|
width: min(720px, calc(100vw - 32px));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-dialog {
|
||||||
|
width: min(1080px, calc(100vw - 32px));
|
||||||
|
height: min(760px, calc(100vh - 48px));
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-dialog-body {
|
||||||
|
overflow: hidden;
|
||||||
|
padding: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-shell {
|
||||||
|
display: flex;
|
||||||
|
height: 100%;
|
||||||
|
min-height: 0;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-set-toolbar {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: minmax(190px, 0.8fr) minmax(300px, 1.2fr) auto;
|
||||||
|
align-items: end;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-set-toolbar select,
|
||||||
|
.dataflow-decision-set-toolbar input,
|
||||||
|
.dataflow-decision-editor textarea {
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-create,
|
||||||
|
.dataflow-decision-counts,
|
||||||
|
.dataflow-results-actions,
|
||||||
|
.dataflow-decision-editor-actions {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-create input {
|
||||||
|
min-width: 0;
|
||||||
|
flex: 1 1 auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-create .btn {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-counts {
|
||||||
|
min-height: 34px;
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 11px;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-counts strong {
|
||||||
|
color: var(--text-strong);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-counts .is-warning strong {
|
||||||
|
color: var(--warning-deep);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-layout {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: minmax(420px, 1.1fr) minmax(340px, 0.9fr);
|
||||||
|
min-height: 0;
|
||||||
|
flex: 1 1 auto;
|
||||||
|
overflow: hidden;
|
||||||
|
border: var(--border-line);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows,
|
||||||
|
.dataflow-decision-editor {
|
||||||
|
min-width: 0;
|
||||||
|
min-height: 0;
|
||||||
|
overflow: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows {
|
||||||
|
border-right: var(--border-line);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-row-heading,
|
||||||
|
.dataflow-decision-rows > button {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: minmax(180px, 1fr) minmax(90px, 0.45fr) 90px;
|
||||||
|
align-items: center;
|
||||||
|
gap: 10px;
|
||||||
|
width: 100%;
|
||||||
|
min-height: 42px;
|
||||||
|
border: 0;
|
||||||
|
border-bottom: var(--border-line);
|
||||||
|
padding: 7px 10px;
|
||||||
|
text-align: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-row-heading {
|
||||||
|
position: sticky;
|
||||||
|
z-index: 2;
|
||||||
|
top: 0;
|
||||||
|
background: var(--panel-header);
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 10px;
|
||||||
|
font-weight: 700;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows > button {
|
||||||
|
background: transparent;
|
||||||
|
color: var(--text);
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows > button:hover,
|
||||||
|
.dataflow-decision-rows > button.is-selected {
|
||||||
|
background: var(--primary-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows > button.is-selected {
|
||||||
|
box-shadow: inset 3px 0 0 var(--accent);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows > button > span {
|
||||||
|
min-width: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-state {
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 10px;
|
||||||
|
font-weight: 700;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-state.is-accept,
|
||||||
|
.dataflow-decision-state.is-correct {
|
||||||
|
color: var(--success-text);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-state.is-reject,
|
||||||
|
.dataflow-decision-state.is-stale {
|
||||||
|
color: var(--danger-text);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-state.is-defer {
|
||||||
|
color: var(--warning-deep);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor {
|
||||||
|
display: grid;
|
||||||
|
grid-auto-rows: max-content;
|
||||||
|
align-content: start;
|
||||||
|
gap: 14px;
|
||||||
|
padding: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor-heading {
|
||||||
|
display: flex;
|
||||||
|
align-items: start;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 12px;
|
||||||
|
padding-bottom: 10px;
|
||||||
|
border-bottom: var(--border-line);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor-heading > div {
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor-heading strong,
|
||||||
|
.dataflow-decision-editor-heading small {
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor-heading small {
|
||||||
|
margin-top: 4px;
|
||||||
|
overflow: hidden;
|
||||||
|
color: var(--muted);
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Consolas, monospace;
|
||||||
|
font-size: 9px;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor .dataflow-json-editor {
|
||||||
|
min-height: 140px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor-actions {
|
||||||
|
justify-content: flex-end;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-history-summary {
|
||||||
|
margin-right: auto;
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 11px;
|
||||||
|
}
|
||||||
|
|
||||||
.dataflow-run-dialog {
|
.dataflow-run-dialog {
|
||||||
width: min(860px, calc(100vw - 32px));
|
width: min(860px, calc(100vw - 32px));
|
||||||
}
|
}
|
||||||
@@ -873,8 +741,6 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-sql-toolbar {
|
.dataflow-sql-toolbar {
|
||||||
min-height: 44px;
|
|
||||||
padding: 7px 10px 7px 14px;
|
|
||||||
color: var(--text-strong);
|
color: var(--text-strong);
|
||||||
font-size: 12px;
|
font-size: 12px;
|
||||||
font-weight: 700;
|
font-weight: 700;
|
||||||
@@ -911,8 +777,6 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-results-toolbar {
|
.dataflow-results-toolbar {
|
||||||
min-height: 42px;
|
|
||||||
padding: 5px 9px;
|
|
||||||
flex-wrap: wrap;
|
flex-wrap: wrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1051,44 +915,7 @@
|
|||||||
white-space: nowrap;
|
white-space: nowrap;
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-workspace-empty {
|
@media (max-width: 1280px) {
|
||||||
display: grid;
|
|
||||||
place-content: center;
|
|
||||||
justify-items: center;
|
|
||||||
gap: 12px;
|
|
||||||
height: 100%;
|
|
||||||
color: var(--muted);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-workspace-empty strong {
|
|
||||||
color: var(--text-strong);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-workspace-empty .btn {
|
|
||||||
display: inline-flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 6px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-working-indicator {
|
|
||||||
position: absolute;
|
|
||||||
z-index: 20;
|
|
||||||
right: 12px;
|
|
||||||
bottom: 10px;
|
|
||||||
border: var(--border-line-dark);
|
|
||||||
border-radius: var(--radius-sm);
|
|
||||||
background: var(--panel);
|
|
||||||
box-shadow: var(--shadow-popover);
|
|
||||||
color: var(--text-strong);
|
|
||||||
font-size: 12px;
|
|
||||||
padding: 7px 10px;
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (max-width: 1180px) {
|
|
||||||
.dataflow-shell {
|
|
||||||
grid-template-columns: 240px minmax(0, 1fr);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-workspace-toolbar {
|
.dataflow-workspace-toolbar {
|
||||||
align-items: stretch;
|
align-items: stretch;
|
||||||
flex-direction: column;
|
flex-direction: column;
|
||||||
@@ -1108,10 +935,6 @@
|
|||||||
}
|
}
|
||||||
|
|
||||||
@media (max-width: 900px) {
|
@media (max-width: 900px) {
|
||||||
.dataflow-shell {
|
|
||||||
grid-template-columns: 210px minmax(0, 1fr);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-editor {
|
.dataflow-editor {
|
||||||
grid-template-columns: 135px minmax(0, 1fr);
|
grid-template-columns: 135px minmax(0, 1fr);
|
||||||
grid-template-rows: minmax(0, 1fr) minmax(170px, 32%);
|
grid-template-rows: minmax(0, 1fr) minmax(170px, 32%);
|
||||||
@@ -1144,16 +967,6 @@
|
|||||||
height: calc(100dvh - 115px);
|
height: calc(100dvh - 115px);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-shell {
|
|
||||||
grid-template-columns: 1fr;
|
|
||||||
grid-template-rows: minmax(150px, 24%) minmax(0, 1fr);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-pipeline-panel {
|
|
||||||
border-right: 0;
|
|
||||||
border-bottom: var(--border-line);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-workspace-toolbar {
|
.dataflow-workspace-toolbar {
|
||||||
overflow-x: auto;
|
overflow-x: auto;
|
||||||
align-items: flex-start;
|
align-items: flex-start;
|
||||||
@@ -1179,41 +992,6 @@
|
|||||||
grid-template-rows: auto minmax(0, 1fr) minmax(160px, 30%);
|
grid-template-rows: auto minmax(0, 1fr) minmax(160px, 30%);
|
||||||
}
|
}
|
||||||
|
|
||||||
.dataflow-palette {
|
|
||||||
border-right: 0;
|
|
||||||
border-bottom: var(--border-line);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette .dataflow-panel-heading {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-items {
|
|
||||||
display: flex;
|
|
||||||
overflow-x: auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-group {
|
|
||||||
display: flex;
|
|
||||||
flex: 0 0 auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-group + .dataflow-palette-group {
|
|
||||||
margin-top: 0;
|
|
||||||
padding-top: 0;
|
|
||||||
padding-left: 6px;
|
|
||||||
border-top: 0;
|
|
||||||
border-left: var(--border-line);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-group h3 {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-palette-items button {
|
|
||||||
min-width: 128px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dataflow-inspector-fields {
|
.dataflow-inspector-fields {
|
||||||
grid-template-columns: 1fr 1fr;
|
grid-template-columns: 1fr 1fr;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user