Add durable reconciliation decisions
This commit is contained in:
@@ -152,6 +152,16 @@ logical row disappeared. It never silently applies a correction to business
|
|||||||
data; a downstream governed transform or Workflow handoff must interpret the
|
data; a downstream governed transform or Workflow handoff must interpret the
|
||||||
recorded action.
|
recorded action.
|
||||||
|
|
||||||
|
For saved reconciliation pipelines, the preview results provide a review
|
||||||
|
dialog for those rows. Reviewers create a tenant-owned decision set and append
|
||||||
|
accept, reject, correct, or defer decisions with a mandatory reason. Writes use
|
||||||
|
optimistic concurrency; updating a decision creates another immutable revision
|
||||||
|
rather than replacing history. The current projection appears in the ordinary
|
||||||
|
Dataflow source catalogue as `dataflow-decision-set:<id>` and carries a content
|
||||||
|
fingerprint. A changed input hash is therefore shown as stale and cannot be
|
||||||
|
silently reused. Corrections remain annotations until an explicit downstream
|
||||||
|
transform applies them.
|
||||||
|
|
||||||
Reporting consumers may either evaluate a pinned pipeline revision or pin one
|
Reporting consumers may either evaluate a pinned pipeline revision or pin one
|
||||||
successful published run. An exact run pin is immutable: it cannot be supplied
|
successful published run. An exact run pin is immutable: it cannot be supplied
|
||||||
new parameters, and Dataflow reads only the recorded Datasource materialization
|
new parameters, and Dataflow reads only the recorded Datasource materialization
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from sqlalchemy import (
|
|||||||
)
|
)
|
||||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||||
|
|
||||||
|
from govoplan_core.core.concurrency import strong_resource_etag
|
||||||
from govoplan_core.db.base import Base, TimestampMixin
|
from govoplan_core.db.base import Base, TimestampMixin
|
||||||
|
|
||||||
|
|
||||||
@@ -121,6 +122,11 @@ class DataflowPipeline(Base, TimestampMixin):
|
|||||||
cascade="all, delete-orphan",
|
cascade="all, delete-orphan",
|
||||||
order_by="DataflowTrigger.created_at",
|
order_by="DataflowTrigger.created_at",
|
||||||
)
|
)
|
||||||
|
decision_sets: Mapped[list["DataflowReconciliationDecisionSet"]] = relationship(
|
||||||
|
back_populates="pipeline",
|
||||||
|
cascade="all, delete-orphan",
|
||||||
|
order_by="DataflowReconciliationDecisionSet.name",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DataflowPipelineRevision(Base, TimestampMixin):
|
class DataflowPipelineRevision(Base, TimestampMixin):
|
||||||
@@ -153,6 +159,99 @@ class DataflowPipelineRevision(Base, TimestampMixin):
|
|||||||
pipeline: Mapped[DataflowPipeline] = relationship(back_populates="revisions")
|
pipeline: Mapped[DataflowPipeline] = relationship(back_populates="revisions")
|
||||||
|
|
||||||
|
|
||||||
|
class DataflowReconciliationDecisionSet(Base, TimestampMixin):
|
||||||
|
__tablename__ = "dataflow_reconciliation_decision_sets"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"pipeline_id",
|
||||||
|
"name",
|
||||||
|
name="uq_dataflow_decision_sets_pipeline_name",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_dataflow_decision_sets_tenant_pipeline",
|
||||||
|
"tenant_id",
|
||||||
|
"pipeline_id",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
pipeline_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey("dataflow_pipelines.id", ondelete="CASCADE"),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
name: Mapped[str] = mapped_column(String(300), nullable=False)
|
||||||
|
node_id: Mapped[str | None] = mapped_column(String(100), nullable=True)
|
||||||
|
resource_revision: Mapped[int] = mapped_column(
|
||||||
|
Integer,
|
||||||
|
default=1,
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
created_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
|
updated_by: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
|
|
||||||
|
pipeline: Mapped[DataflowPipeline] = relationship(back_populates="decision_sets")
|
||||||
|
decisions: Mapped[list["DataflowReconciliationDecision"]] = relationship(
|
||||||
|
back_populates="decision_set",
|
||||||
|
cascade="all, delete-orphan",
|
||||||
|
order_by="DataflowReconciliationDecision.revision",
|
||||||
|
)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def strong_etag(self) -> str:
|
||||||
|
return strong_resource_etag(
|
||||||
|
"dataflow_reconciliation_decision_set",
|
||||||
|
self.id,
|
||||||
|
self.resource_revision,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DataflowReconciliationDecision(Base, TimestampMixin):
|
||||||
|
__tablename__ = "dataflow_reconciliation_decisions"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint(
|
||||||
|
"decision_set_id",
|
||||||
|
"revision",
|
||||||
|
name="uq_dataflow_reconciliation_decision_revision",
|
||||||
|
),
|
||||||
|
Index(
|
||||||
|
"ix_dataflow_reconciliation_decisions_current",
|
||||||
|
"tenant_id",
|
||||||
|
"decision_set_id",
|
||||||
|
"key_hash",
|
||||||
|
"revision",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[str] = mapped_column(String(36), primary_key=True, default=new_uuid)
|
||||||
|
tenant_id: Mapped[str] = mapped_column(String(36), nullable=False, index=True)
|
||||||
|
decision_set_id: Mapped[str] = mapped_column(
|
||||||
|
ForeignKey(
|
||||||
|
"dataflow_reconciliation_decision_sets.id",
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
nullable=False,
|
||||||
|
index=True,
|
||||||
|
)
|
||||||
|
revision: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||||
|
key_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||||
|
input_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
action: Mapped[str] = mapped_column(String(20), nullable=False)
|
||||||
|
reason: Mapped[str] = mapped_column(Text, nullable=False)
|
||||||
|
correction: Mapped[dict[str, Any] | None] = mapped_column(JSON, nullable=True)
|
||||||
|
actor_ref: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
decided_at: Mapped[datetime] = mapped_column(
|
||||||
|
DateTime(timezone=True),
|
||||||
|
nullable=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
decision_set: Mapped[DataflowReconciliationDecisionSet] = relationship(
|
||||||
|
back_populates="decisions"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DataflowRun(Base, TimestampMixin):
|
class DataflowRun(Base, TimestampMixin):
|
||||||
__tablename__ = "dataflow_runs"
|
__tablename__ = "dataflow_runs"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
|
|||||||
@@ -200,7 +200,12 @@ DOCUMENTATION = (
|
|||||||
"Every graph node declares typed inputs, configuration, output schema, and validation rules. "
|
"Every graph node declares typed inputs, configuration, output schema, and validation rules. "
|
||||||
"Source nodes pin inline content or governed Datasource references; combine, filter, transform, "
|
"Source nodes pin inline content or governed Datasource references; combine, filter, transform, "
|
||||||
"quality, reconciliation, reusable-subflow, and output nodes remain explicit in the canonical graph. "
|
"quality, reconciliation, reusable-subflow, and output nodes remain explicit in the canonical graph. "
|
||||||
"Reconciliation rows expose stable key hashes, explicit before/after values, and input hashes. A separate decision-table input can annotate exact matches, invalidate changed inputs, and report orphaned decisions without silently rewriting business data. "
|
"Reconciliation rows expose stable key hashes, explicit before/after values, and input hashes. The "
|
||||||
|
"review dialog records accept, reject, correct, or defer decisions in tenant-owned immutable decision "
|
||||||
|
"sets. Their current projection is a fingerprinted Dataflow source; every superseded revision retains "
|
||||||
|
"the actor, reason, exact input hash, time, and optional correction. A reconcile.decisions node can "
|
||||||
|
"annotate exact matches, invalidate changed inputs, and report orphaned decisions without silently "
|
||||||
|
"rewriting business data. "
|
||||||
"Expressions use the typed Dataflow expression language and never execute arbitrary host or database "
|
"Expressions use the typed Dataflow expression language and never execute arbitrary host or database "
|
||||||
"code. Selecting a node may request a bounded intermediate preview; preview rows are transient, "
|
"code. Selecting a node may request a bounded intermediate preview; preview rows are transient, "
|
||||||
"privacy-filtered for the actor, and are not retained as run output. SQL editing compiles into the same "
|
"privacy-filtered for the actor, and are not retained as run output. SQL editing compiles into the same "
|
||||||
@@ -218,6 +223,7 @@ DOCUMENTATION = (
|
|||||||
"dataflow.field.expression",
|
"dataflow.field.expression",
|
||||||
"dataflow.field.schema",
|
"dataflow.field.schema",
|
||||||
"dataflow.action.preview-node",
|
"dataflow.action.preview-node",
|
||||||
|
"dataflow.action.review-decisions",
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
@@ -232,6 +238,8 @@ DOCUMENTATION = (
|
|||||||
"the revision and authorization grant, then re-evaluate authority for every delivery. Runs create "
|
"the revision and authorization grant, then re-evaluate authority for every delivery. Runs create "
|
||||||
"durable command and recovery evidence. Publishing creates a governed Datasource materialization, and "
|
"durable command and recovery evidence. Publishing creates a governed Datasource materialization, and "
|
||||||
"environment promotion changes which immutable revision is eligible for staging or production runs. "
|
"environment promotion changes which immutable revision is eligible for staging or production runs. "
|
||||||
|
"Recording a reconciliation decision uses optimistic concurrency and appends an immutable revision; "
|
||||||
|
"it never mutates the reviewed business row. "
|
||||||
"Deletion prevents future use while retained run, deployment, lineage, audit, and recovery evidence "
|
"Deletion prevents future use while retained run, deployment, lineage, audit, and recovery evidence "
|
||||||
"continues under its retention policy."
|
"continues under its retention policy."
|
||||||
),
|
),
|
||||||
@@ -247,12 +255,14 @@ DOCUMENTATION = (
|
|||||||
"dataflow.action.save",
|
"dataflow.action.save",
|
||||||
"dataflow.action.derive",
|
"dataflow.action.derive",
|
||||||
"dataflow.action.trigger",
|
"dataflow.action.trigger",
|
||||||
|
"dataflow.action.record-decision",
|
||||||
"dataflow.action.delete",
|
"dataflow.action.delete",
|
||||||
],
|
],
|
||||||
"consequence_classes": {
|
"consequence_classes": {
|
||||||
"save_revision": "Appends an immutable pipeline definition revision.",
|
"save_revision": "Appends an immutable pipeline definition revision.",
|
||||||
"derive_copy": "Creates a separately governed copy pinned to the source revision and hash.",
|
"derive_copy": "Creates a separately governed copy pinned to the source revision and hash.",
|
||||||
"configure_trigger": "Creates or changes an automation command with revision and authorization evidence.",
|
"configure_trigger": "Creates or changes an automation command with revision and authorization evidence.",
|
||||||
|
"record_decision": "Appends an actor-attributed immutable decision revision against an exact input hash.",
|
||||||
"delete_pipeline": "Prevents future use while retained evidence remains governed.",
|
"delete_pipeline": "Prevents future use while retained evidence remains governed.",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -362,6 +372,22 @@ def _tenant_summary(session, tenant_id: str) -> dict[str, int]:
|
|||||||
)
|
)
|
||||||
.count()
|
.count()
|
||||||
),
|
),
|
||||||
|
"dataflow_reconciliation_decision_sets": (
|
||||||
|
session.query(dataflow_models.DataflowReconciliationDecisionSet)
|
||||||
|
.filter(
|
||||||
|
dataflow_models.DataflowReconciliationDecisionSet.tenant_id
|
||||||
|
== tenant_id
|
||||||
|
)
|
||||||
|
.count()
|
||||||
|
),
|
||||||
|
"dataflow_reconciliation_decisions": (
|
||||||
|
session.query(dataflow_models.DataflowReconciliationDecision)
|
||||||
|
.filter(
|
||||||
|
dataflow_models.DataflowReconciliationDecision.tenant_id
|
||||||
|
== tenant_id
|
||||||
|
)
|
||||||
|
.count()
|
||||||
|
),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -532,6 +558,14 @@ manifest = ModuleManifest(
|
|||||||
parent_id="dataflow.page",
|
parent_id="dataflow.page",
|
||||||
order=50,
|
order=50,
|
||||||
),
|
),
|
||||||
|
ViewSurface(
|
||||||
|
id="dataflow.decisions",
|
||||||
|
module_id=MODULE_ID,
|
||||||
|
kind="action",
|
||||||
|
label="Reconciliation decisions",
|
||||||
|
parent_id="dataflow.results",
|
||||||
|
order=55,
|
||||||
|
),
|
||||||
ViewSurface(
|
ViewSurface(
|
||||||
id="dataflow.triggers",
|
id="dataflow.triggers",
|
||||||
module_id=MODULE_ID,
|
module_id=MODULE_ID,
|
||||||
@@ -580,6 +614,8 @@ manifest = ModuleManifest(
|
|||||||
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
script_location=str(Path(__file__).with_name("migrations") / "versions"),
|
||||||
retirement_supported=True,
|
retirement_supported=True,
|
||||||
retirement_provider=drop_table_retirement_provider(
|
retirement_provider=drop_table_retirement_provider(
|
||||||
|
dataflow_models.DataflowReconciliationDecision,
|
||||||
|
dataflow_models.DataflowReconciliationDecisionSet,
|
||||||
dataflow_models.DataflowTriggerDelivery,
|
dataflow_models.DataflowTriggerDelivery,
|
||||||
dataflow_models.DataflowTrigger,
|
dataflow_models.DataflowTrigger,
|
||||||
dataflow_models.DataflowRun,
|
dataflow_models.DataflowRun,
|
||||||
@@ -597,6 +633,8 @@ manifest = ModuleManifest(
|
|||||||
persistent_table_uninstall_guard(
|
persistent_table_uninstall_guard(
|
||||||
dataflow_models.DataflowPipeline,
|
dataflow_models.DataflowPipeline,
|
||||||
dataflow_models.DataflowPipelineRevision,
|
dataflow_models.DataflowPipelineRevision,
|
||||||
|
dataflow_models.DataflowReconciliationDecisionSet,
|
||||||
|
dataflow_models.DataflowReconciliationDecision,
|
||||||
dataflow_models.DataflowPipelineDeployment,
|
dataflow_models.DataflowPipelineDeployment,
|
||||||
dataflow_models.DataflowRun,
|
dataflow_models.DataflowRun,
|
||||||
dataflow_models.DataflowTrigger,
|
dataflow_models.DataflowTrigger,
|
||||||
@@ -614,7 +652,13 @@ manifest = ModuleManifest(
|
|||||||
known_limits=(
|
known_limits=(
|
||||||
"Execution adapters do not yet cover every declared node family.",
|
"Execution adapters do not yet cover every declared node family.",
|
||||||
),
|
),
|
||||||
owned_concepts=("dataflow definition", "dataflow revision", "dataflow run", "transformation graph"),
|
owned_concepts=(
|
||||||
|
"dataflow definition",
|
||||||
|
"dataflow revision",
|
||||||
|
"dataflow run",
|
||||||
|
"reconciliation decision set",
|
||||||
|
"transformation graph",
|
||||||
|
),
|
||||||
non_owned_concepts=("datasource binding", "connector transport", "report presentation", "workflow task"),
|
non_owned_concepts=("datasource binding", "connector transport", "report presentation", "workflow task"),
|
||||||
recovery_docs=("README.md", "docs/DURABLE_RUN_RECOVERY.md"),
|
recovery_docs=("README.md", "docs/DURABLE_RUN_RECOVERY.md"),
|
||||||
security_docs=("README.md",),
|
security_docs=("README.md",),
|
||||||
|
|||||||
+118
@@ -0,0 +1,118 @@
|
|||||||
|
"""Add durable Dataflow reconciliation decision sets.
|
||||||
|
|
||||||
|
Revision ID: a3d7f1c5e9b2
|
||||||
|
Revises: f6c2a9d4e7b1
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "a3d7f1c5e9b2"
|
||||||
|
down_revision = "f6c2a9d4e7b1"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.create_table(
|
||||||
|
"dataflow_reconciliation_decision_sets",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("pipeline_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("name", sa.String(length=300), nullable=False),
|
||||||
|
sa.Column("node_id", sa.String(length=100), nullable=True),
|
||||||
|
sa.Column("resource_revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("created_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("updated_by", sa.String(length=255), nullable=True),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["pipeline_id"],
|
||||||
|
["dataflow_pipelines.id"],
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"tenant_id",
|
||||||
|
"pipeline_id",
|
||||||
|
"name",
|
||||||
|
name="uq_dataflow_decision_sets_pipeline_name",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_decision_sets_tenant_id",
|
||||||
|
"dataflow_reconciliation_decision_sets",
|
||||||
|
["tenant_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_decision_sets_pipeline_id",
|
||||||
|
"dataflow_reconciliation_decision_sets",
|
||||||
|
["pipeline_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_decision_sets_tenant_pipeline",
|
||||||
|
"dataflow_reconciliation_decision_sets",
|
||||||
|
["tenant_id", "pipeline_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_table(
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
|
sa.Column("id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("tenant_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("decision_set_id", sa.String(length=36), nullable=False),
|
||||||
|
sa.Column("revision", sa.Integer(), nullable=False),
|
||||||
|
sa.Column("key_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("input_hash", sa.String(length=64), nullable=False),
|
||||||
|
sa.Column("action", sa.String(length=20), nullable=False),
|
||||||
|
sa.Column("reason", sa.Text(), nullable=False),
|
||||||
|
sa.Column("correction", sa.JSON(), nullable=True),
|
||||||
|
sa.Column("actor_ref", sa.String(length=255), nullable=False),
|
||||||
|
sa.Column("decided_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||||
|
sa.ForeignKeyConstraint(
|
||||||
|
["decision_set_id"],
|
||||||
|
["dataflow_reconciliation_decision_sets.id"],
|
||||||
|
ondelete="CASCADE",
|
||||||
|
),
|
||||||
|
sa.PrimaryKeyConstraint("id"),
|
||||||
|
sa.UniqueConstraint(
|
||||||
|
"decision_set_id",
|
||||||
|
"revision",
|
||||||
|
name="uq_dataflow_reconciliation_decision_revision",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_reconciliation_decisions_tenant_id",
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
|
["tenant_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_reconciliation_decisions_decision_set_id",
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
|
["decision_set_id"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_reconciliation_decisions_key_hash",
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
|
["key_hash"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
op.create_index(
|
||||||
|
"ix_dataflow_reconciliation_decisions_current",
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
|
["tenant_id", "decision_set_id", "key_hash", "revision"],
|
||||||
|
unique=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_table("dataflow_reconciliation_decisions")
|
||||||
|
op.drop_table("dataflow_reconciliation_decision_sets")
|
||||||
@@ -0,0 +1,380 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from collections.abc import Sequence
|
||||||
|
|
||||||
|
from sqlalchemy import and_, func, select
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
from sqlalchemy.orm import Session, joinedload, selectinload
|
||||||
|
|
||||||
|
from govoplan_core.core.concurrency import claim_revision
|
||||||
|
from govoplan_core.security.time import utc_now
|
||||||
|
from govoplan_dataflow.backend.db.models import (
|
||||||
|
DataflowPipeline,
|
||||||
|
DataflowReconciliationDecision,
|
||||||
|
DataflowReconciliationDecisionSet,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
DECISION_SET_REF_PREFIX = "dataflow-decision-set:"
|
||||||
|
DECISION_REF_PREFIX = "dataflow-decision:"
|
||||||
|
DECISION_ACTIONS = frozenset({"accept", "reject", "correct", "defer"})
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionError(ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionNotFoundError(ReconciliationDecisionError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionConflictError(ReconciliationDecisionError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def list_decision_sets(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
pipeline_id: str | None = None,
|
||||||
|
include_decisions: bool = True,
|
||||||
|
) -> tuple[DataflowReconciliationDecisionSet, ...]:
|
||||||
|
options = [joinedload(DataflowReconciliationDecisionSet.pipeline)]
|
||||||
|
if include_decisions:
|
||||||
|
options.append(selectinload(DataflowReconciliationDecisionSet.decisions))
|
||||||
|
statement = (
|
||||||
|
select(DataflowReconciliationDecisionSet)
|
||||||
|
.options(*options)
|
||||||
|
.where(DataflowReconciliationDecisionSet.tenant_id == tenant_id)
|
||||||
|
.order_by(
|
||||||
|
DataflowReconciliationDecisionSet.name,
|
||||||
|
DataflowReconciliationDecisionSet.id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if pipeline_id:
|
||||||
|
statement = statement.where(
|
||||||
|
DataflowReconciliationDecisionSet.pipeline_id == pipeline_id
|
||||||
|
)
|
||||||
|
return tuple(session.scalars(statement).all())
|
||||||
|
|
||||||
|
|
||||||
|
def create_decision_set(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
pipeline_id: str,
|
||||||
|
name: str,
|
||||||
|
node_id: str | None,
|
||||||
|
actor_ref: str,
|
||||||
|
) -> DataflowReconciliationDecisionSet:
|
||||||
|
pipeline = session.get(DataflowPipeline, pipeline_id)
|
||||||
|
if (
|
||||||
|
pipeline is None
|
||||||
|
or pipeline.tenant_id not in {None, tenant_id}
|
||||||
|
or pipeline.deleted_at is not None
|
||||||
|
):
|
||||||
|
raise ReconciliationDecisionNotFoundError("Dataflow pipeline not found.")
|
||||||
|
item = DataflowReconciliationDecisionSet(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
pipeline_id=pipeline_id,
|
||||||
|
name=name.strip(),
|
||||||
|
node_id=node_id,
|
||||||
|
resource_revision=1,
|
||||||
|
created_by=actor_ref,
|
||||||
|
updated_by=actor_ref,
|
||||||
|
)
|
||||||
|
session.add(item)
|
||||||
|
try:
|
||||||
|
session.flush()
|
||||||
|
except IntegrityError as exc:
|
||||||
|
raise ReconciliationDecisionConflictError(
|
||||||
|
"A decision set with this name already exists for the pipeline."
|
||||||
|
) from exc
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def get_decision_set(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
decision_set_id: str,
|
||||||
|
include_decisions: bool = True,
|
||||||
|
) -> DataflowReconciliationDecisionSet:
|
||||||
|
options = [joinedload(DataflowReconciliationDecisionSet.pipeline)]
|
||||||
|
if include_decisions:
|
||||||
|
options.append(selectinload(DataflowReconciliationDecisionSet.decisions))
|
||||||
|
item = session.scalar(
|
||||||
|
select(DataflowReconciliationDecisionSet)
|
||||||
|
.options(*options)
|
||||||
|
.where(
|
||||||
|
DataflowReconciliationDecisionSet.id == decision_set_id,
|
||||||
|
DataflowReconciliationDecisionSet.tenant_id == tenant_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if item is None:
|
||||||
|
raise ReconciliationDecisionNotFoundError(
|
||||||
|
"Reconciliation decision set not found."
|
||||||
|
)
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def record_decision(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
tenant_id: str,
|
||||||
|
decision_set_id: str,
|
||||||
|
expected_revision: int,
|
||||||
|
key_hash: str,
|
||||||
|
input_hash: str,
|
||||||
|
action: str,
|
||||||
|
reason: str,
|
||||||
|
correction: dict[str, object] | None,
|
||||||
|
actor_ref: str,
|
||||||
|
) -> DataflowReconciliationDecisionSet:
|
||||||
|
item = get_decision_set(
|
||||||
|
session,
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
decision_set_id=decision_set_id,
|
||||||
|
)
|
||||||
|
if action not in DECISION_ACTIONS:
|
||||||
|
raise ReconciliationDecisionError("Unsupported reconciliation action.")
|
||||||
|
next_revision = claim_revision(
|
||||||
|
session,
|
||||||
|
model=DataflowReconciliationDecisionSet,
|
||||||
|
filters=(
|
||||||
|
DataflowReconciliationDecisionSet.id == decision_set_id,
|
||||||
|
DataflowReconciliationDecisionSet.tenant_id == tenant_id,
|
||||||
|
),
|
||||||
|
revision_attribute="resource_revision",
|
||||||
|
expected_revision=expected_revision,
|
||||||
|
resource_type="dataflow_reconciliation_decision_set",
|
||||||
|
resource_id=decision_set_id,
|
||||||
|
)
|
||||||
|
item.resource_revision = next_revision
|
||||||
|
item.updated_by = actor_ref
|
||||||
|
item.decisions.append(
|
||||||
|
DataflowReconciliationDecision(
|
||||||
|
tenant_id=tenant_id,
|
||||||
|
revision=next_revision,
|
||||||
|
key_hash=key_hash,
|
||||||
|
input_hash=input_hash,
|
||||||
|
action=action,
|
||||||
|
reason=reason.strip(),
|
||||||
|
correction=dict(correction) if correction else None,
|
||||||
|
actor_ref=actor_ref,
|
||||||
|
decided_at=utc_now(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.flush()
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def current_decisions(
|
||||||
|
item: DataflowReconciliationDecisionSet,
|
||||||
|
) -> tuple[DataflowReconciliationDecision, ...]:
|
||||||
|
current: dict[str, DataflowReconciliationDecision] = {}
|
||||||
|
for decision in sorted(item.decisions, key=lambda value: value.revision):
|
||||||
|
current[decision.key_hash] = decision
|
||||||
|
return tuple(
|
||||||
|
sorted(
|
||||||
|
current.values(),
|
||||||
|
key=lambda value: (value.key_hash, value.revision),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def decision_rows(
|
||||||
|
item: DataflowReconciliationDecisionSet,
|
||||||
|
) -> tuple[dict[str, object], ...]:
|
||||||
|
return tuple(_decision_row(value) for value in current_decisions(item))
|
||||||
|
|
||||||
|
|
||||||
|
def decision_set_fingerprint(
|
||||||
|
item: DataflowReconciliationDecisionSet,
|
||||||
|
) -> str:
|
||||||
|
payload = {
|
||||||
|
"decision_set_id": item.id,
|
||||||
|
"resource_revision": item.resource_revision,
|
||||||
|
}
|
||||||
|
return "sha256:" + hashlib.sha256(
|
||||||
|
json.dumps(
|
||||||
|
payload,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
ensure_ascii=True,
|
||||||
|
).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def decision_set_payload(
|
||||||
|
item: DataflowReconciliationDecisionSet,
|
||||||
|
*,
|
||||||
|
include_decisions: bool = True,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
current = current_decisions(item) if include_decisions else ()
|
||||||
|
history = (
|
||||||
|
sorted(item.decisions, key=lambda value: value.revision)
|
||||||
|
if include_decisions
|
||||||
|
else ()
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"ref": f"{DECISION_SET_REF_PREFIX}{item.id}",
|
||||||
|
"id": item.id,
|
||||||
|
"pipeline_id": item.pipeline_id,
|
||||||
|
"name": item.name,
|
||||||
|
"node_id": item.node_id,
|
||||||
|
"resource_revision": item.resource_revision,
|
||||||
|
"etag": item.strong_etag,
|
||||||
|
"fingerprint": decision_set_fingerprint(item),
|
||||||
|
"decisions_included": include_decisions,
|
||||||
|
"current_decisions": [_decision_payload(value) for value in current],
|
||||||
|
"history": [
|
||||||
|
_decision_payload(value)
|
||||||
|
for value in history
|
||||||
|
],
|
||||||
|
"created_by": item.created_by,
|
||||||
|
"updated_by": item.updated_by,
|
||||||
|
"created_at": item.created_at,
|
||||||
|
"updated_at": item.updated_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def decision_set_source_payload(
|
||||||
|
item: DataflowReconciliationDecisionSet,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"ref": f"{DECISION_SET_REF_PREFIX}{item.id}",
|
||||||
|
"provider": "dataflow.reconciliation_decisions",
|
||||||
|
"source_name": _source_name(item.name, item.id),
|
||||||
|
"name": item.name,
|
||||||
|
"description": "Current immutable reconciliation decisions; prior revisions remain in decision history.",
|
||||||
|
"mode": "static",
|
||||||
|
"columns": [
|
||||||
|
{"name": "key_hash", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "input_hash", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "decision_ref", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "action", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "actor_ref", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "decided_at", "data_type": "datetime", "nullable": False},
|
||||||
|
{"name": "reason", "data_type": "string", "nullable": False},
|
||||||
|
{"name": "correction", "data_type": "object", "nullable": True},
|
||||||
|
],
|
||||||
|
"schema_version": "1",
|
||||||
|
"fingerprint": decision_set_fingerprint(item),
|
||||||
|
"row_count": None,
|
||||||
|
"byte_count": None,
|
||||||
|
"updated_at": item.updated_at,
|
||||||
|
"capabilities": ["preview", "read", "immutable_history"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def current_decision_rows(
|
||||||
|
session: Session,
|
||||||
|
*,
|
||||||
|
decision_set_id: str,
|
||||||
|
limit: int,
|
||||||
|
) -> tuple[tuple[dict[str, object], ...], int]:
|
||||||
|
bounded_limit = max(1, limit)
|
||||||
|
latest = (
|
||||||
|
select(
|
||||||
|
DataflowReconciliationDecision.key_hash.label("key_hash"),
|
||||||
|
func.max(DataflowReconciliationDecision.revision).label("revision"),
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
||||||
|
)
|
||||||
|
.group_by(DataflowReconciliationDecision.key_hash)
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
statement = (
|
||||||
|
select(DataflowReconciliationDecision)
|
||||||
|
.join(
|
||||||
|
latest,
|
||||||
|
and_(
|
||||||
|
DataflowReconciliationDecision.key_hash == latest.c.key_hash,
|
||||||
|
DataflowReconciliationDecision.revision == latest.c.revision,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.where(
|
||||||
|
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
||||||
|
)
|
||||||
|
.order_by(DataflowReconciliationDecision.key_hash)
|
||||||
|
.limit(bounded_limit)
|
||||||
|
)
|
||||||
|
records = tuple(session.scalars(statement))
|
||||||
|
total = int(
|
||||||
|
session.scalar(
|
||||||
|
select(func.count(func.distinct(DataflowReconciliationDecision.key_hash))).where(
|
||||||
|
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
||||||
|
)
|
||||||
|
)
|
||||||
|
or 0
|
||||||
|
)
|
||||||
|
return tuple(_decision_row(value) for value in records), total
|
||||||
|
|
||||||
|
|
||||||
|
def decision_set_id_from_ref(value: str) -> str | None:
|
||||||
|
cleaned = str(value or "").strip()
|
||||||
|
if not cleaned.startswith(DECISION_SET_REF_PREFIX):
|
||||||
|
return None
|
||||||
|
identifier = cleaned[len(DECISION_SET_REF_PREFIX) :]
|
||||||
|
return identifier or None
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_row(
|
||||||
|
value: DataflowReconciliationDecision,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"key_hash": value.key_hash,
|
||||||
|
"input_hash": value.input_hash,
|
||||||
|
"decision_ref": f"{DECISION_REF_PREFIX}{value.id}",
|
||||||
|
"action": value.action,
|
||||||
|
"actor_ref": value.actor_ref,
|
||||||
|
"decided_at": value.decided_at.isoformat(),
|
||||||
|
"reason": value.reason,
|
||||||
|
"correction": dict(value.correction) if value.correction else None,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_payload(
|
||||||
|
value: DataflowReconciliationDecision,
|
||||||
|
) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"ref": f"{DECISION_REF_PREFIX}{value.id}",
|
||||||
|
"id": value.id,
|
||||||
|
"revision": value.revision,
|
||||||
|
"key_hash": value.key_hash,
|
||||||
|
"input_hash": value.input_hash,
|
||||||
|
"action": value.action,
|
||||||
|
"reason": value.reason,
|
||||||
|
"correction": dict(value.correction) if value.correction else None,
|
||||||
|
"actor_ref": value.actor_ref,
|
||||||
|
"decided_at": value.decided_at,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _source_name(name: str, identifier: str) -> str:
|
||||||
|
slug = re.sub(r"[^a-z0-9]+", "_", name.casefold()).strip("_")
|
||||||
|
return (f"decisions_{slug}" if slug else f"decisions_{identifier[:8]}")[:120]
|
||||||
|
|
||||||
|
|
||||||
|
__all__: Sequence[str] = (
|
||||||
|
"DECISION_SET_REF_PREFIX",
|
||||||
|
"ReconciliationDecisionConflictError",
|
||||||
|
"ReconciliationDecisionError",
|
||||||
|
"ReconciliationDecisionNotFoundError",
|
||||||
|
"create_decision_set",
|
||||||
|
"current_decision_rows",
|
||||||
|
"current_decisions",
|
||||||
|
"decision_rows",
|
||||||
|
"decision_set_fingerprint",
|
||||||
|
"decision_set_id_from_ref",
|
||||||
|
"decision_set_payload",
|
||||||
|
"decision_set_source_payload",
|
||||||
|
"get_decision_set",
|
||||||
|
"list_decision_sets",
|
||||||
|
"record_decision",
|
||||||
|
)
|
||||||
@@ -10,6 +10,7 @@ from govoplan_core.api.v1.schemas import (
|
|||||||
from govoplan_core.audit.logging import audit_event
|
from govoplan_core.audit.logging import audit_event
|
||||||
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
|
from govoplan_core.auth import ApiPrincipal, get_api_principal, has_scope
|
||||||
from govoplan_core.core.automation import AutomationInvocation
|
from govoplan_core.core.automation import AutomationInvocation
|
||||||
|
from govoplan_core.core.concurrency import RevisionConflictError
|
||||||
from govoplan_core.core.dataflows import (
|
from govoplan_core.core.dataflows import (
|
||||||
DataflowPublicationTarget,
|
DataflowPublicationTarget,
|
||||||
DataflowRunRequest,
|
DataflowRunRequest,
|
||||||
@@ -71,6 +72,10 @@ from govoplan_dataflow.backend.schemas import (
|
|||||||
PipelineSqlResponse,
|
PipelineSqlResponse,
|
||||||
PipelineUpdateRequest,
|
PipelineUpdateRequest,
|
||||||
PipelineValidationResponse,
|
PipelineValidationResponse,
|
||||||
|
ReconciliationDecisionSetCreateRequest,
|
||||||
|
ReconciliationDecisionSetListResponse,
|
||||||
|
ReconciliationDecisionSetResponse,
|
||||||
|
ReconciliationDecisionWriteRequest,
|
||||||
TabularSnapshotCreateRequest,
|
TabularSnapshotCreateRequest,
|
||||||
TabularSourceColumnResponse,
|
TabularSourceColumnResponse,
|
||||||
TabularSourceListResponse,
|
TabularSourceListResponse,
|
||||||
@@ -110,6 +115,17 @@ from govoplan_dataflow.backend.service import (
|
|||||||
validate_draft,
|
validate_draft,
|
||||||
)
|
)
|
||||||
from govoplan_dataflow.backend.run_worker import run_metrics
|
from govoplan_dataflow.backend.run_worker import run_metrics
|
||||||
|
from govoplan_dataflow.backend.reconciliation_decisions import (
|
||||||
|
ReconciliationDecisionConflictError,
|
||||||
|
ReconciliationDecisionError,
|
||||||
|
ReconciliationDecisionNotFoundError,
|
||||||
|
create_decision_set,
|
||||||
|
decision_set_payload,
|
||||||
|
decision_set_source_payload,
|
||||||
|
get_decision_set,
|
||||||
|
list_decision_sets,
|
||||||
|
record_decision,
|
||||||
|
)
|
||||||
from govoplan_dataflow.backend.recovery import dataflow_run_recovery_states
|
from govoplan_dataflow.backend.recovery import dataflow_run_recovery_states
|
||||||
from govoplan_dataflow.backend.triggers import (
|
from govoplan_dataflow.backend.triggers import (
|
||||||
create_trigger,
|
create_trigger,
|
||||||
@@ -145,6 +161,18 @@ def _actor_id(principal: ApiPrincipal) -> str | None:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_actor_ref(principal: ApiPrincipal) -> str:
|
||||||
|
actor_id = _actor_id(principal)
|
||||||
|
if not actor_id:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_403_FORBIDDEN,
|
||||||
|
detail="A stable actor reference is required to record a decision.",
|
||||||
|
)
|
||||||
|
principal_ref = principal.to_platform_principal()
|
||||||
|
prefix = "service-account" if principal_ref.service_account_id else "account"
|
||||||
|
return f"{prefix}:{actor_id}"
|
||||||
|
|
||||||
|
|
||||||
def _http_error(exc: DataflowError) -> HTTPException:
|
def _http_error(exc: DataflowError) -> HTTPException:
|
||||||
if isinstance(exc, DataflowNotFoundError):
|
if isinstance(exc, DataflowNotFoundError):
|
||||||
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
||||||
@@ -161,6 +189,17 @@ def _http_error(exc: DataflowError) -> HTTPException:
|
|||||||
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
return HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc))
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_http_error(exc: ReconciliationDecisionError) -> HTTPException:
|
||||||
|
if isinstance(exc, ReconciliationDecisionNotFoundError):
|
||||||
|
return HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=str(exc))
|
||||||
|
if isinstance(exc, ReconciliationDecisionConflictError):
|
||||||
|
return HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc))
|
||||||
|
return HTTPException(
|
||||||
|
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
|
||||||
|
detail=str(exc),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _governance_http_error(exc: PermissionError | ValueError) -> HTTPException:
|
def _governance_http_error(exc: PermissionError | ValueError) -> HTTPException:
|
||||||
return HTTPException(
|
return HTTPException(
|
||||||
status_code=(
|
status_code=(
|
||||||
@@ -329,21 +368,42 @@ def api_list_sources(
|
|||||||
registry = get_registry()
|
registry = get_registry()
|
||||||
provider = datasource_catalogue(registry)
|
provider = datasource_catalogue(registry)
|
||||||
writer = datasource_lifecycle(registry)
|
writer = datasource_lifecycle(registry)
|
||||||
if provider is None:
|
sources = ()
|
||||||
return TabularSourceListResponse(available=False, writable=False, sources=[])
|
if provider is not None:
|
||||||
try:
|
try:
|
||||||
sources = provider.list_datasources(
|
sources = provider.list_datasources(
|
||||||
session,
|
session,
|
||||||
principal,
|
principal,
|
||||||
query=query,
|
query=query,
|
||||||
limit=100,
|
limit=100,
|
||||||
)
|
)
|
||||||
except DatasourceError as exc:
|
except DatasourceError as exc:
|
||||||
raise _source_http_error(exc) from exc
|
raise _source_http_error(exc) from exc
|
||||||
|
decision_sources = []
|
||||||
|
for item in list_decision_sets(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
include_decisions=False,
|
||||||
|
):
|
||||||
|
if query and query.casefold() not in item.name.casefold():
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
require_definition_action(
|
||||||
|
item.pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=registry,
|
||||||
|
action="view",
|
||||||
|
)
|
||||||
|
except PermissionError:
|
||||||
|
continue
|
||||||
|
decision_sources.append(decision_set_source_payload(item))
|
||||||
return TabularSourceListResponse(
|
return TabularSourceListResponse(
|
||||||
available=True,
|
available=provider is not None or bool(decision_sources),
|
||||||
writable=writer is not None,
|
writable=writer is not None,
|
||||||
sources=[_source_response(source) for source in sources],
|
sources=[
|
||||||
|
*(_source_response(source) for source in sources),
|
||||||
|
*(TabularSourceResponse.model_validate(item) for item in decision_sources),
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -504,6 +564,212 @@ def api_create_pipeline(
|
|||||||
return response
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/pipelines/{pipeline_id}/decision-sets",
|
||||||
|
response_model=ReconciliationDecisionSetListResponse,
|
||||||
|
)
|
||||||
|
def api_list_reconciliation_decision_sets(
|
||||||
|
pipeline_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ReconciliationDecisionSetListResponse:
|
||||||
|
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
pipeline = get_pipeline(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
pipeline_id=pipeline_id,
|
||||||
|
)
|
||||||
|
require_definition_action(
|
||||||
|
pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=get_registry(),
|
||||||
|
action="view",
|
||||||
|
)
|
||||||
|
except PermissionError as exc:
|
||||||
|
raise _governance_http_error(exc) from exc
|
||||||
|
except DataflowError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
return ReconciliationDecisionSetListResponse(
|
||||||
|
decision_sets=[
|
||||||
|
ReconciliationDecisionSetResponse.model_validate(
|
||||||
|
decision_set_payload(item, include_decisions=False)
|
||||||
|
)
|
||||||
|
for item in list_decision_sets(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
pipeline_id=pipeline_id,
|
||||||
|
include_decisions=False,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/pipelines/{pipeline_id}/decision-sets",
|
||||||
|
response_model=ReconciliationDecisionSetResponse,
|
||||||
|
status_code=status.HTTP_201_CREATED,
|
||||||
|
)
|
||||||
|
def api_create_reconciliation_decision_set(
|
||||||
|
pipeline_id: str,
|
||||||
|
payload: ReconciliationDecisionSetCreateRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ReconciliationDecisionSetResponse:
|
||||||
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
pipeline = get_pipeline(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
pipeline_id=pipeline_id,
|
||||||
|
)
|
||||||
|
require_definition_action(
|
||||||
|
pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=get_registry(),
|
||||||
|
action="edit",
|
||||||
|
)
|
||||||
|
item = create_decision_set(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
pipeline_id=pipeline_id,
|
||||||
|
name=payload.name,
|
||||||
|
node_id=payload.node_id,
|
||||||
|
actor_ref=_decision_actor_ref(principal),
|
||||||
|
)
|
||||||
|
except PermissionError as exc:
|
||||||
|
raise _governance_http_error(exc) from exc
|
||||||
|
except DataflowError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
except ReconciliationDecisionError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _decision_http_error(exc) from exc
|
||||||
|
audit_event(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=getattr(principal.user, "id", None),
|
||||||
|
api_key_id=principal.api_key_id,
|
||||||
|
action="dataflow.reconciliation_decision_set.created",
|
||||||
|
object_type="dataflow_reconciliation_decision_set",
|
||||||
|
object_id=item.id,
|
||||||
|
details={"pipeline_id": pipeline_id, "node_id": payload.node_id},
|
||||||
|
)
|
||||||
|
response = ReconciliationDecisionSetResponse.model_validate(
|
||||||
|
decision_set_payload(item)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/decision-sets/{decision_set_id}",
|
||||||
|
response_model=ReconciliationDecisionSetResponse,
|
||||||
|
)
|
||||||
|
def api_get_reconciliation_decision_set(
|
||||||
|
decision_set_id: str,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ReconciliationDecisionSetResponse:
|
||||||
|
_require_any_scope(principal, READ_SCOPE, WRITE_SCOPE, RUN_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
item = get_decision_set(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
decision_set_id=decision_set_id,
|
||||||
|
)
|
||||||
|
require_definition_action(
|
||||||
|
item.pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=get_registry(),
|
||||||
|
action="view",
|
||||||
|
)
|
||||||
|
except PermissionError as exc:
|
||||||
|
raise _governance_http_error(exc) from exc
|
||||||
|
except ReconciliationDecisionError as exc:
|
||||||
|
raise _decision_http_error(exc) from exc
|
||||||
|
return ReconciliationDecisionSetResponse.model_validate(
|
||||||
|
decision_set_payload(item)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post(
|
||||||
|
"/decision-sets/{decision_set_id}/decisions",
|
||||||
|
response_model=ReconciliationDecisionSetResponse,
|
||||||
|
)
|
||||||
|
def api_record_reconciliation_decision(
|
||||||
|
decision_set_id: str,
|
||||||
|
payload: ReconciliationDecisionWriteRequest,
|
||||||
|
session: Session = Depends(get_session),
|
||||||
|
principal: ApiPrincipal = Depends(get_api_principal),
|
||||||
|
) -> ReconciliationDecisionSetResponse:
|
||||||
|
_require_any_scope(principal, WRITE_SCOPE, ADMIN_SCOPE)
|
||||||
|
try:
|
||||||
|
item = get_decision_set(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
decision_set_id=decision_set_id,
|
||||||
|
)
|
||||||
|
pipeline = get_pipeline(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
pipeline_id=item.pipeline_id,
|
||||||
|
)
|
||||||
|
require_definition_action(
|
||||||
|
pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=get_registry(),
|
||||||
|
action="edit",
|
||||||
|
)
|
||||||
|
item = record_decision(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
decision_set_id=decision_set_id,
|
||||||
|
expected_revision=payload.base_revision,
|
||||||
|
key_hash=payload.key_hash,
|
||||||
|
input_hash=payload.input_hash,
|
||||||
|
action=payload.action,
|
||||||
|
reason=payload.reason,
|
||||||
|
correction=payload.correction,
|
||||||
|
actor_ref=_decision_actor_ref(principal),
|
||||||
|
)
|
||||||
|
except PermissionError as exc:
|
||||||
|
raise _governance_http_error(exc) from exc
|
||||||
|
except DataflowError as exc:
|
||||||
|
raise _http_error(exc) from exc
|
||||||
|
except RevisionConflictError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=status.HTTP_409_CONFLICT,
|
||||||
|
detail=exc.as_dict(),
|
||||||
|
) from exc
|
||||||
|
except ReconciliationDecisionError as exc:
|
||||||
|
session.rollback()
|
||||||
|
raise _decision_http_error(exc) from exc
|
||||||
|
decision = item.decisions[-1]
|
||||||
|
audit_event(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
user_id=getattr(principal.user, "id", None),
|
||||||
|
api_key_id=principal.api_key_id,
|
||||||
|
action="dataflow.reconciliation_decision.recorded",
|
||||||
|
object_type="dataflow_reconciliation_decision",
|
||||||
|
object_id=decision.id,
|
||||||
|
details={
|
||||||
|
"decision_set_id": item.id,
|
||||||
|
"pipeline_id": item.pipeline_id,
|
||||||
|
"revision": decision.revision,
|
||||||
|
"key_hash": decision.key_hash,
|
||||||
|
"input_hash": decision.input_hash,
|
||||||
|
"decision_action": decision.action,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
response = ReconciliationDecisionSetResponse.model_validate(
|
||||||
|
decision_set_payload(item)
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
@router.get("/pipelines/{pipeline_id}", response_model=PipelineResponse)
|
@router.get("/pipelines/{pipeline_id}", response_model=PipelineResponse)
|
||||||
def api_get_pipeline(
|
def api_get_pipeline(
|
||||||
pipeline_id: str,
|
pipeline_id: str,
|
||||||
|
|||||||
@@ -145,6 +145,68 @@ class PipelineListResponse(BaseModel):
|
|||||||
pipelines: list[PipelineResponse]
|
pipelines: list[PipelineResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionSetCreateRequest(BaseModel):
|
||||||
|
name: str = Field(min_length=1, max_length=300)
|
||||||
|
node_id: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
min_length=1,
|
||||||
|
max_length=100,
|
||||||
|
pattern=r"^[A-Za-z0-9_.:-]+$",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionWriteRequest(BaseModel):
|
||||||
|
base_revision: int = Field(ge=1)
|
||||||
|
key_hash: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||||
|
input_hash: str = Field(pattern=r"^[0-9a-f]{64}$")
|
||||||
|
action: Literal["accept", "reject", "correct", "defer"]
|
||||||
|
reason: str = Field(min_length=3, max_length=4_000)
|
||||||
|
correction: dict[str, Any] | None = None
|
||||||
|
|
||||||
|
@model_validator(mode="after")
|
||||||
|
def validate_correction(self) -> "ReconciliationDecisionWriteRequest":
|
||||||
|
if self.action == "correct" and not self.correction:
|
||||||
|
raise ValueError("Correct decisions require corrected field values.")
|
||||||
|
if self.action != "correct" and self.correction:
|
||||||
|
raise ValueError("Only correct decisions may include corrected field values.")
|
||||||
|
return self
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionResponse(BaseModel):
|
||||||
|
ref: str
|
||||||
|
id: str
|
||||||
|
revision: int
|
||||||
|
key_hash: str
|
||||||
|
input_hash: str
|
||||||
|
action: Literal["accept", "reject", "correct", "defer"]
|
||||||
|
reason: str
|
||||||
|
correction: dict[str, Any] | None
|
||||||
|
actor_ref: str
|
||||||
|
decided_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionSetResponse(BaseModel):
|
||||||
|
ref: str
|
||||||
|
id: str
|
||||||
|
pipeline_id: str
|
||||||
|
name: str
|
||||||
|
node_id: str | None
|
||||||
|
resource_revision: int = Field(ge=1)
|
||||||
|
etag: str
|
||||||
|
fingerprint: str
|
||||||
|
decisions_included: bool
|
||||||
|
current_decisions: list[ReconciliationDecisionResponse]
|
||||||
|
history: list[ReconciliationDecisionResponse]
|
||||||
|
created_by: str | None
|
||||||
|
updated_by: str | None
|
||||||
|
created_at: datetime
|
||||||
|
updated_at: datetime
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionSetListResponse(BaseModel):
|
||||||
|
decision_sets: list[ReconciliationDecisionSetResponse]
|
||||||
|
|
||||||
|
|
||||||
class PipelineCreateRequest(BaseModel):
|
class PipelineCreateRequest(BaseModel):
|
||||||
name: str = Field(min_length=1, max_length=300)
|
name: str = Field(min_length=1, max_length=300)
|
||||||
description: str | None = Field(default=None, max_length=4000)
|
description: str | None = Field(default=None, max_length=4000)
|
||||||
|
|||||||
@@ -86,6 +86,12 @@ from govoplan_dataflow.backend.recovery import (
|
|||||||
begin_dataflow_run_recovery,
|
begin_dataflow_run_recovery,
|
||||||
dataflow_run_recovery_state,
|
dataflow_run_recovery_state,
|
||||||
)
|
)
|
||||||
|
from govoplan_dataflow.backend.reconciliation_decisions import (
|
||||||
|
current_decision_rows,
|
||||||
|
decision_set_fingerprint,
|
||||||
|
decision_set_id_from_ref,
|
||||||
|
get_decision_set,
|
||||||
|
)
|
||||||
from govoplan_dataflow.backend.sql_compiler import (
|
from govoplan_dataflow.backend.sql_compiler import (
|
||||||
SqlCompilationError,
|
SqlCompilationError,
|
||||||
compile_sql,
|
compile_sql,
|
||||||
@@ -1352,8 +1358,12 @@ def _run_authorization_payload(
|
|||||||
graph = PipelineGraph.model_validate(revision.graph)
|
graph = PipelineGraph.model_validate(revision.graph)
|
||||||
scopes = {RUN_SCOPE}
|
scopes = {RUN_SCOPE}
|
||||||
if any(
|
if any(
|
||||||
node.type == "source.reference"
|
(
|
||||||
or bool(node.config.get("source_ref"))
|
node.type == "source.reference"
|
||||||
|
or bool(node.config.get("source_ref"))
|
||||||
|
)
|
||||||
|
and decision_set_id_from_ref(str(node.config.get("source_ref") or ""))
|
||||||
|
is None
|
||||||
for node in graph.nodes
|
for node in graph.nodes
|
||||||
):
|
):
|
||||||
scopes.add(DATASOURCE_READ_SCOPE)
|
scopes.add(DATASOURCE_READ_SCOPE)
|
||||||
@@ -1880,6 +1890,49 @@ def _datasource_source_resolver(
|
|||||||
provider = datasource_catalogue(registry)
|
provider = datasource_catalogue(registry)
|
||||||
|
|
||||||
def resolve_source(node: GraphNode, limit: int) -> ResolvedSource:
|
def resolve_source(node: GraphNode, limit: int) -> ResolvedSource:
|
||||||
|
source_ref = str(node.config.get("source_ref") or "")
|
||||||
|
decision_set_id = decision_set_id_from_ref(source_ref)
|
||||||
|
if decision_set_id is not None:
|
||||||
|
try:
|
||||||
|
decision_set = get_decision_set(
|
||||||
|
session,
|
||||||
|
tenant_id=principal.tenant_id,
|
||||||
|
decision_set_id=decision_set_id,
|
||||||
|
include_decisions=False,
|
||||||
|
)
|
||||||
|
require_definition_action(
|
||||||
|
decision_set.pipeline,
|
||||||
|
principal=principal,
|
||||||
|
registry=registry,
|
||||||
|
action="view",
|
||||||
|
)
|
||||||
|
except (PermissionError, ValueError) as exc:
|
||||||
|
raise PipelineExecutionError(
|
||||||
|
str(exc),
|
||||||
|
node_id=node.id,
|
||||||
|
) from exc
|
||||||
|
rows, total_rows = current_decision_rows(
|
||||||
|
session,
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
limit=limit,
|
||||||
|
)
|
||||||
|
fingerprint = decision_set_fingerprint(decision_set)
|
||||||
|
expected_fingerprint = _clean_optional(
|
||||||
|
node.config.get("expected_fingerprint")
|
||||||
|
)
|
||||||
|
if expected_fingerprint and expected_fingerprint != fingerprint:
|
||||||
|
raise PipelineExecutionError(
|
||||||
|
"Reconciliation decisions changed; refresh the decision source before running it.",
|
||||||
|
node_id=node.id,
|
||||||
|
)
|
||||||
|
return ResolvedSource(
|
||||||
|
rows=rows,
|
||||||
|
source_ref=source_ref,
|
||||||
|
provider="dataflow.reconciliation_decisions",
|
||||||
|
fingerprint=fingerprint,
|
||||||
|
total_rows=total_rows,
|
||||||
|
truncated=total_rows > len(rows),
|
||||||
|
)
|
||||||
if provider is None:
|
if provider is None:
|
||||||
raise PipelineExecutionError(
|
raise PipelineExecutionError(
|
||||||
"Datasource-backed execution requires the Datasources "
|
"Datasource-backed execution requires the Datasources "
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ class DataflowInterfaceDocumentationContractTests(unittest.TestCase):
|
|||||||
"dataflow.sql",
|
"dataflow.sql",
|
||||||
"dataflow.inspector",
|
"dataflow.inspector",
|
||||||
"dataflow.results",
|
"dataflow.results",
|
||||||
|
"dataflow.decisions",
|
||||||
"dataflow.triggers",
|
"dataflow.triggers",
|
||||||
"dataflow.runs",
|
"dataflow.runs",
|
||||||
"dataflow.widget.pipelines",
|
"dataflow.widget.pipelines",
|
||||||
@@ -38,6 +39,7 @@ class DataflowInterfaceDocumentationContractTests(unittest.TestCase):
|
|||||||
"dataflow.runs",
|
"dataflow.runs",
|
||||||
):
|
):
|
||||||
self.assertEqual("dataflow.page", surfaces[surface_id].parent_id)
|
self.assertEqual("dataflow.page", surfaces[surface_id].parent_id)
|
||||||
|
self.assertEqual("dataflow.results", surfaces["dataflow.decisions"].parent_id)
|
||||||
|
|
||||||
def test_help_and_consequence_metadata_remain_published(self) -> None:
|
def test_help_and_consequence_metadata_remain_published(self) -> None:
|
||||||
topics = {topic.id: topic for topic in get_manifest().documentation}
|
topics = {topic.id: topic for topic in get_manifest().documentation}
|
||||||
@@ -48,8 +50,10 @@ class DataflowInterfaceDocumentationContractTests(unittest.TestCase):
|
|||||||
|
|
||||||
self.assertIn("dataflow.state.read-only", boundary.metadata["help_contexts"])
|
self.assertIn("dataflow.state.read-only", boundary.metadata["help_contexts"])
|
||||||
self.assertIn("dataflow.field.expression", nodes.metadata["help_contexts"])
|
self.assertIn("dataflow.field.expression", nodes.metadata["help_contexts"])
|
||||||
|
self.assertIn("dataflow.action.review-decisions", nodes.metadata["help_contexts"])
|
||||||
self.assertIn("save_revision", fields.metadata["consequence_classes"])
|
self.assertIn("save_revision", fields.metadata["consequence_classes"])
|
||||||
self.assertIn("delete_pipeline", fields.metadata["consequence_classes"])
|
self.assertIn("delete_pipeline", fields.metadata["consequence_classes"])
|
||||||
|
self.assertIn("record_decision", fields.metadata["consequence_classes"])
|
||||||
self.assertIn("publish_output", execution.metadata["consequence_classes"])
|
self.assertIn("publish_output", execution.metadata["consequence_classes"])
|
||||||
self.assertIn("promote_revision", execution.metadata["consequence_classes"])
|
self.assertIn("promote_revision", execution.metadata["consequence_classes"])
|
||||||
|
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ class DataflowMigrationTests(unittest.TestCase):
|
|||||||
try:
|
try:
|
||||||
with engine.connect() as connection:
|
with engine.connect() as connection:
|
||||||
self.assertIn(
|
self.assertIn(
|
||||||
"f6c2a9d4e7b1",
|
"a3d7f1c5e9b2",
|
||||||
set(MigrationContext.configure(connection).get_current_heads()),
|
set(MigrationContext.configure(connection).get_current_heads()),
|
||||||
)
|
)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@@ -32,6 +32,8 @@ class DataflowMigrationTests(unittest.TestCase):
|
|||||||
"dataflow_pipelines",
|
"dataflow_pipelines",
|
||||||
"dataflow_pipeline_revisions",
|
"dataflow_pipeline_revisions",
|
||||||
"dataflow_pipeline_deployments",
|
"dataflow_pipeline_deployments",
|
||||||
|
"dataflow_reconciliation_decision_sets",
|
||||||
|
"dataflow_reconciliation_decisions",
|
||||||
"dataflow_runs",
|
"dataflow_runs",
|
||||||
"dataflow_triggers",
|
"dataflow_triggers",
|
||||||
"dataflow_trigger_deliveries",
|
"dataflow_trigger_deliveries",
|
||||||
|
|||||||
@@ -0,0 +1,247 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.auth import ApiPrincipal
|
||||||
|
from govoplan_core.core.access import PrincipalRef
|
||||||
|
from govoplan_core.core.concurrency import RevisionConflictError
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_dataflow.backend.db.models import (
|
||||||
|
DataflowPipeline,
|
||||||
|
DataflowReconciliationDecision,
|
||||||
|
DataflowReconciliationDecisionSet,
|
||||||
|
)
|
||||||
|
from govoplan_dataflow.backend.executor import PipelineExecutionError
|
||||||
|
from govoplan_dataflow.backend.reconciliation_decisions import (
|
||||||
|
create_decision_set,
|
||||||
|
current_decision_rows,
|
||||||
|
current_decisions,
|
||||||
|
decision_set_fingerprint,
|
||||||
|
decision_set_payload,
|
||||||
|
decision_set_source_payload,
|
||||||
|
record_decision,
|
||||||
|
)
|
||||||
|
from govoplan_dataflow.backend.schemas import GraphNode, GraphPosition
|
||||||
|
from govoplan_dataflow.backend.service import _datasource_source_resolver
|
||||||
|
|
||||||
|
|
||||||
|
TABLES = (
|
||||||
|
DataflowPipeline.__table__,
|
||||||
|
DataflowReconciliationDecisionSet.__table__,
|
||||||
|
DataflowReconciliationDecision.__table__,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ReconciliationDecisionTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(self.engine, tables=TABLES)
|
||||||
|
self.session = Session(self.engine)
|
||||||
|
self.pipeline = DataflowPipeline(
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scope_type="tenant",
|
||||||
|
scope_id="tenant-1",
|
||||||
|
name="Monthly reconciliation",
|
||||||
|
status="active",
|
||||||
|
created_by="account-1",
|
||||||
|
updated_by="account-1",
|
||||||
|
)
|
||||||
|
self.session.add(self.pipeline)
|
||||||
|
self.session.flush()
|
||||||
|
self.principal = ApiPrincipal(
|
||||||
|
principal=PrincipalRef(
|
||||||
|
account_id="account-1",
|
||||||
|
membership_id="membership-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scopes=frozenset(
|
||||||
|
{
|
||||||
|
"dataflow:pipeline:read",
|
||||||
|
"dataflow:pipeline:write",
|
||||||
|
"dataflow:pipeline:run",
|
||||||
|
}
|
||||||
|
),
|
||||||
|
),
|
||||||
|
account=SimpleNamespace(id="account-1"),
|
||||||
|
user=SimpleNamespace(id="membership-1"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_decisions_are_immutable_current_rows_and_occ_protected(self) -> None:
|
||||||
|
decision_set = create_decision_set(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=self.pipeline.id,
|
||||||
|
name="July review",
|
||||||
|
node_id="reconcile",
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
first = record_decision(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
expected_revision=1,
|
||||||
|
key_hash="a" * 64,
|
||||||
|
input_hash="b" * 64,
|
||||||
|
action="accept",
|
||||||
|
reason="Checked against the source record.",
|
||||||
|
correction=None,
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
first_fingerprint = decision_set_fingerprint(first)
|
||||||
|
second = record_decision(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
expected_revision=2,
|
||||||
|
key_hash="a" * 64,
|
||||||
|
input_hash="c" * 64,
|
||||||
|
action="correct",
|
||||||
|
reason="The monthly source changed after review.",
|
||||||
|
correction={"amount": 25},
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(2, len(second.decisions))
|
||||||
|
self.assertEqual(1, len(current_decisions(second)))
|
||||||
|
self.assertEqual("c" * 64, current_decisions(second)[0].input_hash)
|
||||||
|
self.assertNotEqual(first_fingerprint, decision_set_fingerprint(second))
|
||||||
|
payload = decision_set_payload(second)
|
||||||
|
self.assertEqual(2, len(payload["history"]))
|
||||||
|
self.assertEqual(1, len(payload["current_decisions"]))
|
||||||
|
with self.assertRaises(RevisionConflictError):
|
||||||
|
record_decision(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
expected_revision=2,
|
||||||
|
key_hash="d" * 64,
|
||||||
|
input_hash="e" * 64,
|
||||||
|
action="defer",
|
||||||
|
reason="Needs another source.",
|
||||||
|
correction=None,
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_decision_set_is_a_fingerprinted_reference_source(self) -> None:
|
||||||
|
decision_set = create_decision_set(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=self.pipeline.id,
|
||||||
|
name="July review",
|
||||||
|
node_id="reconcile",
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
record_decision(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
expected_revision=1,
|
||||||
|
key_hash="a" * 64,
|
||||||
|
input_hash="b" * 64,
|
||||||
|
action="reject",
|
||||||
|
reason="The observed record belongs to another case.",
|
||||||
|
correction=None,
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
source = decision_set_source_payload(decision_set)
|
||||||
|
resolver = _datasource_source_resolver(
|
||||||
|
session=self.session,
|
||||||
|
principal=self.principal,
|
||||||
|
registry=None,
|
||||||
|
)
|
||||||
|
node = GraphNode(
|
||||||
|
id="decisions",
|
||||||
|
type="source.reference",
|
||||||
|
label="Review decisions",
|
||||||
|
position=GraphPosition(x=0, y=0),
|
||||||
|
config={
|
||||||
|
"source_ref": source["ref"],
|
||||||
|
"expected_fingerprint": source["fingerprint"],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
resolved = resolver(node, 100)
|
||||||
|
|
||||||
|
self.assertEqual("dataflow.reconciliation_decisions", resolved.provider)
|
||||||
|
self.assertEqual(1, resolved.total_rows)
|
||||||
|
self.assertEqual("reject", resolved.rows[0]["action"])
|
||||||
|
stale = node.model_copy(deep=True)
|
||||||
|
stale.config["expected_fingerprint"] = "sha256:" + "0" * 64
|
||||||
|
with self.assertRaisesRegex(PipelineExecutionError, "changed"):
|
||||||
|
resolver(stale, 100)
|
||||||
|
|
||||||
|
def test_tenant_can_keep_decisions_for_visible_system_pipeline(self) -> None:
|
||||||
|
system_pipeline = DataflowPipeline(
|
||||||
|
tenant_id=None,
|
||||||
|
scope_type="system",
|
||||||
|
scope_id=None,
|
||||||
|
name="Governed monthly reconciliation",
|
||||||
|
status="active",
|
||||||
|
created_by="system-admin",
|
||||||
|
updated_by="system-admin",
|
||||||
|
)
|
||||||
|
self.session.add(system_pipeline)
|
||||||
|
self.session.flush()
|
||||||
|
|
||||||
|
decision_set = create_decision_set(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=system_pipeline.id,
|
||||||
|
name="Tenant July review",
|
||||||
|
node_id=None,
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual("tenant-1", decision_set.tenant_id)
|
||||||
|
self.assertEqual(system_pipeline.id, decision_set.pipeline_id)
|
||||||
|
|
||||||
|
def test_summary_and_current_projection_do_not_require_full_history(self) -> None:
|
||||||
|
decision_set = create_decision_set(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
pipeline_id=self.pipeline.id,
|
||||||
|
name="Bounded review",
|
||||||
|
node_id="reconcile",
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
for expected_revision, key_hash, input_hash in (
|
||||||
|
(1, "a" * 64, "b" * 64),
|
||||||
|
(2, "c" * 64, "d" * 64),
|
||||||
|
(3, "a" * 64, "e" * 64),
|
||||||
|
):
|
||||||
|
record_decision(
|
||||||
|
self.session,
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
expected_revision=expected_revision,
|
||||||
|
key_hash=key_hash,
|
||||||
|
input_hash=input_hash,
|
||||||
|
action="accept",
|
||||||
|
reason="Reviewed against the current input.",
|
||||||
|
correction=None,
|
||||||
|
actor_ref="account:account-1",
|
||||||
|
)
|
||||||
|
|
||||||
|
summary = decision_set_payload(decision_set, include_decisions=False)
|
||||||
|
rows, total = current_decision_rows(
|
||||||
|
self.session,
|
||||||
|
decision_set_id=decision_set.id,
|
||||||
|
limit=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertFalse(summary["decisions_included"])
|
||||||
|
self.assertEqual([], summary["current_decisions"])
|
||||||
|
self.assertEqual([], summary["history"])
|
||||||
|
self.assertEqual(2, total)
|
||||||
|
self.assertEqual(1, len(rows))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -205,6 +205,7 @@ export type TabularSource = {
|
|||||||
columns: TabularSourceColumn[];
|
columns: TabularSourceColumn[];
|
||||||
schema_version: string;
|
schema_version: string;
|
||||||
fingerprint: string;
|
fingerprint: string;
|
||||||
|
decisions_included: boolean;
|
||||||
row_count?: number | null;
|
row_count?: number | null;
|
||||||
byte_count?: number | null;
|
byte_count?: number | null;
|
||||||
updated_at?: string | null;
|
updated_at?: string | null;
|
||||||
@@ -217,6 +218,38 @@ export type TabularSourceCatalogue = {
|
|||||||
sources: TabularSource[];
|
sources: TabularSource[];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type ReconciliationDecisionAction = "accept" | "reject" | "correct" | "defer";
|
||||||
|
|
||||||
|
export type ReconciliationDecision = {
|
||||||
|
ref: string;
|
||||||
|
id: string;
|
||||||
|
revision: number;
|
||||||
|
key_hash: string;
|
||||||
|
input_hash: string;
|
||||||
|
action: ReconciliationDecisionAction;
|
||||||
|
reason: string;
|
||||||
|
correction?: Record<string, unknown> | null;
|
||||||
|
actor_ref: string;
|
||||||
|
decided_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type ReconciliationDecisionSet = {
|
||||||
|
ref: string;
|
||||||
|
id: string;
|
||||||
|
pipeline_id: string;
|
||||||
|
name: string;
|
||||||
|
node_id?: string | null;
|
||||||
|
resource_revision: number;
|
||||||
|
etag: string;
|
||||||
|
fingerprint: string;
|
||||||
|
current_decisions: ReconciliationDecision[];
|
||||||
|
history: ReconciliationDecision[];
|
||||||
|
created_by?: string | null;
|
||||||
|
updated_by?: string | null;
|
||||||
|
created_at: string;
|
||||||
|
updated_at: string;
|
||||||
|
};
|
||||||
|
|
||||||
export type PipelineRun = {
|
export type PipelineRun = {
|
||||||
ref: string;
|
ref: string;
|
||||||
pipeline_id: string;
|
pipeline_id: string;
|
||||||
@@ -363,6 +396,58 @@ export function createDataflowSourceSnapshot(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function listDataflowDecisionSets(
|
||||||
|
settings: ApiSettings,
|
||||||
|
pipelineId: string
|
||||||
|
): Promise<ReconciliationDecisionSet[]> {
|
||||||
|
const response = await apiFetch<{ decision_sets: ReconciliationDecisionSet[] }>(
|
||||||
|
settings,
|
||||||
|
`/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/decision-sets`
|
||||||
|
);
|
||||||
|
return response.decision_sets;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createDataflowDecisionSet(
|
||||||
|
settings: ApiSettings,
|
||||||
|
pipelineId: string,
|
||||||
|
payload: { name: string; node_id?: string | null }
|
||||||
|
): Promise<ReconciliationDecisionSet> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`/api/v1/dataflow/pipelines/${encodeURIComponent(pipelineId)}/decision-sets`,
|
||||||
|
{ method: "POST", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getDataflowDecisionSet(
|
||||||
|
settings: ApiSettings,
|
||||||
|
decisionSetId: string
|
||||||
|
): Promise<ReconciliationDecisionSet> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`/api/v1/dataflow/decision-sets/${encodeURIComponent(decisionSetId)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function recordDataflowDecision(
|
||||||
|
settings: ApiSettings,
|
||||||
|
decisionSetId: string,
|
||||||
|
payload: {
|
||||||
|
base_revision: number;
|
||||||
|
key_hash: string;
|
||||||
|
input_hash: string;
|
||||||
|
action: ReconciliationDecisionAction;
|
||||||
|
reason: string;
|
||||||
|
correction?: Record<string, unknown> | null;
|
||||||
|
}
|
||||||
|
): Promise<ReconciliationDecisionSet> {
|
||||||
|
return apiFetch(
|
||||||
|
settings,
|
||||||
|
`/api/v1/dataflow/decision-sets/${encodeURIComponent(decisionSetId)}/decisions`,
|
||||||
|
{ method: "POST", body: JSON.stringify(payload) }
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export async function listDataflowPipelines(settings: ApiSettings): Promise<Pipeline[]> {
|
export async function listDataflowPipelines(settings: ApiSettings): Promise<Pipeline[]> {
|
||||||
const response = await apiFetch<{ pipelines: Pipeline[] }>(settings, "/api/v1/dataflow/pipelines");
|
const response = await apiFetch<{ pipelines: Pipeline[] }>(settings, "/api/v1/dataflow/pipelines");
|
||||||
return response.pipelines;
|
return response.pipelines;
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import {
|
|||||||
Code2,
|
Code2,
|
||||||
CopyPlus,
|
CopyPlus,
|
||||||
DatabaseZap,
|
DatabaseZap,
|
||||||
|
ListChecks,
|
||||||
Network,
|
Network,
|
||||||
Play,
|
Play,
|
||||||
Plus,
|
Plus,
|
||||||
@@ -51,6 +52,7 @@ import { useLocation } from "react-router";
|
|||||||
import {
|
import {
|
||||||
compileDataflowSql,
|
compileDataflowSql,
|
||||||
cancelDataflowPipelineRun,
|
cancelDataflowPipelineRun,
|
||||||
|
createDataflowDecisionSet,
|
||||||
createDataflowTrigger,
|
createDataflowTrigger,
|
||||||
createDataflowPipeline,
|
createDataflowPipeline,
|
||||||
createDataflowSourceSnapshot,
|
createDataflowSourceSnapshot,
|
||||||
@@ -58,7 +60,9 @@ import {
|
|||||||
deleteDataflowTrigger,
|
deleteDataflowTrigger,
|
||||||
dataflowScopeReferenceProvider,
|
dataflowScopeReferenceProvider,
|
||||||
deriveDataflowPipeline,
|
deriveDataflowPipeline,
|
||||||
|
getDataflowDecisionSet,
|
||||||
listDataflowNodeTypes,
|
listDataflowNodeTypes,
|
||||||
|
listDataflowDecisionSets,
|
||||||
listDataflowPipelineRuns,
|
listDataflowPipelineRuns,
|
||||||
listDataflowPipelineDeployments,
|
listDataflowPipelineDeployments,
|
||||||
listDataflowPipelines,
|
listDataflowPipelines,
|
||||||
@@ -66,6 +70,7 @@ import {
|
|||||||
listDataflowTriggers,
|
listDataflowTriggers,
|
||||||
previewDataflowPipeline,
|
previewDataflowPipeline,
|
||||||
promoteDataflowPipeline,
|
promoteDataflowPipeline,
|
||||||
|
recordDataflowDecision,
|
||||||
runDataflowPipeline,
|
runDataflowPipeline,
|
||||||
renderDataflowSql,
|
renderDataflowSql,
|
||||||
updateDataflowPipeline,
|
updateDataflowPipeline,
|
||||||
@@ -84,6 +89,8 @@ import {
|
|||||||
type PipelinePreview,
|
type PipelinePreview,
|
||||||
type PipelineDeployment,
|
type PipelineDeployment,
|
||||||
type PipelineRun,
|
type PipelineRun,
|
||||||
|
type ReconciliationDecisionAction,
|
||||||
|
type ReconciliationDecisionSet,
|
||||||
type TabularSource
|
type TabularSource
|
||||||
} from "../../api/dataflow";
|
} from "../../api/dataflow";
|
||||||
import DataflowCanvas, { updateGraphNode } from "./DataflowCanvas";
|
import DataflowCanvas, { updateGraphNode } from "./DataflowCanvas";
|
||||||
@@ -141,6 +148,7 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
const [definitionSettingsOpen, setDefinitionSettingsOpen] = useState(false);
|
const [definitionSettingsOpen, setDefinitionSettingsOpen] = useState(false);
|
||||||
const [deriveOpen, setDeriveOpen] = useState(false);
|
const [deriveOpen, setDeriveOpen] = useState(false);
|
||||||
const [triggersOpen, setTriggersOpen] = useState(false);
|
const [triggersOpen, setTriggersOpen] = useState(false);
|
||||||
|
const [decisionReviewOpen, setDecisionReviewOpen] = useState(false);
|
||||||
const [nodeLibrary, setNodeLibrary] = useState<NodeTypeDefinition[]>(FALLBACK_NODE_LIBRARY);
|
const [nodeLibrary, setNodeLibrary] = useState<NodeTypeDefinition[]>(FALLBACK_NODE_LIBRARY);
|
||||||
const [sources, setSources] = useState<TabularSource[]>([]);
|
const [sources, setSources] = useState<TabularSource[]>([]);
|
||||||
const [sourceCatalogueAvailable, setSourceCatalogueAvailable] = useState(false);
|
const [sourceCatalogueAvailable, setSourceCatalogueAvailable] = useState(false);
|
||||||
@@ -932,6 +940,16 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
diagnostics={diagnostics}
|
diagnostics={diagnostics}
|
||||||
nodeDiagnostics={nodeDiagnostics}
|
nodeDiagnostics={nodeDiagnostics}
|
||||||
selectedNodeId={selectedNodeId}
|
selectedNodeId={selectedNodeId}
|
||||||
|
decisionReviewDisabledReason={
|
||||||
|
!draft.id
|
||||||
|
? "Save the pipeline before recording review decisions."
|
||||||
|
: dirty
|
||||||
|
? "Save the current pipeline revision before recording review decisions."
|
||||||
|
: !canEdit
|
||||||
|
? editBlockedReason
|
||||||
|
: undefined
|
||||||
|
}
|
||||||
|
onReviewDecisions={() => setDecisionReviewOpen(true)}
|
||||||
onClose={() => setResultOpen(false)}
|
onClose={() => setResultOpen(false)}
|
||||||
/>
|
/>
|
||||||
) : null}
|
) : null}
|
||||||
@@ -1068,6 +1086,22 @@ export default function DataflowPage({ settings, auth }: { settings: ApiSettings
|
|||||||
editable={canManageTriggers}
|
editable={canManageTriggers}
|
||||||
onClose={() => setTriggersOpen(false)}
|
onClose={() => setTriggersOpen(false)}
|
||||||
/>
|
/>
|
||||||
|
<ReconciliationDecisionDialog
|
||||||
|
open={decisionReviewOpen}
|
||||||
|
settings={settings}
|
||||||
|
pipeline={draft?.id ? { id: draft.id, name: draft.name } : null}
|
||||||
|
node={selectedPreviewNode(draft?.graph.nodes ?? [], preview, selectedNodeId)}
|
||||||
|
preview={preview}
|
||||||
|
editable={canEdit && !dirty}
|
||||||
|
onClose={() => setDecisionReviewOpen(false)}
|
||||||
|
onChanged={() => {
|
||||||
|
void listDataflowSources(settings).then((catalogue) => {
|
||||||
|
setSources(catalogue.sources);
|
||||||
|
setSourceCatalogueAvailable(catalogue.available);
|
||||||
|
setSourceCatalogueWritable(catalogue.writable);
|
||||||
|
}).catch((sourceError) => setError(apiErrorMessage(sourceError)));
|
||||||
|
}}
|
||||||
|
/>
|
||||||
</main>
|
</main>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -2448,6 +2482,380 @@ function SourceSnapshotDialog({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ReviewableReconciliationRow = {
|
||||||
|
keyHash: string;
|
||||||
|
inputHash: string;
|
||||||
|
row: Record<string, unknown>;
|
||||||
|
};
|
||||||
|
|
||||||
|
function ReconciliationDecisionDialog({
|
||||||
|
open,
|
||||||
|
settings,
|
||||||
|
pipeline,
|
||||||
|
node,
|
||||||
|
preview,
|
||||||
|
editable,
|
||||||
|
onClose,
|
||||||
|
onChanged
|
||||||
|
}: {
|
||||||
|
open: boolean;
|
||||||
|
settings: ApiSettings;
|
||||||
|
pipeline: { id: string; name: string } | null;
|
||||||
|
node: PipelineGraphNode | null;
|
||||||
|
preview: PipelinePreview | null;
|
||||||
|
editable: boolean;
|
||||||
|
onClose: () => void;
|
||||||
|
onChanged: () => void;
|
||||||
|
}) {
|
||||||
|
const [decisionSets, setDecisionSets] = useState<ReconciliationDecisionSet[]>([]);
|
||||||
|
const [selectedSetId, setSelectedSetId] = useState("");
|
||||||
|
const [selectedKeyHash, setSelectedKeyHash] = useState("");
|
||||||
|
const [newSetName, setNewSetName] = useState("");
|
||||||
|
const [action, setAction] = useState<ReconciliationDecisionAction>("accept");
|
||||||
|
const [reason, setReason] = useState("");
|
||||||
|
const [correctionText, setCorrectionText] = useState("{}");
|
||||||
|
const [loading, setLoading] = useState(false);
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [error, setError] = useState("");
|
||||||
|
const [success, setSuccess] = useState("");
|
||||||
|
|
||||||
|
const rows = useMemo<ReviewableReconciliationRow[]>(() => {
|
||||||
|
const result = preview?.node_preview ?? preview;
|
||||||
|
if (!result || node?.type !== "reconcile.compare") return [];
|
||||||
|
return result.rows.flatMap((row) => {
|
||||||
|
const keyHash = row._reconciliation_key_hash;
|
||||||
|
const inputHash = row._reconciliation_input_hash;
|
||||||
|
if (!isSha256Hex(keyHash) || !isSha256Hex(inputHash)) return [];
|
||||||
|
return [{ keyHash, inputHash, row }];
|
||||||
|
});
|
||||||
|
}, [node?.type, preview]);
|
||||||
|
|
||||||
|
const selectedSet = decisionSets.find((item) => item.id === selectedSetId) ?? null;
|
||||||
|
const selectedRow = rows.find((item) => item.keyHash === selectedKeyHash) ?? null;
|
||||||
|
const currentDecision = selectedSet?.current_decisions.find(
|
||||||
|
(item) => item.key_hash === selectedRow?.keyHash
|
||||||
|
) ?? null;
|
||||||
|
const exactDecision = currentDecision?.input_hash === selectedRow?.inputHash
|
||||||
|
? currentDecision
|
||||||
|
: null;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open) return;
|
||||||
|
setSelectedKeyHash((current) => rows.some((item) => item.keyHash === current)
|
||||||
|
? current
|
||||||
|
: rows[0]?.keyHash ?? "");
|
||||||
|
}, [open, rows]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open || !pipeline) {
|
||||||
|
setDecisionSets([]);
|
||||||
|
setSelectedSetId("");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let cancelled = false;
|
||||||
|
setLoading(true);
|
||||||
|
setError("");
|
||||||
|
void listDataflowDecisionSets(settings, pipeline.id)
|
||||||
|
.then((items) => {
|
||||||
|
if (cancelled) return;
|
||||||
|
const relevant = items.filter((item) => !item.node_id || item.node_id === node?.id);
|
||||||
|
setDecisionSets(relevant);
|
||||||
|
setSelectedSetId((current) => relevant.some((item) => item.id === current)
|
||||||
|
? current
|
||||||
|
: relevant[0]?.id ?? "");
|
||||||
|
})
|
||||||
|
.catch((loadError) => {
|
||||||
|
if (!cancelled) setError(apiErrorMessage(loadError));
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (!cancelled) setLoading(false);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [node?.id, open, pipeline?.id, settings]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!open || !selectedSetId) return;
|
||||||
|
let cancelled = false;
|
||||||
|
setLoading(true);
|
||||||
|
void getDataflowDecisionSet(settings, selectedSetId)
|
||||||
|
.then((item) => {
|
||||||
|
if (cancelled) return;
|
||||||
|
setDecisionSets((current) => current.map(
|
||||||
|
(candidate) => candidate.id === item.id ? item : candidate
|
||||||
|
));
|
||||||
|
})
|
||||||
|
.catch((loadError) => {
|
||||||
|
if (!cancelled) setError(apiErrorMessage(loadError));
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (!cancelled) setLoading(false);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [open, selectedSetId, settings]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setAction(exactDecision?.action ?? "accept");
|
||||||
|
setReason(exactDecision?.reason ?? "");
|
||||||
|
setCorrectionText(JSON.stringify(exactDecision?.correction ?? {}, null, 2));
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
}, [exactDecision?.id, selectedKeyHash, selectedSetId]);
|
||||||
|
|
||||||
|
const createSet = async () => {
|
||||||
|
if (!pipeline || !node || !newSetName.trim() || !editable) return;
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
try {
|
||||||
|
const created = await createDataflowDecisionSet(settings, pipeline.id, {
|
||||||
|
name: newSetName.trim(),
|
||||||
|
node_id: node.id
|
||||||
|
});
|
||||||
|
setDecisionSets((current) => [created, ...current.filter((item) => item.id !== created.id)]);
|
||||||
|
setSelectedSetId(created.id);
|
||||||
|
setNewSetName("");
|
||||||
|
setSuccess("Decision set created. It is now available as a governed Dataflow source.");
|
||||||
|
onChanged();
|
||||||
|
} catch (createError) {
|
||||||
|
setError(apiErrorMessage(createError));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const saveDecision = async () => {
|
||||||
|
if (!selectedSet || !selectedRow || !editable) return;
|
||||||
|
if (reason.trim().length < 3) {
|
||||||
|
setError("Record a reason of at least three characters.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let correction: Record<string, unknown> | null = null;
|
||||||
|
if (action === "correct") {
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(correctionText) as unknown;
|
||||||
|
if (!isRecord(parsed) || !Object.keys(parsed).length) {
|
||||||
|
setError("Correct decisions require a JSON object with at least one corrected field.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
correction = parsed;
|
||||||
|
} catch {
|
||||||
|
setError("Correction must be a valid JSON object.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setBusy(true);
|
||||||
|
setError("");
|
||||||
|
setSuccess("");
|
||||||
|
try {
|
||||||
|
const updated = await recordDataflowDecision(settings, selectedSet.id, {
|
||||||
|
base_revision: selectedSet.resource_revision,
|
||||||
|
key_hash: selectedRow.keyHash,
|
||||||
|
input_hash: selectedRow.inputHash,
|
||||||
|
action,
|
||||||
|
reason: reason.trim(),
|
||||||
|
correction
|
||||||
|
});
|
||||||
|
setDecisionSets((current) => current.map((item) => item.id === updated.id ? updated : item));
|
||||||
|
setSuccess(`Recorded decision revision ${updated.resource_revision}.`);
|
||||||
|
onChanged();
|
||||||
|
} catch (saveError) {
|
||||||
|
setError(apiErrorMessage(saveError));
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const reviewedCount = selectedSet
|
||||||
|
? rows.filter((row) => selectedSet.current_decisions.some(
|
||||||
|
(decision) => decision.key_hash === row.keyHash && decision.input_hash === row.inputHash
|
||||||
|
)).length
|
||||||
|
: 0;
|
||||||
|
const staleCount = selectedSet
|
||||||
|
? rows.filter((row) => selectedSet.current_decisions.some(
|
||||||
|
(decision) => decision.key_hash === row.keyHash && decision.input_hash !== row.inputHash
|
||||||
|
)).length
|
||||||
|
: 0;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<Dialog
|
||||||
|
open={open}
|
||||||
|
title={`Review reconciliation${node ? `: ${node.label}` : ""}`}
|
||||||
|
className="dataflow-decision-dialog"
|
||||||
|
bodyClassName="dataflow-decision-dialog-body"
|
||||||
|
closeDisabled={busy}
|
||||||
|
onClose={onClose}
|
||||||
|
footer={(
|
||||||
|
<>
|
||||||
|
<span className="dataflow-decision-history-summary">
|
||||||
|
{selectedSet ? `${selectedSet.history.length} immutable decision revision(s)` : "No decision set selected"}
|
||||||
|
</span>
|
||||||
|
<Button onClick={onClose} disabled={busy}>Close</Button>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
<div className="dataflow-decision-shell">
|
||||||
|
{error ? <DismissibleAlert tone="danger" resetKey={error}>{error}</DismissibleAlert> : null}
|
||||||
|
{success ? <DismissibleAlert tone="success" resetKey={success}>{success}</DismissibleAlert> : null}
|
||||||
|
<div className="dataflow-decision-set-toolbar">
|
||||||
|
<FormField
|
||||||
|
label="Decision set"
|
||||||
|
help="The current projection is consumable as a source; every prior decision remains in immutable history."
|
||||||
|
documentation={DATAFLOW_FIELDS_DOCUMENTATION}
|
||||||
|
>
|
||||||
|
<select
|
||||||
|
value={selectedSetId}
|
||||||
|
onChange={(event) => setSelectedSetId(event.target.value)}
|
||||||
|
disabled={loading || !decisionSets.length}
|
||||||
|
>
|
||||||
|
{!decisionSets.length ? <option value="">No decision sets</option> : null}
|
||||||
|
{decisionSets.map((item) => (
|
||||||
|
<option key={item.id} value={item.id}>{item.name}</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</FormField>
|
||||||
|
<FormField
|
||||||
|
label="New decision set"
|
||||||
|
help="Create one set per review purpose or review round."
|
||||||
|
documentation={DATAFLOW_FIELDS_DOCUMENTATION}
|
||||||
|
>
|
||||||
|
<div className="dataflow-decision-create">
|
||||||
|
<input
|
||||||
|
value={newSetName}
|
||||||
|
onChange={(event) => setNewSetName(event.target.value)}
|
||||||
|
placeholder={`${pipeline?.name ?? "Pipeline"} review`}
|
||||||
|
disabled={!editable || busy}
|
||||||
|
/>
|
||||||
|
<Button
|
||||||
|
variant="primary"
|
||||||
|
onClick={() => void createSet()}
|
||||||
|
disabled={!editable || busy || !newSetName.trim()}
|
||||||
|
>
|
||||||
|
<Plus size={16} /> Create
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</FormField>
|
||||||
|
<div className="dataflow-decision-counts" aria-label="Review progress">
|
||||||
|
<span><strong>{reviewedCount}</strong> reviewed</span>
|
||||||
|
<span className={staleCount ? "is-warning" : ""}><strong>{staleCount}</strong> stale</span>
|
||||||
|
<span><strong>{Math.max(0, rows.length - reviewedCount - staleCount)}</strong> open</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{!rows.length ? (
|
||||||
|
<div className="dataflow-results-empty">
|
||||||
|
Run a preview of a reconciliation comparison that emits stable key and input hashes.
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<div className="dataflow-decision-layout">
|
||||||
|
<div className="dataflow-decision-rows" role="listbox" aria-label="Reconciliation rows">
|
||||||
|
<div className="dataflow-decision-row-heading">
|
||||||
|
<span>Record</span><span>Comparison</span><span>Decision</span>
|
||||||
|
</div>
|
||||||
|
{rows.map((item) => {
|
||||||
|
const decision = selectedSet?.current_decisions.find(
|
||||||
|
(candidate) => candidate.key_hash === item.keyHash
|
||||||
|
);
|
||||||
|
const stale = Boolean(decision && decision.input_hash !== item.inputHash);
|
||||||
|
const statusLabel = stale ? "stale" : decision?.action ?? "open";
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
key={item.keyHash}
|
||||||
|
type="button"
|
||||||
|
role="option"
|
||||||
|
aria-selected={item.keyHash === selectedKeyHash}
|
||||||
|
className={item.keyHash === selectedKeyHash ? "is-selected" : ""}
|
||||||
|
onClick={() => setSelectedKeyHash(item.keyHash)}
|
||||||
|
>
|
||||||
|
<span title={item.keyHash}>{reviewRowLabel(item.row)}</span>
|
||||||
|
<span>{formatCell(item.row._reconciliation_status)}</span>
|
||||||
|
<span className={`dataflow-decision-state is-${statusLabel}`}>{statusLabel}</span>
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
<div className="dataflow-decision-editor">
|
||||||
|
{selectedRow && selectedSet ? (
|
||||||
|
<>
|
||||||
|
<div className="dataflow-decision-editor-heading">
|
||||||
|
<div>
|
||||||
|
<strong>{reviewRowLabel(selectedRow.row)}</strong>
|
||||||
|
<small title={selectedRow.keyHash}>{selectedRow.keyHash}</small>
|
||||||
|
</div>
|
||||||
|
{currentDecision && !exactDecision ? (
|
||||||
|
<StatusBadge status="warning" label="Prior decision is stale" />
|
||||||
|
) : exactDecision ? (
|
||||||
|
<StatusBadge status="success" label={`Current: ${exactDecision.action}`} />
|
||||||
|
) : (
|
||||||
|
<StatusBadge status="inactive" label="Unreviewed" />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<FormField label="Decision" documentation={DATAFLOW_FIELDS_DOCUMENTATION}>
|
||||||
|
<SegmentedControl<ReconciliationDecisionAction>
|
||||||
|
ariaLabel="Reconciliation decision"
|
||||||
|
options={[
|
||||||
|
{ id: "accept", label: "Accept" },
|
||||||
|
{ id: "reject", label: "Reject" },
|
||||||
|
{ id: "correct", label: "Correct" },
|
||||||
|
{ id: "defer", label: "Defer" }
|
||||||
|
]}
|
||||||
|
value={action}
|
||||||
|
onChange={setAction}
|
||||||
|
disabled={!editable || busy}
|
||||||
|
/>
|
||||||
|
</FormField>
|
||||||
|
<FormField
|
||||||
|
label="Reason"
|
||||||
|
help="The reason is retained with the actor, exact input hash, and immutable revision."
|
||||||
|
documentation={DATAFLOW_FIELDS_DOCUMENTATION}
|
||||||
|
>
|
||||||
|
<textarea
|
||||||
|
rows={4}
|
||||||
|
value={reason}
|
||||||
|
onChange={(event) => setReason(event.target.value)}
|
||||||
|
disabled={!editable || busy}
|
||||||
|
/>
|
||||||
|
</FormField>
|
||||||
|
{action === "correct" ? (
|
||||||
|
<FormField
|
||||||
|
label="Corrected fields (JSON)"
|
||||||
|
help="Corrections are annotations. A downstream governed transform decides whether to apply them."
|
||||||
|
documentation={DATAFLOW_FIELDS_DOCUMENTATION}
|
||||||
|
>
|
||||||
|
<textarea
|
||||||
|
className="dataflow-json-editor"
|
||||||
|
value={correctionText}
|
||||||
|
onChange={(event) => setCorrectionText(event.target.value)}
|
||||||
|
spellCheck={false}
|
||||||
|
disabled={!editable || busy}
|
||||||
|
/>
|
||||||
|
</FormField>
|
||||||
|
) : null}
|
||||||
|
<div className="dataflow-decision-editor-actions">
|
||||||
|
<Button
|
||||||
|
variant="primary"
|
||||||
|
onClick={() => void saveDecision()}
|
||||||
|
disabled={!editable || busy || reason.trim().length < 3}
|
||||||
|
>
|
||||||
|
<Save size={16} /> {busy ? "Recording..." : "Record revision"}
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<div className="dataflow-results-empty">
|
||||||
|
{loading ? "Loading decision sets..." : "Create or select a decision set to review this row."}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</Dialog>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function ResultPanel({
|
function ResultPanel({
|
||||||
tab,
|
tab,
|
||||||
onTabChange,
|
onTabChange,
|
||||||
@@ -2456,6 +2864,8 @@ function ResultPanel({
|
|||||||
diagnostics,
|
diagnostics,
|
||||||
nodeDiagnostics,
|
nodeDiagnostics,
|
||||||
selectedNodeId,
|
selectedNodeId,
|
||||||
|
decisionReviewDisabledReason,
|
||||||
|
onReviewDecisions,
|
||||||
onClose
|
onClose
|
||||||
}: {
|
}: {
|
||||||
tab: ResultTab;
|
tab: ResultTab;
|
||||||
@@ -2465,6 +2875,8 @@ function ResultPanel({
|
|||||||
diagnostics: DataflowDiagnostic[];
|
diagnostics: DataflowDiagnostic[];
|
||||||
nodeDiagnostics: NodePreviewDiagnostic[];
|
nodeDiagnostics: NodePreviewDiagnostic[];
|
||||||
selectedNodeId: string | null;
|
selectedNodeId: string | null;
|
||||||
|
decisionReviewDisabledReason?: string;
|
||||||
|
onReviewDecisions: () => void;
|
||||||
onClose: () => void;
|
onClose: () => void;
|
||||||
}) {
|
}) {
|
||||||
const outputNodeId = nodes.find((node) => node.type === "output")?.id ?? "";
|
const outputNodeId = nodes.find((node) => node.type === "output")?.id ?? "";
|
||||||
@@ -2491,7 +2903,18 @@ function ResultPanel({
|
|||||||
</span>
|
</span>
|
||||||
) : null}
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
<Button variant="ghost" onClick={onClose}>Close</Button>
|
<div className="dataflow-results-actions">
|
||||||
|
{tab === "preview" && previewNode?.type === "reconcile.compare" ? (
|
||||||
|
<Button
|
||||||
|
onClick={onReviewDecisions}
|
||||||
|
disabled={!preview || Boolean(decisionReviewDisabledReason)}
|
||||||
|
disabledReason={decisionReviewDisabledReason}
|
||||||
|
>
|
||||||
|
<ListChecks size={16} /> Review decisions
|
||||||
|
</Button>
|
||||||
|
) : null}
|
||||||
|
<Button variant="ghost" onClick={onClose}>Close</Button>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
{tab === "preview" ? (
|
{tab === "preview" ? (
|
||||||
<PreviewTable preview={preview} />
|
<PreviewTable preview={preview} />
|
||||||
@@ -2625,6 +3048,27 @@ function formatCell(value: unknown): string {
|
|||||||
return String(value);
|
return String(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function selectedPreviewNode(
|
||||||
|
nodes: PipelineGraphNode[],
|
||||||
|
preview: PipelinePreview | null,
|
||||||
|
selectedNodeId: string | null
|
||||||
|
): PipelineGraphNode | null {
|
||||||
|
const nodeId = preview?.node_preview?.node_id ?? selectedNodeId;
|
||||||
|
return nodes.find((node) => node.id === nodeId) ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isSha256Hex(value: unknown): value is string {
|
||||||
|
return typeof value === "string" && /^[0-9a-f]{64}$/.test(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function reviewRowLabel(row: Record<string, unknown>): string {
|
||||||
|
const values = Object.entries(row)
|
||||||
|
.filter(([key, value]) => !key.startsWith("_") && value !== null && value !== undefined)
|
||||||
|
.slice(0, 3)
|
||||||
|
.map(([key, value]) => `${key}: ${formatCell(value)}`);
|
||||||
|
return values.join(" · ") || "Reconciliation row";
|
||||||
|
}
|
||||||
|
|
||||||
function sourceNameFromFile(value: string): string {
|
function sourceNameFromFile(value: string): string {
|
||||||
const normalized = value
|
const normalized = value
|
||||||
.normalize("NFKD")
|
.normalize("NFKD")
|
||||||
|
|||||||
@@ -736,6 +736,210 @@
|
|||||||
width: min(720px, calc(100vw - 32px));
|
width: min(720px, calc(100vw - 32px));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-dialog {
|
||||||
|
width: min(1080px, calc(100vw - 32px));
|
||||||
|
height: min(760px, calc(100vh - 48px));
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-dialog-body {
|
||||||
|
overflow: hidden;
|
||||||
|
padding: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-shell {
|
||||||
|
display: flex;
|
||||||
|
height: 100%;
|
||||||
|
min-height: 0;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-set-toolbar {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: minmax(190px, 0.8fr) minmax(300px, 1.2fr) auto;
|
||||||
|
align-items: end;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-set-toolbar select,
|
||||||
|
.dataflow-decision-set-toolbar input,
|
||||||
|
.dataflow-decision-editor textarea {
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-create,
|
||||||
|
.dataflow-decision-counts,
|
||||||
|
.dataflow-results-actions,
|
||||||
|
.dataflow-decision-editor-actions {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-create input {
|
||||||
|
min-width: 0;
|
||||||
|
flex: 1 1 auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-create .btn {
|
||||||
|
flex: 0 0 auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-counts {
|
||||||
|
min-height: 34px;
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 11px;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-counts strong {
|
||||||
|
color: var(--text-strong);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-counts .is-warning strong {
|
||||||
|
color: var(--warning-deep);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-layout {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: minmax(420px, 1.1fr) minmax(340px, 0.9fr);
|
||||||
|
min-height: 0;
|
||||||
|
flex: 1 1 auto;
|
||||||
|
overflow: hidden;
|
||||||
|
border: var(--border-line);
|
||||||
|
border-radius: var(--radius-sm);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows,
|
||||||
|
.dataflow-decision-editor {
|
||||||
|
min-width: 0;
|
||||||
|
min-height: 0;
|
||||||
|
overflow: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows {
|
||||||
|
border-right: var(--border-line);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-row-heading,
|
||||||
|
.dataflow-decision-rows > button {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: minmax(180px, 1fr) minmax(90px, 0.45fr) 90px;
|
||||||
|
align-items: center;
|
||||||
|
gap: 10px;
|
||||||
|
width: 100%;
|
||||||
|
min-height: 42px;
|
||||||
|
border: 0;
|
||||||
|
border-bottom: var(--border-line);
|
||||||
|
padding: 7px 10px;
|
||||||
|
text-align: left;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-row-heading {
|
||||||
|
position: sticky;
|
||||||
|
z-index: 2;
|
||||||
|
top: 0;
|
||||||
|
background: var(--panel-header);
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 10px;
|
||||||
|
font-weight: 700;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows > button {
|
||||||
|
background: transparent;
|
||||||
|
color: var(--text);
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows > button:hover,
|
||||||
|
.dataflow-decision-rows > button.is-selected {
|
||||||
|
background: var(--primary-soft);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows > button.is-selected {
|
||||||
|
box-shadow: inset 3px 0 0 var(--accent);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-rows > button > span {
|
||||||
|
min-width: 0;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-state {
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 10px;
|
||||||
|
font-weight: 700;
|
||||||
|
text-transform: uppercase;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-state.is-accept,
|
||||||
|
.dataflow-decision-state.is-correct {
|
||||||
|
color: var(--success-text);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-state.is-reject,
|
||||||
|
.dataflow-decision-state.is-stale {
|
||||||
|
color: var(--danger-text);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-state.is-defer {
|
||||||
|
color: var(--warning-deep);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor {
|
||||||
|
display: grid;
|
||||||
|
grid-auto-rows: max-content;
|
||||||
|
align-content: start;
|
||||||
|
gap: 14px;
|
||||||
|
padding: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor-heading {
|
||||||
|
display: flex;
|
||||||
|
align-items: start;
|
||||||
|
justify-content: space-between;
|
||||||
|
gap: 12px;
|
||||||
|
padding-bottom: 10px;
|
||||||
|
border-bottom: var(--border-line);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor-heading > div {
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor-heading strong,
|
||||||
|
.dataflow-decision-editor-heading small {
|
||||||
|
display: block;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor-heading small {
|
||||||
|
margin-top: 4px;
|
||||||
|
overflow: hidden;
|
||||||
|
color: var(--muted);
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Consolas, monospace;
|
||||||
|
font-size: 9px;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor .dataflow-json-editor {
|
||||||
|
min-height: 140px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-editor-actions {
|
||||||
|
justify-content: flex-end;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dataflow-decision-history-summary {
|
||||||
|
margin-right: auto;
|
||||||
|
color: var(--muted);
|
||||||
|
font-size: 11px;
|
||||||
|
}
|
||||||
|
|
||||||
.dataflow-run-dialog {
|
.dataflow-run-dialog {
|
||||||
width: min(860px, calc(100vw - 32px));
|
width: min(860px, calc(100vw - 32px));
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user