Add durable reconciliation decisions
This commit is contained in:
@@ -0,0 +1,380 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
|
||||
from sqlalchemy import and_, func, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session, joinedload, selectinload
|
||||
|
||||
from govoplan_core.core.concurrency import claim_revision
|
||||
from govoplan_core.security.time import utc_now
|
||||
from govoplan_dataflow.backend.db.models import (
|
||||
DataflowPipeline,
|
||||
DataflowReconciliationDecision,
|
||||
DataflowReconciliationDecisionSet,
|
||||
)
|
||||
|
||||
|
||||
DECISION_SET_REF_PREFIX = "dataflow-decision-set:"
|
||||
DECISION_REF_PREFIX = "dataflow-decision:"
|
||||
DECISION_ACTIONS = frozenset({"accept", "reject", "correct", "defer"})
|
||||
|
||||
|
||||
class ReconciliationDecisionError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
class ReconciliationDecisionNotFoundError(ReconciliationDecisionError):
|
||||
pass
|
||||
|
||||
|
||||
class ReconciliationDecisionConflictError(ReconciliationDecisionError):
|
||||
pass
|
||||
|
||||
|
||||
def list_decision_sets(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
pipeline_id: str | None = None,
|
||||
include_decisions: bool = True,
|
||||
) -> tuple[DataflowReconciliationDecisionSet, ...]:
|
||||
options = [joinedload(DataflowReconciliationDecisionSet.pipeline)]
|
||||
if include_decisions:
|
||||
options.append(selectinload(DataflowReconciliationDecisionSet.decisions))
|
||||
statement = (
|
||||
select(DataflowReconciliationDecisionSet)
|
||||
.options(*options)
|
||||
.where(DataflowReconciliationDecisionSet.tenant_id == tenant_id)
|
||||
.order_by(
|
||||
DataflowReconciliationDecisionSet.name,
|
||||
DataflowReconciliationDecisionSet.id,
|
||||
)
|
||||
)
|
||||
if pipeline_id:
|
||||
statement = statement.where(
|
||||
DataflowReconciliationDecisionSet.pipeline_id == pipeline_id
|
||||
)
|
||||
return tuple(session.scalars(statement).all())
|
||||
|
||||
|
||||
def create_decision_set(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
pipeline_id: str,
|
||||
name: str,
|
||||
node_id: str | None,
|
||||
actor_ref: str,
|
||||
) -> DataflowReconciliationDecisionSet:
|
||||
pipeline = session.get(DataflowPipeline, pipeline_id)
|
||||
if (
|
||||
pipeline is None
|
||||
or pipeline.tenant_id not in {None, tenant_id}
|
||||
or pipeline.deleted_at is not None
|
||||
):
|
||||
raise ReconciliationDecisionNotFoundError("Dataflow pipeline not found.")
|
||||
item = DataflowReconciliationDecisionSet(
|
||||
tenant_id=tenant_id,
|
||||
pipeline_id=pipeline_id,
|
||||
name=name.strip(),
|
||||
node_id=node_id,
|
||||
resource_revision=1,
|
||||
created_by=actor_ref,
|
||||
updated_by=actor_ref,
|
||||
)
|
||||
session.add(item)
|
||||
try:
|
||||
session.flush()
|
||||
except IntegrityError as exc:
|
||||
raise ReconciliationDecisionConflictError(
|
||||
"A decision set with this name already exists for the pipeline."
|
||||
) from exc
|
||||
return item
|
||||
|
||||
|
||||
def get_decision_set(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
decision_set_id: str,
|
||||
include_decisions: bool = True,
|
||||
) -> DataflowReconciliationDecisionSet:
|
||||
options = [joinedload(DataflowReconciliationDecisionSet.pipeline)]
|
||||
if include_decisions:
|
||||
options.append(selectinload(DataflowReconciliationDecisionSet.decisions))
|
||||
item = session.scalar(
|
||||
select(DataflowReconciliationDecisionSet)
|
||||
.options(*options)
|
||||
.where(
|
||||
DataflowReconciliationDecisionSet.id == decision_set_id,
|
||||
DataflowReconciliationDecisionSet.tenant_id == tenant_id,
|
||||
)
|
||||
)
|
||||
if item is None:
|
||||
raise ReconciliationDecisionNotFoundError(
|
||||
"Reconciliation decision set not found."
|
||||
)
|
||||
return item
|
||||
|
||||
|
||||
def record_decision(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
decision_set_id: str,
|
||||
expected_revision: int,
|
||||
key_hash: str,
|
||||
input_hash: str,
|
||||
action: str,
|
||||
reason: str,
|
||||
correction: dict[str, object] | None,
|
||||
actor_ref: str,
|
||||
) -> DataflowReconciliationDecisionSet:
|
||||
item = get_decision_set(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
decision_set_id=decision_set_id,
|
||||
)
|
||||
if action not in DECISION_ACTIONS:
|
||||
raise ReconciliationDecisionError("Unsupported reconciliation action.")
|
||||
next_revision = claim_revision(
|
||||
session,
|
||||
model=DataflowReconciliationDecisionSet,
|
||||
filters=(
|
||||
DataflowReconciliationDecisionSet.id == decision_set_id,
|
||||
DataflowReconciliationDecisionSet.tenant_id == tenant_id,
|
||||
),
|
||||
revision_attribute="resource_revision",
|
||||
expected_revision=expected_revision,
|
||||
resource_type="dataflow_reconciliation_decision_set",
|
||||
resource_id=decision_set_id,
|
||||
)
|
||||
item.resource_revision = next_revision
|
||||
item.updated_by = actor_ref
|
||||
item.decisions.append(
|
||||
DataflowReconciliationDecision(
|
||||
tenant_id=tenant_id,
|
||||
revision=next_revision,
|
||||
key_hash=key_hash,
|
||||
input_hash=input_hash,
|
||||
action=action,
|
||||
reason=reason.strip(),
|
||||
correction=dict(correction) if correction else None,
|
||||
actor_ref=actor_ref,
|
||||
decided_at=utc_now(),
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
return item
|
||||
|
||||
|
||||
def current_decisions(
|
||||
item: DataflowReconciliationDecisionSet,
|
||||
) -> tuple[DataflowReconciliationDecision, ...]:
|
||||
current: dict[str, DataflowReconciliationDecision] = {}
|
||||
for decision in sorted(item.decisions, key=lambda value: value.revision):
|
||||
current[decision.key_hash] = decision
|
||||
return tuple(
|
||||
sorted(
|
||||
current.values(),
|
||||
key=lambda value: (value.key_hash, value.revision),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def decision_rows(
|
||||
item: DataflowReconciliationDecisionSet,
|
||||
) -> tuple[dict[str, object], ...]:
|
||||
return tuple(_decision_row(value) for value in current_decisions(item))
|
||||
|
||||
|
||||
def decision_set_fingerprint(
|
||||
item: DataflowReconciliationDecisionSet,
|
||||
) -> str:
|
||||
payload = {
|
||||
"decision_set_id": item.id,
|
||||
"resource_revision": item.resource_revision,
|
||||
}
|
||||
return "sha256:" + hashlib.sha256(
|
||||
json.dumps(
|
||||
payload,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def decision_set_payload(
|
||||
item: DataflowReconciliationDecisionSet,
|
||||
*,
|
||||
include_decisions: bool = True,
|
||||
) -> dict[str, object]:
|
||||
current = current_decisions(item) if include_decisions else ()
|
||||
history = (
|
||||
sorted(item.decisions, key=lambda value: value.revision)
|
||||
if include_decisions
|
||||
else ()
|
||||
)
|
||||
return {
|
||||
"ref": f"{DECISION_SET_REF_PREFIX}{item.id}",
|
||||
"id": item.id,
|
||||
"pipeline_id": item.pipeline_id,
|
||||
"name": item.name,
|
||||
"node_id": item.node_id,
|
||||
"resource_revision": item.resource_revision,
|
||||
"etag": item.strong_etag,
|
||||
"fingerprint": decision_set_fingerprint(item),
|
||||
"decisions_included": include_decisions,
|
||||
"current_decisions": [_decision_payload(value) for value in current],
|
||||
"history": [
|
||||
_decision_payload(value)
|
||||
for value in history
|
||||
],
|
||||
"created_by": item.created_by,
|
||||
"updated_by": item.updated_by,
|
||||
"created_at": item.created_at,
|
||||
"updated_at": item.updated_at,
|
||||
}
|
||||
|
||||
|
||||
def decision_set_source_payload(
|
||||
item: DataflowReconciliationDecisionSet,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"ref": f"{DECISION_SET_REF_PREFIX}{item.id}",
|
||||
"provider": "dataflow.reconciliation_decisions",
|
||||
"source_name": _source_name(item.name, item.id),
|
||||
"name": item.name,
|
||||
"description": "Current immutable reconciliation decisions; prior revisions remain in decision history.",
|
||||
"mode": "static",
|
||||
"columns": [
|
||||
{"name": "key_hash", "data_type": "string", "nullable": False},
|
||||
{"name": "input_hash", "data_type": "string", "nullable": False},
|
||||
{"name": "decision_ref", "data_type": "string", "nullable": False},
|
||||
{"name": "action", "data_type": "string", "nullable": False},
|
||||
{"name": "actor_ref", "data_type": "string", "nullable": False},
|
||||
{"name": "decided_at", "data_type": "datetime", "nullable": False},
|
||||
{"name": "reason", "data_type": "string", "nullable": False},
|
||||
{"name": "correction", "data_type": "object", "nullable": True},
|
||||
],
|
||||
"schema_version": "1",
|
||||
"fingerprint": decision_set_fingerprint(item),
|
||||
"row_count": None,
|
||||
"byte_count": None,
|
||||
"updated_at": item.updated_at,
|
||||
"capabilities": ["preview", "read", "immutable_history"],
|
||||
}
|
||||
|
||||
|
||||
def current_decision_rows(
|
||||
session: Session,
|
||||
*,
|
||||
decision_set_id: str,
|
||||
limit: int,
|
||||
) -> tuple[tuple[dict[str, object], ...], int]:
|
||||
bounded_limit = max(1, limit)
|
||||
latest = (
|
||||
select(
|
||||
DataflowReconciliationDecision.key_hash.label("key_hash"),
|
||||
func.max(DataflowReconciliationDecision.revision).label("revision"),
|
||||
)
|
||||
.where(
|
||||
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
||||
)
|
||||
.group_by(DataflowReconciliationDecision.key_hash)
|
||||
.subquery()
|
||||
)
|
||||
statement = (
|
||||
select(DataflowReconciliationDecision)
|
||||
.join(
|
||||
latest,
|
||||
and_(
|
||||
DataflowReconciliationDecision.key_hash == latest.c.key_hash,
|
||||
DataflowReconciliationDecision.revision == latest.c.revision,
|
||||
),
|
||||
)
|
||||
.where(
|
||||
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
||||
)
|
||||
.order_by(DataflowReconciliationDecision.key_hash)
|
||||
.limit(bounded_limit)
|
||||
)
|
||||
records = tuple(session.scalars(statement))
|
||||
total = int(
|
||||
session.scalar(
|
||||
select(func.count(func.distinct(DataflowReconciliationDecision.key_hash))).where(
|
||||
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
||||
)
|
||||
)
|
||||
or 0
|
||||
)
|
||||
return tuple(_decision_row(value) for value in records), total
|
||||
|
||||
|
||||
def decision_set_id_from_ref(value: str) -> str | None:
|
||||
cleaned = str(value or "").strip()
|
||||
if not cleaned.startswith(DECISION_SET_REF_PREFIX):
|
||||
return None
|
||||
identifier = cleaned[len(DECISION_SET_REF_PREFIX) :]
|
||||
return identifier or None
|
||||
|
||||
|
||||
def _decision_row(
|
||||
value: DataflowReconciliationDecision,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"key_hash": value.key_hash,
|
||||
"input_hash": value.input_hash,
|
||||
"decision_ref": f"{DECISION_REF_PREFIX}{value.id}",
|
||||
"action": value.action,
|
||||
"actor_ref": value.actor_ref,
|
||||
"decided_at": value.decided_at.isoformat(),
|
||||
"reason": value.reason,
|
||||
"correction": dict(value.correction) if value.correction else None,
|
||||
}
|
||||
|
||||
|
||||
def _decision_payload(
|
||||
value: DataflowReconciliationDecision,
|
||||
) -> dict[str, object]:
|
||||
return {
|
||||
"ref": f"{DECISION_REF_PREFIX}{value.id}",
|
||||
"id": value.id,
|
||||
"revision": value.revision,
|
||||
"key_hash": value.key_hash,
|
||||
"input_hash": value.input_hash,
|
||||
"action": value.action,
|
||||
"reason": value.reason,
|
||||
"correction": dict(value.correction) if value.correction else None,
|
||||
"actor_ref": value.actor_ref,
|
||||
"decided_at": value.decided_at,
|
||||
}
|
||||
|
||||
|
||||
def _source_name(name: str, identifier: str) -> str:
|
||||
slug = re.sub(r"[^a-z0-9]+", "_", name.casefold()).strip("_")
|
||||
return (f"decisions_{slug}" if slug else f"decisions_{identifier[:8]}")[:120]
|
||||
|
||||
|
||||
__all__: Sequence[str] = (
|
||||
"DECISION_SET_REF_PREFIX",
|
||||
"ReconciliationDecisionConflictError",
|
||||
"ReconciliationDecisionError",
|
||||
"ReconciliationDecisionNotFoundError",
|
||||
"create_decision_set",
|
||||
"current_decision_rows",
|
||||
"current_decisions",
|
||||
"decision_rows",
|
||||
"decision_set_fingerprint",
|
||||
"decision_set_id_from_ref",
|
||||
"decision_set_payload",
|
||||
"decision_set_source_payload",
|
||||
"get_decision_set",
|
||||
"list_decision_sets",
|
||||
"record_decision",
|
||||
)
|
||||
Reference in New Issue
Block a user