381 lines
12 KiB
Python
381 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from collections.abc import Sequence
|
|
|
|
from sqlalchemy import and_, func, select
|
|
from sqlalchemy.exc import IntegrityError
|
|
from sqlalchemy.orm import Session, joinedload, selectinload
|
|
|
|
from govoplan_core.core.concurrency import claim_revision
|
|
from govoplan_core.security.time import utc_now
|
|
from govoplan_dataflow.backend.db.models import (
|
|
DataflowPipeline,
|
|
DataflowReconciliationDecision,
|
|
DataflowReconciliationDecisionSet,
|
|
)
|
|
|
|
|
|
DECISION_SET_REF_PREFIX = "dataflow-decision-set:"
|
|
DECISION_REF_PREFIX = "dataflow-decision:"
|
|
DECISION_ACTIONS = frozenset({"accept", "reject", "correct", "defer"})
|
|
|
|
|
|
class ReconciliationDecisionError(ValueError):
|
|
pass
|
|
|
|
|
|
class ReconciliationDecisionNotFoundError(ReconciliationDecisionError):
|
|
pass
|
|
|
|
|
|
class ReconciliationDecisionConflictError(ReconciliationDecisionError):
|
|
pass
|
|
|
|
|
|
def list_decision_sets(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
pipeline_id: str | None = None,
|
|
include_decisions: bool = True,
|
|
) -> tuple[DataflowReconciliationDecisionSet, ...]:
|
|
options = [joinedload(DataflowReconciliationDecisionSet.pipeline)]
|
|
if include_decisions:
|
|
options.append(selectinload(DataflowReconciliationDecisionSet.decisions))
|
|
statement = (
|
|
select(DataflowReconciliationDecisionSet)
|
|
.options(*options)
|
|
.where(DataflowReconciliationDecisionSet.tenant_id == tenant_id)
|
|
.order_by(
|
|
DataflowReconciliationDecisionSet.name,
|
|
DataflowReconciliationDecisionSet.id,
|
|
)
|
|
)
|
|
if pipeline_id:
|
|
statement = statement.where(
|
|
DataflowReconciliationDecisionSet.pipeline_id == pipeline_id
|
|
)
|
|
return tuple(session.scalars(statement).all())
|
|
|
|
|
|
def create_decision_set(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
pipeline_id: str,
|
|
name: str,
|
|
node_id: str | None,
|
|
actor_ref: str,
|
|
) -> DataflowReconciliationDecisionSet:
|
|
pipeline = session.get(DataflowPipeline, pipeline_id)
|
|
if (
|
|
pipeline is None
|
|
or pipeline.tenant_id not in {None, tenant_id}
|
|
or pipeline.deleted_at is not None
|
|
):
|
|
raise ReconciliationDecisionNotFoundError("Dataflow pipeline not found.")
|
|
item = DataflowReconciliationDecisionSet(
|
|
tenant_id=tenant_id,
|
|
pipeline_id=pipeline_id,
|
|
name=name.strip(),
|
|
node_id=node_id,
|
|
resource_revision=1,
|
|
created_by=actor_ref,
|
|
updated_by=actor_ref,
|
|
)
|
|
session.add(item)
|
|
try:
|
|
session.flush()
|
|
except IntegrityError as exc:
|
|
raise ReconciliationDecisionConflictError(
|
|
"A decision set with this name already exists for the pipeline."
|
|
) from exc
|
|
return item
|
|
|
|
|
|
def get_decision_set(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
decision_set_id: str,
|
|
include_decisions: bool = True,
|
|
) -> DataflowReconciliationDecisionSet:
|
|
options = [joinedload(DataflowReconciliationDecisionSet.pipeline)]
|
|
if include_decisions:
|
|
options.append(selectinload(DataflowReconciliationDecisionSet.decisions))
|
|
item = session.scalar(
|
|
select(DataflowReconciliationDecisionSet)
|
|
.options(*options)
|
|
.where(
|
|
DataflowReconciliationDecisionSet.id == decision_set_id,
|
|
DataflowReconciliationDecisionSet.tenant_id == tenant_id,
|
|
)
|
|
)
|
|
if item is None:
|
|
raise ReconciliationDecisionNotFoundError(
|
|
"Reconciliation decision set not found."
|
|
)
|
|
return item
|
|
|
|
|
|
def record_decision(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
decision_set_id: str,
|
|
expected_revision: int,
|
|
key_hash: str,
|
|
input_hash: str,
|
|
action: str,
|
|
reason: str,
|
|
correction: dict[str, object] | None,
|
|
actor_ref: str,
|
|
) -> DataflowReconciliationDecisionSet:
|
|
item = get_decision_set(
|
|
session,
|
|
tenant_id=tenant_id,
|
|
decision_set_id=decision_set_id,
|
|
)
|
|
if action not in DECISION_ACTIONS:
|
|
raise ReconciliationDecisionError("Unsupported reconciliation action.")
|
|
next_revision = claim_revision(
|
|
session,
|
|
model=DataflowReconciliationDecisionSet,
|
|
filters=(
|
|
DataflowReconciliationDecisionSet.id == decision_set_id,
|
|
DataflowReconciliationDecisionSet.tenant_id == tenant_id,
|
|
),
|
|
revision_attribute="resource_revision",
|
|
expected_revision=expected_revision,
|
|
resource_type="dataflow_reconciliation_decision_set",
|
|
resource_id=decision_set_id,
|
|
)
|
|
item.resource_revision = next_revision
|
|
item.updated_by = actor_ref
|
|
item.decisions.append(
|
|
DataflowReconciliationDecision(
|
|
tenant_id=tenant_id,
|
|
revision=next_revision,
|
|
key_hash=key_hash,
|
|
input_hash=input_hash,
|
|
action=action,
|
|
reason=reason.strip(),
|
|
correction=dict(correction) if correction else None,
|
|
actor_ref=actor_ref,
|
|
decided_at=utc_now(),
|
|
)
|
|
)
|
|
session.flush()
|
|
return item
|
|
|
|
|
|
def current_decisions(
|
|
item: DataflowReconciliationDecisionSet,
|
|
) -> tuple[DataflowReconciliationDecision, ...]:
|
|
current: dict[str, DataflowReconciliationDecision] = {}
|
|
for decision in sorted(item.decisions, key=lambda value: value.revision):
|
|
current[decision.key_hash] = decision
|
|
return tuple(
|
|
sorted(
|
|
current.values(),
|
|
key=lambda value: (value.key_hash, value.revision),
|
|
)
|
|
)
|
|
|
|
|
|
def decision_rows(
|
|
item: DataflowReconciliationDecisionSet,
|
|
) -> tuple[dict[str, object], ...]:
|
|
return tuple(_decision_row(value) for value in current_decisions(item))
|
|
|
|
|
|
def decision_set_fingerprint(
|
|
item: DataflowReconciliationDecisionSet,
|
|
) -> str:
|
|
payload = {
|
|
"decision_set_id": item.id,
|
|
"resource_revision": item.resource_revision,
|
|
}
|
|
return "sha256:" + hashlib.sha256(
|
|
json.dumps(
|
|
payload,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
ensure_ascii=True,
|
|
).encode("utf-8")
|
|
).hexdigest()
|
|
|
|
|
|
def decision_set_payload(
|
|
item: DataflowReconciliationDecisionSet,
|
|
*,
|
|
include_decisions: bool = True,
|
|
) -> dict[str, object]:
|
|
current = current_decisions(item) if include_decisions else ()
|
|
history = (
|
|
sorted(item.decisions, key=lambda value: value.revision)
|
|
if include_decisions
|
|
else ()
|
|
)
|
|
return {
|
|
"ref": f"{DECISION_SET_REF_PREFIX}{item.id}",
|
|
"id": item.id,
|
|
"pipeline_id": item.pipeline_id,
|
|
"name": item.name,
|
|
"node_id": item.node_id,
|
|
"resource_revision": item.resource_revision,
|
|
"etag": item.strong_etag,
|
|
"fingerprint": decision_set_fingerprint(item),
|
|
"decisions_included": include_decisions,
|
|
"current_decisions": [_decision_payload(value) for value in current],
|
|
"history": [
|
|
_decision_payload(value)
|
|
for value in history
|
|
],
|
|
"created_by": item.created_by,
|
|
"updated_by": item.updated_by,
|
|
"created_at": item.created_at,
|
|
"updated_at": item.updated_at,
|
|
}
|
|
|
|
|
|
def decision_set_source_payload(
|
|
item: DataflowReconciliationDecisionSet,
|
|
) -> dict[str, object]:
|
|
return {
|
|
"ref": f"{DECISION_SET_REF_PREFIX}{item.id}",
|
|
"provider": "dataflow.reconciliation_decisions",
|
|
"source_name": _source_name(item.name, item.id),
|
|
"name": item.name,
|
|
"description": "Current immutable reconciliation decisions; prior revisions remain in decision history.",
|
|
"mode": "static",
|
|
"columns": [
|
|
{"name": "key_hash", "data_type": "string", "nullable": False},
|
|
{"name": "input_hash", "data_type": "string", "nullable": False},
|
|
{"name": "decision_ref", "data_type": "string", "nullable": False},
|
|
{"name": "action", "data_type": "string", "nullable": False},
|
|
{"name": "actor_ref", "data_type": "string", "nullable": False},
|
|
{"name": "decided_at", "data_type": "datetime", "nullable": False},
|
|
{"name": "reason", "data_type": "string", "nullable": False},
|
|
{"name": "correction", "data_type": "object", "nullable": True},
|
|
],
|
|
"schema_version": "1",
|
|
"fingerprint": decision_set_fingerprint(item),
|
|
"row_count": None,
|
|
"byte_count": None,
|
|
"updated_at": item.updated_at,
|
|
"capabilities": ["preview", "read", "immutable_history"],
|
|
}
|
|
|
|
|
|
def current_decision_rows(
|
|
session: Session,
|
|
*,
|
|
decision_set_id: str,
|
|
limit: int,
|
|
) -> tuple[tuple[dict[str, object], ...], int]:
|
|
bounded_limit = max(1, limit)
|
|
latest = (
|
|
select(
|
|
DataflowReconciliationDecision.key_hash.label("key_hash"),
|
|
func.max(DataflowReconciliationDecision.revision).label("revision"),
|
|
)
|
|
.where(
|
|
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
|
)
|
|
.group_by(DataflowReconciliationDecision.key_hash)
|
|
.subquery()
|
|
)
|
|
statement = (
|
|
select(DataflowReconciliationDecision)
|
|
.join(
|
|
latest,
|
|
and_(
|
|
DataflowReconciliationDecision.key_hash == latest.c.key_hash,
|
|
DataflowReconciliationDecision.revision == latest.c.revision,
|
|
),
|
|
)
|
|
.where(
|
|
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
|
)
|
|
.order_by(DataflowReconciliationDecision.key_hash)
|
|
.limit(bounded_limit)
|
|
)
|
|
records = tuple(session.scalars(statement))
|
|
total = int(
|
|
session.scalar(
|
|
select(func.count(func.distinct(DataflowReconciliationDecision.key_hash))).where(
|
|
DataflowReconciliationDecision.decision_set_id == decision_set_id
|
|
)
|
|
)
|
|
or 0
|
|
)
|
|
return tuple(_decision_row(value) for value in records), total
|
|
|
|
|
|
def decision_set_id_from_ref(value: str) -> str | None:
|
|
cleaned = str(value or "").strip()
|
|
if not cleaned.startswith(DECISION_SET_REF_PREFIX):
|
|
return None
|
|
identifier = cleaned[len(DECISION_SET_REF_PREFIX) :]
|
|
return identifier or None
|
|
|
|
|
|
def _decision_row(
|
|
value: DataflowReconciliationDecision,
|
|
) -> dict[str, object]:
|
|
return {
|
|
"key_hash": value.key_hash,
|
|
"input_hash": value.input_hash,
|
|
"decision_ref": f"{DECISION_REF_PREFIX}{value.id}",
|
|
"action": value.action,
|
|
"actor_ref": value.actor_ref,
|
|
"decided_at": value.decided_at.isoformat(),
|
|
"reason": value.reason,
|
|
"correction": dict(value.correction) if value.correction else None,
|
|
}
|
|
|
|
|
|
def _decision_payload(
|
|
value: DataflowReconciliationDecision,
|
|
) -> dict[str, object]:
|
|
return {
|
|
"ref": f"{DECISION_REF_PREFIX}{value.id}",
|
|
"id": value.id,
|
|
"revision": value.revision,
|
|
"key_hash": value.key_hash,
|
|
"input_hash": value.input_hash,
|
|
"action": value.action,
|
|
"reason": value.reason,
|
|
"correction": dict(value.correction) if value.correction else None,
|
|
"actor_ref": value.actor_ref,
|
|
"decided_at": value.decided_at,
|
|
}
|
|
|
|
|
|
def _source_name(name: str, identifier: str) -> str:
|
|
slug = re.sub(r"[^a-z0-9]+", "_", name.casefold()).strip("_")
|
|
return (f"decisions_{slug}" if slug else f"decisions_{identifier[:8]}")[:120]
|
|
|
|
|
|
__all__: Sequence[str] = (
|
|
"DECISION_SET_REF_PREFIX",
|
|
"ReconciliationDecisionConflictError",
|
|
"ReconciliationDecisionError",
|
|
"ReconciliationDecisionNotFoundError",
|
|
"create_decision_set",
|
|
"current_decision_rows",
|
|
"current_decisions",
|
|
"decision_rows",
|
|
"decision_set_fingerprint",
|
|
"decision_set_id_from_ref",
|
|
"decision_set_payload",
|
|
"decision_set_source_payload",
|
|
"get_decision_set",
|
|
"list_decision_sets",
|
|
"record_decision",
|
|
)
|