from __future__ import annotations import hashlib import json import re from collections.abc import Sequence from sqlalchemy import and_, func, select from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, joinedload, selectinload from govoplan_core.core.concurrency import claim_revision from govoplan_core.security.time import utc_now from govoplan_dataflow.backend.db.models import ( DataflowPipeline, DataflowReconciliationDecision, DataflowReconciliationDecisionSet, ) DECISION_SET_REF_PREFIX = "dataflow-decision-set:" DECISION_REF_PREFIX = "dataflow-decision:" DECISION_ACTIONS = frozenset({"accept", "reject", "correct", "defer"}) class ReconciliationDecisionError(ValueError): pass class ReconciliationDecisionNotFoundError(ReconciliationDecisionError): pass class ReconciliationDecisionConflictError(ReconciliationDecisionError): pass def list_decision_sets( session: Session, *, tenant_id: str, pipeline_id: str | None = None, include_decisions: bool = True, ) -> tuple[DataflowReconciliationDecisionSet, ...]: options = [joinedload(DataflowReconciliationDecisionSet.pipeline)] if include_decisions: options.append(selectinload(DataflowReconciliationDecisionSet.decisions)) statement = ( select(DataflowReconciliationDecisionSet) .options(*options) .where(DataflowReconciliationDecisionSet.tenant_id == tenant_id) .order_by( DataflowReconciliationDecisionSet.name, DataflowReconciliationDecisionSet.id, ) ) if pipeline_id: statement = statement.where( DataflowReconciliationDecisionSet.pipeline_id == pipeline_id ) return tuple(session.scalars(statement).all()) def create_decision_set( session: Session, *, tenant_id: str, pipeline_id: str, name: str, node_id: str | None, actor_ref: str, ) -> DataflowReconciliationDecisionSet: pipeline = session.get(DataflowPipeline, pipeline_id) if ( pipeline is None or pipeline.tenant_id not in {None, tenant_id} or pipeline.deleted_at is not None ): raise ReconciliationDecisionNotFoundError("Dataflow pipeline not found.") item = DataflowReconciliationDecisionSet( tenant_id=tenant_id, pipeline_id=pipeline_id, name=name.strip(), node_id=node_id, resource_revision=1, created_by=actor_ref, updated_by=actor_ref, ) session.add(item) try: session.flush() except IntegrityError as exc: raise ReconciliationDecisionConflictError( "A decision set with this name already exists for the pipeline." ) from exc return item def get_decision_set( session: Session, *, tenant_id: str, decision_set_id: str, include_decisions: bool = True, ) -> DataflowReconciliationDecisionSet: options = [joinedload(DataflowReconciliationDecisionSet.pipeline)] if include_decisions: options.append(selectinload(DataflowReconciliationDecisionSet.decisions)) item = session.scalar( select(DataflowReconciliationDecisionSet) .options(*options) .where( DataflowReconciliationDecisionSet.id == decision_set_id, DataflowReconciliationDecisionSet.tenant_id == tenant_id, ) ) if item is None: raise ReconciliationDecisionNotFoundError( "Reconciliation decision set not found." ) return item def record_decision( session: Session, *, tenant_id: str, decision_set_id: str, expected_revision: int, key_hash: str, input_hash: str, action: str, reason: str, correction: dict[str, object] | None, actor_ref: str, ) -> DataflowReconciliationDecisionSet: item = get_decision_set( session, tenant_id=tenant_id, decision_set_id=decision_set_id, ) if action not in DECISION_ACTIONS: raise ReconciliationDecisionError("Unsupported reconciliation action.") next_revision = claim_revision( session, model=DataflowReconciliationDecisionSet, filters=( DataflowReconciliationDecisionSet.id == decision_set_id, DataflowReconciliationDecisionSet.tenant_id == tenant_id, ), revision_attribute="resource_revision", expected_revision=expected_revision, resource_type="dataflow_reconciliation_decision_set", resource_id=decision_set_id, ) item.resource_revision = next_revision item.updated_by = actor_ref item.decisions.append( DataflowReconciliationDecision( tenant_id=tenant_id, revision=next_revision, key_hash=key_hash, input_hash=input_hash, action=action, reason=reason.strip(), correction=dict(correction) if correction else None, actor_ref=actor_ref, decided_at=utc_now(), ) ) session.flush() return item def current_decisions( item: DataflowReconciliationDecisionSet, ) -> tuple[DataflowReconciliationDecision, ...]: current: dict[str, DataflowReconciliationDecision] = {} for decision in sorted(item.decisions, key=lambda value: value.revision): current[decision.key_hash] = decision return tuple( sorted( current.values(), key=lambda value: (value.key_hash, value.revision), ) ) def decision_rows( item: DataflowReconciliationDecisionSet, ) -> tuple[dict[str, object], ...]: return tuple(_decision_row(value) for value in current_decisions(item)) def decision_set_fingerprint( item: DataflowReconciliationDecisionSet, ) -> str: payload = { "decision_set_id": item.id, "resource_revision": item.resource_revision, } return "sha256:" + hashlib.sha256( json.dumps( payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True, ).encode("utf-8") ).hexdigest() def decision_set_payload( item: DataflowReconciliationDecisionSet, *, include_decisions: bool = True, ) -> dict[str, object]: current = current_decisions(item) if include_decisions else () history = ( sorted(item.decisions, key=lambda value: value.revision) if include_decisions else () ) return { "ref": f"{DECISION_SET_REF_PREFIX}{item.id}", "id": item.id, "pipeline_id": item.pipeline_id, "name": item.name, "node_id": item.node_id, "resource_revision": item.resource_revision, "etag": item.strong_etag, "fingerprint": decision_set_fingerprint(item), "decisions_included": include_decisions, "current_decisions": [_decision_payload(value) for value in current], "history": [ _decision_payload(value) for value in history ], "created_by": item.created_by, "updated_by": item.updated_by, "created_at": item.created_at, "updated_at": item.updated_at, } def decision_set_source_payload( item: DataflowReconciliationDecisionSet, ) -> dict[str, object]: return { "ref": f"{DECISION_SET_REF_PREFIX}{item.id}", "provider": "dataflow.reconciliation_decisions", "source_name": _source_name(item.name, item.id), "name": item.name, "description": "Current immutable reconciliation decisions; prior revisions remain in decision history.", "mode": "static", "columns": [ {"name": "key_hash", "data_type": "string", "nullable": False}, {"name": "input_hash", "data_type": "string", "nullable": False}, {"name": "decision_ref", "data_type": "string", "nullable": False}, {"name": "action", "data_type": "string", "nullable": False}, {"name": "actor_ref", "data_type": "string", "nullable": False}, {"name": "decided_at", "data_type": "datetime", "nullable": False}, {"name": "reason", "data_type": "string", "nullable": False}, {"name": "correction", "data_type": "object", "nullable": True}, ], "schema_version": "1", "fingerprint": decision_set_fingerprint(item), "row_count": None, "byte_count": None, "updated_at": item.updated_at, "capabilities": ["preview", "read", "immutable_history"], } def current_decision_rows( session: Session, *, decision_set_id: str, limit: int, ) -> tuple[tuple[dict[str, object], ...], int]: bounded_limit = max(1, limit) latest = ( select( DataflowReconciliationDecision.key_hash.label("key_hash"), func.max(DataflowReconciliationDecision.revision).label("revision"), ) .where( DataflowReconciliationDecision.decision_set_id == decision_set_id ) .group_by(DataflowReconciliationDecision.key_hash) .subquery() ) statement = ( select(DataflowReconciliationDecision) .join( latest, and_( DataflowReconciliationDecision.key_hash == latest.c.key_hash, DataflowReconciliationDecision.revision == latest.c.revision, ), ) .where( DataflowReconciliationDecision.decision_set_id == decision_set_id ) .order_by(DataflowReconciliationDecision.key_hash) .limit(bounded_limit) ) records = tuple(session.scalars(statement)) total = int( session.scalar( select(func.count(func.distinct(DataflowReconciliationDecision.key_hash))).where( DataflowReconciliationDecision.decision_set_id == decision_set_id ) ) or 0 ) return tuple(_decision_row(value) for value in records), total def decision_set_id_from_ref(value: str) -> str | None: cleaned = str(value or "").strip() if not cleaned.startswith(DECISION_SET_REF_PREFIX): return None identifier = cleaned[len(DECISION_SET_REF_PREFIX) :] return identifier or None def _decision_row( value: DataflowReconciliationDecision, ) -> dict[str, object]: return { "key_hash": value.key_hash, "input_hash": value.input_hash, "decision_ref": f"{DECISION_REF_PREFIX}{value.id}", "action": value.action, "actor_ref": value.actor_ref, "decided_at": value.decided_at.isoformat(), "reason": value.reason, "correction": dict(value.correction) if value.correction else None, } def _decision_payload( value: DataflowReconciliationDecision, ) -> dict[str, object]: return { "ref": f"{DECISION_REF_PREFIX}{value.id}", "id": value.id, "revision": value.revision, "key_hash": value.key_hash, "input_hash": value.input_hash, "action": value.action, "reason": value.reason, "correction": dict(value.correction) if value.correction else None, "actor_ref": value.actor_ref, "decided_at": value.decided_at, } def _source_name(name: str, identifier: str) -> str: slug = re.sub(r"[^a-z0-9]+", "_", name.casefold()).strip("_") return (f"decisions_{slug}" if slug else f"decisions_{identifier[:8]}")[:120] __all__: Sequence[str] = ( "DECISION_SET_REF_PREFIX", "ReconciliationDecisionConflictError", "ReconciliationDecisionError", "ReconciliationDecisionNotFoundError", "create_decision_set", "current_decision_rows", "current_decisions", "decision_rows", "decision_set_fingerprint", "decision_set_id_from_ref", "decision_set_payload", "decision_set_source_payload", "get_decision_set", "list_decision_sets", "record_decision", )