meubility-workbench/app/decision_replay.py

from __future__ import annotations

import hashlib
import json
from datetime import datetime, timezone
from typing import Any

from sqlalchemy import or_, select
from sqlalchemy.orm import Session

from app.models import Dataset, GtfsFeedDiffItem, GtfsRoute, GtfsStop, MapGtfsDecision, OsmFeature
from app.workbench import _upsert_review_item


def capture_gtfs_decision_replay(session: Session, previous_dataset: Dataset | None, *, diff_run_id: int | None = None) -> dict[str, Any]:
    if previous_dataset is None or previous_dataset.kind != "gtfs":
        return {"previous_dataset_id": None, "diff_run_id": diff_run_id, "decisions": []}
    route_by_id = {
        int(route.id): route
        for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == previous_dataset.id)).all()
    }
    stop_by_id = {
        int(stop.id): stop
        for stop in session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == previous_dataset.id)).all()
    }
    route_ids = list(route_by_id)
    stop_ids = list(stop_by_id)
    filters = [MapGtfsDecision.gtfs_dataset_id == previous_dataset.id]
    if route_ids:
        filters.append(MapGtfsDecision.gtfs_route_id.in_(route_ids))
    if stop_ids:
        filters.append(MapGtfsDecision.gtfs_stop_id.in_(stop_ids))
    decisions = session.scalars(
        select(MapGtfsDecision)
        .where(MapGtfsDecision.active.is_(True), or_(*filters))
        .order_by(MapGtfsDecision.updated_at.desc(), MapGtfsDecision.id.desc())
    ).all()
    captured: list[dict[str, Any]] = []
    for decision in decisions:
        object_type = None
        object_key = None
        route = route_by_id.get(int(decision.gtfs_route_id or 0))
        stop = stop_by_id.get(int(decision.gtfs_stop_id or 0))
        if route is not None:
            object_type = "route"
            object_key = route.route_id
        elif stop is not None:
            object_type = "stop"
            object_key = stop.stop_id
        if not object_type or not object_key:
            continue
        captured.append(
            {
                "decision_id": decision.id,
                "decision_key": decision.decision_key,
                "decision_type": decision.decision_type,
                "status": decision.status,
                "source_id": decision.source_id,
                "gtfs_dataset_id": decision.gtfs_dataset_id,
                "object_type": object_type,
                "object_key": object_key,
                "selector": _json_object(decision.selector_json),
                "action": _json_object(decision.action_json),
                "note": decision.note,
                "reviewer": decision.reviewer,
                "confidence": decision.confidence,
                "osm_dataset_id": decision.osm_dataset_id,
                "osm_feature_id": decision.osm_feature_id,
                "metadata": _json_object(decision.metadata_json),
            }
        )
    return {
        "previous_dataset_id": previous_dataset.id,
        "source_id": previous_dataset.source_id,
        "diff_run_id": diff_run_id,
        "decisions": captured,
    }


def apply_gtfs_decision_replay(
    session: Session,
    replay_plan: dict[str, Any] | None,
    *,
    new_dataset: Dataset,
    diff_run_id: int | None = None,
) -> dict[str, int]:
    if not replay_plan:
        return {"captured": 0, "replayed": 0, "skipped": 0, "review_items": 0}
    decisions = replay_plan.get("decisions") if isinstance(replay_plan.get("decisions"), list) else []
    if not decisions and diff_run_id is None:
        return {"captured": 0, "replayed": 0, "skipped": 0, "review_items": 0}

    current_diff_run_id = diff_run_id or _optional_int(replay_plan.get("diff_run_id"))
    changed_keys = _changed_or_removed_keys(session, current_diff_run_id)
    routes = {
        route.route_id: route
        for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == new_dataset.id)).all()
    }
    previous_decisions_by_key: dict[tuple[str, str], list[dict[str, Any]]] = {}
    replayed = 0
    skipped = 0
    for decision in decisions:
        object_type = str(decision.get("object_type") or "")
        object_key = str(decision.get("object_key") or "")
        if not object_type or not object_key:
            skipped += 1
            continue
        previous_decisions_by_key.setdefault((object_type, object_key), []).append(decision)
        if (object_type, object_key) in changed_keys:
            skipped += 1
            continue
        if object_type != "route":
            # Stop-level operational replay is handled by durable canonical-stop
            # MatchRule rows during route-layer rebuild. Keep this audit layer
            # route-focused to avoid stale canonical-stop foreign keys.
            skipped += 1
            continue
        route = routes.get(object_key)
        if route is None:
            skipped += 1
            continue
        if _replay_route_decision(session, decision, new_dataset=new_dataset, route=route):
            replayed += 1
        else:
            skipped += 1

    review_items = create_gtfs_update_decision_review_items(
        session,
        diff_run_id=current_diff_run_id,
        new_dataset=new_dataset,
        previous_decisions_by_key=previous_decisions_by_key,
    )
    session.flush()
    return {"captured": len(decisions), "replayed": replayed, "skipped": skipped, "review_items": review_items}


def create_gtfs_update_decision_review_items(
    session: Session,
    *,
    diff_run_id: int | None,
    new_dataset: Dataset,
    previous_decisions_by_key: dict[tuple[str, str], list[dict[str, Any]]] | None = None,
) -> int:
    if diff_run_id is None:
        return 0
    previous_decisions_by_key = previous_decisions_by_key or {}
    routes = {
        route.route_id: route
        for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == new_dataset.id)).all()
    }
    stops = {
        stop.stop_id: stop
        for stop in session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == new_dataset.id)).all()
    }
    items = session.scalars(
        select(GtfsFeedDiffItem)
        .where(GtfsFeedDiffItem.diff_run_id == diff_run_id, GtfsFeedDiffItem.change_type.in_(["changed", "removed"]))
        .order_by(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.change_type, GtfsFeedDiffItem.object_key)
    ).all()
    created_or_updated = 0
    for item in items:
        object_type = "route" if item.item_type == "route" else "stop" if item.item_type == "stop" else str(item.item_type)
        object_key = str(item.object_key)
        previous_decisions = previous_decisions_by_key.get((object_type, object_key), [])
        severity = "error" if previous_decisions and item.change_type == "removed" else "warn"
        route = routes.get(object_key) if object_type == "route" and item.change_type != "removed" else None
        stop = stops.get(object_key) if object_type == "stop" and item.change_type != "removed" else None
        title = _diff_review_title(item, previous_decision_count=len(previous_decisions))
        metadata = {
            "diff_run_id": diff_run_id,
            "item_id": item.id,
            "item_type": item.item_type,
            "object_key": object_key,
            "change_type": item.change_type,
            "old": _json_object(item.old_payload_json),
            "new": _json_object(item.new_payload_json),
            "diff": _json_object(item.diff_json),
            "previous_decisions": previous_decisions,
        }
        result = _upsert_review_item(
            session,
            fingerprint=f"gtfs-update-diff:{diff_run_id}:{item.item_type}:{item.object_key}:{item.change_type}",
            queue="gtfs_update_diff",
            item_type=f"gtfs_{item.item_type}_update",
            severity=severity,
            title=title,
            description=_diff_review_description(item, len(previous_decisions)),
            source_id=new_dataset.source_id,
            dataset_id=new_dataset.id,
            gtfs_route_id=None if route is None else route.id,
            gtfs_stop_id=None if stop is None else stop.id,
            evidence_payload=metadata,
        )
        if result in {"created", "updated", "reopened"}:
            created_or_updated += 1
    return created_or_updated


def _replay_route_decision(session: Session, decision: dict[str, Any], *, new_dataset: Dataset, route: GtfsRoute) -> bool:
    selector = _updated_route_selector(decision.get("selector"), new_dataset=new_dataset, route=route)
    action = decision.get("action") if isinstance(decision.get("action"), dict) else {}
    evidence = {
        "selector": selector,
        "action": action,
        "confidence": decision.get("confidence"),
        "replayed_from_decision_id": decision.get("decision_id"),
    }
    decision_key = _stable_key(str(decision.get("decision_type") or "route_osm_relation"), selector, action)
    existing = session.scalar(select(MapGtfsDecision).where(MapGtfsDecision.decision_key == decision_key))
    now = datetime.now(timezone.utc)
    osm_dataset_id, osm_feature_id = _existing_osm_reference(session, decision)
    metadata = decision.get("metadata") if isinstance(decision.get("metadata"), dict) else {}
    replay_metadata = {
        **metadata,
        "replay": {
            "from_decision_id": decision.get("decision_id"),
            "from_decision_key": decision.get("decision_key"),
            "previous_dataset_id": decision.get("gtfs_dataset_id"),
            "new_dataset_id": new_dataset.id,
            "object_key": route.route_id,
        },
    }
    if existing is None:
        session.add(
            MapGtfsDecision(
                decision_key=decision_key,
                decision_type=str(decision.get("decision_type") or "route_osm_relation"),
                status=str(decision.get("status") or "accepted"),
                active=True,
                source_id=new_dataset.source_id,
                gtfs_dataset_id=new_dataset.id,
                gtfs_route_id=route.id,
                osm_dataset_id=osm_dataset_id,
                osm_feature_id=osm_feature_id,
                route_match_id=None,
                confidence=_optional_float(decision.get("confidence")),
                evidence_hash=_hash_payload(evidence),
                selector_json=_json_dumps(selector),
                action_json=_json_dumps(action),
                note=decision.get("note"),
                reviewer="decision_replay",
                metadata_json=_json_dumps(replay_metadata),
                created_at=now,
                updated_at=now,
            )
        )
        return True
    existing.status = str(decision.get("status") or existing.status)
    existing.active = True
    existing.source_id = new_dataset.source_id
    existing.gtfs_dataset_id = new_dataset.id
    existing.gtfs_route_id = route.id
    existing.osm_dataset_id = osm_dataset_id
    existing.osm_feature_id = osm_feature_id
    existing.route_match_id = None
    existing.confidence = _optional_float(decision.get("confidence"))
    existing.evidence_hash = _hash_payload(evidence)
    existing.selector_json = _json_dumps(selector)
    existing.action_json = _json_dumps(action)
    existing.reviewer = "decision_replay"
    existing.metadata_json = _json_dumps(replay_metadata)
    existing.updated_at = now
    return True


def _updated_route_selector(value: object, *, new_dataset: Dataset, route: GtfsRoute) -> dict[str, Any]:
    selector = dict(value) if isinstance(value, dict) else {}
    selector["gtfs"] = {
        "source_id": new_dataset.source_id,
        "dataset_id": new_dataset.id,
        "route_id": route.route_id,
        "route_key": route.route_key,
        "ref": route.short_name,
        "mode": route.mode,
    }
    selector.pop("route_match_id", None)
    selector.pop("gtfs_route_id", None)
    return selector


def _existing_osm_reference(session: Session, decision: dict[str, Any]) -> tuple[int | None, int | None]:
    dataset_id = _optional_int(decision.get("osm_dataset_id"))
    feature_id = _optional_int(decision.get("osm_feature_id"))
    feature = session.get(OsmFeature, feature_id) if feature_id is not None else None
    if feature is not None:
        return feature.dataset_id, feature.id
    if dataset_id is not None and session.get(Dataset, dataset_id) is not None:
        return dataset_id, None
    return None, None


def _changed_or_removed_keys(session: Session, diff_run_id: int | None) -> set[tuple[str, str]]:
    if diff_run_id is None:
        return set()
    rows = session.execute(
        select(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.object_key)
        .where(GtfsFeedDiffItem.diff_run_id == diff_run_id, GtfsFeedDiffItem.change_type.in_(["changed", "removed"]))
    ).all()
    return {
        ("route" if item_type == "route" else "stop" if item_type == "stop" else str(item_type), str(object_key))
        for item_type, object_key in rows
    }


def _diff_review_title(item: GtfsFeedDiffItem, *, previous_decision_count: int) -> str:
    prefix = "Recheck previous decision for" if previous_decision_count else "Review updated"
    return f"{prefix} {item.item_type} {item.object_key}: {item.change_type}"


def _diff_review_description(item: GtfsFeedDiffItem, previous_decision_count: int) -> str:
    if previous_decision_count:
        return f"{item.item_type.title()} changed after {previous_decision_count} previous decision(s); manual review is needed before replay."
    return f"{item.item_type.title()} was {item.change_type} in the latest feed update."


def _stable_key(decision_type: str, selector: dict[str, Any], action: dict[str, Any]) -> str:
    payload = {"decision_type": decision_type, "selector": selector, "action": action}
    return f"{decision_type}:{_hash_payload(payload)[:40]}"


def _hash_payload(payload: dict[str, Any]) -> str:
    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
    return hashlib.sha256(encoded).hexdigest()


def _json_object(value: str | None) -> dict[str, Any]:
    if not value:
        return {}
    try:
        payload = json.loads(value)
    except json.JSONDecodeError:
        return {}
    return payload if isinstance(payload, dict) else {}


def _json_dumps(value: dict[str, Any]) -> str:
    return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)


def _optional_int(value: object) -> int | None:
    try:
        return None if value is None else int(value)
    except (TypeError, ValueError):
        return None


def _optional_float(value: object) -> float | None:
    try:
        return None if value is None else float(value)
    except (TypeError, ValueError):
        return None