Next alpha stage commit

app/decision_replay.py

@@ -0,0 +1,351 @@
+from __future__ import annotations
+
+import hashlib
+import json
+from datetime import datetime, timezone
+from typing import Any
+
+from sqlalchemy import or_, select
+from sqlalchemy.orm import Session
+
+from app.models import Dataset, GtfsFeedDiffItem, GtfsRoute, GtfsStop, MapGtfsDecision, OsmFeature
+from app.workbench import _upsert_review_item
+
+
+def capture_gtfs_decision_replay(session: Session, previous_dataset: Dataset | None, *, diff_run_id: int | None = None) -> dict[str, Any]:
+    if previous_dataset is None or previous_dataset.kind != "gtfs":
+        return {"previous_dataset_id": None, "diff_run_id": diff_run_id, "decisions": []}
+    route_by_id = {
+        int(route.id): route
+        for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == previous_dataset.id)).all()
+    }
+    stop_by_id = {
+        int(stop.id): stop
+        for stop in session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == previous_dataset.id)).all()
+    }
+    route_ids = list(route_by_id)
+    stop_ids = list(stop_by_id)
+    filters = [MapGtfsDecision.gtfs_dataset_id == previous_dataset.id]
+    if route_ids:
+        filters.append(MapGtfsDecision.gtfs_route_id.in_(route_ids))
+    if stop_ids:
+        filters.append(MapGtfsDecision.gtfs_stop_id.in_(stop_ids))
+    decisions = session.scalars(
+        select(MapGtfsDecision)
+        .where(MapGtfsDecision.active.is_(True), or_(*filters))
+        .order_by(MapGtfsDecision.updated_at.desc(), MapGtfsDecision.id.desc())
+    ).all()
+    captured: list[dict[str, Any]] = []
+    for decision in decisions:
+        object_type = None
+        object_key = None
+        route = route_by_id.get(int(decision.gtfs_route_id or 0))
+        stop = stop_by_id.get(int(decision.gtfs_stop_id or 0))
+        if route is not None:
+            object_type = "route"
+            object_key = route.route_id
+        elif stop is not None:
+            object_type = "stop"
+            object_key = stop.stop_id
+        if not object_type or not object_key:
+            continue
+        captured.append(
+            {
+                "decision_id": decision.id,
+                "decision_key": decision.decision_key,
+                "decision_type": decision.decision_type,
+                "status": decision.status,
+                "source_id": decision.source_id,
+                "gtfs_dataset_id": decision.gtfs_dataset_id,
+                "object_type": object_type,
+                "object_key": object_key,
+                "selector": _json_object(decision.selector_json),
+                "action": _json_object(decision.action_json),
+                "note": decision.note,
+                "reviewer": decision.reviewer,
+                "confidence": decision.confidence,
+                "osm_dataset_id": decision.osm_dataset_id,
+                "osm_feature_id": decision.osm_feature_id,
+                "metadata": _json_object(decision.metadata_json),
+            }
+        )
+    return {
+        "previous_dataset_id": previous_dataset.id,
+        "source_id": previous_dataset.source_id,
+        "diff_run_id": diff_run_id,
+        "decisions": captured,
+    }
+
+
+def apply_gtfs_decision_replay(
+    session: Session,
+    replay_plan: dict[str, Any] | None,
+    *,
+    new_dataset: Dataset,
+    diff_run_id: int | None = None,
+) -> dict[str, int]:
+    if not replay_plan:
+        return {"captured": 0, "replayed": 0, "skipped": 0, "review_items": 0}
+    decisions = replay_plan.get("decisions") if isinstance(replay_plan.get("decisions"), list) else []
+    if not decisions and diff_run_id is None:
+        return {"captured": 0, "replayed": 0, "skipped": 0, "review_items": 0}
+
+    current_diff_run_id = diff_run_id or _optional_int(replay_plan.get("diff_run_id"))
+    changed_keys = _changed_or_removed_keys(session, current_diff_run_id)
+    routes = {
+        route.route_id: route
+        for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == new_dataset.id)).all()
+    }
+    previous_decisions_by_key: dict[tuple[str, str], list[dict[str, Any]]] = {}
+    replayed = 0
+    skipped = 0
+    for decision in decisions:
+        object_type = str(decision.get("object_type") or "")
+        object_key = str(decision.get("object_key") or "")
+        if not object_type or not object_key:
+            skipped += 1
+            continue
+        previous_decisions_by_key.setdefault((object_type, object_key), []).append(decision)
+        if (object_type, object_key) in changed_keys:
+            skipped += 1
+            continue
+        if object_type != "route":
+            # Stop-level operational replay is handled by durable canonical-stop
+            # MatchRule rows during route-layer rebuild. Keep this audit layer
+            # route-focused to avoid stale canonical-stop foreign keys.
+            skipped += 1
+            continue
+        route = routes.get(object_key)
+        if route is None:
+            skipped += 1
+            continue
+        if _replay_route_decision(session, decision, new_dataset=new_dataset, route=route):
+            replayed += 1
+        else:
+            skipped += 1
+
+    review_items = create_gtfs_update_decision_review_items(
+        session,
+        diff_run_id=current_diff_run_id,
+        new_dataset=new_dataset,
+        previous_decisions_by_key=previous_decisions_by_key,
+    )
+    session.flush()
+    return {"captured": len(decisions), "replayed": replayed, "skipped": skipped, "review_items": review_items}
+
+
+def create_gtfs_update_decision_review_items(
+    session: Session,
+    *,
+    diff_run_id: int | None,
+    new_dataset: Dataset,
+    previous_decisions_by_key: dict[tuple[str, str], list[dict[str, Any]]] | None = None,
+) -> int:
+    if diff_run_id is None:
+        return 0
+    previous_decisions_by_key = previous_decisions_by_key or {}
+    routes = {
+        route.route_id: route
+        for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == new_dataset.id)).all()
+    }
+    stops = {
+        stop.stop_id: stop
+        for stop in session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == new_dataset.id)).all()
+    }
+    items = session.scalars(
+        select(GtfsFeedDiffItem)
+        .where(GtfsFeedDiffItem.diff_run_id == diff_run_id, GtfsFeedDiffItem.change_type.in_(["changed", "removed"]))
+        .order_by(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.change_type, GtfsFeedDiffItem.object_key)
+    ).all()
+    created_or_updated = 0
+    for item in items:
+        object_type = "route" if item.item_type == "route" else "stop" if item.item_type == "stop" else str(item.item_type)
+        object_key = str(item.object_key)
+        previous_decisions = previous_decisions_by_key.get((object_type, object_key), [])
+        severity = "error" if previous_decisions and item.change_type == "removed" else "warn"
+        route = routes.get(object_key) if object_type == "route" and item.change_type != "removed" else None
+        stop = stops.get(object_key) if object_type == "stop" and item.change_type != "removed" else None
+        title = _diff_review_title(item, previous_decision_count=len(previous_decisions))
+        metadata = {
+            "diff_run_id": diff_run_id,
+            "item_id": item.id,
+            "item_type": item.item_type,
+            "object_key": object_key,
+            "change_type": item.change_type,
+            "old": _json_object(item.old_payload_json),
+            "new": _json_object(item.new_payload_json),
+            "diff": _json_object(item.diff_json),
+            "previous_decisions": previous_decisions,
+        }
+        result = _upsert_review_item(
+            session,
+            fingerprint=f"gtfs-update-diff:{diff_run_id}:{item.item_type}:{item.object_key}:{item.change_type}",
+            queue="gtfs_update_diff",
+            item_type=f"gtfs_{item.item_type}_update",
+            severity=severity,
+            title=title,
+            description=_diff_review_description(item, len(previous_decisions)),
+            source_id=new_dataset.source_id,
+            dataset_id=new_dataset.id,
+            gtfs_route_id=None if route is None else route.id,
+            gtfs_stop_id=None if stop is None else stop.id,
+            evidence_payload=metadata,
+        )
+        if result in {"created", "updated", "reopened"}:
+            created_or_updated += 1
+    return created_or_updated
+
+
+def _replay_route_decision(session: Session, decision: dict[str, Any], *, new_dataset: Dataset, route: GtfsRoute) -> bool:
+    selector = _updated_route_selector(decision.get("selector"), new_dataset=new_dataset, route=route)
+    action = decision.get("action") if isinstance(decision.get("action"), dict) else {}
+    evidence = {
+        "selector": selector,
+        "action": action,
+        "confidence": decision.get("confidence"),
+        "replayed_from_decision_id": decision.get("decision_id"),
+    }
+    decision_key = _stable_key(str(decision.get("decision_type") or "route_osm_relation"), selector, action)
+    existing = session.scalar(select(MapGtfsDecision).where(MapGtfsDecision.decision_key == decision_key))
+    now = datetime.now(timezone.utc)
+    osm_dataset_id, osm_feature_id = _existing_osm_reference(session, decision)
+    metadata = decision.get("metadata") if isinstance(decision.get("metadata"), dict) else {}
+    replay_metadata = {
+        **metadata,
+        "replay": {
+            "from_decision_id": decision.get("decision_id"),
+            "from_decision_key": decision.get("decision_key"),
+            "previous_dataset_id": decision.get("gtfs_dataset_id"),
+            "new_dataset_id": new_dataset.id,
+            "object_key": route.route_id,
+        },
+    }
+    if existing is None:
+        session.add(
+            MapGtfsDecision(
+                decision_key=decision_key,
+                decision_type=str(decision.get("decision_type") or "route_osm_relation"),
+                status=str(decision.get("status") or "accepted"),
+                active=True,
+                source_id=new_dataset.source_id,
+                gtfs_dataset_id=new_dataset.id,
+                gtfs_route_id=route.id,
+                osm_dataset_id=osm_dataset_id,
+                osm_feature_id=osm_feature_id,
+                route_match_id=None,
+                confidence=_optional_float(decision.get("confidence")),
+                evidence_hash=_hash_payload(evidence),
+                selector_json=_json_dumps(selector),
+                action_json=_json_dumps(action),
+                note=decision.get("note"),
+                reviewer="decision_replay",
+                metadata_json=_json_dumps(replay_metadata),
+                created_at=now,
+                updated_at=now,
+            )
+        )
+        return True
+    existing.status = str(decision.get("status") or existing.status)
+    existing.active = True
+    existing.source_id = new_dataset.source_id
+    existing.gtfs_dataset_id = new_dataset.id
+    existing.gtfs_route_id = route.id
+    existing.osm_dataset_id = osm_dataset_id
+    existing.osm_feature_id = osm_feature_id
+    existing.route_match_id = None
+    existing.confidence = _optional_float(decision.get("confidence"))
+    existing.evidence_hash = _hash_payload(evidence)
+    existing.selector_json = _json_dumps(selector)
+    existing.action_json = _json_dumps(action)
+    existing.reviewer = "decision_replay"
+    existing.metadata_json = _json_dumps(replay_metadata)
+    existing.updated_at = now
+    return True
+
+
+def _updated_route_selector(value: object, *, new_dataset: Dataset, route: GtfsRoute) -> dict[str, Any]:
+    selector = dict(value) if isinstance(value, dict) else {}
+    selector["gtfs"] = {
+        "source_id": new_dataset.source_id,
+        "dataset_id": new_dataset.id,
+        "route_id": route.route_id,
+        "route_key": route.route_key,
+        "ref": route.short_name,
+        "mode": route.mode,
+    }
+    selector.pop("route_match_id", None)
+    selector.pop("gtfs_route_id", None)
+    return selector
+
+
+def _existing_osm_reference(session: Session, decision: dict[str, Any]) -> tuple[int | None, int | None]:
+    dataset_id = _optional_int(decision.get("osm_dataset_id"))
+    feature_id = _optional_int(decision.get("osm_feature_id"))
+    feature = session.get(OsmFeature, feature_id) if feature_id is not None else None
+    if feature is not None:
+        return feature.dataset_id, feature.id
+    if dataset_id is not None and session.get(Dataset, dataset_id) is not None:
+        return dataset_id, None
+    return None, None
+
+
+def _changed_or_removed_keys(session: Session, diff_run_id: int | None) -> set[tuple[str, str]]:
+    if diff_run_id is None:
+        return set()
+    rows = session.execute(
+        select(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.object_key)
+        .where(GtfsFeedDiffItem.diff_run_id == diff_run_id, GtfsFeedDiffItem.change_type.in_(["changed", "removed"]))
+    ).all()
+    return {
+        ("route" if item_type == "route" else "stop" if item_type == "stop" else str(item_type), str(object_key))
+        for item_type, object_key in rows
+    }
+
+
+def _diff_review_title(item: GtfsFeedDiffItem, *, previous_decision_count: int) -> str:
+    prefix = "Recheck previous decision for" if previous_decision_count else "Review updated"
+    return f"{prefix} {item.item_type} {item.object_key}: {item.change_type}"
+
+
+def _diff_review_description(item: GtfsFeedDiffItem, previous_decision_count: int) -> str:
+    if previous_decision_count:
+        return f"{item.item_type.title()} changed after {previous_decision_count} previous decision(s); manual review is needed before replay."
+    return f"{item.item_type.title()} was {item.change_type} in the latest feed update."
+
+
+def _stable_key(decision_type: str, selector: dict[str, Any], action: dict[str, Any]) -> str:
+    payload = {"decision_type": decision_type, "selector": selector, "action": action}
+    return f"{decision_type}:{_hash_payload(payload)[:40]}"
+
+
+def _hash_payload(payload: dict[str, Any]) -> str:
+    encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
+    return hashlib.sha256(encoded).hexdigest()
+
+
+def _json_object(value: str | None) -> dict[str, Any]:
+    if not value:
+        return {}
+    try:
+        payload = json.loads(value)
+    except json.JSONDecodeError:
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def _json_dumps(value: dict[str, Any]) -> str:
+    return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
+
+
+def _optional_int(value: object) -> int | None:
+    try:
+        return None if value is None else int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _optional_float(value: object) -> float | None:
+    try:
+        return None if value is None else float(value)
+    except (TypeError, ValueError):
+        return None