from __future__ import annotations import hashlib import json from datetime import datetime, timezone from typing import Any from sqlalchemy import or_, select from sqlalchemy.orm import Session from app.models import Dataset, GtfsFeedDiffItem, GtfsRoute, GtfsStop, MapGtfsDecision, OsmFeature from app.workbench import _upsert_review_item def capture_gtfs_decision_replay(session: Session, previous_dataset: Dataset | None, *, diff_run_id: int | None = None) -> dict[str, Any]: if previous_dataset is None or previous_dataset.kind != "gtfs": return {"previous_dataset_id": None, "diff_run_id": diff_run_id, "decisions": []} route_by_id = { int(route.id): route for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == previous_dataset.id)).all() } stop_by_id = { int(stop.id): stop for stop in session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == previous_dataset.id)).all() } route_ids = list(route_by_id) stop_ids = list(stop_by_id) filters = [MapGtfsDecision.gtfs_dataset_id == previous_dataset.id] if route_ids: filters.append(MapGtfsDecision.gtfs_route_id.in_(route_ids)) if stop_ids: filters.append(MapGtfsDecision.gtfs_stop_id.in_(stop_ids)) decisions = session.scalars( select(MapGtfsDecision) .where(MapGtfsDecision.active.is_(True), or_(*filters)) .order_by(MapGtfsDecision.updated_at.desc(), MapGtfsDecision.id.desc()) ).all() captured: list[dict[str, Any]] = [] for decision in decisions: object_type = None object_key = None route = route_by_id.get(int(decision.gtfs_route_id or 0)) stop = stop_by_id.get(int(decision.gtfs_stop_id or 0)) if route is not None: object_type = "route" object_key = route.route_id elif stop is not None: object_type = "stop" object_key = stop.stop_id if not object_type or not object_key: continue captured.append( { "decision_id": decision.id, "decision_key": decision.decision_key, "decision_type": decision.decision_type, "status": decision.status, "source_id": decision.source_id, "gtfs_dataset_id": decision.gtfs_dataset_id, "object_type": object_type, "object_key": object_key, "selector": _json_object(decision.selector_json), "action": _json_object(decision.action_json), "note": decision.note, "reviewer": decision.reviewer, "confidence": decision.confidence, "osm_dataset_id": decision.osm_dataset_id, "osm_feature_id": decision.osm_feature_id, "metadata": _json_object(decision.metadata_json), } ) return { "previous_dataset_id": previous_dataset.id, "source_id": previous_dataset.source_id, "diff_run_id": diff_run_id, "decisions": captured, } def apply_gtfs_decision_replay( session: Session, replay_plan: dict[str, Any] | None, *, new_dataset: Dataset, diff_run_id: int | None = None, ) -> dict[str, int]: if not replay_plan: return {"captured": 0, "replayed": 0, "skipped": 0, "review_items": 0} decisions = replay_plan.get("decisions") if isinstance(replay_plan.get("decisions"), list) else [] if not decisions and diff_run_id is None: return {"captured": 0, "replayed": 0, "skipped": 0, "review_items": 0} current_diff_run_id = diff_run_id or _optional_int(replay_plan.get("diff_run_id")) changed_keys = _changed_or_removed_keys(session, current_diff_run_id) routes = { route.route_id: route for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == new_dataset.id)).all() } previous_decisions_by_key: dict[tuple[str, str], list[dict[str, Any]]] = {} replayed = 0 skipped = 0 for decision in decisions: object_type = str(decision.get("object_type") or "") object_key = str(decision.get("object_key") or "") if not object_type or not object_key: skipped += 1 continue previous_decisions_by_key.setdefault((object_type, object_key), []).append(decision) if (object_type, object_key) in changed_keys: skipped += 1 continue if object_type != "route": # Stop-level operational replay is handled by durable canonical-stop # MatchRule rows during route-layer rebuild. Keep this audit layer # route-focused to avoid stale canonical-stop foreign keys. skipped += 1 continue route = routes.get(object_key) if route is None: skipped += 1 continue if _replay_route_decision(session, decision, new_dataset=new_dataset, route=route): replayed += 1 else: skipped += 1 review_items = create_gtfs_update_decision_review_items( session, diff_run_id=current_diff_run_id, new_dataset=new_dataset, previous_decisions_by_key=previous_decisions_by_key, ) session.flush() return {"captured": len(decisions), "replayed": replayed, "skipped": skipped, "review_items": review_items} def create_gtfs_update_decision_review_items( session: Session, *, diff_run_id: int | None, new_dataset: Dataset, previous_decisions_by_key: dict[tuple[str, str], list[dict[str, Any]]] | None = None, ) -> int: if diff_run_id is None: return 0 previous_decisions_by_key = previous_decisions_by_key or {} routes = { route.route_id: route for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == new_dataset.id)).all() } stops = { stop.stop_id: stop for stop in session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == new_dataset.id)).all() } items = session.scalars( select(GtfsFeedDiffItem) .where(GtfsFeedDiffItem.diff_run_id == diff_run_id, GtfsFeedDiffItem.change_type.in_(["changed", "removed"])) .order_by(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.change_type, GtfsFeedDiffItem.object_key) ).all() created_or_updated = 0 for item in items: object_type = "route" if item.item_type == "route" else "stop" if item.item_type == "stop" else str(item.item_type) object_key = str(item.object_key) previous_decisions = previous_decisions_by_key.get((object_type, object_key), []) severity = "error" if previous_decisions and item.change_type == "removed" else "warn" route = routes.get(object_key) if object_type == "route" and item.change_type != "removed" else None stop = stops.get(object_key) if object_type == "stop" and item.change_type != "removed" else None title = _diff_review_title(item, previous_decision_count=len(previous_decisions)) metadata = { "diff_run_id": diff_run_id, "item_id": item.id, "item_type": item.item_type, "object_key": object_key, "change_type": item.change_type, "old": _json_object(item.old_payload_json), "new": _json_object(item.new_payload_json), "diff": _json_object(item.diff_json), "previous_decisions": previous_decisions, } result = _upsert_review_item( session, fingerprint=f"gtfs-update-diff:{diff_run_id}:{item.item_type}:{item.object_key}:{item.change_type}", queue="gtfs_update_diff", item_type=f"gtfs_{item.item_type}_update", severity=severity, title=title, description=_diff_review_description(item, len(previous_decisions)), source_id=new_dataset.source_id, dataset_id=new_dataset.id, gtfs_route_id=None if route is None else route.id, gtfs_stop_id=None if stop is None else stop.id, evidence_payload=metadata, ) if result in {"created", "updated", "reopened"}: created_or_updated += 1 return created_or_updated def _replay_route_decision(session: Session, decision: dict[str, Any], *, new_dataset: Dataset, route: GtfsRoute) -> bool: selector = _updated_route_selector(decision.get("selector"), new_dataset=new_dataset, route=route) action = decision.get("action") if isinstance(decision.get("action"), dict) else {} evidence = { "selector": selector, "action": action, "confidence": decision.get("confidence"), "replayed_from_decision_id": decision.get("decision_id"), } decision_key = _stable_key(str(decision.get("decision_type") or "route_osm_relation"), selector, action) existing = session.scalar(select(MapGtfsDecision).where(MapGtfsDecision.decision_key == decision_key)) now = datetime.now(timezone.utc) osm_dataset_id, osm_feature_id = _existing_osm_reference(session, decision) metadata = decision.get("metadata") if isinstance(decision.get("metadata"), dict) else {} replay_metadata = { **metadata, "replay": { "from_decision_id": decision.get("decision_id"), "from_decision_key": decision.get("decision_key"), "previous_dataset_id": decision.get("gtfs_dataset_id"), "new_dataset_id": new_dataset.id, "object_key": route.route_id, }, } if existing is None: session.add( MapGtfsDecision( decision_key=decision_key, decision_type=str(decision.get("decision_type") or "route_osm_relation"), status=str(decision.get("status") or "accepted"), active=True, source_id=new_dataset.source_id, gtfs_dataset_id=new_dataset.id, gtfs_route_id=route.id, osm_dataset_id=osm_dataset_id, osm_feature_id=osm_feature_id, route_match_id=None, confidence=_optional_float(decision.get("confidence")), evidence_hash=_hash_payload(evidence), selector_json=_json_dumps(selector), action_json=_json_dumps(action), note=decision.get("note"), reviewer="decision_replay", metadata_json=_json_dumps(replay_metadata), created_at=now, updated_at=now, ) ) return True existing.status = str(decision.get("status") or existing.status) existing.active = True existing.source_id = new_dataset.source_id existing.gtfs_dataset_id = new_dataset.id existing.gtfs_route_id = route.id existing.osm_dataset_id = osm_dataset_id existing.osm_feature_id = osm_feature_id existing.route_match_id = None existing.confidence = _optional_float(decision.get("confidence")) existing.evidence_hash = _hash_payload(evidence) existing.selector_json = _json_dumps(selector) existing.action_json = _json_dumps(action) existing.reviewer = "decision_replay" existing.metadata_json = _json_dumps(replay_metadata) existing.updated_at = now return True def _updated_route_selector(value: object, *, new_dataset: Dataset, route: GtfsRoute) -> dict[str, Any]: selector = dict(value) if isinstance(value, dict) else {} selector["gtfs"] = { "source_id": new_dataset.source_id, "dataset_id": new_dataset.id, "route_id": route.route_id, "route_key": route.route_key, "ref": route.short_name, "mode": route.mode, } selector.pop("route_match_id", None) selector.pop("gtfs_route_id", None) return selector def _existing_osm_reference(session: Session, decision: dict[str, Any]) -> tuple[int | None, int | None]: dataset_id = _optional_int(decision.get("osm_dataset_id")) feature_id = _optional_int(decision.get("osm_feature_id")) feature = session.get(OsmFeature, feature_id) if feature_id is not None else None if feature is not None: return feature.dataset_id, feature.id if dataset_id is not None and session.get(Dataset, dataset_id) is not None: return dataset_id, None return None, None def _changed_or_removed_keys(session: Session, diff_run_id: int | None) -> set[tuple[str, str]]: if diff_run_id is None: return set() rows = session.execute( select(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.object_key) .where(GtfsFeedDiffItem.diff_run_id == diff_run_id, GtfsFeedDiffItem.change_type.in_(["changed", "removed"])) ).all() return { ("route" if item_type == "route" else "stop" if item_type == "stop" else str(item_type), str(object_key)) for item_type, object_key in rows } def _diff_review_title(item: GtfsFeedDiffItem, *, previous_decision_count: int) -> str: prefix = "Recheck previous decision for" if previous_decision_count else "Review updated" return f"{prefix} {item.item_type} {item.object_key}: {item.change_type}" def _diff_review_description(item: GtfsFeedDiffItem, previous_decision_count: int) -> str: if previous_decision_count: return f"{item.item_type.title()} changed after {previous_decision_count} previous decision(s); manual review is needed before replay." return f"{item.item_type.title()} was {item.change_type} in the latest feed update." def _stable_key(decision_type: str, selector: dict[str, Any], action: dict[str, Any]) -> str: payload = {"decision_type": decision_type, "selector": selector, "action": action} return f"{decision_type}:{_hash_payload(payload)[:40]}" def _hash_payload(payload: dict[str, Any]) -> str: encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") return hashlib.sha256(encoded).hexdigest() def _json_object(value: str | None) -> dict[str, Any]: if not value: return {} try: payload = json.loads(value) except json.JSONDecodeError: return {} return payload if isinstance(payload, dict) else {} def _json_dumps(value: dict[str, Any]) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) def _optional_int(value: object) -> int | None: try: return None if value is None else int(value) except (TypeError, ValueError): return None def _optional_float(value: object) -> float | None: try: return None if value is None else float(value) except (TypeError, ValueError): return None