from __future__ import annotations import hashlib import json from datetime import datetime, timezone from typing import Any, Callable from sqlalchemy import and_, case, exists, func, or_, select from sqlalchemy.orm import Session, aliased, joinedload from app.harmonization import active_harmonized_gtfs_route_shadow_map, gtfs_harmonized_snapshot, gtfs_route_overlap_key from app.models import ( CanonicalStopLink, Dataset, GtfsRoute, GtfsStop, GtfsTrip, MapGtfsDecision, MapGtfsReviewItem, OsmFeature, RouteMatch, Source, ) ProgressCallback = Callable[[str, str, int | None, int | None, dict[str, Any] | None], None] OPEN_REVIEW_STATUSES = {"open", "in_review"} TERMINAL_REVIEW_STATUSES = {"resolved", "dismissed"} ROUTE_REVIEW_STATUSES = {"matched", "probable", "weak", "missing"} def generate_map_gtfs_review_items( session: Session, *, source_id: int | None = None, dataset_id: int | None = None, limit: int = 1000, progress_callback: ProgressCallback | None = None, ) -> dict[str, Any]: selected_limit = max(1, min(int(limit), 50_000)) result = { "created": 0, "updated": 0, "reopened": 0, "unchanged": 0, "resolved": 0, "route_items": 0, "dedup_items": 0, "stop_items": 0, "limit": selected_limit, } route_limit = max(1, selected_limit // 2) dedup_limit = max(1, selected_limit // 4) _emit(progress_callback, "map_gtfs_review_routes_started", "Scanning GTFS/OSM route matches.", 0, route_limit, result) route_result = _generate_route_review_items(session, source_id=source_id, dataset_id=dataset_id, limit=route_limit) _merge_counts(result, route_result) result["route_items"] = route_result["items"] remaining = max(0, selected_limit - int(route_result["items"])) _emit( progress_callback, "map_gtfs_review_routes_completed", "Scanned GTFS/OSM route matches.", int(route_result["items"]), route_limit, route_result, ) if remaining and dedup_limit: current_dedup_limit = min(dedup_limit, remaining) _emit(progress_callback, "map_gtfs_review_dedup_started", "Scanning shadowed GTFS routes for duplicate/conflict review.", 0, current_dedup_limit, result) dedup_result = _generate_gtfs_route_dedup_review_items(session, source_id=source_id, dataset_id=dataset_id, limit=current_dedup_limit) _merge_counts(result, dedup_result) result["dedup_items"] = dedup_result["items"] remaining = max(0, remaining - int(dedup_result["items"])) _emit( progress_callback, "map_gtfs_review_dedup_completed", "Scanned GTFS route duplicates.", int(dedup_result["items"]), current_dedup_limit, dedup_result, ) if remaining: _emit(progress_callback, "map_gtfs_review_stops_started", "Scanning GTFS stops needing concrete map links.", 0, remaining, result) stop_result = _generate_stop_review_items(session, source_id=source_id, dataset_id=dataset_id, limit=remaining) _merge_counts(result, stop_result) result["stop_items"] = stop_result["items"] _emit( progress_callback, "map_gtfs_review_stops_completed", "Scanned GTFS stops needing concrete map links.", int(stop_result["items"]), remaining, stop_result, ) session.flush() _emit(progress_callback, "map_gtfs_review_completed", "Map/GTFS review queue generated.", None, None, result) return result def list_map_gtfs_review_items( session: Session, *, queue: str | None = None, status: str | None = "open", severity: str | None = None, mode: str | None = None, q: str | None = None, source_id: int | None = None, dataset_id: int | None = None, limit: int = 100, ) -> list[MapGtfsReviewItem]: stmt = ( select(MapGtfsReviewItem) .outerjoin(GtfsRoute, GtfsRoute.id == MapGtfsReviewItem.gtfs_route_id) .outerjoin(GtfsStop, GtfsStop.id == MapGtfsReviewItem.gtfs_stop_id) .outerjoin(OsmFeature, OsmFeature.id == MapGtfsReviewItem.osm_feature_id) .outerjoin(Source, Source.id == MapGtfsReviewItem.source_id) .options( joinedload(MapGtfsReviewItem.source), joinedload(MapGtfsReviewItem.dataset), joinedload(MapGtfsReviewItem.gtfs_route), joinedload(MapGtfsReviewItem.gtfs_stop), joinedload(MapGtfsReviewItem.osm_feature), joinedload(MapGtfsReviewItem.route_match), joinedload(MapGtfsReviewItem.decision), ) .order_by( _severity_order_expression(), MapGtfsReviewItem.updated_at.desc(), MapGtfsReviewItem.id.desc(), ) ) if queue: stmt = stmt.where(MapGtfsReviewItem.queue == queue) if status: stmt = stmt.where(MapGtfsReviewItem.status == status) if severity: stmt = stmt.where(MapGtfsReviewItem.severity == severity) mode_text = " ".join(str(mode or "").strip().split()) if mode_text: stmt = stmt.where(or_(GtfsRoute.mode == mode_text, OsmFeature.mode == mode_text)) query_text = " ".join(str(q or "").strip().split()) if query_text: pattern = f"%{query_text}%" stmt = stmt.where( or_( MapGtfsReviewItem.fingerprint.ilike(pattern), MapGtfsReviewItem.title.ilike(pattern), MapGtfsReviewItem.description.ilike(pattern), GtfsRoute.route_id.ilike(pattern), GtfsRoute.short_name.ilike(pattern), GtfsRoute.long_name.ilike(pattern), GtfsRoute.operator_name.ilike(pattern), GtfsStop.stop_id.ilike(pattern), GtfsStop.name.ilike(pattern), OsmFeature.ref.ilike(pattern), OsmFeature.name.ilike(pattern), Source.name.ilike(pattern), ) ) if source_id is not None: stmt = stmt.where(MapGtfsReviewItem.source_id == source_id) if dataset_id is not None: stmt = stmt.where(MapGtfsReviewItem.dataset_id == dataset_id) return session.scalars(stmt.limit(max(1, min(int(limit), 1000)))).all() def map_gtfs_review_summary(session: Session) -> dict[str, Any]: rows = session.execute( select(MapGtfsReviewItem.queue, MapGtfsReviewItem.status, MapGtfsReviewItem.severity, func.count()) .select_from(MapGtfsReviewItem) .group_by(MapGtfsReviewItem.queue, MapGtfsReviewItem.status, MapGtfsReviewItem.severity) ).all() by_queue: dict[str, dict[str, int]] = {} by_status: dict[str, int] = {} by_severity: dict[str, int] = {} total = 0 for queue, status, severity, count in rows: count = int(count or 0) total += count queue_counts = by_queue.setdefault(str(queue), {}) queue_counts[str(status)] = queue_counts.get(str(status), 0) + count by_status[str(status)] = by_status.get(str(status), 0) + count by_severity[str(severity)] = by_severity.get(str(severity), 0) + count decisions = session.scalar(select(func.count()).select_from(MapGtfsDecision)) or 0 active_decisions = session.scalar(select(func.count()).select_from(MapGtfsDecision).where(MapGtfsDecision.active.is_(True))) or 0 return { "total": total, "by_queue": by_queue, "by_status": by_status, "by_severity": by_severity, "decisions": int(decisions), "active_decisions": int(active_decisions), } def update_map_gtfs_review_item( session: Session, item_id: int, *, status: str | None = None, note: str | None = None, ) -> MapGtfsReviewItem: item = session.get(MapGtfsReviewItem, item_id) if item is None: raise ValueError("review item not found") now = datetime.now(timezone.utc) if status is not None: if status not in {"open", "in_review", "resolved", "dismissed"}: raise ValueError("review status must be open, in_review, resolved, or dismissed") item.status = status item.resolved_at = now if status in TERMINAL_REVIEW_STATUSES else None if note is not None: metadata = _json_object(item.metadata_json) metadata["review_note"] = note item.metadata_json = _json_dumps(metadata) item.updated_at = now session.flush() return item def record_route_match_decision( session: Session, match: RouteMatch, *, decision_type: str = "route_osm_relation", reviewer: str | None = "manual", note: str | None = None, ) -> MapGtfsDecision | None: route = session.get(GtfsRoute, match.gtfs_route_id) if route is None: return None route_dataset = session.get(Dataset, route.dataset_id) feature = session.get(OsmFeature, match.osm_feature_id) if match.osm_feature_id else None feature_dataset = session.get(Dataset, feature.dataset_id) if feature is not None else None selector = { "gtfs": { "source_id": None if route_dataset is None else route_dataset.source_id, "dataset_id": route.dataset_id, "route_id": route.route_id, "route_key": route.route_key, "ref": route.short_name, "mode": route.mode, }, "route_match_id": match.id, } action: dict[str, Any] = {"status": match.status} if feature is not None: action["osm"] = { "source_id": None if feature_dataset is None else feature_dataset.source_id, "dataset_id": feature.dataset_id, "osm_type": feature.osm_type, "osm_id": feature.osm_id, "route_key": feature.route_key, "ref": feature.ref, "mode": feature.mode, } evidence = { "selector": selector, "action": action, "confidence": match.confidence, "reasons": _json_object(match.reasons_json), } decision_key = _stable_key(decision_type, selector, action) evidence_hash = _hash_payload(evidence) decision = session.scalar(select(MapGtfsDecision).where(MapGtfsDecision.decision_key == decision_key)) now = datetime.now(timezone.utc) if decision is None: decision = MapGtfsDecision( decision_key=decision_key, decision_type=decision_type, status=match.status, active=True, source_id=None if route_dataset is None else route_dataset.source_id, gtfs_dataset_id=route.dataset_id, gtfs_route_id=route.id, osm_dataset_id=None if feature is None else feature.dataset_id, osm_feature_id=None if feature is None else feature.id, route_match_id=match.id, confidence=match.confidence, evidence_hash=evidence_hash, selector_json=_json_dumps(selector), action_json=_json_dumps(action), note=note, reviewer=reviewer, metadata_json=_json_dumps({"reasons": _json_object(match.reasons_json)}), created_at=now, updated_at=now, ) session.add(decision) else: decision.status = match.status decision.active = True decision.osm_dataset_id = None if feature is None else feature.dataset_id decision.osm_feature_id = None if feature is None else feature.id decision.route_match_id = match.id decision.confidence = match.confidence decision.evidence_hash = evidence_hash decision.selector_json = _json_dumps(selector) decision.action_json = _json_dumps(action) decision.note = note if note is not None else decision.note decision.reviewer = reviewer if reviewer is not None else decision.reviewer decision.metadata_json = _json_dumps({"reasons": _json_object(match.reasons_json)}) decision.updated_at = now _resolve_review_item_by_fingerprint(session, _route_fingerprint(route), decision=decision) session.flush() return decision def review_item_payload(item: MapGtfsReviewItem) -> dict[str, Any]: route = item.gtfs_route stop = item.gtfs_stop osm = item.osm_feature return { "id": item.id, "fingerprint": item.fingerprint, "queue": item.queue, "item_type": item.item_type, "status": item.status, "severity": item.severity, "title": item.title, "description": item.description, "source": None if item.source is None else {"id": item.source.id, "name": item.source.name, "kind": item.source.kind}, "dataset": None if item.dataset is None else {"id": item.dataset.id, "kind": item.dataset.kind, "sha256": item.dataset.sha256}, "gtfs_route": None if route is None else { "id": route.id, "dataset_id": route.dataset_id, "route_id": route.route_id, "ref": route.short_name, "name": route.long_name, "mode": route.mode, "operator": route.operator_name, }, "gtfs_stop": None if stop is None else { "id": stop.id, "dataset_id": stop.dataset_id, "stop_id": stop.stop_id, "name": stop.name, "lat": stop.lat, "lon": stop.lon, }, "osm_feature": None if osm is None else { "id": osm.id, "dataset_id": osm.dataset_id, "osm_type": osm.osm_type, "osm_id": osm.osm_id, "kind": osm.kind, "mode": osm.mode, "ref": osm.ref, "name": osm.name, }, "route_match_id": item.route_match_id, "canonical_stop_id": item.canonical_stop_id, "canonical_stop_link_id": item.canonical_stop_link_id, "decision_id": item.decision_id, "evidence_hash": item.evidence_hash, "metadata": _json_object(item.metadata_json), "created_at": item.created_at.isoformat(), "updated_at": item.updated_at.isoformat(), "resolved_at": None if item.resolved_at is None else item.resolved_at.isoformat(), } def decision_payload(decision: MapGtfsDecision) -> dict[str, Any]: return { "id": decision.id, "decision_key": decision.decision_key, "decision_type": decision.decision_type, "status": decision.status, "active": decision.active, "source_id": decision.source_id, "gtfs_dataset_id": decision.gtfs_dataset_id, "gtfs_route_id": decision.gtfs_route_id, "gtfs_stop_id": decision.gtfs_stop_id, "osm_dataset_id": decision.osm_dataset_id, "osm_feature_id": decision.osm_feature_id, "route_match_id": decision.route_match_id, "canonical_stop_id": decision.canonical_stop_id, "canonical_stop_link_id": decision.canonical_stop_link_id, "confidence": decision.confidence, "evidence_hash": decision.evidence_hash, "selector": _json_object(decision.selector_json), "action": _json_object(decision.action_json), "note": decision.note, "reviewer": decision.reviewer, "metadata": _json_object(decision.metadata_json), "created_at": decision.created_at.isoformat(), "updated_at": decision.updated_at.isoformat(), } def _generate_route_review_items( session: Session, *, source_id: int | None, dataset_id: int | None, limit: int, ) -> dict[str, int]: stmt = ( select(RouteMatch) .options(joinedload(RouteMatch.gtfs_route), joinedload(RouteMatch.osm_feature)) .join(RouteMatch.gtfs_route) .join(Dataset, Dataset.id == GtfsRoute.dataset_id) .where(Dataset.is_active.is_(True), Dataset.kind == "gtfs", RouteMatch.status.in_(ROUTE_REVIEW_STATUSES)) .order_by(RouteMatch.status.desc(), RouteMatch.confidence.asc(), RouteMatch.id) ) if source_id is not None: stmt = stmt.where(Dataset.source_id == source_id) if dataset_id is not None: stmt = stmt.where(GtfsRoute.dataset_id == dataset_id) matches = session.scalars(stmt.limit(limit)).all() counts = _empty_counts() counts["items"] = len(matches) for match in matches: route = match.gtfs_route if route is None: continue dataset = session.get(Dataset, route.dataset_id) metadata = { "route_match_status": match.status, "confidence": match.confidence, "reasons": _json_object(match.reasons_json), "gtfs": { "route_id": route.route_id, "route_key": route.route_key, "ref": route.short_name, "mode": route.mode, }, "osm_feature_id": match.osm_feature_id, } status_label = match.status.replace("_", " ") item = _upsert_review_item( session, fingerprint=_route_fingerprint(route), queue="route_matching", item_type="gtfs_route_osm_route", severity=_route_review_severity(match.status), title=f"Review route {route.short_name or route.route_id}: {status_label}", description=_route_review_description(route, match), source_id=None if dataset is None else dataset.source_id, dataset_id=route.dataset_id, gtfs_route_id=route.id, osm_feature_id=match.osm_feature_id, route_match_id=match.id, evidence_payload=metadata, ) counts[item] += 1 return counts def _generate_gtfs_route_dedup_review_items( session: Session, *, source_id: int | None, dataset_id: int | None, limit: int, ) -> dict[str, int]: snapshot = gtfs_harmonized_snapshot(session) snapshot_items_by_dataset = { int(item["dataset_id"]): item for item in snapshot.get("datasets", []) if item.get("dataset_id") is not None } included_dataset_ids = [int(item["dataset_id"]) for item in snapshot.get("included", []) if item.get("dataset_active", True)] shadowed_items = { int(item["dataset_id"]): item for item in snapshot.get("shadowed", []) if item.get("dataset_id") is not None } if source_id is not None: shadowed_items = {dataset_id: item for dataset_id, item in shadowed_items.items() if int(item.get("source_id") or 0) == int(source_id)} if dataset_id is not None: shadowed_items = {int(dataset_id): item} if int(dataset_id) in shadowed_items else {} route_shadow_map = active_harmonized_gtfs_route_shadow_map(session) if not included_dataset_ids and not route_shadow_map: counts = _empty_counts() counts["items"] = 0 return counts pairs: list[tuple[GtfsRoute, GtfsRoute, dict[str, Any]]] = [] if included_dataset_ids and shadowed_items: included_by_key = _routes_by_overlap_key(session, included_dataset_ids) shadowed_routes = session.scalars( select(GtfsRoute) .where(GtfsRoute.dataset_id.in_(list(shadowed_items))) .order_by(GtfsRoute.dataset_id, GtfsRoute.mode, GtfsRoute.short_name, GtfsRoute.route_id) .limit(max(1, int(limit)) * 3) ).all() for route in shadowed_routes: key = gtfs_route_overlap_key(route) if not key: continue candidates = included_by_key.get(key, []) if not candidates: continue shadowed_item = shadowed_items[int(route.dataset_id)] preferred_dataset_id = _optional_int(shadowed_item.get("shadowed_by_dataset_id")) candidate = _best_included_duplicate_candidate(candidates, preferred_dataset_id=preferred_dataset_id) pairs.append((route, candidate, shadowed_item)) if len(pairs) >= max(1, int(limit)): break if len(pairs) < max(1, int(limit)) and route_shadow_map: pairs.extend( _partial_route_shadow_pairs( session, route_shadow_map=route_shadow_map, snapshot_items_by_dataset=snapshot_items_by_dataset, source_id=source_id, dataset_id=dataset_id, limit=max(0, int(limit) - len(pairs)), ) ) trip_counts = _trip_counts_for_routes(session, [route for pair in pairs for route in pair[:2]]) counts = _empty_counts() counts["items"] = len(pairs) for shadowed_route, included_route, shadowed_item in pairs: dataset = session.get(Dataset, shadowed_route.dataset_id) evidence = _route_dedup_evidence( shadowed_route, included_route, shadowed_item=shadowed_item, trip_counts=trip_counts, ) severity = "warn" if evidence["data_differs"] else "info" item = _upsert_review_item( session, fingerprint=_route_dedup_fingerprint(shadowed_route, included_route), queue="gtfs_deduplication", item_type="gtfs_route_duplicate", severity=severity, title=_route_dedup_title(shadowed_route, included_route, shadowed_item), description=_route_dedup_description(evidence), source_id=None if dataset is None else dataset.source_id, dataset_id=shadowed_route.dataset_id, gtfs_route_id=shadowed_route.id, evidence_payload=evidence, ) counts[item] += 1 return counts def _partial_route_shadow_pairs( session: Session, *, route_shadow_map: dict[int, int], snapshot_items_by_dataset: dict[int, dict[str, Any]], source_id: int | None, dataset_id: int | None, limit: int, ) -> list[tuple[GtfsRoute, GtfsRoute, dict[str, Any]]]: if limit <= 0 or not route_shadow_map: return [] route_ids = sorted({*route_shadow_map.keys(), *route_shadow_map.values()}) routes = {int(route.id): route for route in session.scalars(select(GtfsRoute).where(GtfsRoute.id.in_(route_ids))).all()} dataset_ids = sorted({int(route.dataset_id) for route in routes.values()}) datasets = {int(dataset.id): dataset for dataset in session.scalars(select(Dataset).where(Dataset.id.in_(dataset_ids))).all()} pairs: list[tuple[GtfsRoute, GtfsRoute, dict[str, Any]]] = [] for shadowed_route_id, included_route_id in sorted(route_shadow_map.items()): shadowed_route = routes.get(int(shadowed_route_id)) included_route = routes.get(int(included_route_id)) if shadowed_route is None or included_route is None: continue shadowed_dataset = datasets.get(int(shadowed_route.dataset_id)) included_dataset = datasets.get(int(included_route.dataset_id)) if source_id is not None and (shadowed_dataset is None or int(shadowed_dataset.source_id) != int(source_id)): continue if dataset_id is not None and int(shadowed_route.dataset_id) != int(dataset_id): continue shadowed_item = { **snapshot_items_by_dataset.get(int(shadowed_route.dataset_id), {}), "reason": "route_covered_by_higher_precedence_dataset", "shadowed_by_dataset_id": included_route.dataset_id, "shadowed_by_source_id": None if included_dataset is None else included_dataset.source_id, "route_key_overlap": 1, "route_key_overlap_ratio": 1.0, } pairs.append((shadowed_route, included_route, shadowed_item)) if len(pairs) >= limit: break return pairs def _generate_stop_review_items( session: Session, *, source_id: int | None, dataset_id: int | None, limit: int, ) -> dict[str, int]: gtfs_link = aliased(CanonicalStopLink) visual_link = aliased(CanonicalStopLink) visual_link_exists = ( select(visual_link.id) .where( visual_link.canonical_stop_id == gtfs_link.canonical_stop_id, visual_link.object_type == "osm_feature", ) .exists() ) stmt = ( select(GtfsStop, gtfs_link.id, gtfs_link.canonical_stop_id) .join(Dataset, Dataset.id == GtfsStop.dataset_id) .outerjoin( gtfs_link, and_( gtfs_link.object_type == "gtfs_stop", gtfs_link.dataset_id == GtfsStop.dataset_id, gtfs_link.object_id == GtfsStop.id, ), ) .where( Dataset.is_active.is_(True), Dataset.kind == "gtfs", or_(gtfs_link.id.is_(None), ~visual_link_exists), ) .order_by(GtfsStop.dataset_id, GtfsStop.name, GtfsStop.stop_id) ) if source_id is not None: stmt = stmt.where(Dataset.source_id == source_id) if dataset_id is not None: stmt = stmt.where(GtfsStop.dataset_id == dataset_id) rows = session.execute(stmt.limit(limit)).all() counts = _empty_counts() counts["items"] = len(rows) for stop, link_id, canonical_stop_id in rows: dataset = session.get(Dataset, stop.dataset_id) missing_canonical = link_id is None metadata = { "missing_canonical_stop": missing_canonical, "missing_visual_stop": not missing_canonical, "gtfs": {"stop_id": stop.stop_id, "name": stop.name, "lat": stop.lat, "lon": stop.lon}, "canonical_stop_id": canonical_stop_id, } item = _upsert_review_item( session, fingerprint=_stop_fingerprint(stop), queue="stop_matching", item_type="gtfs_stop_osm_stop", severity="warn" if missing_canonical else "info", title=f"Review stop {stop.name or stop.stop_id}", description="GTFS stop has no canonical stop link." if missing_canonical else "GTFS stop has no concrete OSM stop/platform link.", source_id=None if dataset is None else dataset.source_id, dataset_id=stop.dataset_id, gtfs_stop_id=stop.id, canonical_stop_id=canonical_stop_id, canonical_stop_link_id=link_id, evidence_payload=metadata, ) counts[item] += 1 return counts def _routes_by_overlap_key(session: Session, dataset_ids: list[int]) -> dict[str, list[GtfsRoute]]: if not dataset_ids: return {} rows = session.scalars( select(GtfsRoute) .where(GtfsRoute.dataset_id.in_(dataset_ids)) .order_by(GtfsRoute.dataset_id, GtfsRoute.mode, GtfsRoute.short_name, GtfsRoute.route_id) ).all() routes: dict[str, list[GtfsRoute]] = {} for route in rows: key = gtfs_route_overlap_key(route) if key: routes.setdefault(key, []).append(route) return routes def _best_included_duplicate_candidate(routes: list[GtfsRoute], *, preferred_dataset_id: int | None) -> GtfsRoute: return sorted( routes, key=lambda route: ( 0 if preferred_dataset_id is not None and int(route.dataset_id) == preferred_dataset_id else 1, route.dataset_id, route.route_id, ), )[0] def _trip_counts_for_routes(session: Session, routes: list[GtfsRoute]) -> dict[tuple[int, str], int]: keys = {(int(route.dataset_id), str(route.route_id)) for route in routes} if not keys: return {} dataset_ids = sorted({dataset_id for dataset_id, _route_id in keys}) route_ids = sorted({route_id for _dataset_id, route_id in keys}) rows = session.execute( select(GtfsTrip.dataset_id, GtfsTrip.route_id, func.count()) .where(GtfsTrip.dataset_id.in_(dataset_ids), GtfsTrip.route_id.in_(route_ids)) .group_by(GtfsTrip.dataset_id, GtfsTrip.route_id) ).all() return { (int(dataset_id), str(route_id)): int(count or 0) for dataset_id, route_id, count in rows if (int(dataset_id), str(route_id)) in keys } def _route_dedup_evidence( shadowed_route: GtfsRoute, included_route: GtfsRoute, *, shadowed_item: dict[str, Any], trip_counts: dict[tuple[int, str], int], ) -> dict[str, Any]: shadowed_trip_count = trip_counts.get((int(shadowed_route.dataset_id), str(shadowed_route.route_id)), 0) included_trip_count = trip_counts.get((int(included_route.dataset_id), str(included_route.route_id)), 0) differences = [] for field, left, right in [ ("name", shadowed_route.long_name, included_route.long_name), ("operator", shadowed_route.operator_name, included_route.operator_name), ("route_scope", shadowed_route.route_scope, included_route.route_scope), ]: if _dedup_text(left) != _dedup_text(right): differences.append(field) if shadowed_trip_count != included_trip_count: differences.append("trip_count") if _geometry_digest(shadowed_route.geometry_geojson) != _geometry_digest(included_route.geometry_geojson): differences.append("geometry") return { "data_differs": bool(differences), "differences": differences, "overlap_key": gtfs_route_overlap_key(shadowed_route), "shadowed_dataset": { "dataset_id": shadowed_route.dataset_id, "source_id": shadowed_item.get("source_id"), "source_name": shadowed_item.get("source_name"), "route_id": shadowed_route.route_id, "ref": shadowed_route.short_name, "name": shadowed_route.long_name, "operator": shadowed_route.operator_name, "mode": shadowed_route.mode, "trip_count": shadowed_trip_count, }, "included_dataset": { "dataset_id": included_route.dataset_id, "source_id": shadowed_item.get("shadowed_by_source_id"), "route_id": included_route.route_id, "ref": included_route.short_name, "name": included_route.long_name, "operator": included_route.operator_name, "mode": included_route.mode, "trip_count": included_trip_count, }, "snapshot": { "reason": shadowed_item.get("reason"), "route_key_overlap": shadowed_item.get("route_key_overlap"), "route_key_overlap_ratio": shadowed_item.get("route_key_overlap_ratio"), }, } def _route_dedup_title(shadowed_route: GtfsRoute, included_route: GtfsRoute, shadowed_item: dict[str, Any]) -> str: ref = shadowed_route.short_name or shadowed_route.route_id source = shadowed_item.get("source_name") or f"dataset #{shadowed_route.dataset_id}" return f"Review duplicate GTFS route {ref}: {source} -> dataset #{included_route.dataset_id}" def _route_dedup_description(evidence: dict[str, Any]) -> str: differences = evidence.get("differences") or [] if differences: return f"Route is shadowed by the active snapshot but differs in {', '.join(differences)}." return "Route is shadowed by the active snapshot and looks equivalent on first-pass route evidence." def _route_dedup_fingerprint(shadowed_route: GtfsRoute, included_route: GtfsRoute) -> str: return f"gtfs-dedup:{shadowed_route.dataset_id}:{shadowed_route.route_id}:{included_route.dataset_id}:{included_route.route_id}" def _geometry_digest(value: str | None) -> str: if not value: return "" return hashlib.sha256(str(value).encode("utf-8")).hexdigest() def _dedup_text(value: str | None) -> str: return " ".join(str(value or "").casefold().replace("ß", "ss").split()) def _upsert_review_item( session: Session, *, fingerprint: str, queue: str, item_type: str, severity: str, title: str, description: str, source_id: int | None = None, dataset_id: int | None = None, gtfs_route_id: int | None = None, gtfs_stop_id: int | None = None, osm_feature_id: int | None = None, route_match_id: int | None = None, canonical_stop_id: int | None = None, canonical_stop_link_id: int | None = None, decision_id: int | None = None, evidence_payload: dict[str, Any] | None = None, ) -> str: evidence_hash = _hash_payload(evidence_payload or {}) metadata_json = _json_dumps(evidence_payload or {}) item = session.scalar(select(MapGtfsReviewItem).where(MapGtfsReviewItem.fingerprint == fingerprint)) now = datetime.now(timezone.utc) if item is None: session.add( MapGtfsReviewItem( fingerprint=fingerprint, queue=queue, item_type=item_type, status="open", severity=severity, title=title, description=description, source_id=source_id, dataset_id=dataset_id, gtfs_route_id=gtfs_route_id, gtfs_stop_id=gtfs_stop_id, osm_feature_id=osm_feature_id, route_match_id=route_match_id, canonical_stop_id=canonical_stop_id, canonical_stop_link_id=canonical_stop_link_id, decision_id=decision_id, evidence_hash=evidence_hash, metadata_json=metadata_json, created_at=now, updated_at=now, ) ) return "created" if item.evidence_hash == evidence_hash and item.status in TERMINAL_REVIEW_STATUSES: return "unchanged" changed = ( item.queue != queue or item.item_type != item_type or item.severity != severity or item.title != title or item.description != description or item.source_id != source_id or item.dataset_id != dataset_id or item.gtfs_route_id != gtfs_route_id or item.gtfs_stop_id != gtfs_stop_id or item.osm_feature_id != osm_feature_id or item.route_match_id != route_match_id or item.canonical_stop_id != canonical_stop_id or item.canonical_stop_link_id != canonical_stop_link_id or item.decision_id != decision_id or item.evidence_hash != evidence_hash or item.metadata_json != metadata_json ) was_terminal = item.status in TERMINAL_REVIEW_STATUSES item.queue = queue item.item_type = item_type item.severity = severity item.title = title item.description = description item.source_id = source_id item.dataset_id = dataset_id item.gtfs_route_id = gtfs_route_id item.gtfs_stop_id = gtfs_stop_id item.osm_feature_id = osm_feature_id item.route_match_id = route_match_id item.canonical_stop_id = canonical_stop_id item.canonical_stop_link_id = canonical_stop_link_id item.decision_id = decision_id item.evidence_hash = evidence_hash item.metadata_json = metadata_json if was_terminal: item.status = "open" item.resolved_at = None item.updated_at = now return "reopened" if changed: item.updated_at = now return "updated" return "unchanged" def _resolve_review_item_by_fingerprint( session: Session, fingerprint: str, *, decision: MapGtfsDecision | None = None, ) -> None: item = session.scalar(select(MapGtfsReviewItem).where(MapGtfsReviewItem.fingerprint == fingerprint)) if item is None: return now = datetime.now(timezone.utc) item.status = "resolved" item.resolved_at = now item.updated_at = now if decision is not None: item.decision = decision def _route_review_severity(status: str) -> str: if status == "missing": return "error" if status in {"weak", "probable"}: return "warn" return "info" def _route_review_description(route: GtfsRoute, match: RouteMatch) -> str: if match.status == "missing": return "No plausible OSM route relation is linked yet." if match.status == "weak": return "The best OSM route candidate has weak evidence." if match.status == "probable": return "The OSM route candidate is plausible but should be reviewed." return "Automatic route match is available for review." def _route_fingerprint(route: GtfsRoute) -> str: return f"route:{route.dataset_id}:{route.route_id}" def _stop_fingerprint(stop: GtfsStop) -> str: return f"stop:{stop.dataset_id}:{stop.stop_id}" def _stable_key(decision_type: str, selector: dict[str, Any], action: dict[str, Any]) -> str: payload = {"decision_type": decision_type, "selector": selector, "action": action} return f"{decision_type}:{_hash_payload(payload)[:40]}" def _hash_payload(payload: dict[str, Any]) -> str: encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") return hashlib.sha256(encoded).hexdigest() def _json_object(value: str | None) -> dict[str, Any]: if not value: return {} try: payload = json.loads(value) except json.JSONDecodeError: return {} return payload if isinstance(payload, dict) else {} def _json_dumps(value: dict[str, Any]) -> str: return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) def _empty_counts() -> dict[str, int]: return {"created": 0, "updated": 0, "reopened": 0, "unchanged": 0, "resolved": 0, "items": 0} def _merge_counts(target: dict[str, Any], source: dict[str, int]) -> None: for key in ["created", "updated", "reopened", "unchanged", "resolved"]: target[key] = int(target.get(key, 0)) + int(source.get(key, 0)) def _optional_int(value: object) -> int | None: try: return None if value is None else int(value) except (TypeError, ValueError): return None def _emit( progress_callback: ProgressCallback | None, event_type: str, message: str, progress_current: int | None, progress_total: int | None, metadata: dict[str, Any] | None = None, ) -> None: if progress_callback is not None: progress_callback(event_type, message, progress_current, progress_total, metadata) def _severity_order_expression(): return case( (MapGtfsReviewItem.severity == "error", 0), (MapGtfsReviewItem.severity == "warn", 1), (MapGtfsReviewItem.severity == "info", 2), else_=9, )