986 lines
38 KiB
Python
986 lines
38 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Callable
|
|
|
|
from sqlalchemy import and_, case, exists, func, or_, select
|
|
from sqlalchemy.orm import Session, aliased, joinedload
|
|
|
|
from app.harmonization import active_harmonized_gtfs_route_shadow_map, gtfs_harmonized_snapshot, gtfs_route_overlap_key
|
|
from app.models import (
|
|
CanonicalStopLink,
|
|
Dataset,
|
|
GtfsRoute,
|
|
GtfsStop,
|
|
GtfsTrip,
|
|
MapGtfsDecision,
|
|
MapGtfsReviewItem,
|
|
OsmFeature,
|
|
RouteMatch,
|
|
Source,
|
|
)
|
|
|
|
ProgressCallback = Callable[[str, str, int | None, int | None, dict[str, Any] | None], None]
|
|
|
|
OPEN_REVIEW_STATUSES = {"open", "in_review"}
|
|
TERMINAL_REVIEW_STATUSES = {"resolved", "dismissed"}
|
|
ROUTE_REVIEW_STATUSES = {"matched", "probable", "weak", "missing"}
|
|
|
|
|
|
def generate_map_gtfs_review_items(
|
|
session: Session,
|
|
*,
|
|
source_id: int | None = None,
|
|
dataset_id: int | None = None,
|
|
limit: int = 1000,
|
|
progress_callback: ProgressCallback | None = None,
|
|
) -> dict[str, Any]:
|
|
selected_limit = max(1, min(int(limit), 50_000))
|
|
result = {
|
|
"created": 0,
|
|
"updated": 0,
|
|
"reopened": 0,
|
|
"unchanged": 0,
|
|
"resolved": 0,
|
|
"route_items": 0,
|
|
"dedup_items": 0,
|
|
"stop_items": 0,
|
|
"limit": selected_limit,
|
|
}
|
|
route_limit = max(1, selected_limit // 2)
|
|
dedup_limit = max(1, selected_limit // 4)
|
|
_emit(progress_callback, "map_gtfs_review_routes_started", "Scanning GTFS/OSM route matches.", 0, route_limit, result)
|
|
route_result = _generate_route_review_items(session, source_id=source_id, dataset_id=dataset_id, limit=route_limit)
|
|
_merge_counts(result, route_result)
|
|
result["route_items"] = route_result["items"]
|
|
remaining = max(0, selected_limit - int(route_result["items"]))
|
|
_emit(
|
|
progress_callback,
|
|
"map_gtfs_review_routes_completed",
|
|
"Scanned GTFS/OSM route matches.",
|
|
int(route_result["items"]),
|
|
route_limit,
|
|
route_result,
|
|
)
|
|
if remaining and dedup_limit:
|
|
current_dedup_limit = min(dedup_limit, remaining)
|
|
_emit(progress_callback, "map_gtfs_review_dedup_started", "Scanning shadowed GTFS routes for duplicate/conflict review.", 0, current_dedup_limit, result)
|
|
dedup_result = _generate_gtfs_route_dedup_review_items(session, source_id=source_id, dataset_id=dataset_id, limit=current_dedup_limit)
|
|
_merge_counts(result, dedup_result)
|
|
result["dedup_items"] = dedup_result["items"]
|
|
remaining = max(0, remaining - int(dedup_result["items"]))
|
|
_emit(
|
|
progress_callback,
|
|
"map_gtfs_review_dedup_completed",
|
|
"Scanned GTFS route duplicates.",
|
|
int(dedup_result["items"]),
|
|
current_dedup_limit,
|
|
dedup_result,
|
|
)
|
|
if remaining:
|
|
_emit(progress_callback, "map_gtfs_review_stops_started", "Scanning GTFS stops needing concrete map links.", 0, remaining, result)
|
|
stop_result = _generate_stop_review_items(session, source_id=source_id, dataset_id=dataset_id, limit=remaining)
|
|
_merge_counts(result, stop_result)
|
|
result["stop_items"] = stop_result["items"]
|
|
_emit(
|
|
progress_callback,
|
|
"map_gtfs_review_stops_completed",
|
|
"Scanned GTFS stops needing concrete map links.",
|
|
int(stop_result["items"]),
|
|
remaining,
|
|
stop_result,
|
|
)
|
|
session.flush()
|
|
_emit(progress_callback, "map_gtfs_review_completed", "Map/GTFS review queue generated.", None, None, result)
|
|
return result
|
|
|
|
|
|
def list_map_gtfs_review_items(
|
|
session: Session,
|
|
*,
|
|
queue: str | None = None,
|
|
status: str | None = "open",
|
|
severity: str | None = None,
|
|
mode: str | None = None,
|
|
q: str | None = None,
|
|
source_id: int | None = None,
|
|
dataset_id: int | None = None,
|
|
limit: int = 100,
|
|
) -> list[MapGtfsReviewItem]:
|
|
stmt = (
|
|
select(MapGtfsReviewItem)
|
|
.outerjoin(GtfsRoute, GtfsRoute.id == MapGtfsReviewItem.gtfs_route_id)
|
|
.outerjoin(GtfsStop, GtfsStop.id == MapGtfsReviewItem.gtfs_stop_id)
|
|
.outerjoin(OsmFeature, OsmFeature.id == MapGtfsReviewItem.osm_feature_id)
|
|
.outerjoin(Source, Source.id == MapGtfsReviewItem.source_id)
|
|
.options(
|
|
joinedload(MapGtfsReviewItem.source),
|
|
joinedload(MapGtfsReviewItem.dataset),
|
|
joinedload(MapGtfsReviewItem.gtfs_route),
|
|
joinedload(MapGtfsReviewItem.gtfs_stop),
|
|
joinedload(MapGtfsReviewItem.osm_feature),
|
|
joinedload(MapGtfsReviewItem.route_match),
|
|
joinedload(MapGtfsReviewItem.decision),
|
|
)
|
|
.order_by(
|
|
_severity_order_expression(),
|
|
MapGtfsReviewItem.updated_at.desc(),
|
|
MapGtfsReviewItem.id.desc(),
|
|
)
|
|
)
|
|
if queue:
|
|
stmt = stmt.where(MapGtfsReviewItem.queue == queue)
|
|
if status:
|
|
stmt = stmt.where(MapGtfsReviewItem.status == status)
|
|
if severity:
|
|
stmt = stmt.where(MapGtfsReviewItem.severity == severity)
|
|
mode_text = " ".join(str(mode or "").strip().split())
|
|
if mode_text:
|
|
stmt = stmt.where(or_(GtfsRoute.mode == mode_text, OsmFeature.mode == mode_text))
|
|
query_text = " ".join(str(q or "").strip().split())
|
|
if query_text:
|
|
pattern = f"%{query_text}%"
|
|
stmt = stmt.where(
|
|
or_(
|
|
MapGtfsReviewItem.fingerprint.ilike(pattern),
|
|
MapGtfsReviewItem.title.ilike(pattern),
|
|
MapGtfsReviewItem.description.ilike(pattern),
|
|
GtfsRoute.route_id.ilike(pattern),
|
|
GtfsRoute.short_name.ilike(pattern),
|
|
GtfsRoute.long_name.ilike(pattern),
|
|
GtfsRoute.operator_name.ilike(pattern),
|
|
GtfsStop.stop_id.ilike(pattern),
|
|
GtfsStop.name.ilike(pattern),
|
|
OsmFeature.ref.ilike(pattern),
|
|
OsmFeature.name.ilike(pattern),
|
|
Source.name.ilike(pattern),
|
|
)
|
|
)
|
|
if source_id is not None:
|
|
stmt = stmt.where(MapGtfsReviewItem.source_id == source_id)
|
|
if dataset_id is not None:
|
|
stmt = stmt.where(MapGtfsReviewItem.dataset_id == dataset_id)
|
|
return session.scalars(stmt.limit(max(1, min(int(limit), 1000)))).all()
|
|
|
|
|
|
def map_gtfs_review_summary(session: Session) -> dict[str, Any]:
|
|
rows = session.execute(
|
|
select(MapGtfsReviewItem.queue, MapGtfsReviewItem.status, MapGtfsReviewItem.severity, func.count())
|
|
.select_from(MapGtfsReviewItem)
|
|
.group_by(MapGtfsReviewItem.queue, MapGtfsReviewItem.status, MapGtfsReviewItem.severity)
|
|
).all()
|
|
by_queue: dict[str, dict[str, int]] = {}
|
|
by_status: dict[str, int] = {}
|
|
by_severity: dict[str, int] = {}
|
|
total = 0
|
|
for queue, status, severity, count in rows:
|
|
count = int(count or 0)
|
|
total += count
|
|
queue_counts = by_queue.setdefault(str(queue), {})
|
|
queue_counts[str(status)] = queue_counts.get(str(status), 0) + count
|
|
by_status[str(status)] = by_status.get(str(status), 0) + count
|
|
by_severity[str(severity)] = by_severity.get(str(severity), 0) + count
|
|
decisions = session.scalar(select(func.count()).select_from(MapGtfsDecision)) or 0
|
|
active_decisions = session.scalar(select(func.count()).select_from(MapGtfsDecision).where(MapGtfsDecision.active.is_(True))) or 0
|
|
return {
|
|
"total": total,
|
|
"by_queue": by_queue,
|
|
"by_status": by_status,
|
|
"by_severity": by_severity,
|
|
"decisions": int(decisions),
|
|
"active_decisions": int(active_decisions),
|
|
}
|
|
|
|
|
|
def update_map_gtfs_review_item(
|
|
session: Session,
|
|
item_id: int,
|
|
*,
|
|
status: str | None = None,
|
|
note: str | None = None,
|
|
) -> MapGtfsReviewItem:
|
|
item = session.get(MapGtfsReviewItem, item_id)
|
|
if item is None:
|
|
raise ValueError("review item not found")
|
|
now = datetime.now(timezone.utc)
|
|
if status is not None:
|
|
if status not in {"open", "in_review", "resolved", "dismissed"}:
|
|
raise ValueError("review status must be open, in_review, resolved, or dismissed")
|
|
item.status = status
|
|
item.resolved_at = now if status in TERMINAL_REVIEW_STATUSES else None
|
|
if note is not None:
|
|
metadata = _json_object(item.metadata_json)
|
|
metadata["review_note"] = note
|
|
item.metadata_json = _json_dumps(metadata)
|
|
item.updated_at = now
|
|
session.flush()
|
|
return item
|
|
|
|
|
|
def record_route_match_decision(
|
|
session: Session,
|
|
match: RouteMatch,
|
|
*,
|
|
decision_type: str = "route_osm_relation",
|
|
reviewer: str | None = "manual",
|
|
note: str | None = None,
|
|
) -> MapGtfsDecision | None:
|
|
route = session.get(GtfsRoute, match.gtfs_route_id)
|
|
if route is None:
|
|
return None
|
|
route_dataset = session.get(Dataset, route.dataset_id)
|
|
feature = session.get(OsmFeature, match.osm_feature_id) if match.osm_feature_id else None
|
|
feature_dataset = session.get(Dataset, feature.dataset_id) if feature is not None else None
|
|
selector = {
|
|
"gtfs": {
|
|
"source_id": None if route_dataset is None else route_dataset.source_id,
|
|
"dataset_id": route.dataset_id,
|
|
"route_id": route.route_id,
|
|
"route_key": route.route_key,
|
|
"ref": route.short_name,
|
|
"mode": route.mode,
|
|
},
|
|
"route_match_id": match.id,
|
|
}
|
|
action: dict[str, Any] = {"status": match.status}
|
|
if feature is not None:
|
|
action["osm"] = {
|
|
"source_id": None if feature_dataset is None else feature_dataset.source_id,
|
|
"dataset_id": feature.dataset_id,
|
|
"osm_type": feature.osm_type,
|
|
"osm_id": feature.osm_id,
|
|
"route_key": feature.route_key,
|
|
"ref": feature.ref,
|
|
"mode": feature.mode,
|
|
}
|
|
evidence = {
|
|
"selector": selector,
|
|
"action": action,
|
|
"confidence": match.confidence,
|
|
"reasons": _json_object(match.reasons_json),
|
|
}
|
|
decision_key = _stable_key(decision_type, selector, action)
|
|
evidence_hash = _hash_payload(evidence)
|
|
decision = session.scalar(select(MapGtfsDecision).where(MapGtfsDecision.decision_key == decision_key))
|
|
now = datetime.now(timezone.utc)
|
|
if decision is None:
|
|
decision = MapGtfsDecision(
|
|
decision_key=decision_key,
|
|
decision_type=decision_type,
|
|
status=match.status,
|
|
active=True,
|
|
source_id=None if route_dataset is None else route_dataset.source_id,
|
|
gtfs_dataset_id=route.dataset_id,
|
|
gtfs_route_id=route.id,
|
|
osm_dataset_id=None if feature is None else feature.dataset_id,
|
|
osm_feature_id=None if feature is None else feature.id,
|
|
route_match_id=match.id,
|
|
confidence=match.confidence,
|
|
evidence_hash=evidence_hash,
|
|
selector_json=_json_dumps(selector),
|
|
action_json=_json_dumps(action),
|
|
note=note,
|
|
reviewer=reviewer,
|
|
metadata_json=_json_dumps({"reasons": _json_object(match.reasons_json)}),
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
session.add(decision)
|
|
else:
|
|
decision.status = match.status
|
|
decision.active = True
|
|
decision.osm_dataset_id = None if feature is None else feature.dataset_id
|
|
decision.osm_feature_id = None if feature is None else feature.id
|
|
decision.route_match_id = match.id
|
|
decision.confidence = match.confidence
|
|
decision.evidence_hash = evidence_hash
|
|
decision.selector_json = _json_dumps(selector)
|
|
decision.action_json = _json_dumps(action)
|
|
decision.note = note if note is not None else decision.note
|
|
decision.reviewer = reviewer if reviewer is not None else decision.reviewer
|
|
decision.metadata_json = _json_dumps({"reasons": _json_object(match.reasons_json)})
|
|
decision.updated_at = now
|
|
_resolve_review_item_by_fingerprint(session, _route_fingerprint(route), decision=decision)
|
|
session.flush()
|
|
return decision
|
|
|
|
|
|
def review_item_payload(item: MapGtfsReviewItem) -> dict[str, Any]:
|
|
route = item.gtfs_route
|
|
stop = item.gtfs_stop
|
|
osm = item.osm_feature
|
|
return {
|
|
"id": item.id,
|
|
"fingerprint": item.fingerprint,
|
|
"queue": item.queue,
|
|
"item_type": item.item_type,
|
|
"status": item.status,
|
|
"severity": item.severity,
|
|
"title": item.title,
|
|
"description": item.description,
|
|
"source": None if item.source is None else {"id": item.source.id, "name": item.source.name, "kind": item.source.kind},
|
|
"dataset": None if item.dataset is None else {"id": item.dataset.id, "kind": item.dataset.kind, "sha256": item.dataset.sha256},
|
|
"gtfs_route": None
|
|
if route is None
|
|
else {
|
|
"id": route.id,
|
|
"dataset_id": route.dataset_id,
|
|
"route_id": route.route_id,
|
|
"ref": route.short_name,
|
|
"name": route.long_name,
|
|
"mode": route.mode,
|
|
"operator": route.operator_name,
|
|
},
|
|
"gtfs_stop": None
|
|
if stop is None
|
|
else {
|
|
"id": stop.id,
|
|
"dataset_id": stop.dataset_id,
|
|
"stop_id": stop.stop_id,
|
|
"name": stop.name,
|
|
"lat": stop.lat,
|
|
"lon": stop.lon,
|
|
},
|
|
"osm_feature": None
|
|
if osm is None
|
|
else {
|
|
"id": osm.id,
|
|
"dataset_id": osm.dataset_id,
|
|
"osm_type": osm.osm_type,
|
|
"osm_id": osm.osm_id,
|
|
"kind": osm.kind,
|
|
"mode": osm.mode,
|
|
"ref": osm.ref,
|
|
"name": osm.name,
|
|
},
|
|
"route_match_id": item.route_match_id,
|
|
"canonical_stop_id": item.canonical_stop_id,
|
|
"canonical_stop_link_id": item.canonical_stop_link_id,
|
|
"decision_id": item.decision_id,
|
|
"evidence_hash": item.evidence_hash,
|
|
"metadata": _json_object(item.metadata_json),
|
|
"created_at": item.created_at.isoformat(),
|
|
"updated_at": item.updated_at.isoformat(),
|
|
"resolved_at": None if item.resolved_at is None else item.resolved_at.isoformat(),
|
|
}
|
|
|
|
|
|
def decision_payload(decision: MapGtfsDecision) -> dict[str, Any]:
|
|
return {
|
|
"id": decision.id,
|
|
"decision_key": decision.decision_key,
|
|
"decision_type": decision.decision_type,
|
|
"status": decision.status,
|
|
"active": decision.active,
|
|
"source_id": decision.source_id,
|
|
"gtfs_dataset_id": decision.gtfs_dataset_id,
|
|
"gtfs_route_id": decision.gtfs_route_id,
|
|
"gtfs_stop_id": decision.gtfs_stop_id,
|
|
"osm_dataset_id": decision.osm_dataset_id,
|
|
"osm_feature_id": decision.osm_feature_id,
|
|
"route_match_id": decision.route_match_id,
|
|
"canonical_stop_id": decision.canonical_stop_id,
|
|
"canonical_stop_link_id": decision.canonical_stop_link_id,
|
|
"confidence": decision.confidence,
|
|
"evidence_hash": decision.evidence_hash,
|
|
"selector": _json_object(decision.selector_json),
|
|
"action": _json_object(decision.action_json),
|
|
"note": decision.note,
|
|
"reviewer": decision.reviewer,
|
|
"metadata": _json_object(decision.metadata_json),
|
|
"created_at": decision.created_at.isoformat(),
|
|
"updated_at": decision.updated_at.isoformat(),
|
|
}
|
|
|
|
|
|
def _generate_route_review_items(
|
|
session: Session,
|
|
*,
|
|
source_id: int | None,
|
|
dataset_id: int | None,
|
|
limit: int,
|
|
) -> dict[str, int]:
|
|
stmt = (
|
|
select(RouteMatch)
|
|
.options(joinedload(RouteMatch.gtfs_route), joinedload(RouteMatch.osm_feature))
|
|
.join(RouteMatch.gtfs_route)
|
|
.join(Dataset, Dataset.id == GtfsRoute.dataset_id)
|
|
.where(Dataset.is_active.is_(True), Dataset.kind == "gtfs", RouteMatch.status.in_(ROUTE_REVIEW_STATUSES))
|
|
.order_by(RouteMatch.status.desc(), RouteMatch.confidence.asc(), RouteMatch.id)
|
|
)
|
|
if source_id is not None:
|
|
stmt = stmt.where(Dataset.source_id == source_id)
|
|
if dataset_id is not None:
|
|
stmt = stmt.where(GtfsRoute.dataset_id == dataset_id)
|
|
matches = session.scalars(stmt.limit(limit)).all()
|
|
counts = _empty_counts()
|
|
counts["items"] = len(matches)
|
|
for match in matches:
|
|
route = match.gtfs_route
|
|
if route is None:
|
|
continue
|
|
dataset = session.get(Dataset, route.dataset_id)
|
|
metadata = {
|
|
"route_match_status": match.status,
|
|
"confidence": match.confidence,
|
|
"reasons": _json_object(match.reasons_json),
|
|
"gtfs": {
|
|
"route_id": route.route_id,
|
|
"route_key": route.route_key,
|
|
"ref": route.short_name,
|
|
"mode": route.mode,
|
|
},
|
|
"osm_feature_id": match.osm_feature_id,
|
|
}
|
|
status_label = match.status.replace("_", " ")
|
|
item = _upsert_review_item(
|
|
session,
|
|
fingerprint=_route_fingerprint(route),
|
|
queue="route_matching",
|
|
item_type="gtfs_route_osm_route",
|
|
severity=_route_review_severity(match.status),
|
|
title=f"Review route {route.short_name or route.route_id}: {status_label}",
|
|
description=_route_review_description(route, match),
|
|
source_id=None if dataset is None else dataset.source_id,
|
|
dataset_id=route.dataset_id,
|
|
gtfs_route_id=route.id,
|
|
osm_feature_id=match.osm_feature_id,
|
|
route_match_id=match.id,
|
|
evidence_payload=metadata,
|
|
)
|
|
counts[item] += 1
|
|
return counts
|
|
|
|
|
|
def _generate_gtfs_route_dedup_review_items(
|
|
session: Session,
|
|
*,
|
|
source_id: int | None,
|
|
dataset_id: int | None,
|
|
limit: int,
|
|
) -> dict[str, int]:
|
|
snapshot = gtfs_harmonized_snapshot(session)
|
|
snapshot_items_by_dataset = {
|
|
int(item["dataset_id"]): item
|
|
for item in snapshot.get("datasets", [])
|
|
if item.get("dataset_id") is not None
|
|
}
|
|
included_dataset_ids = [int(item["dataset_id"]) for item in snapshot.get("included", []) if item.get("dataset_active", True)]
|
|
shadowed_items = {
|
|
int(item["dataset_id"]): item
|
|
for item in snapshot.get("shadowed", [])
|
|
if item.get("dataset_id") is not None
|
|
}
|
|
if source_id is not None:
|
|
shadowed_items = {dataset_id: item for dataset_id, item in shadowed_items.items() if int(item.get("source_id") or 0) == int(source_id)}
|
|
if dataset_id is not None:
|
|
shadowed_items = {int(dataset_id): item} if int(dataset_id) in shadowed_items else {}
|
|
route_shadow_map = active_harmonized_gtfs_route_shadow_map(session)
|
|
if not included_dataset_ids and not route_shadow_map:
|
|
counts = _empty_counts()
|
|
counts["items"] = 0
|
|
return counts
|
|
|
|
pairs: list[tuple[GtfsRoute, GtfsRoute, dict[str, Any]]] = []
|
|
if included_dataset_ids and shadowed_items:
|
|
included_by_key = _routes_by_overlap_key(session, included_dataset_ids)
|
|
shadowed_routes = session.scalars(
|
|
select(GtfsRoute)
|
|
.where(GtfsRoute.dataset_id.in_(list(shadowed_items)))
|
|
.order_by(GtfsRoute.dataset_id, GtfsRoute.mode, GtfsRoute.short_name, GtfsRoute.route_id)
|
|
.limit(max(1, int(limit)) * 3)
|
|
).all()
|
|
for route in shadowed_routes:
|
|
key = gtfs_route_overlap_key(route)
|
|
if not key:
|
|
continue
|
|
candidates = included_by_key.get(key, [])
|
|
if not candidates:
|
|
continue
|
|
shadowed_item = shadowed_items[int(route.dataset_id)]
|
|
preferred_dataset_id = _optional_int(shadowed_item.get("shadowed_by_dataset_id"))
|
|
candidate = _best_included_duplicate_candidate(candidates, preferred_dataset_id=preferred_dataset_id)
|
|
pairs.append((route, candidate, shadowed_item))
|
|
if len(pairs) >= max(1, int(limit)):
|
|
break
|
|
if len(pairs) < max(1, int(limit)) and route_shadow_map:
|
|
pairs.extend(
|
|
_partial_route_shadow_pairs(
|
|
session,
|
|
route_shadow_map=route_shadow_map,
|
|
snapshot_items_by_dataset=snapshot_items_by_dataset,
|
|
source_id=source_id,
|
|
dataset_id=dataset_id,
|
|
limit=max(0, int(limit) - len(pairs)),
|
|
)
|
|
)
|
|
|
|
trip_counts = _trip_counts_for_routes(session, [route for pair in pairs for route in pair[:2]])
|
|
counts = _empty_counts()
|
|
counts["items"] = len(pairs)
|
|
for shadowed_route, included_route, shadowed_item in pairs:
|
|
dataset = session.get(Dataset, shadowed_route.dataset_id)
|
|
evidence = _route_dedup_evidence(
|
|
shadowed_route,
|
|
included_route,
|
|
shadowed_item=shadowed_item,
|
|
trip_counts=trip_counts,
|
|
)
|
|
severity = "warn" if evidence["data_differs"] else "info"
|
|
item = _upsert_review_item(
|
|
session,
|
|
fingerprint=_route_dedup_fingerprint(shadowed_route, included_route),
|
|
queue="gtfs_deduplication",
|
|
item_type="gtfs_route_duplicate",
|
|
severity=severity,
|
|
title=_route_dedup_title(shadowed_route, included_route, shadowed_item),
|
|
description=_route_dedup_description(evidence),
|
|
source_id=None if dataset is None else dataset.source_id,
|
|
dataset_id=shadowed_route.dataset_id,
|
|
gtfs_route_id=shadowed_route.id,
|
|
evidence_payload=evidence,
|
|
)
|
|
counts[item] += 1
|
|
return counts
|
|
|
|
|
|
def _partial_route_shadow_pairs(
|
|
session: Session,
|
|
*,
|
|
route_shadow_map: dict[int, int],
|
|
snapshot_items_by_dataset: dict[int, dict[str, Any]],
|
|
source_id: int | None,
|
|
dataset_id: int | None,
|
|
limit: int,
|
|
) -> list[tuple[GtfsRoute, GtfsRoute, dict[str, Any]]]:
|
|
if limit <= 0 or not route_shadow_map:
|
|
return []
|
|
route_ids = sorted({*route_shadow_map.keys(), *route_shadow_map.values()})
|
|
routes = {int(route.id): route for route in session.scalars(select(GtfsRoute).where(GtfsRoute.id.in_(route_ids))).all()}
|
|
dataset_ids = sorted({int(route.dataset_id) for route in routes.values()})
|
|
datasets = {int(dataset.id): dataset for dataset in session.scalars(select(Dataset).where(Dataset.id.in_(dataset_ids))).all()}
|
|
pairs: list[tuple[GtfsRoute, GtfsRoute, dict[str, Any]]] = []
|
|
for shadowed_route_id, included_route_id in sorted(route_shadow_map.items()):
|
|
shadowed_route = routes.get(int(shadowed_route_id))
|
|
included_route = routes.get(int(included_route_id))
|
|
if shadowed_route is None or included_route is None:
|
|
continue
|
|
shadowed_dataset = datasets.get(int(shadowed_route.dataset_id))
|
|
included_dataset = datasets.get(int(included_route.dataset_id))
|
|
if source_id is not None and (shadowed_dataset is None or int(shadowed_dataset.source_id) != int(source_id)):
|
|
continue
|
|
if dataset_id is not None and int(shadowed_route.dataset_id) != int(dataset_id):
|
|
continue
|
|
shadowed_item = {
|
|
**snapshot_items_by_dataset.get(int(shadowed_route.dataset_id), {}),
|
|
"reason": "route_covered_by_higher_precedence_dataset",
|
|
"shadowed_by_dataset_id": included_route.dataset_id,
|
|
"shadowed_by_source_id": None if included_dataset is None else included_dataset.source_id,
|
|
"route_key_overlap": 1,
|
|
"route_key_overlap_ratio": 1.0,
|
|
}
|
|
pairs.append((shadowed_route, included_route, shadowed_item))
|
|
if len(pairs) >= limit:
|
|
break
|
|
return pairs
|
|
|
|
|
|
def _generate_stop_review_items(
|
|
session: Session,
|
|
*,
|
|
source_id: int | None,
|
|
dataset_id: int | None,
|
|
limit: int,
|
|
) -> dict[str, int]:
|
|
gtfs_link = aliased(CanonicalStopLink)
|
|
visual_link = aliased(CanonicalStopLink)
|
|
visual_link_exists = (
|
|
select(visual_link.id)
|
|
.where(
|
|
visual_link.canonical_stop_id == gtfs_link.canonical_stop_id,
|
|
visual_link.object_type == "osm_feature",
|
|
)
|
|
.exists()
|
|
)
|
|
stmt = (
|
|
select(GtfsStop, gtfs_link.id, gtfs_link.canonical_stop_id)
|
|
.join(Dataset, Dataset.id == GtfsStop.dataset_id)
|
|
.outerjoin(
|
|
gtfs_link,
|
|
and_(
|
|
gtfs_link.object_type == "gtfs_stop",
|
|
gtfs_link.dataset_id == GtfsStop.dataset_id,
|
|
gtfs_link.object_id == GtfsStop.id,
|
|
),
|
|
)
|
|
.where(
|
|
Dataset.is_active.is_(True),
|
|
Dataset.kind == "gtfs",
|
|
or_(gtfs_link.id.is_(None), ~visual_link_exists),
|
|
)
|
|
.order_by(GtfsStop.dataset_id, GtfsStop.name, GtfsStop.stop_id)
|
|
)
|
|
if source_id is not None:
|
|
stmt = stmt.where(Dataset.source_id == source_id)
|
|
if dataset_id is not None:
|
|
stmt = stmt.where(GtfsStop.dataset_id == dataset_id)
|
|
rows = session.execute(stmt.limit(limit)).all()
|
|
counts = _empty_counts()
|
|
counts["items"] = len(rows)
|
|
for stop, link_id, canonical_stop_id in rows:
|
|
dataset = session.get(Dataset, stop.dataset_id)
|
|
missing_canonical = link_id is None
|
|
metadata = {
|
|
"missing_canonical_stop": missing_canonical,
|
|
"missing_visual_stop": not missing_canonical,
|
|
"gtfs": {"stop_id": stop.stop_id, "name": stop.name, "lat": stop.lat, "lon": stop.lon},
|
|
"canonical_stop_id": canonical_stop_id,
|
|
}
|
|
item = _upsert_review_item(
|
|
session,
|
|
fingerprint=_stop_fingerprint(stop),
|
|
queue="stop_matching",
|
|
item_type="gtfs_stop_osm_stop",
|
|
severity="warn" if missing_canonical else "info",
|
|
title=f"Review stop {stop.name or stop.stop_id}",
|
|
description="GTFS stop has no canonical stop link." if missing_canonical else "GTFS stop has no concrete OSM stop/platform link.",
|
|
source_id=None if dataset is None else dataset.source_id,
|
|
dataset_id=stop.dataset_id,
|
|
gtfs_stop_id=stop.id,
|
|
canonical_stop_id=canonical_stop_id,
|
|
canonical_stop_link_id=link_id,
|
|
evidence_payload=metadata,
|
|
)
|
|
counts[item] += 1
|
|
return counts
|
|
|
|
|
|
def _routes_by_overlap_key(session: Session, dataset_ids: list[int]) -> dict[str, list[GtfsRoute]]:
|
|
if not dataset_ids:
|
|
return {}
|
|
rows = session.scalars(
|
|
select(GtfsRoute)
|
|
.where(GtfsRoute.dataset_id.in_(dataset_ids))
|
|
.order_by(GtfsRoute.dataset_id, GtfsRoute.mode, GtfsRoute.short_name, GtfsRoute.route_id)
|
|
).all()
|
|
routes: dict[str, list[GtfsRoute]] = {}
|
|
for route in rows:
|
|
key = gtfs_route_overlap_key(route)
|
|
if key:
|
|
routes.setdefault(key, []).append(route)
|
|
return routes
|
|
|
|
|
|
def _best_included_duplicate_candidate(routes: list[GtfsRoute], *, preferred_dataset_id: int | None) -> GtfsRoute:
|
|
return sorted(
|
|
routes,
|
|
key=lambda route: (
|
|
0 if preferred_dataset_id is not None and int(route.dataset_id) == preferred_dataset_id else 1,
|
|
route.dataset_id,
|
|
route.route_id,
|
|
),
|
|
)[0]
|
|
|
|
|
|
def _trip_counts_for_routes(session: Session, routes: list[GtfsRoute]) -> dict[tuple[int, str], int]:
|
|
keys = {(int(route.dataset_id), str(route.route_id)) for route in routes}
|
|
if not keys:
|
|
return {}
|
|
dataset_ids = sorted({dataset_id for dataset_id, _route_id in keys})
|
|
route_ids = sorted({route_id for _dataset_id, route_id in keys})
|
|
rows = session.execute(
|
|
select(GtfsTrip.dataset_id, GtfsTrip.route_id, func.count())
|
|
.where(GtfsTrip.dataset_id.in_(dataset_ids), GtfsTrip.route_id.in_(route_ids))
|
|
.group_by(GtfsTrip.dataset_id, GtfsTrip.route_id)
|
|
).all()
|
|
return {
|
|
(int(dataset_id), str(route_id)): int(count or 0)
|
|
for dataset_id, route_id, count in rows
|
|
if (int(dataset_id), str(route_id)) in keys
|
|
}
|
|
|
|
|
|
def _route_dedup_evidence(
|
|
shadowed_route: GtfsRoute,
|
|
included_route: GtfsRoute,
|
|
*,
|
|
shadowed_item: dict[str, Any],
|
|
trip_counts: dict[tuple[int, str], int],
|
|
) -> dict[str, Any]:
|
|
shadowed_trip_count = trip_counts.get((int(shadowed_route.dataset_id), str(shadowed_route.route_id)), 0)
|
|
included_trip_count = trip_counts.get((int(included_route.dataset_id), str(included_route.route_id)), 0)
|
|
differences = []
|
|
for field, left, right in [
|
|
("name", shadowed_route.long_name, included_route.long_name),
|
|
("operator", shadowed_route.operator_name, included_route.operator_name),
|
|
("route_scope", shadowed_route.route_scope, included_route.route_scope),
|
|
]:
|
|
if _dedup_text(left) != _dedup_text(right):
|
|
differences.append(field)
|
|
if shadowed_trip_count != included_trip_count:
|
|
differences.append("trip_count")
|
|
if _geometry_digest(shadowed_route.geometry_geojson) != _geometry_digest(included_route.geometry_geojson):
|
|
differences.append("geometry")
|
|
return {
|
|
"data_differs": bool(differences),
|
|
"differences": differences,
|
|
"overlap_key": gtfs_route_overlap_key(shadowed_route),
|
|
"shadowed_dataset": {
|
|
"dataset_id": shadowed_route.dataset_id,
|
|
"source_id": shadowed_item.get("source_id"),
|
|
"source_name": shadowed_item.get("source_name"),
|
|
"route_id": shadowed_route.route_id,
|
|
"ref": shadowed_route.short_name,
|
|
"name": shadowed_route.long_name,
|
|
"operator": shadowed_route.operator_name,
|
|
"mode": shadowed_route.mode,
|
|
"trip_count": shadowed_trip_count,
|
|
},
|
|
"included_dataset": {
|
|
"dataset_id": included_route.dataset_id,
|
|
"source_id": shadowed_item.get("shadowed_by_source_id"),
|
|
"route_id": included_route.route_id,
|
|
"ref": included_route.short_name,
|
|
"name": included_route.long_name,
|
|
"operator": included_route.operator_name,
|
|
"mode": included_route.mode,
|
|
"trip_count": included_trip_count,
|
|
},
|
|
"snapshot": {
|
|
"reason": shadowed_item.get("reason"),
|
|
"route_key_overlap": shadowed_item.get("route_key_overlap"),
|
|
"route_key_overlap_ratio": shadowed_item.get("route_key_overlap_ratio"),
|
|
},
|
|
}
|
|
|
|
|
|
def _route_dedup_title(shadowed_route: GtfsRoute, included_route: GtfsRoute, shadowed_item: dict[str, Any]) -> str:
|
|
ref = shadowed_route.short_name or shadowed_route.route_id
|
|
source = shadowed_item.get("source_name") or f"dataset #{shadowed_route.dataset_id}"
|
|
return f"Review duplicate GTFS route {ref}: {source} -> dataset #{included_route.dataset_id}"
|
|
|
|
|
|
def _route_dedup_description(evidence: dict[str, Any]) -> str:
|
|
differences = evidence.get("differences") or []
|
|
if differences:
|
|
return f"Route is shadowed by the active snapshot but differs in {', '.join(differences)}."
|
|
return "Route is shadowed by the active snapshot and looks equivalent on first-pass route evidence."
|
|
|
|
|
|
def _route_dedup_fingerprint(shadowed_route: GtfsRoute, included_route: GtfsRoute) -> str:
|
|
return f"gtfs-dedup:{shadowed_route.dataset_id}:{shadowed_route.route_id}:{included_route.dataset_id}:{included_route.route_id}"
|
|
|
|
|
|
def _geometry_digest(value: str | None) -> str:
|
|
if not value:
|
|
return ""
|
|
return hashlib.sha256(str(value).encode("utf-8")).hexdigest()
|
|
|
|
|
|
def _dedup_text(value: str | None) -> str:
|
|
return " ".join(str(value or "").casefold().replace("ß", "ss").split())
|
|
|
|
|
|
def _upsert_review_item(
|
|
session: Session,
|
|
*,
|
|
fingerprint: str,
|
|
queue: str,
|
|
item_type: str,
|
|
severity: str,
|
|
title: str,
|
|
description: str,
|
|
source_id: int | None = None,
|
|
dataset_id: int | None = None,
|
|
gtfs_route_id: int | None = None,
|
|
gtfs_stop_id: int | None = None,
|
|
osm_feature_id: int | None = None,
|
|
route_match_id: int | None = None,
|
|
canonical_stop_id: int | None = None,
|
|
canonical_stop_link_id: int | None = None,
|
|
decision_id: int | None = None,
|
|
evidence_payload: dict[str, Any] | None = None,
|
|
) -> str:
|
|
evidence_hash = _hash_payload(evidence_payload or {})
|
|
metadata_json = _json_dumps(evidence_payload or {})
|
|
item = session.scalar(select(MapGtfsReviewItem).where(MapGtfsReviewItem.fingerprint == fingerprint))
|
|
now = datetime.now(timezone.utc)
|
|
if item is None:
|
|
session.add(
|
|
MapGtfsReviewItem(
|
|
fingerprint=fingerprint,
|
|
queue=queue,
|
|
item_type=item_type,
|
|
status="open",
|
|
severity=severity,
|
|
title=title,
|
|
description=description,
|
|
source_id=source_id,
|
|
dataset_id=dataset_id,
|
|
gtfs_route_id=gtfs_route_id,
|
|
gtfs_stop_id=gtfs_stop_id,
|
|
osm_feature_id=osm_feature_id,
|
|
route_match_id=route_match_id,
|
|
canonical_stop_id=canonical_stop_id,
|
|
canonical_stop_link_id=canonical_stop_link_id,
|
|
decision_id=decision_id,
|
|
evidence_hash=evidence_hash,
|
|
metadata_json=metadata_json,
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
)
|
|
return "created"
|
|
if item.evidence_hash == evidence_hash and item.status in TERMINAL_REVIEW_STATUSES:
|
|
return "unchanged"
|
|
changed = (
|
|
item.queue != queue
|
|
or item.item_type != item_type
|
|
or item.severity != severity
|
|
or item.title != title
|
|
or item.description != description
|
|
or item.source_id != source_id
|
|
or item.dataset_id != dataset_id
|
|
or item.gtfs_route_id != gtfs_route_id
|
|
or item.gtfs_stop_id != gtfs_stop_id
|
|
or item.osm_feature_id != osm_feature_id
|
|
or item.route_match_id != route_match_id
|
|
or item.canonical_stop_id != canonical_stop_id
|
|
or item.canonical_stop_link_id != canonical_stop_link_id
|
|
or item.decision_id != decision_id
|
|
or item.evidence_hash != evidence_hash
|
|
or item.metadata_json != metadata_json
|
|
)
|
|
was_terminal = item.status in TERMINAL_REVIEW_STATUSES
|
|
item.queue = queue
|
|
item.item_type = item_type
|
|
item.severity = severity
|
|
item.title = title
|
|
item.description = description
|
|
item.source_id = source_id
|
|
item.dataset_id = dataset_id
|
|
item.gtfs_route_id = gtfs_route_id
|
|
item.gtfs_stop_id = gtfs_stop_id
|
|
item.osm_feature_id = osm_feature_id
|
|
item.route_match_id = route_match_id
|
|
item.canonical_stop_id = canonical_stop_id
|
|
item.canonical_stop_link_id = canonical_stop_link_id
|
|
item.decision_id = decision_id
|
|
item.evidence_hash = evidence_hash
|
|
item.metadata_json = metadata_json
|
|
if was_terminal:
|
|
item.status = "open"
|
|
item.resolved_at = None
|
|
item.updated_at = now
|
|
return "reopened"
|
|
if changed:
|
|
item.updated_at = now
|
|
return "updated"
|
|
return "unchanged"
|
|
|
|
|
|
def _resolve_review_item_by_fingerprint(
|
|
session: Session,
|
|
fingerprint: str,
|
|
*,
|
|
decision: MapGtfsDecision | None = None,
|
|
) -> None:
|
|
item = session.scalar(select(MapGtfsReviewItem).where(MapGtfsReviewItem.fingerprint == fingerprint))
|
|
if item is None:
|
|
return
|
|
now = datetime.now(timezone.utc)
|
|
item.status = "resolved"
|
|
item.resolved_at = now
|
|
item.updated_at = now
|
|
if decision is not None:
|
|
item.decision = decision
|
|
|
|
|
|
def _route_review_severity(status: str) -> str:
|
|
if status == "missing":
|
|
return "error"
|
|
if status in {"weak", "probable"}:
|
|
return "warn"
|
|
return "info"
|
|
|
|
|
|
def _route_review_description(route: GtfsRoute, match: RouteMatch) -> str:
|
|
if match.status == "missing":
|
|
return "No plausible OSM route relation is linked yet."
|
|
if match.status == "weak":
|
|
return "The best OSM route candidate has weak evidence."
|
|
if match.status == "probable":
|
|
return "The OSM route candidate is plausible but should be reviewed."
|
|
return "Automatic route match is available for review."
|
|
|
|
|
|
def _route_fingerprint(route: GtfsRoute) -> str:
|
|
return f"route:{route.dataset_id}:{route.route_id}"
|
|
|
|
|
|
def _stop_fingerprint(stop: GtfsStop) -> str:
|
|
return f"stop:{stop.dataset_id}:{stop.stop_id}"
|
|
|
|
|
|
def _stable_key(decision_type: str, selector: dict[str, Any], action: dict[str, Any]) -> str:
|
|
payload = {"decision_type": decision_type, "selector": selector, "action": action}
|
|
return f"{decision_type}:{_hash_payload(payload)[:40]}"
|
|
|
|
|
|
def _hash_payload(payload: dict[str, Any]) -> str:
|
|
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
|
|
return hashlib.sha256(encoded).hexdigest()
|
|
|
|
|
|
def _json_object(value: str | None) -> dict[str, Any]:
|
|
if not value:
|
|
return {}
|
|
try:
|
|
payload = json.loads(value)
|
|
except json.JSONDecodeError:
|
|
return {}
|
|
return payload if isinstance(payload, dict) else {}
|
|
|
|
|
|
def _json_dumps(value: dict[str, Any]) -> str:
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
|
|
|
|
|
def _empty_counts() -> dict[str, int]:
|
|
return {"created": 0, "updated": 0, "reopened": 0, "unchanged": 0, "resolved": 0, "items": 0}
|
|
|
|
|
|
def _merge_counts(target: dict[str, Any], source: dict[str, int]) -> None:
|
|
for key in ["created", "updated", "reopened", "unchanged", "resolved"]:
|
|
target[key] = int(target.get(key, 0)) + int(source.get(key, 0))
|
|
|
|
|
|
def _optional_int(value: object) -> int | None:
|
|
try:
|
|
return None if value is None else int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _emit(
|
|
progress_callback: ProgressCallback | None,
|
|
event_type: str,
|
|
message: str,
|
|
progress_current: int | None,
|
|
progress_total: int | None,
|
|
metadata: dict[str, Any] | None = None,
|
|
) -> None:
|
|
if progress_callback is not None:
|
|
progress_callback(event_type, message, progress_current, progress_total, metadata)
|
|
|
|
|
|
def _severity_order_expression():
|
|
return case(
|
|
(MapGtfsReviewItem.severity == "error", 0),
|
|
(MapGtfsReviewItem.severity == "warn", 1),
|
|
(MapGtfsReviewItem.severity == "info", 2),
|
|
else_=9,
|
|
)
|