Next alpha stage commit

app/pipeline/route_layer.py

@@ -11,6 +11,7 @@ from sqlalchemy.orm import Session
 
 from app.config import settings
 from app.gtfs_storage import all_scheduled_stop_ids, stop_times_by_trip as storage_stop_times_by_trip
+from app.harmonization import active_harmonized_gtfs_shadowed_route_ids
 from app.models import (
     CanonicalStop,
     CanonicalStopLink,
@@ -36,7 +37,7 @@ from app.pipeline.utils import bbox_overlap, geometry_json_and_bbox, norm_ref, n
 from app.spatial import analyze_postgresql_tables, refresh_postgis_geometries, using_postgresql
 
 
-ROUTE_LAYER_VERSION = "route_layer_v3_stop_alias_matching"
+ROUTE_LAYER_VERSION = "route_layer_v4_harmonized_route_ownership"
 GTFS_ROUTE_PATTERN_NULL_SHAPE = "__route__"
 OSM_STOP_LINK_RADIUS_DEG = 0.0018
 OSM_STOP_NAME_LINK_RADIUS_DEG = 0.0032
@@ -144,6 +145,7 @@ def rebuild_route_layer(
         "route_patterns_updated": pattern_result.get("route_patterns_updated", 0),
         "route_patterns_reused": pattern_result.get("route_patterns_reused", 0),
         "route_patterns_removed": pattern_result.get("route_patterns_removed", 0),
+        "shadowed_routes_skipped": pattern_result.get("shadowed_routes_skipped", 0),
         "route_pattern_links": pattern_result["route_pattern_links"],
         "trip_pattern_links": pattern_result["trip_pattern_links"],
         "route_pattern_stops": pattern_result["route_pattern_stops"],
@@ -161,6 +163,7 @@ def _route_layer_dependency(session: Session) -> dict[str, object]:
         for dataset in session.scalars(select(Dataset).where(Dataset.is_active.is_(True)).order_by(Dataset.kind, Dataset.id)).all()
     ]
     match_rows = session.execute(select(RouteMatch.id, RouteMatch.gtfs_route_id, RouteMatch.osm_feature_id, RouteMatch.status, RouteMatch.updated_at).order_by(RouteMatch.id)).all()
+    shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session)
     match_signature = dependency_hash(
         [
             [
@@ -177,6 +180,10 @@ def _route_layer_dependency(session: Session) -> dict[str, object]:
         "version": ROUTE_LAYER_VERSION,
         "active_datasets": active_datasets,
         "route_matches": {"count": len(match_rows), "signature": match_signature},
+        "shadowed_routes": {
+            "count": len(shadowed_route_ids),
+            "signature": dependency_hash(sorted(shadowed_route_ids)) if shadowed_route_ids else "",
+        },
     }
 
 
@@ -945,14 +952,15 @@ def _build_route_patterns(
 ) -> dict[str, int]:
     osm_candidates = _osm_route_candidates(session, progress_callback=progress_callback)
     overrides = _route_layer_overrides(session)
-    seeds = _gtfs_pattern_seeds(session)
+    shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session)
+    seeds, shadowed_routes_skipped = _gtfs_pattern_seeds(session, shadowed_route_ids=shadowed_route_ids)
     _emit_progress(
         progress_callback,
         "route_layer_pattern_seeds",
         f"Loaded {len(seeds)} GTFS route-pattern seeds.",
         0,
         len(seeds),
-        {"seeds": len(seeds)},
+        {"seeds": len(seeds), "shadowed_routes_skipped": shadowed_routes_skipped},
     )
     link_count = 0
     stop_count = 0
@@ -1256,6 +1264,7 @@ def _build_route_patterns(
         "trip_pattern_links": trip_link_count,
         "route_pattern_stops": stop_count,
         "gtfs_proposed_patterns": proposed_count,
+        "shadowed_routes_skipped": shadowed_routes_skipped,
     }
     _emit_progress(
         progress_callback,
@@ -1494,10 +1503,11 @@ def _route_layer_overrides(session: Session) -> _RouteLayerOverrides:
     return _RouteLayerOverrides(accepted_by_gtfs_route_id=accepted, rejected_by_gtfs_route_id=rejected)
 
 
-def _gtfs_pattern_seeds(session: Session) -> list[_GtfsPatternSeed]:
+def _gtfs_pattern_seeds(session: Session, *, shadowed_route_ids: set[int] | None = None) -> tuple[list[_GtfsPatternSeed], int]:
     active_gtfs_dataset_ids = _active_dataset_ids(session, "gtfs")
     if not active_gtfs_dataset_ids:
-        return []
+        return [], 0
+    shadowed_route_ids = shadowed_route_ids or set()
     rows = session.execute(
         select(GtfsRoute, GtfsTrip.shape_id, func.min(GtfsTrip.trip_id))
         .join(GtfsTrip, and_(GtfsTrip.dataset_id == GtfsRoute.dataset_id, GtfsTrip.route_id == GtfsRoute.route_id))
@@ -1525,7 +1535,11 @@ def _gtfs_pattern_seeds(session: Session) -> list[_GtfsPatternSeed]:
         for dataset_id, shape_id, geometry, min_lon, min_lat, max_lon, max_lat in shape_rows
     }
     seeds = []
+    skipped = 0
     for route, shape_id, trip_id in rows:
+        if int(route.id) in shadowed_route_ids:
+            skipped += 1
+            continue
         geometry_text = None
         geometry_source = "none"
         bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat)
@@ -1556,7 +1570,7 @@ def _gtfs_pattern_seeds(session: Session) -> list[_GtfsPatternSeed]:
                 center_point=center_point,
             )
         )
-    return seeds
+    return seeds, skipped
 
 
 def _choose_osm_candidate(