add-ideas/meubility-workbench
1
0
Fork 0
Code Issues 122 Pull Requests Actions Packages Projects Releases Wiki Activity

Next alpha stage commit

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-06 20:54:46 +02:00
parent e23387738b
commit 1d7ec956d2
42 changed files with 11603 additions and 2837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
app/pipeline/matcher.py
View File

@@ -2,13 +2,15 @@ from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import json
from typing import Callable, Optional
from typing import Callable, Optional, Sequence
from shapely.geometry import LineString, MultiLineString, Point, shape
from sqlalchemy import delete, select
from sqlalchemy.orm import Session
from app.harmonization import active_harmonized_gtfs_shadowed_route_ids
from app.config import settings
from app.models import Dataset, GtfsRoute, MatchRule, OsmFeature, RouteMatch
from app.osm_storage import ensure_main_osm_feature, osm_feature_bbox, query_osm_features
@@ -34,7 +36,7 @@ MAX_EXACT_REF_CANDIDATES = 120
OSM_SCOPE_NEAR_DISTANCE_DEG = 0.15
GEOMETRY_PROXIMITY_DEG = 0.0035
GEOMETRY_SAMPLE_POINTS = 24
MATCHER_VERSION = "matcher_v4_scope_spatial_manual_rules"
MATCHER_VERSION = "matcher_v5_spatial_candidate_gate"
ProgressCallback = Callable[[str, str, int | None, int | None, dict[str, object] | None], None]
@@ -78,6 +80,7 @@ def run_route_matching(
*,
progress_callback: ProgressCallback | None = None,
batch_size: int | None = None,
gtfs_dataset_ids: Sequence[int] | None = None,
) -> dict[str, object]:
"""Match active GTFS routes against active OSM route features."""
active_datasets = session.execute(
@@ -86,22 +89,37 @@ def run_route_matching(
if not active_datasets:
return {"routes": 0, "matches": 0, "missing": 0}
dataset_source_ids = {int(dataset_id): int(source_id) for dataset_id, _, source_id in active_datasets}
gtfs_dataset_ids = [int(dataset_id) for dataset_id, kind, _ in active_datasets if kind == "gtfs"]
active_gtfs_dataset_ids = [int(dataset_id) for dataset_id, kind, _ in active_datasets if kind == "gtfs"]
osm_dataset_ids = [int(dataset_id) for dataset_id, kind, _ in active_datasets if kind == "osm_geojson"]
if not gtfs_dataset_ids:
if gtfs_dataset_ids is None:
selected_gtfs_dataset_ids = active_gtfs_dataset_ids
else:
requested_gtfs_dataset_ids = {int(dataset_id) for dataset_id in gtfs_dataset_ids}
selected_gtfs_dataset_ids = [
dataset_id
for dataset_id in active_gtfs_dataset_ids
if dataset_id in requested_gtfs_dataset_ids
]
if not selected_gtfs_dataset_ids:
return {"routes": 0, "matches": 0, "missing": 0}
route_row_ids = session.scalars(
select(GtfsRoute.id)
.where(GtfsRoute.dataset_id.in_(gtfs_dataset_ids))
.order_by(GtfsRoute.dataset_id, GtfsRoute.route_id, GtfsRoute.id)
).all()
all_route_row_ids = [
int(route_id)
for route_id in session.scalars(
select(GtfsRoute.id)
.where(GtfsRoute.dataset_id.in_(selected_gtfs_dataset_ids))
.order_by(GtfsRoute.dataset_id, GtfsRoute.route_id, GtfsRoute.id)
).all()
]
shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session)
route_row_ids = [route_id for route_id in all_route_row_ids if route_id not in shadowed_route_ids]
shadowed_skipped = len(all_route_row_ids) - len(route_row_ids)
# Reconcile current match rows from auto scoring plus durable manual rules.
total_routes = len(route_row_ids)
if total_routes == 0:
return {"routes": 0, "matches": 0, "missing": 0}
return {"routes": 0, "matches": 0, "missing": 0, "shadowed_skipped": shadowed_skipped}
dependency = _route_matching_dependency(session, active_datasets)
dependency = _route_matching_dependency(session, active_datasets, shadowed_route_ids=shadowed_route_ids)
run = start_pipeline_run(
session,
stage=STAGE_MATCH_ROUTES,
@@ -117,11 +135,49 @@ def run_route_matching(
f"Matching {total_routes} GTFS routes in batches of {effective_batch_size}.",
0,
total_routes,
{"gtfs_datasets": gtfs_dataset_ids, "osm_datasets": osm_dataset_ids, "batch_size": effective_batch_size},
{
"gtfs_datasets": selected_gtfs_dataset_ids,
"osm_datasets": osm_dataset_ids,
"batch_size": effective_batch_size,
"shadowed_skipped": shadowed_skipped,
},
)
manual_rules = _manual_match_rules(session)
osm_scope_bbox = osm_feature_bbox(session, osm_dataset_ids, kinds=["route"])
counts = {"routes": total_routes, "matches": 0, "missing": 0, "manual": 0, "created": 0, "updated": 0, "unchanged": 0}
_emit_progress(
progress_callback,
"route_matching_osm_index_started",
"Loading OSM route candidates for matcher.",
0,
None,
{"osm_datasets": osm_dataset_ids},
)
osm_routes = query_osm_features(
session,
osm_dataset_ids,
kinds=["route"],
prefer_materialized_ids=False,
)
osm_route_index = _build_osm_route_index(osm_routes)
feature_profile_cache: dict[tuple[int, str, str], _GeometryProfile | None] = {}
_emit_progress(
progress_callback,
"route_matching_osm_index_completed",
f"Loaded {len(osm_routes)} OSM route candidates.",
len(osm_routes),
None,
{"osm_routes": len(osm_routes)},
)
counts = {
"routes": total_routes,
"matches": 0,
"missing": 0,
"manual": 0,
"created": 0,
"updated": 0,
"unchanged": 0,
"shadowed_skipped": shadowed_skipped,
}
scoped_counts = {"in_osm_scope": 0, "near_osm_scope": 0, "outside_osm_scope": 0, "unknown_scope": 0}
processed = 0
for chunk in _chunks_int(route_row_ids, effective_batch_size):
@@ -137,6 +193,8 @@ def run_route_matching(
dataset_source_ids=dataset_source_ids,
manual_rules=manual_rules,
osm_scope_bbox=osm_scope_bbox,
osm_route_index=osm_route_index,
feature_profile_cache=feature_profile_cache,
scoped_counts=scoped_counts,
)
counts["matches"] += batch_counts["matches"]
@@ -178,7 +236,12 @@ def run_route_matching(
return result
def _route_matching_dependency(session: Session, active_datasets) -> dict[str, object]:
def _route_matching_dependency(
session: Session,
active_datasets,
*,
shadowed_route_ids: set[int] | None = None,
) -> dict[str, object]:
datasets = [
{"id": int(dataset_id), "kind": str(kind), "source_id": int(source_id), "sha256": _dataset_sha(session, int(dataset_id))}
for dataset_id, kind, source_id in active_datasets
@@ -193,7 +256,15 @@ def _route_matching_dependency(session: Session, active_datasets) -> dict[str, o
}
for rule in session.scalars(select(MatchRule).order_by(MatchRule.id)).all()
]
return {"version": MATCHER_VERSION, "active_datasets": datasets, "manual_rules": rules}
return {
"version": MATCHER_VERSION,
"active_datasets": datasets,
"manual_rules": rules,
"shadowed_routes": {
"count": len(shadowed_route_ids or set()),
"digest": _int_set_digest(shadowed_route_ids or set()),
},
}
def _dataset_sha(session: Session, dataset_id: int) -> str | None:
@@ -201,6 +272,13 @@ def _dataset_sha(session: Session, dataset_id: int) -> str | None:
return None if dataset is None else dataset.sha256
def _int_set_digest(values: set[int]) -> str:
if not values:
return ""
payload = ",".join(str(value) for value in sorted(values))
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
def _match_route_batch(
*,
session: Session,
@@ -209,6 +287,8 @@ def _match_route_batch(
dataset_source_ids: dict[int, int],
manual_rules: list[_ManualMatchRule],
osm_scope_bbox: tuple[float | None, float | None, float | None, float | None],
osm_route_index: _OsmRouteIndex,
feature_profile_cache: dict[tuple[int, str, str], _GeometryProfile | None],
scoped_counts: dict[str, int],
) -> dict[str, int]:
matches = 0
@@ -265,10 +345,13 @@ def _match_route_batch(
best_score = 0.0
best_reasons: dict[str, object] = {}
route_geometry_profile = _geometry_profile(route.geometry_geojson)
for feature in candidate_osm_routes_for_route(session, route, osm_dataset_ids):
for feature in _candidate_osm_routes(route, osm_route_index):
if _is_rejected_pair(manual_rules, route, route_source_id, feature, dataset_source_ids.get(feature.dataset_id)):
continue
feature_geometry_profile = _geometry_profile(feature.geometry_geojson)
feature_key = _feature_identity_key(feature)
if feature_key not in feature_profile_cache:
feature_profile_cache[feature_key] = _geometry_profile(feature.geometry_geojson)
feature_geometry_profile = feature_profile_cache[feature_key]
score, reasons = score_route_pair(
route,
feature,
@@ -393,22 +476,26 @@ def _build_osm_route_index(osm_routes: list[OsmFeature]) -> _OsmRouteIndex:
def _candidate_osm_routes(route: GtfsRoute, index: _OsmRouteIndex) -> list[OsmFeature]:
selected: list[OsmFeature] = []
seen: set[int] = set()
seen: set[tuple[int, str, str]] = set()
def add(features: list[OsmFeature], *, require_compatible_mode: bool = True) -> None:
def add(features: list[OsmFeature], *, require_compatible_mode: bool = True, require_spatial: bool = False) -> None:
for feature in features:
if feature.id in seen:
key = (feature.dataset_id, feature.osm_type, feature.osm_id)
if key in seen:
continue
if require_compatible_mode and not _mode_compatible(route.mode or "", feature.mode or ""):
continue
seen.add(feature.id)
if require_spatial and not _candidate_is_spatially_plausible(route, feature):
continue
seen.add(key)
selected.append(feature)
route_ref = norm_ref(route.short_name or route.route_id)
route_has_bbox = _has_known_bbox((route.min_lon, route.min_lat, route.max_lon, route.max_lat))
if route_ref:
add(index.by_ref.get(route_ref, []))
add(index.by_ref.get(route_ref, []), require_spatial=route_has_bbox)
if route.route_key:
add(index.by_route_key.get(route.route_key, []))
add(index.by_route_key.get(route.route_key, []), require_spatial=route_has_bbox)
if selected:
return _spatially_ranked_candidates(route, selected, MAX_EXACT_REF_CANDIDATES)
@@ -431,7 +518,7 @@ def _candidate_osm_routes(route: GtfsRoute, index: _OsmRouteIndex) -> list[OsmFe
near_candidates.append((distance, feature))
fallback_limit = MAX_FALLBACK_CANDIDATES_WITH_REF if route_ref else MAX_FALLBACK_CANDIDATES_WITHOUT_REF
fallback = [feature for _, feature in sorted(near_candidates, key=lambda item: item[0])[:fallback_limit]]
if not fallback:
if not fallback and not route_has_bbox:
fallback = mode_candidates[:fallback_limit]
add(fallback)
return _spatially_ranked_candidates(route, selected, fallback_limit)
@@ -443,17 +530,22 @@ def candidate_osm_routes_for_route(session: Session, route: GtfsRoute, osm_datas
selected: list[OsmFeature] = []
seen: set[tuple[int, str, str]] = set()
def add(features: list[OsmFeature], *, require_compatible_mode: bool = True) -> None:
def add(features: list[OsmFeature], *, require_compatible_mode: bool = True, require_spatial: bool = False) -> None:
for feature in features:
key = (feature.dataset_id, feature.osm_type, feature.osm_id)
if key in seen:
continue
if require_compatible_mode and not _mode_compatible(route.mode or "", feature.mode or ""):
continue
if require_spatial and not _candidate_is_spatially_plausible(route, feature):
continue
seen.add(key)
selected.append(feature)
route_ref = norm_ref(route.short_name or route.route_id)
gtfs_bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat)
route_has_bbox = _has_known_bbox(gtfs_bbox)
exact_ref_bbox = _expanded_bbox(gtfs_bbox, OSM_SCOPE_NEAR_DISTANCE_DEG) if route_has_bbox else None
route_keys = [key for key in [route.route_key, route_ref] if key]
for route_key in dict.fromkeys(route_keys):
add(
@@ -462,14 +554,15 @@ def candidate_osm_routes_for_route(session: Session, route: GtfsRoute, osm_datas
osm_dataset_ids,
kinds=["route"],
route_key=route_key,
)
bbox=exact_ref_bbox,
),
require_spatial=route_has_bbox,
)
if selected:
return _spatially_ranked_candidates(route, selected, MAX_EXACT_REF_CANDIDATES)
gtfs_bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat)
compatible_modes = sorted(MODE_GROUPS.get(route.mode or "", {route.mode or ""}) - {""})
if not any(value is None for value in gtfs_bbox):
if route_has_bbox:
bbox = _expanded_bbox(gtfs_bbox, 0.10)
add(
query_osm_features(
@@ -482,7 +575,7 @@ def candidate_osm_routes_for_route(session: Session, route: GtfsRoute, osm_datas
),
require_compatible_mode=False,
)
if not selected:
if not selected and not route_has_bbox:
add(
query_osm_features(
session,
@@ -634,6 +727,27 @@ def route_match_scope(route: GtfsRoute, osm_scope_bbox: tuple[float | None, floa
return "outside_osm_scope"
def _candidate_is_spatially_plausible(route: GtfsRoute, feature: OsmFeature) -> bool:
route_bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat)
feature_bbox = (feature.min_lon, feature.min_lat, feature.max_lon, feature.max_lat)
if not _has_known_bbox(route_bbox):
return True
if not _has_known_bbox(feature_bbox):
return False
if bbox_overlap(route_bbox, feature_bbox):
return True
distance = approx_bbox_center_distance_deg(route_bbox, feature_bbox)
return distance is not None and distance < OSM_SCOPE_NEAR_DISTANCE_DEG
def _has_known_bbox(bbox: tuple[float | None, float | None, float | None, float | None]) -> bool:
return not any(value is None for value in bbox)
def _feature_identity_key(feature: OsmFeature) -> tuple[int, str, str]:
return (int(feature.dataset_id), str(feature.osm_type), str(feature.osm_id))
def _combined_bbox(features: list[OsmFeature]) -> tuple[float | None, float | None, float | None, float | None]:
boxes = [
(feature.min_lon, feature.min_lat, feature.max_lon, feature.max_lat)