add-ideas/meubility-workbench
1
0
Fork 0
Code Issues 122 Pull Requests Actions Packages Projects Releases Wiki Activity

Next alpha stage commit

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-06 20:54:46 +02:00
parent e23387738b
commit 1d7ec956d2
42 changed files with 11603 additions and 2837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
app/pipeline/gtfs.py
View File

@@ -15,6 +15,8 @@ from sqlalchemy import func, select, text
from sqlalchemy.orm import Session
from app.config import settings
from app.decision_replay import apply_gtfs_decision_replay, capture_gtfs_decision_replay
from app.gtfs_diff import create_gtfs_update_diff_from_stage
from app.gtfs_storage import GTFS_STORAGE_MAIN, GTFS_STORAGE_METADATA_KEY, GTFS_STORAGE_SIDECAR_STOP_TIMES, effective_gtfs_timetable_storage
from app.models import (
Dataset,
@@ -64,6 +66,7 @@ GTFS_EXTENDED_MODE_RANGES = [
GTFS_IMPORTER_VERSION = "gtfs_import_v6_sidecar_stop_times"
REQUIRED_FILES = {"agency.txt", "stops.txt", "routes.txt", "trips.txt", "stop_times.txt"}
GTFS_CALENDAR_WEEKDAY_COLUMNS = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]
GTFS_STAGE_BATCH_SIZE = 50_000
ProgressCallback = Callable[[str, str, int | None, int | None, dict[str, Any] | None], None]
@@ -82,7 +85,7 @@ def run_gtfs_source(session: Session, source: Source, progress_callback: Progres
)
.order_by(Dataset.id.desc())
)
if existing is not None and _dataset_importer_version(existing) == GTFS_IMPORTER_VERSION:
if existing is not None and _dataset_import_can_reuse(existing):
return existing
return import_gtfs_zip(session=session, source=source, zip_path=local_path, source_hash=source_hash, progress_callback=progress_callback)
@@ -395,15 +398,96 @@ def _activate_staged_gtfs(
dataset = session.get(Dataset, dataset.id) or dataset
source = session.get(Source, source.id) or source
replaced_datasets = [existing for existing in list(source.datasets) if existing.id != dataset.id and existing.kind == "gtfs"]
previous_dataset = next(
(
existing
for existing in sorted(replaced_datasets, key=lambda item: (not item.is_active, item.created_at, item.id))
if existing.status == "imported"
),
None,
)
for existing in source.datasets:
if existing.id != dataset.id:
existing.is_active = False
copy_stop_times = _copy_stop_times_to_main(summary)
heavy_index_drop = copy_stop_times and _should_drop_indexes_for_activation(stage_path)
diff_result: dict[str, Any] | None = None
decision_replay_plan: dict[str, Any] | None = None
if heavy_index_drop:
_emit_progress(progress_callback, "gtfs_activation_indexes_dropped", "Dropping heavy GTFS lookup indexes before bulk activation.", None, None, None)
_drop_gtfs_bulk_indexes(session.connection())
try:
if previous_dataset is not None:
_emit_progress(
progress_callback,
"gtfs_update_diff_started",
f"Comparing staged GTFS dataset #{dataset.id} with previous dataset #{previous_dataset.id}.",
None,
None,
{"previous_dataset_id": previous_dataset.id, "new_dataset_id": dataset.id},
)
try:
diff_result = create_gtfs_update_diff_from_stage(
session,
source=source,
previous_dataset=previous_dataset,
new_dataset=dataset,
stage_path=stage_path,
)
except Exception as exc: # noqa: BLE001 - update diff must not block activation
diff_result = None
summary["gtfs_update_diff_error"] = str(exc)
_emit_progress(
progress_callback,
"gtfs_update_diff_failed",
"GTFS update diff failed; continuing dataset activation.",
None,
None,
{"error": str(exc), "previous_dataset_id": previous_dataset.id, "new_dataset_id": dataset.id},
)
if diff_result is not None:
summary["gtfs_update_diff"] = {
"id": diff_result["id"],
"summary": diff_result["summary"],
}
_emit_progress(
progress_callback,
"gtfs_update_diff_completed",
"Stored GTFS update diff before replacing the previous dataset.",
None,
None,
diff_result,
)
try:
decision_replay_plan = capture_gtfs_decision_replay(
session,
previous_dataset,
diff_run_id=None if diff_result is None else int(diff_result["id"]),
)
summary["gtfs_decision_replay"] = {
"captured": len(decision_replay_plan.get("decisions", [])),
"diff_run_id": decision_replay_plan.get("diff_run_id"),
"previous_dataset_id": previous_dataset.id,
}
_emit_progress(
progress_callback,
"gtfs_decision_replay_captured",
"Captured existing GTFS-to-map decisions for update replay.",
None,
None,
summary["gtfs_decision_replay"],
)
except Exception as exc: # noqa: BLE001 - replay capture must not block activation
decision_replay_plan = None
summary["gtfs_decision_replay_error"] = str(exc)
_emit_progress(
progress_callback,
"gtfs_decision_replay_capture_failed",
"GTFS decision replay capture failed; continuing dataset activation.",
None,
None,
{"error": str(exc), "previous_dataset_id": previous_dataset.id, "new_dataset_id": dataset.id},
)
if replaced_datasets:
_emit_progress(
progress_callback,
@@ -511,6 +595,33 @@ def _activate_staged_gtfs(
],
progress_callback,
)
if decision_replay_plan is not None:
try:
replay_result = apply_gtfs_decision_replay(
session,
decision_replay_plan,
new_dataset=dataset,
diff_run_id=None if diff_result is None else int(diff_result["id"]),
)
summary["gtfs_decision_replay"] = replay_result
_emit_progress(
progress_callback,
"gtfs_decision_replay_completed",
"Replayed unchanged GTFS-to-map decisions and queued changed decisions for review.",
None,
None,
replay_result,
)
except Exception as exc: # noqa: BLE001 - replay must not block activation
summary["gtfs_decision_replay_error"] = str(exc)
_emit_progress(
progress_callback,
"gtfs_decision_replay_failed",
"GTFS decision replay failed; continuing dataset activation.",
None,
None,
{"error": str(exc), "new_dataset_id": dataset.id},
)
finally:
if heavy_index_drop:
_emit_progress(progress_callback, "gtfs_activation_indexes_rebuilding", "Rebuilding GTFS lookup indexes after bulk activation.", None, None, None)
@@ -557,10 +668,7 @@ def _copy_stage_table(
rows = cursor.fetchmany(GTFS_STAGE_BATCH_SIZE)
if not rows:
break
payload = [
{"dataset_id": dataset_id, **{column: row[index] for index, column in enumerate(columns)}}
for row in rows
]
payload = [_copy_stage_row_payload(table, dataset_id, columns, row) for row in rows]
session.execute(text(insert_sql), payload)
copied += len(rows)
_emit_progress(
@@ -573,6 +681,15 @@ def _copy_stage_table(
)
def _copy_stage_row_payload(table: str, dataset_id: int, columns: list[str], row: sqlite3.Row | tuple[Any, ...]) -> dict[str, Any]:
payload = {"dataset_id": dataset_id, **{column: row[index] for index, column in enumerate(columns)}}
if table == "gtfs_calendars":
for column in GTFS_CALENDAR_WEEKDAY_COLUMNS:
if column in payload:
payload[column] = _bool_flag(payload[column])
return payload
def _should_drop_indexes_for_activation(stage_path: Path) -> bool:
if settings.is_postgresql_database:
return False
@@ -1325,3 +1442,13 @@ def _dataset_importer_version(dataset: Dataset) -> str:
return str(json.loads(dataset.metadata_json or "{}").get("importer") or "")
except json.JSONDecodeError:
return ""
def _dataset_import_can_reuse(dataset: Dataset) -> bool:
try:
metadata = json.loads(dataset.metadata_json or "{}")
except json.JSONDecodeError:
return False
if str(metadata.get("importer") or "") != GTFS_IMPORTER_VERSION:
return False
return int(metadata.get("stop_times_import_limit") or 0) == int(settings.gtfs_stop_times_import_limit or 0)

172
app/pipeline/matcher.py
View File

@@ -2,13 +2,15 @@ from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
import hashlib
import json
from typing import Callable, Optional
from typing import Callable, Optional, Sequence
from shapely.geometry import LineString, MultiLineString, Point, shape
from sqlalchemy import delete, select
from sqlalchemy.orm import Session
from app.harmonization import active_harmonized_gtfs_shadowed_route_ids
from app.config import settings
from app.models import Dataset, GtfsRoute, MatchRule, OsmFeature, RouteMatch
from app.osm_storage import ensure_main_osm_feature, osm_feature_bbox, query_osm_features
@@ -34,7 +36,7 @@ MAX_EXACT_REF_CANDIDATES = 120
OSM_SCOPE_NEAR_DISTANCE_DEG = 0.15
GEOMETRY_PROXIMITY_DEG = 0.0035
GEOMETRY_SAMPLE_POINTS = 24
MATCHER_VERSION = "matcher_v4_scope_spatial_manual_rules"
MATCHER_VERSION = "matcher_v5_spatial_candidate_gate"
ProgressCallback = Callable[[str, str, int | None, int | None, dict[str, object] | None], None]
@@ -78,6 +80,7 @@ def run_route_matching(
*,
progress_callback: ProgressCallback | None = None,
batch_size: int | None = None,
gtfs_dataset_ids: Sequence[int] | None = None,
) -> dict[str, object]:
"""Match active GTFS routes against active OSM route features."""
active_datasets = session.execute(
@@ -86,22 +89,37 @@ def run_route_matching(
if not active_datasets:
return {"routes": 0, "matches": 0, "missing": 0}
dataset_source_ids = {int(dataset_id): int(source_id) for dataset_id, _, source_id in active_datasets}
gtfs_dataset_ids = [int(dataset_id) for dataset_id, kind, _ in active_datasets if kind == "gtfs"]
active_gtfs_dataset_ids = [int(dataset_id) for dataset_id, kind, _ in active_datasets if kind == "gtfs"]
osm_dataset_ids = [int(dataset_id) for dataset_id, kind, _ in active_datasets if kind == "osm_geojson"]
if not gtfs_dataset_ids:
if gtfs_dataset_ids is None:
selected_gtfs_dataset_ids = active_gtfs_dataset_ids
else:
requested_gtfs_dataset_ids = {int(dataset_id) for dataset_id in gtfs_dataset_ids}
selected_gtfs_dataset_ids = [
dataset_id
for dataset_id in active_gtfs_dataset_ids
if dataset_id in requested_gtfs_dataset_ids
]
if not selected_gtfs_dataset_ids:
return {"routes": 0, "matches": 0, "missing": 0}
route_row_ids = session.scalars(
select(GtfsRoute.id)
.where(GtfsRoute.dataset_id.in_(gtfs_dataset_ids))
.order_by(GtfsRoute.dataset_id, GtfsRoute.route_id, GtfsRoute.id)
).all()
all_route_row_ids = [
int(route_id)
for route_id in session.scalars(
select(GtfsRoute.id)
.where(GtfsRoute.dataset_id.in_(selected_gtfs_dataset_ids))
.order_by(GtfsRoute.dataset_id, GtfsRoute.route_id, GtfsRoute.id)
).all()
]
shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session)
route_row_ids = [route_id for route_id in all_route_row_ids if route_id not in shadowed_route_ids]
shadowed_skipped = len(all_route_row_ids) - len(route_row_ids)
# Reconcile current match rows from auto scoring plus durable manual rules.
total_routes = len(route_row_ids)
if total_routes == 0:
return {"routes": 0, "matches": 0, "missing": 0}
return {"routes": 0, "matches": 0, "missing": 0, "shadowed_skipped": shadowed_skipped}
dependency = _route_matching_dependency(session, active_datasets)
dependency = _route_matching_dependency(session, active_datasets, shadowed_route_ids=shadowed_route_ids)
run = start_pipeline_run(
session,
stage=STAGE_MATCH_ROUTES,
@@ -117,11 +135,49 @@ def run_route_matching(
f"Matching {total_routes} GTFS routes in batches of {effective_batch_size}.",
0,
total_routes,
{"gtfs_datasets": gtfs_dataset_ids, "osm_datasets": osm_dataset_ids, "batch_size": effective_batch_size},
{
"gtfs_datasets": selected_gtfs_dataset_ids,
"osm_datasets": osm_dataset_ids,
"batch_size": effective_batch_size,
"shadowed_skipped": shadowed_skipped,
},
)
manual_rules = _manual_match_rules(session)
osm_scope_bbox = osm_feature_bbox(session, osm_dataset_ids, kinds=["route"])
counts = {"routes": total_routes, "matches": 0, "missing": 0, "manual": 0, "created": 0, "updated": 0, "unchanged": 0}
_emit_progress(
progress_callback,
"route_matching_osm_index_started",
"Loading OSM route candidates for matcher.",
0,
None,
{"osm_datasets": osm_dataset_ids},
)
osm_routes = query_osm_features(
session,
osm_dataset_ids,
kinds=["route"],
prefer_materialized_ids=False,
)
osm_route_index = _build_osm_route_index(osm_routes)
feature_profile_cache: dict[tuple[int, str, str], _GeometryProfile | None] = {}
_emit_progress(
progress_callback,
"route_matching_osm_index_completed",
f"Loaded {len(osm_routes)} OSM route candidates.",
len(osm_routes),
None,
{"osm_routes": len(osm_routes)},
)
counts = {
"routes": total_routes,
"matches": 0,
"missing": 0,
"manual": 0,
"created": 0,
"updated": 0,
"unchanged": 0,
"shadowed_skipped": shadowed_skipped,
}
scoped_counts = {"in_osm_scope": 0, "near_osm_scope": 0, "outside_osm_scope": 0, "unknown_scope": 0}
processed = 0
for chunk in _chunks_int(route_row_ids, effective_batch_size):
@@ -137,6 +193,8 @@ def run_route_matching(
dataset_source_ids=dataset_source_ids,
manual_rules=manual_rules,
osm_scope_bbox=osm_scope_bbox,
osm_route_index=osm_route_index,
feature_profile_cache=feature_profile_cache,
scoped_counts=scoped_counts,
)
counts["matches"] += batch_counts["matches"]
@@ -178,7 +236,12 @@ def run_route_matching(
return result
def _route_matching_dependency(session: Session, active_datasets) -> dict[str, object]:
def _route_matching_dependency(
session: Session,
active_datasets,
*,
shadowed_route_ids: set[int] | None = None,
) -> dict[str, object]:
datasets = [
{"id": int(dataset_id), "kind": str(kind), "source_id": int(source_id), "sha256": _dataset_sha(session, int(dataset_id))}
for dataset_id, kind, source_id in active_datasets
@@ -193,7 +256,15 @@ def _route_matching_dependency(session: Session, active_datasets) -> dict[str, o
}
for rule in session.scalars(select(MatchRule).order_by(MatchRule.id)).all()
]
return {"version": MATCHER_VERSION, "active_datasets": datasets, "manual_rules": rules}
return {
"version": MATCHER_VERSION,
"active_datasets": datasets,
"manual_rules": rules,
"shadowed_routes": {
"count": len(shadowed_route_ids or set()),
"digest": _int_set_digest(shadowed_route_ids or set()),
},
}
def _dataset_sha(session: Session, dataset_id: int) -> str | None:
@@ -201,6 +272,13 @@ def _dataset_sha(session: Session, dataset_id: int) -> str | None:
return None if dataset is None else dataset.sha256
def _int_set_digest(values: set[int]) -> str:
if not values:
return ""
payload = ",".join(str(value) for value in sorted(values))
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
def _match_route_batch(
*,
session: Session,
@@ -209,6 +287,8 @@ def _match_route_batch(
dataset_source_ids: dict[int, int],
manual_rules: list[_ManualMatchRule],
osm_scope_bbox: tuple[float | None, float | None, float | None, float | None],
osm_route_index: _OsmRouteIndex,
feature_profile_cache: dict[tuple[int, str, str], _GeometryProfile | None],
scoped_counts: dict[str, int],
) -> dict[str, int]:
matches = 0
@@ -265,10 +345,13 @@ def _match_route_batch(
best_score = 0.0
best_reasons: dict[str, object] = {}
route_geometry_profile = _geometry_profile(route.geometry_geojson)
for feature in candidate_osm_routes_for_route(session, route, osm_dataset_ids):
for feature in _candidate_osm_routes(route, osm_route_index):
if _is_rejected_pair(manual_rules, route, route_source_id, feature, dataset_source_ids.get(feature.dataset_id)):
continue
feature_geometry_profile = _geometry_profile(feature.geometry_geojson)
feature_key = _feature_identity_key(feature)
if feature_key not in feature_profile_cache:
feature_profile_cache[feature_key] = _geometry_profile(feature.geometry_geojson)
feature_geometry_profile = feature_profile_cache[feature_key]
score, reasons = score_route_pair(
route,
feature,
@@ -393,22 +476,26 @@ def _build_osm_route_index(osm_routes: list[OsmFeature]) -> _OsmRouteIndex:
def _candidate_osm_routes(route: GtfsRoute, index: _OsmRouteIndex) -> list[OsmFeature]:
selected: list[OsmFeature] = []
seen: set[int] = set()
seen: set[tuple[int, str, str]] = set()
def add(features: list[OsmFeature], *, require_compatible_mode: bool = True) -> None:
def add(features: list[OsmFeature], *, require_compatible_mode: bool = True, require_spatial: bool = False) -> None:
for feature in features:
if feature.id in seen:
key = (feature.dataset_id, feature.osm_type, feature.osm_id)
if key in seen:
continue
if require_compatible_mode and not _mode_compatible(route.mode or "", feature.mode or ""):
continue
seen.add(feature.id)
if require_spatial and not _candidate_is_spatially_plausible(route, feature):
continue
seen.add(key)
selected.append(feature)
route_ref = norm_ref(route.short_name or route.route_id)
route_has_bbox = _has_known_bbox((route.min_lon, route.min_lat, route.max_lon, route.max_lat))
if route_ref:
add(index.by_ref.get(route_ref, []))
add(index.by_ref.get(route_ref, []), require_spatial=route_has_bbox)
if route.route_key:
add(index.by_route_key.get(route.route_key, []))
add(index.by_route_key.get(route.route_key, []), require_spatial=route_has_bbox)
if selected:
return _spatially_ranked_candidates(route, selected, MAX_EXACT_REF_CANDIDATES)
@@ -431,7 +518,7 @@ def _candidate_osm_routes(route: GtfsRoute, index: _OsmRouteIndex) -> list[OsmFe
near_candidates.append((distance, feature))
fallback_limit = MAX_FALLBACK_CANDIDATES_WITH_REF if route_ref else MAX_FALLBACK_CANDIDATES_WITHOUT_REF
fallback = [feature for _, feature in sorted(near_candidates, key=lambda item: item[0])[:fallback_limit]]
if not fallback:
if not fallback and not route_has_bbox:
fallback = mode_candidates[:fallback_limit]
add(fallback)
return _spatially_ranked_candidates(route, selected, fallback_limit)
@@ -443,17 +530,22 @@ def candidate_osm_routes_for_route(session: Session, route: GtfsRoute, osm_datas
selected: list[OsmFeature] = []
seen: set[tuple[int, str, str]] = set()
def add(features: list[OsmFeature], *, require_compatible_mode: bool = True) -> None:
def add(features: list[OsmFeature], *, require_compatible_mode: bool = True, require_spatial: bool = False) -> None:
for feature in features:
key = (feature.dataset_id, feature.osm_type, feature.osm_id)
if key in seen:
continue
if require_compatible_mode and not _mode_compatible(route.mode or "", feature.mode or ""):
continue
if require_spatial and not _candidate_is_spatially_plausible(route, feature):
continue
seen.add(key)
selected.append(feature)
route_ref = norm_ref(route.short_name or route.route_id)
gtfs_bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat)
route_has_bbox = _has_known_bbox(gtfs_bbox)
exact_ref_bbox = _expanded_bbox(gtfs_bbox, OSM_SCOPE_NEAR_DISTANCE_DEG) if route_has_bbox else None
route_keys = [key for key in [route.route_key, route_ref] if key]
for route_key in dict.fromkeys(route_keys):
add(
@@ -462,14 +554,15 @@ def candidate_osm_routes_for_route(session: Session, route: GtfsRoute, osm_datas
osm_dataset_ids,
kinds=["route"],
route_key=route_key,
)
bbox=exact_ref_bbox,
),
require_spatial=route_has_bbox,
)
if selected:
return _spatially_ranked_candidates(route, selected, MAX_EXACT_REF_CANDIDATES)
gtfs_bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat)
compatible_modes = sorted(MODE_GROUPS.get(route.mode or "", {route.mode or ""}) - {""})
if not any(value is None for value in gtfs_bbox):
if route_has_bbox:
bbox = _expanded_bbox(gtfs_bbox, 0.10)
add(
query_osm_features(
@@ -482,7 +575,7 @@ def candidate_osm_routes_for_route(session: Session, route: GtfsRoute, osm_datas
),
require_compatible_mode=False,
)
if not selected:
if not selected and not route_has_bbox:
add(
query_osm_features(
session,
@@ -634,6 +727,27 @@ def route_match_scope(route: GtfsRoute, osm_scope_bbox: tuple[float | None, floa
return "outside_osm_scope"
def _candidate_is_spatially_plausible(route: GtfsRoute, feature: OsmFeature) -> bool:
route_bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat)
feature_bbox = (feature.min_lon, feature.min_lat, feature.max_lon, feature.max_lat)
if not _has_known_bbox(route_bbox):
return True
if not _has_known_bbox(feature_bbox):
return False
if bbox_overlap(route_bbox, feature_bbox):
return True
distance = approx_bbox_center_distance_deg(route_bbox, feature_bbox)
return distance is not None and distance < OSM_SCOPE_NEAR_DISTANCE_DEG
def _has_known_bbox(bbox: tuple[float | None, float | None, float | None, float | None]) -> bool:
return not any(value is None for value in bbox)
def _feature_identity_key(feature: OsmFeature) -> tuple[int, str, str]:
return (int(feature.dataset_id), str(feature.osm_type), str(feature.osm_id))
def _combined_bbox(features: list[OsmFeature]) -> tuple[float | None, float | None, float | None, float | None]:
boxes = [
(feature.min_lon, feature.min_lat, feature.max_lon, feature.max_lat)

26
app/pipeline/route_layer.py
View File

@@ -11,6 +11,7 @@ from sqlalchemy.orm import Session
from app.config import settings
from app.gtfs_storage import all_scheduled_stop_ids, stop_times_by_trip as storage_stop_times_by_trip
from app.harmonization import active_harmonized_gtfs_shadowed_route_ids
from app.models import (
CanonicalStop,
CanonicalStopLink,
@@ -36,7 +37,7 @@ from app.pipeline.utils import bbox_overlap, geometry_json_and_bbox, norm_ref, n
from app.spatial import analyze_postgresql_tables, refresh_postgis_geometries, using_postgresql
ROUTE_LAYER_VERSION = "route_layer_v3_stop_alias_matching"
ROUTE_LAYER_VERSION = "route_layer_v4_harmonized_route_ownership"
GTFS_ROUTE_PATTERN_NULL_SHAPE = "__route__"
OSM_STOP_LINK_RADIUS_DEG = 0.0018
OSM_STOP_NAME_LINK_RADIUS_DEG = 0.0032
@@ -144,6 +145,7 @@ def rebuild_route_layer(
"route_patterns_updated": pattern_result.get("route_patterns_updated", 0),
"route_patterns_reused": pattern_result.get("route_patterns_reused", 0),
"route_patterns_removed": pattern_result.get("route_patterns_removed", 0),
"shadowed_routes_skipped": pattern_result.get("shadowed_routes_skipped", 0),
"route_pattern_links": pattern_result["route_pattern_links"],
"trip_pattern_links": pattern_result["trip_pattern_links"],
"route_pattern_stops": pattern_result["route_pattern_stops"],
@@ -161,6 +163,7 @@ def _route_layer_dependency(session: Session) -> dict[str, object]:
for dataset in session.scalars(select(Dataset).where(Dataset.is_active.is_(True)).order_by(Dataset.kind, Dataset.id)).all()
]
match_rows = session.execute(select(RouteMatch.id, RouteMatch.gtfs_route_id, RouteMatch.osm_feature_id, RouteMatch.status, RouteMatch.updated_at).order_by(RouteMatch.id)).all()
shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session)
match_signature = dependency_hash(
[
[
@@ -177,6 +180,10 @@ def _route_layer_dependency(session: Session) -> dict[str, object]:
"version": ROUTE_LAYER_VERSION,
"active_datasets": active_datasets,
"route_matches": {"count": len(match_rows), "signature": match_signature},
"shadowed_routes": {
"count": len(shadowed_route_ids),
"signature": dependency_hash(sorted(shadowed_route_ids)) if shadowed_route_ids else "",
},
}
@@ -945,14 +952,15 @@ def _build_route_patterns(
) -> dict[str, int]:
osm_candidates = _osm_route_candidates(session, progress_callback=progress_callback)
overrides = _route_layer_overrides(session)
seeds = _gtfs_pattern_seeds(session)
shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session)
seeds, shadowed_routes_skipped = _gtfs_pattern_seeds(session, shadowed_route_ids=shadowed_route_ids)
_emit_progress(
progress_callback,
"route_layer_pattern_seeds",
f"Loaded {len(seeds)} GTFS route-pattern seeds.",
0,
len(seeds),
{"seeds": len(seeds)},
{"seeds": len(seeds), "shadowed_routes_skipped": shadowed_routes_skipped},
)
link_count = 0
stop_count = 0
@@ -1256,6 +1264,7 @@ def _build_route_patterns(
"trip_pattern_links": trip_link_count,
"route_pattern_stops": stop_count,
"gtfs_proposed_patterns": proposed_count,
"shadowed_routes_skipped": shadowed_routes_skipped,
}
_emit_progress(
progress_callback,
@@ -1494,10 +1503,11 @@ def _route_layer_overrides(session: Session) -> _RouteLayerOverrides:
return _RouteLayerOverrides(accepted_by_gtfs_route_id=accepted, rejected_by_gtfs_route_id=rejected)
def _gtfs_pattern_seeds(session: Session) -> list[_GtfsPatternSeed]:
def _gtfs_pattern_seeds(session: Session, *, shadowed_route_ids: set[int] | None = None) -> tuple[list[_GtfsPatternSeed], int]:
active_gtfs_dataset_ids = _active_dataset_ids(session, "gtfs")
if not active_gtfs_dataset_ids:
return []
return [], 0
shadowed_route_ids = shadowed_route_ids or set()
rows = session.execute(
select(GtfsRoute, GtfsTrip.shape_id, func.min(GtfsTrip.trip_id))
.join(GtfsTrip, and_(GtfsTrip.dataset_id == GtfsRoute.dataset_id, GtfsTrip.route_id == GtfsRoute.route_id))
@@ -1525,7 +1535,11 @@ def _gtfs_pattern_seeds(session: Session) -> list[_GtfsPatternSeed]:
for dataset_id, shape_id, geometry, min_lon, min_lat, max_lon, max_lat in shape_rows
}
seeds = []
skipped = 0
for route, shape_id, trip_id in rows:
if int(route.id) in shadowed_route_ids:
skipped += 1
continue
geometry_text = None
geometry_source = "none"
bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat)
@@ -1556,7 +1570,7 @@ def _gtfs_pattern_seeds(session: Session) -> list[_GtfsPatternSeed]:
center_point=center_point,
)
)
return seeds
return seeds, skipped
def _choose_osm_candidate(

14
app/pipeline/sample_data.py
View File

@@ -19,6 +19,11 @@ from app.models import (
GtfsAgency,
GtfsCalendar,
GtfsCalendarDate,
GtfsFeedDiffItem,
GtfsFeedDiffRun,
GtfsHarmonizedSnapshot,
GtfsHarmonizedSnapshotDataset,
GtfsHarmonizedSnapshotRoute,
GtfsRoute,
GtfsRoutePatternLink,
GtfsShape,
@@ -30,6 +35,8 @@ from app.models import (
ItineraryLeg,
Job,
JobEvent,
MapGtfsDecision,
MapGtfsReviewItem,
MatchRule,
OsmDiffState,
OsmFeature,
@@ -96,8 +103,15 @@ def clear_project_data(
ItineraryLeg,
Itinerary,
TravelRequest,
GtfsFeedDiffItem,
GtfsFeedDiffRun,
GtfsHarmonizedSnapshotRoute,
GtfsHarmonizedSnapshotDataset,
GtfsHarmonizedSnapshot,
SourceUpdateCheck,
OsmDiffState,
MapGtfsReviewItem,
MapGtfsDecision,
MatchRule,
RouteMatch,
GtfsTripRoutePatternLink,