From 1d7ec956d26b0a21b175747a3f1e115321d6ae51 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Mon, 6 Jul 2026 20:54:46 +0200 Subject: [PATCH] Next alpha stage commit --- .env.example | 3 +- MVP_ROADMAP.md | 220 -- app/cli.py | 4 + app/config.py | 1 + app/data_management.py | 116 +- app/db.py | 17 + app/decision_replay.py | 351 ++++ app/feed_discovery.py | 640 ++++++ app/gtfs_diff.py | 440 ++++ app/harmonization.py | 1278 ++++++++++- app/jobs.py | 131 +- app/journey.py | 1116 +++++++++- app/journey_search.py | 254 ++- app/main.py | 418 +++- app/models.py | 186 ++ app/osm_storage.py | 7 +- app/pipeline/gtfs.py | 137 +- app/pipeline/matcher.py | 172 +- app/pipeline/route_layer.py | 26 +- app/pipeline/sample_data.py | 14 + app/routing.py | 309 ++- app/static/app.js | 988 ++++++++- app/static/style.css | 244 +++ app/templates/index.html | 81 +- app/workbench.py | 985 +++++++++ app/worker_supervisor.py | 159 +- docs/backlog.md | 202 -- docs/generated/gtfs_discovery_report.json | 15 +- docs/generated/gtfs_feed_candidates.csv | 2223 ++++++++++---------- docs/generated/gtfs_ingestable_sources.csv | 1988 ++++++++--------- docs/generated/gtfs_test_run_sources.csv | 48 +- docs/gtfs_harmonization.md | 85 +- docs/product_vision.md | 88 + docs/source_acquisition.md | 11 +- docs/source_catalog_seed.csv | 1 + scripts/discover_gtfs_sources.py | 6 +- tests/test_api.py | 430 +++- tests/test_feed_discovery.py | 81 + tests/test_gtfs_import.py | 329 ++- tests/test_journey_search.py | 147 ++ tests/test_pipeline.py | 74 +- tests/test_route_layer.py | 415 +++- 42 files changed, 11603 insertions(+), 2837 deletions(-) delete mode 100644 MVP_ROADMAP.md create mode 100644 app/decision_replay.py create mode 100644 app/gtfs_diff.py create mode 100644 app/workbench.py delete mode 100644 docs/backlog.md create mode 100644 docs/product_vision.md create mode 100644 tests/test_journey_search.py diff --git a/.env.example b/.env.example index 41df7e3..46d7966 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,7 @@ DATABASE_URL=sqlite:///./data/workbench.sqlite # DATABASE_URL=postgresql://USER:PASSWORD@localhost:5432/meubility # POSTGRES_USE_SIDECARS=false DATA_DIR=./data -GTFS_STOP_TIMES_IMPORT_LIMIT=250000 +GTFS_STOP_TIMES_IMPORT_LIMIT=0 # Start separate queue worker processes from the API server lifespan. # Workers survive normal server restarts by default; stale leases are recovered. @@ -13,6 +13,7 @@ QUEUE_WORKER_COUNT=1 QUEUE_WORKER_POLL_INTERVAL_SECONDS=2 QUEUE_JOB_LEASE_SECONDS=7200 QUEUE_WORKER_STOP_ON_SHUTDOWN=false +JOURNEY_SEARCH_WORKER_COUNT=4 # Chunk sizes for queued data-preparation jobs. ROUTE_MATCHING_BATCH_SIZE=100 diff --git a/MVP_ROADMAP.md b/MVP_ROADMAP.md deleted file mode 100644 index a9a1326..0000000 --- a/MVP_ROADMAP.md +++ /dev/null @@ -1,220 +0,0 @@ -# MVP roadmap - -Last updated: 2026-07-01 - -See also `docs/backlog.md` for the prioritized engineering backlog, caveats, and open optimization list. - -## Objective - -Build an internal management workbench that turns public mobility data into a normalized, auditable, coverage-scored dataset for a future traveller-facing web/native app. - -The workbench stays distinct from the public app. Its users are data engineers, analysts, and operations staff who need to ingest, inspect, link, correct, route against, and publish mobility data. - -## Current prototype: implemented - -The repository has moved beyond the original SQLite/Berlin prototype. The current development path is Germany-scale and PostGIS-first, while SQLite remains useful as a legacy/test fallback. - -Implemented: - -```text -source registry and source catalog -local source cache -job queue with job events and worker process -PostgreSQL/PostGIS runtime support with SQLite fallback -GTFS static importer for large national feeds -OSM PBF import path for Germany-scale extracts -OSM address index and address-aware journey endpoints -canonical stop/station linking from GTFS and OSM -automatic GTFS <-> OSM route matching -manual route and canonical-stop rule persistence -visual route-layer builder from OSM routes and GTFS shapes -walk/drive routing layer from OSM-derived routing graph -progressive journey-search API and UI polling -map right-click "from here" / "to here" -management UI with map, sources, stats, jobs, matches, search, and journeys -separate GTFS Harmonization and Mapping Data source modules in the UI -generic job-details overlay with phase timeline, event log, and queue snapshot -QA dashboard skeleton for source/import/link/route/publication health -GTFS harmonization concept and service-boundary decision -CLI commands -tests and syntax checks for changed modules -``` - -Recent fixes: - -```text -PostgreSQL startup avoids unnecessary DDL when PostGIS columns/indexes already exist. -Queue route-layer rebuild can be claimed by a real worker instead of staying queued behind a stale worker pid. -Timetable routing no longer requires visual route-pattern trip links. -Walk-leg route geometry has a short-lived in-process cache. -Address search is bbox-aware without being bbox-limited. -Job rows expose a details overlay that polls job events only while open. -Journey routing consumes the active harmonized GTFS snapshot instead of a raw feed picker. -``` - -## Current prototype: known limits - -The app can import and inspect Germany-scale OSM and GTFS, but the routing and route-layer rebuild paths are still prototype-grade. - -Important limits: - -```text -journey search is not yet RAPTOR/CSA or connection-scan based -address endpoints can multiply transit searches through several nearby access/egress stops -progressive transfer stages still recompute too much -route-layer rebuild is coarse-grained and rewrites derived link tables -visual route-pattern links are not yet incrementally updated -canonical stop extraction is CPU/memory heavy on national feeds -route geometry cannot yet classify temporary GTFS detours as separate variants -local-transport-only routing is not a first-class query mode -route-search caches are process-local and not persisted -Alembic migrations are still missing -``` - -## MVP 1: stable Germany data workbench - -### Backend - -- Add proper Alembic migrations for PostgreSQL and keep SQLite test support. -- Add source-run history and dataset-version comparison. -- Make route-layer rebuild incremental: update only affected matches/patterns/stops. -- Keep old route-layer tables readable while a rebuild prepares replacement rows. -- Add source health checks: download success, hash change, feed freshness, calendar validity. -- Expand the QA dashboard into drill-down review queues for source health, GTFS validation, canonical stop conflicts, route conflicts, and publication blockers. -- Add GTFS validation summary reports: service dates, route direction coverage, stop coordinate outliers, bad stop_times, missing shapes. -- Add database maintenance jobs: analyze, vacuum, stale job recovery, orphan cleanup. -- Add durable cache tables for journey stages, nearest stops, address access candidates, and common station-to-station searches. - -### Routing - -- Replace the demo round-expansion router with a GTFS-appropriate algorithm such as RAPTOR or CSA. -- Precompute transfer graph edges: station-internal transfers, nearby walking transfers, and access/egress stop candidates. -- Add routing profiles: - -```text -fastest public transport -fewest transfers -local transport only / Deutschlandticket-like -walk only -drive -car comparison -``` - -- Treat access/egress walking as access legs, not as public-transport transfers. -- Add bounded hub-aware long-distance routing for city-to-city requests: local access to likely hubs, long-distance/regional trunk, local egress. -- Add arrive-by search and better stop conditions for "good enough" results. -- Add route diagnostics that explain why a route was found or pruned. - -### Frontend - -- Add source detail page. -- Add dataset detail page. -- Add match-review queue with filters by mode, operator, country, confidence, and source scope. -- Add route detail inspection: GTFS geometry, OSM geometry, candidate matches, stops, evidence, and route-pattern provenance. -- Add canonical stop/station detail overlay. -- Add persistent rule editor. -- Add routing controls for profile, transfer buffer, avoid/prefer modes, arrive-by, via, and local-only. -- Show partial/progressive route results with clear stage labels. - -### Data outputs - -- GeoJSON exports for small regions. -- GeoParquet exports for analysis. -- PMTiles/vector-tile export for map display. -- Coverage CSV/API for downstream services. - -## MVP 2: Europe-scale coverage map - -- Use Geofabrik country/Europe extracts and reproducible OSM PBF jobs. -- Store OSM transport features, addresses, and routing graph in PostGIS. -- Generate ranked/generalized transport route layers by zoom level. -- Serve tiles with Martin or export PMTiles. -- Add coverage statuses: - -```text -existing_in_osm -static_timetable_covered -live_data_covered -fare_data_covered -booking_covered -missing_static -stale_feed -restricted_license -low_confidence_match -detour_or_temporary_variant -``` - -- Add coverage metrics: - -```text -operator coverage -route coverage -route-km coverage -stop coverage -live-data coverage -feed freshness -license confidence -booking coverage -route-layer provenance coverage -``` - -## MVP 3: more source formats - -Add importers: - -```text -NeTEx -TransXChange -SIRI discovery/live endpoints -GTFS-Realtime -GBFS for shared mobility, optional -operator CSV/API adapters -``` - -Target data model: - -```text -canonical operators -canonical stops/stations/terminals -canonical routes -route variants -trip patterns -calendar/service validity -transfers -access/egress legs -coverage observations -source evidence -manual rules -``` - -## MVP 4: production journey-planning dataset - -- Build a canonical stop/station graph with transfer rules and transfer-time profiles. -- Generate timetable-routing input for RAPTOR/CSA. -- Add first/last-mile routing from OSM walk/drive graph. -- Add emissions factors per mode/operator/country. -- Add fare/ticket placeholders and booking/deep-link metadata. -- Add confidence and provenance to every derived route/journey. - -## MVP 5: booking-readiness layer - -- Track booking availability separately from timetable coverage. -- Add deep-link metadata per operator/route. -- Add partner API adapters later. -- Distinguish clearly: - -```text -travel-plausible itinerary -bookable itinerary -single-interface multi-booking -protected through-ticket -``` - -## Recommended next implementation sprint - -1. Finish route-layer rebuild resilience: incremental updates, shadow tables, and detour/provenance classification. -2. Replace or heavily optimize journey routing: precomputed transfers, hub-aware long-distance routing, local-only profile, and bounded search. -3. Add durable PostgreSQL-backed journey caches for address access, stop pairs, and repeated stage searches. -4. Add Alembic migrations and remove runtime DDL from normal request/worker startup. -5. Add route/journey diagnostics so slow or failed requests explain what was searched and pruned. -6. Add vector-tile output for route layers and large map rendering. diff --git a/app/cli.py b/app/cli.py index 01ec81c..5db0656 100644 --- a/app/cli.py +++ b/app/cli.py @@ -227,7 +227,9 @@ def discover_gtfs_sources_command( no_mobility_database: bool = typer.Option(False, help="Skip Mobility Database feeds_v2.csv"), no_acceptance_test_list: bool = typer.Option(False, help="Skip MobilityData validator acceptance-test feed list"), no_ptna: bool = typer.Option(False, help="Skip PTNA GTFS analysis pages"), + no_opendata_oepnv: bool = typer.Option(False, help="Skip OpenData ÖPNV Germany dataset catalog"), max_ptna_details: int = typer.Option(80, help="Maximum PTNA detail pages to fetch for license/crosswalk metadata"), + max_opendata_oepnv_details: int = typer.Option(80, help="Maximum OpenData ÖPNV detail pages to fetch"), test_limit: int = typer.Option(24, help="Rows to write to the focused test-run ingestable CSV"), check_urls: bool = typer.Option(False, help="Run HEAD/range checks for ingestable feed URLs"), ) -> None: @@ -237,7 +239,9 @@ def discover_gtfs_sources_command( include_mobility_database=not no_mobility_database, include_acceptance_test_list=not no_acceptance_test_list, include_ptna=not no_ptna, + include_opendata_oepnv=not no_opendata_oepnv, max_ptna_details=max_ptna_details, + max_opendata_oepnv_details=max_opendata_oepnv_details, test_limit=test_limit, check_urls=check_urls, ) diff --git a/app/config.py b/app/config.py index 1325451..cc8dc54 100644 --- a/app/config.py +++ b/app/config.py @@ -52,6 +52,7 @@ class Settings(BaseSettings): # Keep supervised workers alive across API server restarts. Stale workers are # detected by PID files at the next startup; stale job leases are requeued. queue_worker_stop_on_shutdown: bool = False + journey_search_worker_count: int = 4 model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8") diff --git a/app/data_management.py b/app/data_management.py index bc24c8d..d43614c 100644 --- a/app/data_management.py +++ b/app/data_management.py @@ -13,6 +13,8 @@ from app.models import ( GtfsAgency, GtfsCalendar, GtfsCalendarDate, + GtfsHarmonizedSnapshotDataset, + GtfsHarmonizedSnapshotRoute, GtfsRoute, GtfsRoutePatternLink, GtfsShape, @@ -20,6 +22,8 @@ from app.models import ( GtfsStopTime, GtfsTripRoutePatternLink, GtfsTrip, + MapGtfsDecision, + MapGtfsReviewItem, OsmDiffState, OsmFeature, RouteMatch, @@ -136,6 +140,8 @@ def delete_source(session: Session, source_id: int) -> dict[str, object]: _delete_dataset_files(dataset) session.delete(dataset) session.execute(delete(OsmDiffState).where(OsmDiffState.source_id == source.id)) + session.execute(delete(MapGtfsReviewItem).where(MapGtfsReviewItem.source_id == source.id)) + session.execute(delete(MapGtfsDecision).where(MapGtfsDecision.source_id == source.id)) session.delete(source) session.flush() return {"deleted": True, "source_id": source_id, "datasets": dataset_results} @@ -217,14 +223,41 @@ def prune_inactive_datasets(session: Session, dry_run: bool = True) -> dict[str, for dataset_id in dataset_ids: _detach_update_checks_for_dataset(session, dataset_id) + for dataset in session.scalars(select(Dataset).where(Dataset.id.in_(dataset_ids))).all(): + _delete_workbench_rows_for_dataset(session, dataset) if match_filters: session.execute(delete(RouteMatch).where(or_(*match_filters))) if gtfs_ids: route_ids = select(GtfsRoute.id).where(GtfsRoute.dataset_id.in_(gtfs_ids)) pattern_ids = select(RoutePattern.id).where(RoutePattern.gtfs_route_id.in_(route_ids)) + session.execute( + delete(GtfsHarmonizedSnapshotRoute).where( + or_( + GtfsHarmonizedSnapshotRoute.dataset_id.in_(gtfs_ids), + GtfsHarmonizedSnapshotRoute.shadowed_by_dataset_id.in_(gtfs_ids), + GtfsHarmonizedSnapshotRoute.gtfs_route_id.in_(route_ids), + GtfsHarmonizedSnapshotRoute.shadowed_by_gtfs_route_id.in_(route_ids), + ) + ) + ) + session.execute(delete(GtfsHarmonizedSnapshotDataset).where(GtfsHarmonizedSnapshotDataset.dataset_id.in_(gtfs_ids))) session.execute(delete(RoutePatternStop).where(RoutePatternStop.route_pattern_id.in_(pattern_ids))) - session.execute(delete(GtfsTripRoutePatternLink).where(GtfsTripRoutePatternLink.dataset_id.in_(gtfs_ids))) - session.execute(delete(GtfsRoutePatternLink).where(GtfsRoutePatternLink.dataset_id.in_(gtfs_ids))) + session.execute( + delete(GtfsTripRoutePatternLink).where( + or_( + GtfsTripRoutePatternLink.dataset_id.in_(gtfs_ids), + GtfsTripRoutePatternLink.route_pattern_id.in_(pattern_ids), + ) + ) + ) + session.execute( + delete(GtfsRoutePatternLink).where( + or_( + GtfsRoutePatternLink.dataset_id.in_(gtfs_ids), + GtfsRoutePatternLink.route_pattern_id.in_(pattern_ids), + ) + ) + ) session.execute(delete(RoutePattern).where(RoutePattern.gtfs_route_id.in_(route_ids))) session.execute(delete(CanonicalStopLink).where(CanonicalStopLink.dataset_id.in_(gtfs_ids), CanonicalStopLink.object_type == "gtfs_stop")) for model in [GtfsStopTime, GtfsShape, GtfsTrip, GtfsCalendarDate, GtfsCalendar, GtfsRoute, GtfsStop, GtfsAgency]: @@ -246,13 +279,39 @@ def prune_inactive_datasets(session: Session, dry_run: bool = True) -> dict[str, def _delete_dataset_rows(session: Session, dataset: Dataset) -> None: + _delete_workbench_rows_for_dataset(session, dataset) if dataset.kind == "gtfs": route_ids = select(GtfsRoute.id).where(GtfsRoute.dataset_id == dataset.id) pattern_ids = select(RoutePattern.id).where(RoutePattern.gtfs_route_id.in_(route_ids)) + session.execute( + delete(GtfsHarmonizedSnapshotRoute).where( + or_( + GtfsHarmonizedSnapshotRoute.dataset_id == dataset.id, + GtfsHarmonizedSnapshotRoute.shadowed_by_dataset_id == dataset.id, + GtfsHarmonizedSnapshotRoute.gtfs_route_id.in_(route_ids), + GtfsHarmonizedSnapshotRoute.shadowed_by_gtfs_route_id.in_(route_ids), + ) + ) + ) + session.execute(delete(GtfsHarmonizedSnapshotDataset).where(GtfsHarmonizedSnapshotDataset.dataset_id == dataset.id)) session.execute(delete(RouteMatch).where(RouteMatch.gtfs_route_id.in_(route_ids))) session.execute(delete(RoutePatternStop).where(RoutePatternStop.route_pattern_id.in_(pattern_ids))) - session.execute(delete(GtfsTripRoutePatternLink).where(GtfsTripRoutePatternLink.dataset_id == dataset.id)) - session.execute(delete(GtfsRoutePatternLink).where(GtfsRoutePatternLink.dataset_id == dataset.id)) + session.execute( + delete(GtfsTripRoutePatternLink).where( + or_( + GtfsTripRoutePatternLink.dataset_id == dataset.id, + GtfsTripRoutePatternLink.route_pattern_id.in_(pattern_ids), + ) + ) + ) + session.execute( + delete(GtfsRoutePatternLink).where( + or_( + GtfsRoutePatternLink.dataset_id == dataset.id, + GtfsRoutePatternLink.route_pattern_id.in_(pattern_ids), + ) + ) + ) session.execute(delete(RoutePattern).where(RoutePattern.gtfs_route_id.in_(route_ids))) session.execute(delete(CanonicalStopLink).where(CanonicalStopLink.dataset_id == dataset.id, CanonicalStopLink.object_type == "gtfs_stop")) for model in [GtfsStopTime, GtfsShape, GtfsTrip, GtfsCalendarDate, GtfsCalendar, GtfsRoute, GtfsStop, GtfsAgency]: @@ -269,6 +328,55 @@ def _delete_dataset_rows(session: Session, dataset: Dataset) -> None: session.execute(delete(OsmFeature).where(OsmFeature.dataset_id == dataset.id)) +def _delete_workbench_rows_for_dataset(session: Session, dataset: Dataset) -> None: + review_filters = [MapGtfsReviewItem.dataset_id == dataset.id] + decision_filters = [ + MapGtfsDecision.gtfs_dataset_id == dataset.id, + MapGtfsDecision.osm_dataset_id == dataset.id, + ] + if dataset.kind == "gtfs": + route_ids = select(GtfsRoute.id).where(GtfsRoute.dataset_id == dataset.id) + stop_ids = select(GtfsStop.id).where(GtfsStop.dataset_id == dataset.id) + match_ids = select(RouteMatch.id).where(RouteMatch.gtfs_route_id.in_(route_ids)) + link_ids = select(CanonicalStopLink.id).where(CanonicalStopLink.dataset_id == dataset.id, CanonicalStopLink.object_type == "gtfs_stop") + review_filters.extend( + [ + MapGtfsReviewItem.gtfs_route_id.in_(route_ids), + MapGtfsReviewItem.gtfs_stop_id.in_(stop_ids), + MapGtfsReviewItem.route_match_id.in_(match_ids), + MapGtfsReviewItem.canonical_stop_link_id.in_(link_ids), + ] + ) + decision_filters.extend( + [ + MapGtfsDecision.gtfs_route_id.in_(route_ids), + MapGtfsDecision.gtfs_stop_id.in_(stop_ids), + MapGtfsDecision.route_match_id.in_(match_ids), + MapGtfsDecision.canonical_stop_link_id.in_(link_ids), + ] + ) + elif dataset.kind == "osm_geojson": + osm_feature_ids = select(OsmFeature.id).where(OsmFeature.dataset_id == dataset.id) + match_ids = select(RouteMatch.id).where(RouteMatch.osm_feature_id.in_(osm_feature_ids)) + link_ids = select(CanonicalStopLink.id).where(CanonicalStopLink.dataset_id == dataset.id, CanonicalStopLink.object_type == "osm_feature") + review_filters.extend( + [ + MapGtfsReviewItem.osm_feature_id.in_(osm_feature_ids), + MapGtfsReviewItem.route_match_id.in_(match_ids), + MapGtfsReviewItem.canonical_stop_link_id.in_(link_ids), + ] + ) + decision_filters.extend( + [ + MapGtfsDecision.osm_feature_id.in_(osm_feature_ids), + MapGtfsDecision.route_match_id.in_(match_ids), + MapGtfsDecision.canonical_stop_link_id.in_(link_ids), + ] + ) + session.execute(delete(MapGtfsReviewItem).where(or_(*review_filters))) + session.execute(delete(MapGtfsDecision).where(or_(*decision_filters))) + + def _delete_dataset_files(dataset: Dataset) -> None: for path in dataset_sidecar_paths(dataset): try: diff --git a/app/db.py b/app/db.py index a5abafa..93c5834 100644 --- a/app/db.py +++ b/app/db.py @@ -184,6 +184,16 @@ def _ensure_runtime_indexes() -> None: "CREATE INDEX IF NOT EXISTS ix_gtfs_shapes_dataset_shape ON gtfs_shapes (dataset_id, shape_id)", "CREATE INDEX IF NOT EXISTS ix_gtfs_calendars_dataset_service_dates ON gtfs_calendars (dataset_id, service_id, start_date, end_date)", "CREATE INDEX IF NOT EXISTS ix_gtfs_calendar_dates_dataset_date ON gtfs_calendar_dates (dataset_id, date, service_id, exception_type)", + "CREATE INDEX IF NOT EXISTS ix_gtfs_harmonized_snapshots_status_created ON gtfs_harmonized_snapshots (status, created_at, id)", + "CREATE INDEX IF NOT EXISTS ix_gtfs_harmonized_snapshot_datasets_role ON gtfs_harmonized_snapshot_datasets (snapshot_id, role, dataset_id)", + "CREATE INDEX IF NOT EXISTS ix_gtfs_harmonized_snapshot_datasets_source ON gtfs_harmonized_snapshot_datasets (source_id, role)", + "CREATE INDEX IF NOT EXISTS ix_gtfs_harmonized_snapshot_routes_role ON gtfs_harmonized_snapshot_routes (snapshot_id, role, dataset_id)", + "CREATE INDEX IF NOT EXISTS ix_gtfs_harmonized_snapshot_routes_overlap ON gtfs_harmonized_snapshot_routes (snapshot_id, overlap_key, role)", + "CREATE INDEX IF NOT EXISTS ix_gtfs_harmonized_snapshot_routes_shadow ON gtfs_harmonized_snapshot_routes (snapshot_id, shadowed_by_gtfs_route_id)", + "CREATE INDEX IF NOT EXISTS ix_gtfs_feed_diff_runs_source_created ON gtfs_feed_diff_runs (source_id, created_at, id)", + "CREATE INDEX IF NOT EXISTS ix_gtfs_feed_diff_runs_new_dataset ON gtfs_feed_diff_runs (new_dataset_id)", + "CREATE INDEX IF NOT EXISTS ix_gtfs_feed_diff_items_run_type ON gtfs_feed_diff_items (diff_run_id, item_type, change_type, severity)", + "CREATE INDEX IF NOT EXISTS ix_gtfs_feed_diff_items_source_status ON gtfs_feed_diff_items (source_id, status, severity)", "CREATE INDEX IF NOT EXISTS ix_canonical_stop_links_object ON canonical_stop_links (object_type, dataset_id, object_id)", "CREATE INDEX IF NOT EXISTS ix_canonical_stop_links_external ON canonical_stop_links (object_type, dataset_id, external_id)", "CREATE INDEX IF NOT EXISTS ix_route_patterns_ref_mode ON route_patterns (route_ref, mode, source_kind)", @@ -210,6 +220,13 @@ def _ensure_runtime_indexes() -> None: "CREATE INDEX IF NOT EXISTS ix_pipeline_runs_stage_source_hash ON pipeline_runs (stage, source_id, dependency_hash, status, started_at)", "CREATE INDEX IF NOT EXISTS ix_pipeline_runs_job ON pipeline_runs (job_id, stage, status)", "CREATE INDEX IF NOT EXISTS ix_match_rules_type_active ON match_rules (rule_type, active)", + "CREATE INDEX IF NOT EXISTS ix_map_gtfs_decisions_type_status ON map_gtfs_decisions (decision_type, status, active)", + "CREATE INDEX IF NOT EXISTS ix_map_gtfs_decisions_route ON map_gtfs_decisions (gtfs_route_id, status, active)", + "CREATE INDEX IF NOT EXISTS ix_map_gtfs_decisions_stop ON map_gtfs_decisions (gtfs_stop_id, status, active)", + "CREATE INDEX IF NOT EXISTS ix_map_gtfs_review_items_queue_status ON map_gtfs_review_items (queue, status, severity, updated_at)", + "CREATE INDEX IF NOT EXISTS ix_map_gtfs_review_items_dataset_status ON map_gtfs_review_items (dataset_id, status, queue)", + "CREATE INDEX IF NOT EXISTS ix_map_gtfs_review_items_route ON map_gtfs_review_items (gtfs_route_id, status)", + "CREATE INDEX IF NOT EXISTS ix_map_gtfs_review_items_stop ON map_gtfs_review_items (gtfs_stop_id, status)", "CREATE INDEX IF NOT EXISTS ix_journey_search_cache_type_expires ON journey_search_cache (cache_type, expires_at)", "CREATE INDEX IF NOT EXISTS ix_travel_requests_created ON travel_requests (created_at)", "CREATE INDEX IF NOT EXISTS ix_itineraries_request_saved ON itineraries (request_id, saved, created_at)", diff --git a/app/decision_replay.py b/app/decision_replay.py new file mode 100644 index 0000000..7596bc1 --- /dev/null +++ b/app/decision_replay.py @@ -0,0 +1,351 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from typing import Any + +from sqlalchemy import or_, select +from sqlalchemy.orm import Session + +from app.models import Dataset, GtfsFeedDiffItem, GtfsRoute, GtfsStop, MapGtfsDecision, OsmFeature +from app.workbench import _upsert_review_item + + +def capture_gtfs_decision_replay(session: Session, previous_dataset: Dataset | None, *, diff_run_id: int | None = None) -> dict[str, Any]: + if previous_dataset is None or previous_dataset.kind != "gtfs": + return {"previous_dataset_id": None, "diff_run_id": diff_run_id, "decisions": []} + route_by_id = { + int(route.id): route + for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == previous_dataset.id)).all() + } + stop_by_id = { + int(stop.id): stop + for stop in session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == previous_dataset.id)).all() + } + route_ids = list(route_by_id) + stop_ids = list(stop_by_id) + filters = [MapGtfsDecision.gtfs_dataset_id == previous_dataset.id] + if route_ids: + filters.append(MapGtfsDecision.gtfs_route_id.in_(route_ids)) + if stop_ids: + filters.append(MapGtfsDecision.gtfs_stop_id.in_(stop_ids)) + decisions = session.scalars( + select(MapGtfsDecision) + .where(MapGtfsDecision.active.is_(True), or_(*filters)) + .order_by(MapGtfsDecision.updated_at.desc(), MapGtfsDecision.id.desc()) + ).all() + captured: list[dict[str, Any]] = [] + for decision in decisions: + object_type = None + object_key = None + route = route_by_id.get(int(decision.gtfs_route_id or 0)) + stop = stop_by_id.get(int(decision.gtfs_stop_id or 0)) + if route is not None: + object_type = "route" + object_key = route.route_id + elif stop is not None: + object_type = "stop" + object_key = stop.stop_id + if not object_type or not object_key: + continue + captured.append( + { + "decision_id": decision.id, + "decision_key": decision.decision_key, + "decision_type": decision.decision_type, + "status": decision.status, + "source_id": decision.source_id, + "gtfs_dataset_id": decision.gtfs_dataset_id, + "object_type": object_type, + "object_key": object_key, + "selector": _json_object(decision.selector_json), + "action": _json_object(decision.action_json), + "note": decision.note, + "reviewer": decision.reviewer, + "confidence": decision.confidence, + "osm_dataset_id": decision.osm_dataset_id, + "osm_feature_id": decision.osm_feature_id, + "metadata": _json_object(decision.metadata_json), + } + ) + return { + "previous_dataset_id": previous_dataset.id, + "source_id": previous_dataset.source_id, + "diff_run_id": diff_run_id, + "decisions": captured, + } + + +def apply_gtfs_decision_replay( + session: Session, + replay_plan: dict[str, Any] | None, + *, + new_dataset: Dataset, + diff_run_id: int | None = None, +) -> dict[str, int]: + if not replay_plan: + return {"captured": 0, "replayed": 0, "skipped": 0, "review_items": 0} + decisions = replay_plan.get("decisions") if isinstance(replay_plan.get("decisions"), list) else [] + if not decisions and diff_run_id is None: + return {"captured": 0, "replayed": 0, "skipped": 0, "review_items": 0} + + current_diff_run_id = diff_run_id or _optional_int(replay_plan.get("diff_run_id")) + changed_keys = _changed_or_removed_keys(session, current_diff_run_id) + routes = { + route.route_id: route + for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == new_dataset.id)).all() + } + previous_decisions_by_key: dict[tuple[str, str], list[dict[str, Any]]] = {} + replayed = 0 + skipped = 0 + for decision in decisions: + object_type = str(decision.get("object_type") or "") + object_key = str(decision.get("object_key") or "") + if not object_type or not object_key: + skipped += 1 + continue + previous_decisions_by_key.setdefault((object_type, object_key), []).append(decision) + if (object_type, object_key) in changed_keys: + skipped += 1 + continue + if object_type != "route": + # Stop-level operational replay is handled by durable canonical-stop + # MatchRule rows during route-layer rebuild. Keep this audit layer + # route-focused to avoid stale canonical-stop foreign keys. + skipped += 1 + continue + route = routes.get(object_key) + if route is None: + skipped += 1 + continue + if _replay_route_decision(session, decision, new_dataset=new_dataset, route=route): + replayed += 1 + else: + skipped += 1 + + review_items = create_gtfs_update_decision_review_items( + session, + diff_run_id=current_diff_run_id, + new_dataset=new_dataset, + previous_decisions_by_key=previous_decisions_by_key, + ) + session.flush() + return {"captured": len(decisions), "replayed": replayed, "skipped": skipped, "review_items": review_items} + + +def create_gtfs_update_decision_review_items( + session: Session, + *, + diff_run_id: int | None, + new_dataset: Dataset, + previous_decisions_by_key: dict[tuple[str, str], list[dict[str, Any]]] | None = None, +) -> int: + if diff_run_id is None: + return 0 + previous_decisions_by_key = previous_decisions_by_key or {} + routes = { + route.route_id: route + for route in session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == new_dataset.id)).all() + } + stops = { + stop.stop_id: stop + for stop in session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == new_dataset.id)).all() + } + items = session.scalars( + select(GtfsFeedDiffItem) + .where(GtfsFeedDiffItem.diff_run_id == diff_run_id, GtfsFeedDiffItem.change_type.in_(["changed", "removed"])) + .order_by(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.change_type, GtfsFeedDiffItem.object_key) + ).all() + created_or_updated = 0 + for item in items: + object_type = "route" if item.item_type == "route" else "stop" if item.item_type == "stop" else str(item.item_type) + object_key = str(item.object_key) + previous_decisions = previous_decisions_by_key.get((object_type, object_key), []) + severity = "error" if previous_decisions and item.change_type == "removed" else "warn" + route = routes.get(object_key) if object_type == "route" and item.change_type != "removed" else None + stop = stops.get(object_key) if object_type == "stop" and item.change_type != "removed" else None + title = _diff_review_title(item, previous_decision_count=len(previous_decisions)) + metadata = { + "diff_run_id": diff_run_id, + "item_id": item.id, + "item_type": item.item_type, + "object_key": object_key, + "change_type": item.change_type, + "old": _json_object(item.old_payload_json), + "new": _json_object(item.new_payload_json), + "diff": _json_object(item.diff_json), + "previous_decisions": previous_decisions, + } + result = _upsert_review_item( + session, + fingerprint=f"gtfs-update-diff:{diff_run_id}:{item.item_type}:{item.object_key}:{item.change_type}", + queue="gtfs_update_diff", + item_type=f"gtfs_{item.item_type}_update", + severity=severity, + title=title, + description=_diff_review_description(item, len(previous_decisions)), + source_id=new_dataset.source_id, + dataset_id=new_dataset.id, + gtfs_route_id=None if route is None else route.id, + gtfs_stop_id=None if stop is None else stop.id, + evidence_payload=metadata, + ) + if result in {"created", "updated", "reopened"}: + created_or_updated += 1 + return created_or_updated + + +def _replay_route_decision(session: Session, decision: dict[str, Any], *, new_dataset: Dataset, route: GtfsRoute) -> bool: + selector = _updated_route_selector(decision.get("selector"), new_dataset=new_dataset, route=route) + action = decision.get("action") if isinstance(decision.get("action"), dict) else {} + evidence = { + "selector": selector, + "action": action, + "confidence": decision.get("confidence"), + "replayed_from_decision_id": decision.get("decision_id"), + } + decision_key = _stable_key(str(decision.get("decision_type") or "route_osm_relation"), selector, action) + existing = session.scalar(select(MapGtfsDecision).where(MapGtfsDecision.decision_key == decision_key)) + now = datetime.now(timezone.utc) + osm_dataset_id, osm_feature_id = _existing_osm_reference(session, decision) + metadata = decision.get("metadata") if isinstance(decision.get("metadata"), dict) else {} + replay_metadata = { + **metadata, + "replay": { + "from_decision_id": decision.get("decision_id"), + "from_decision_key": decision.get("decision_key"), + "previous_dataset_id": decision.get("gtfs_dataset_id"), + "new_dataset_id": new_dataset.id, + "object_key": route.route_id, + }, + } + if existing is None: + session.add( + MapGtfsDecision( + decision_key=decision_key, + decision_type=str(decision.get("decision_type") or "route_osm_relation"), + status=str(decision.get("status") or "accepted"), + active=True, + source_id=new_dataset.source_id, + gtfs_dataset_id=new_dataset.id, + gtfs_route_id=route.id, + osm_dataset_id=osm_dataset_id, + osm_feature_id=osm_feature_id, + route_match_id=None, + confidence=_optional_float(decision.get("confidence")), + evidence_hash=_hash_payload(evidence), + selector_json=_json_dumps(selector), + action_json=_json_dumps(action), + note=decision.get("note"), + reviewer="decision_replay", + metadata_json=_json_dumps(replay_metadata), + created_at=now, + updated_at=now, + ) + ) + return True + existing.status = str(decision.get("status") or existing.status) + existing.active = True + existing.source_id = new_dataset.source_id + existing.gtfs_dataset_id = new_dataset.id + existing.gtfs_route_id = route.id + existing.osm_dataset_id = osm_dataset_id + existing.osm_feature_id = osm_feature_id + existing.route_match_id = None + existing.confidence = _optional_float(decision.get("confidence")) + existing.evidence_hash = _hash_payload(evidence) + existing.selector_json = _json_dumps(selector) + existing.action_json = _json_dumps(action) + existing.reviewer = "decision_replay" + existing.metadata_json = _json_dumps(replay_metadata) + existing.updated_at = now + return True + + +def _updated_route_selector(value: object, *, new_dataset: Dataset, route: GtfsRoute) -> dict[str, Any]: + selector = dict(value) if isinstance(value, dict) else {} + selector["gtfs"] = { + "source_id": new_dataset.source_id, + "dataset_id": new_dataset.id, + "route_id": route.route_id, + "route_key": route.route_key, + "ref": route.short_name, + "mode": route.mode, + } + selector.pop("route_match_id", None) + selector.pop("gtfs_route_id", None) + return selector + + +def _existing_osm_reference(session: Session, decision: dict[str, Any]) -> tuple[int | None, int | None]: + dataset_id = _optional_int(decision.get("osm_dataset_id")) + feature_id = _optional_int(decision.get("osm_feature_id")) + feature = session.get(OsmFeature, feature_id) if feature_id is not None else None + if feature is not None: + return feature.dataset_id, feature.id + if dataset_id is not None and session.get(Dataset, dataset_id) is not None: + return dataset_id, None + return None, None + + +def _changed_or_removed_keys(session: Session, diff_run_id: int | None) -> set[tuple[str, str]]: + if diff_run_id is None: + return set() + rows = session.execute( + select(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.object_key) + .where(GtfsFeedDiffItem.diff_run_id == diff_run_id, GtfsFeedDiffItem.change_type.in_(["changed", "removed"])) + ).all() + return { + ("route" if item_type == "route" else "stop" if item_type == "stop" else str(item_type), str(object_key)) + for item_type, object_key in rows + } + + +def _diff_review_title(item: GtfsFeedDiffItem, *, previous_decision_count: int) -> str: + prefix = "Recheck previous decision for" if previous_decision_count else "Review updated" + return f"{prefix} {item.item_type} {item.object_key}: {item.change_type}" + + +def _diff_review_description(item: GtfsFeedDiffItem, previous_decision_count: int) -> str: + if previous_decision_count: + return f"{item.item_type.title()} changed after {previous_decision_count} previous decision(s); manual review is needed before replay." + return f"{item.item_type.title()} was {item.change_type} in the latest feed update." + + +def _stable_key(decision_type: str, selector: dict[str, Any], action: dict[str, Any]) -> str: + payload = {"decision_type": decision_type, "selector": selector, "action": action} + return f"{decision_type}:{_hash_payload(payload)[:40]}" + + +def _hash_payload(payload: dict[str, Any]) -> str: + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + +def _json_object(value: str | None) -> dict[str, Any]: + if not value: + return {} + try: + payload = json.loads(value) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def _json_dumps(value: dict[str, Any]) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) + + +def _optional_int(value: object) -> int | None: + try: + return None if value is None else int(value) + except (TypeError, ValueError): + return None + + +def _optional_float(value: object) -> float | None: + try: + return None if value is None else float(value) + except (TypeError, ValueError): + return None diff --git a/app/feed_discovery.py b/app/feed_discovery.py index 3aa609d..f14f7fe 100644 --- a/app/feed_discovery.py +++ b/app/feed_discovery.py @@ -22,6 +22,7 @@ MOBILITY_DATABASE_ACCEPTANCE_TEST_URL = ( ) PTNA_GTFS_INDEX_URL = "https://ptna.openstreetmap.de/gtfs/index.html" PTNA_COUNTRY_URL_TEMPLATE = "https://ptna.openstreetmap.de/gtfs/{country}/index.php" +OPENDATA_OEPNV_DATASETS_URL = "https://www.opendata-oepnv.de/ht/de/datensaetze" DEFAULT_DISCOVERY_COUNTRIES = ["DE", "AT", "CH", "NL", "DK", "FR", "BE", "LU", "NO", "SE", "FI", "IE", "GB"] CURATED_TEST_COUNTRIES = ["DE", "CH", "AT", "NL", "DK", "FI", "NO", "SE", "IE", "GB", "FR", "BE", "LU"] @@ -33,6 +34,11 @@ CANONICAL_HEADERS = [ "subdivision", "provider", "feed_name", + "authority_level", + "coverage_scope", + "overlap_role", + "import_policy", + "overlap_group", "stable_id", "ptna_feed_id", "data_type", @@ -71,6 +77,11 @@ class FeedCandidate: subdivision: str = "" provider: str = "" feed_name: str = "" + authority_level: str = "" + coverage_scope: str = "" + overlap_role: str = "" + import_policy: str = "" + overlap_group: str = "" stable_id: str = "" ptna_feed_id: str = "" data_type: str = "gtfs" @@ -142,6 +153,11 @@ class FeedCandidate: notes = _join_notes(notes, f"Mobility Database mirror: {self.latest_url}") if self.osm_license_text: notes = _join_notes(notes, f"OSM permission note: {_truncate(self.osm_license_text, 240)}") + if self.import_policy or self.overlap_role: + notes = _join_notes( + notes, + f"Import policy: {self.import_policy or 'review'}; overlap role: {self.overlap_role or 'review'}.", + ) return { "name": _truncate(name, 240), "kind": "gtfs", @@ -166,7 +182,9 @@ def build_gtfs_discovery_manifests( include_mobility_database: bool = True, include_acceptance_test_list: bool = True, include_ptna: bool = True, + include_opendata_oepnv: bool = True, max_ptna_details: int = 80, + max_opendata_oepnv_details: int = 80, test_limit: int = 24, check_urls: bool = False, timeout: float = 30.0, @@ -183,8 +201,18 @@ def build_gtfs_discovery_manifests( candidates.extend(fetch_mobility_acceptance_candidates(countries=selected_countries, timeout=timeout)) if include_ptna: candidates.extend(fetch_ptna_candidates(countries=selected_countries, max_details=max_ptna_details, timeout=timeout)) + if include_opendata_oepnv: + candidates.extend( + fetch_opendata_oepnv_candidates( + countries=selected_countries, + max_details=max_opendata_oepnv_details, + timeout=timeout, + ) + ) merged = merge_candidates(candidates) + for candidate in merged: + apply_source_hierarchy(candidate) ingestable = [candidate for candidate in merged if candidate.selected_url and candidate.data_type == "gtfs"] if check_urls: for candidate in ingestable: @@ -209,6 +237,7 @@ def build_gtfs_discovery_manifests( "mobility_database": MOBILITY_DATABASE_FEEDS_URL if include_mobility_database else None, "mobility_acceptance_test_list": MOBILITY_DATABASE_ACCEPTANCE_TEST_URL if include_acceptance_test_list else None, "ptna": PTNA_GTFS_INDEX_URL if include_ptna else None, + "opendata_oepnv": OPENDATA_OEPNV_DATASETS_URL if include_opendata_oepnv else None, }, "counts": { "candidates": len(merged), @@ -266,6 +295,7 @@ def fetch_mobility_database_candidates( ) normalize_candidate_geography(candidate) apply_known_download_overrides(candidate) + apply_source_hierarchy(candidate) candidate.priority = _candidate_priority(candidate) candidates.append(candidate) return candidates @@ -303,6 +333,7 @@ def fetch_mobility_acceptance_candidates( ) normalize_candidate_geography(candidate) apply_known_download_overrides(candidate) + apply_source_hierarchy(candidate) candidates.append(candidate) return candidates @@ -332,6 +363,7 @@ def fetch_ptna_candidates( detail_fetches += 1 except requests.RequestException: candidate.notes = _join_notes(candidate.notes, "PTNA detail page could not be fetched during discovery.") + apply_source_hierarchy(candidate) candidate.priority = _candidate_priority(candidate) candidates.append(candidate) return candidates @@ -383,6 +415,7 @@ def parse_ptna_country_page(html: str, *, country: str, page_url: str) -> list[F ) normalize_candidate_geography(candidate) apply_known_download_overrides(candidate) + apply_source_hierarchy(candidate) candidates.append(candidate) return candidates @@ -421,6 +454,134 @@ def parse_ptna_detail_fields(html: str, page_url: str) -> dict[str, str]: return parsed +@dataclass +class OpenDataOepnvDatasetLink: + slug: str + url: str + title: str = "" + + +def fetch_opendata_oepnv_candidates( + *, + countries: list[str] | None = None, + max_details: int = 80, + timeout: float = 30.0, + url: str = OPENDATA_OEPNV_DATASETS_URL, +) -> list[FeedCandidate]: + if countries and "DE" not in countries: + return [] + try: + html = _fetch_text(url, timeout=timeout) + except requests.RequestException: + return [] + dataset_links = parse_opendata_oepnv_dataset_links(html, url) + candidates: list[FeedCandidate] = [] + for dataset_link in dataset_links[:max_details]: + try: + detail_html = _fetch_text(dataset_link.url, timeout=timeout) + except requests.RequestException: + candidate = FeedCandidate( + discovery_source="opendata_oepnv", + country="DE", + provider=_opendata_oepnv_provider(dataset_link.url, dataset_link.slug), + feed_name=dataset_link.title or _title_from_slug(dataset_link.slug), + stable_id=f"opendata-oepnv:{dataset_link.slug}", + status="detail_fetch_failed", + details_url=dataset_link.url, + source_basis="OpenData ÖPNV dataset catalog", + notes="OpenData ÖPNV detail page could not be fetched during discovery.", + priority="P4", + ) + apply_source_hierarchy(candidate) + candidates.append(candidate) + continue + candidate = parse_opendata_oepnv_detail_page( + detail_html, + detail_url=dataset_link.url, + slug=dataset_link.slug, + title_hint=dataset_link.title, + ) + normalize_candidate_geography(candidate) + apply_known_download_overrides(candidate) + apply_source_hierarchy(candidate) + candidate.priority = candidate.priority or _candidate_priority(candidate) + candidates.append(candidate) + return candidates + + +def parse_opendata_oepnv_dataset_links(html: str, page_url: str = OPENDATA_OEPNV_DATASETS_URL) -> list[OpenDataOepnvDatasetLink]: + titles_by_slug: dict[str, str] = {} + title_pattern = re.compile( + r"]*href=[\"'](?P[^\"']*(?:dataset_name%5D|dataset_name\])=[^\"']+)[\"'][^>]*>\s*" + r"]*itemprop=[\"']headline[\"'][^>]*>(?P.*?)</span>", + re.IGNORECASE | re.DOTALL, + ) + for match in title_pattern.finditer(html): + href = urljoin(page_url, unescape(match.group("href"))) + slug = _opendata_oepnv_dataset_slug_from_url(href) + if slug: + titles_by_slug[slug] = _html_text(match.group("title")) + + by_slug: dict[str, OpenDataOepnvDatasetLink] = {} + for link in _all_links(html, page_url): + slug = _opendata_oepnv_dataset_slug_from_url(link) + if not slug: + continue + existing = by_slug.get(slug) + candidate = OpenDataOepnvDatasetLink(slug=slug, url=link, title=titles_by_slug.get(slug, "")) + if existing is None or _opendata_oepnv_link_score(candidate.url) > _opendata_oepnv_link_score(existing.url): + by_slug[slug] = candidate + return sorted(by_slug.values(), key=lambda item: (item.title.lower(), item.slug)) + + +def parse_opendata_oepnv_detail_page( + html: str, + *, + detail_url: str, + slug: str = "", + title_hint: str = "", +) -> FeedCandidate: + text = _html_text(html) + title = _opendata_oepnv_title(html) or title_hint or _title_from_slug(slug) + provider = _opendata_oepnv_provider(detail_url, slug) + data_type = _opendata_oepnv_data_type(title, text) + download_urls = _opendata_oepnv_download_urls(html, detail_url) + selected_url = _choose_opendata_oepnv_download_url(download_urls, data_type) + login_required = _opendata_oepnv_requires_login(text) + license_url, license_text = _opendata_oepnv_license(html, detail_url) + status = "active" if selected_url else "review_required" + if login_required: + status = "registration_required" + notes = "Discovered from OpenData ÖPNV; review per-dataset license and authority before publication." + if selected_url: + notes = _join_notes(notes, "Selected the first current GTFS download URL exposed on the detail page.") + if login_required: + notes = _join_notes(notes, "Download requires an OpenData ÖPNV account; do not auto-import without credentials.") + candidate = FeedCandidate( + discovery_source="opendata_oepnv", + country="DE", + subdivision=_opendata_oepnv_subdivision(detail_url, slug), + provider=provider, + feed_name=title, + stable_id=f"opendata-oepnv:{slug}" if slug else "", + data_type=data_type, + status=status, + is_official="", + selected_url=selected_url, + direct_download_url=selected_url, + original_release_url=detail_url, + license_url=license_url, + license_text=license_text, + details_url=detail_url, + features=_opendata_oepnv_features(data_type, title, text), + priority=_opendata_oepnv_priority(provider, data_type, selected_url), + source_basis="OpenData ÖPNV dataset catalog", + notes=notes, + ) + apply_source_hierarchy(candidate) + return candidate + + def load_curated_ingestable_seed( *, countries: list[str] | None = None, @@ -452,6 +613,7 @@ def load_curated_ingestable_seed( ) normalize_candidate_geography(candidate) apply_known_download_overrides(candidate) + apply_source_hierarchy(candidate) candidates.append(candidate) return candidates @@ -609,6 +771,275 @@ def apply_known_download_overrides(candidate: FeedCandidate) -> None: candidate.notes, "Selected Mobility Database latest.zip mirror because the catalog direct URL is known to be stale.", ) + if _opendata_oepnv_download_is_registration_gated(candidate): + gated_url = candidate.selected_url or candidate.direct_download_url + candidate.original_release_url = candidate.original_release_url or gated_url + candidate.selected_url = "" + candidate.direct_download_url = "" + candidate.status = "registration_required" + candidate.notes = _join_notes( + candidate.notes, + "OpenData ÖPNV marks this download as registration-required; kept as a review candidate instead of direct ingest.", + ) + + +def apply_source_hierarchy(candidate: FeedCandidate) -> None: + text = _hierarchy_text(candidate) + authority_level = _infer_authority_level(candidate, text) + coverage_scope = _infer_coverage_scope(candidate, text) + overlap_role = _infer_overlap_role(authority_level, coverage_scope, candidate, text) + import_policy = _infer_import_policy(authority_level, overlap_role, candidate) + overlap_group = _infer_overlap_group(candidate, authority_level, coverage_scope, text) + + candidate.authority_level = authority_level + candidate.coverage_scope = coverage_scope + candidate.overlap_role = overlap_role + candidate.import_policy = import_policy + candidate.overlap_group = overlap_group + + +def _hierarchy_text(candidate: FeedCandidate) -> str: + return " ".join( + [ + candidate.discovery_source, + candidate.country, + candidate.subdivision, + candidate.provider, + candidate.feed_name, + candidate.stable_id, + candidate.ptna_feed_id, + candidate.status, + _url_hierarchy_text(candidate.selected_url), + _url_hierarchy_text(candidate.direct_download_url), + _url_hierarchy_text(candidate.latest_url), + _url_hierarchy_text(candidate.original_release_url), + _url_hierarchy_text(candidate.details_url), + candidate.source_basis, + candidate.notes, + ] + ).lower() + + +def _infer_authority_level(candidate: FeedCandidate, text: str) -> str: + if _is_mirror_candidate(candidate, text): + return "mirror" + discovery_sources = {part.strip() for part in candidate.discovery_source.split(";") if part.strip()} + if discovery_sources and discovery_sources <= {"ptna", "mobility_validator_acceptance"}: + return "secondary_discovery" + if _contains_any( + text, + [ + "delfi", + "gtfs.de", + "mobilithek", + "germany-wide", + "deutschlandweite", + "entur", + "bod national", + "national gtfs", + "national feed", + "national timetable", + "opentransportdata.swiss", + "trafiklab", + "samtrafiken", + "transport for ireland", + "transportforireland", + "rejseplanen", + "gtfs.ovapi.nl", + ], + ): + return "national_official" + if _contains_any(text, ["operator", "verkehrsunternehmen", "rnv", "flixbus", "flixtrain", "sncf", "tfl"]): + return "operator_feed" + if _contains_any( + text, + [ + "verkehrsverbund", + "transport association", + "regional authority", + "authority feed", + "vbb", + "vrr", + "vvs", + "vrs", + "mvv", + "hvv", + "rmv", + "nvv", + "nwl", + "avv", + "nah.sh", + "nah-sh", + "bayern", + "baden-württemberg", + "baden-wuerttemberg", + "nordrhein-westfalen", + "nrw", + "thüringen", + "thueringen", + ], + ): + return "regional_authority" + if "mobility_database" in candidate.discovery_source: + return "aggregator_catalog" + return "unknown" + + +def _infer_coverage_scope(candidate: FeedCandidate, text: str) -> str: + if _contains_any( + text, + [ + "germany-wide", + "deutschlandweite", + "national gtfs", + "national feed", + "national timetable", + "delfi", + "gtfs.de", + "entur", + "transport for ireland", + "transportforireland", + "trafiklab", + "samtrafiken", + "opentransportdata.swiss", + "gtfs.ovapi.nl", + "rejseplanen", + ], + ): + return "national" + if _contains_any(text, ["europe", "eu/eea", "pan-europe", "flixbus", "flixtrain"]): + return "multi_country" + if _contains_any( + text, + [ + "verkehrsverbund", + "regional", + "bundesland", + "vbb", + "vrr", + "vvs", + "vrs", + "mvv", + "hvv", + "rmv", + "nvv", + "nwl", + "avv", + "nah.sh", + "nah-sh", + ], + ): + return "regional" + if _contains_any(text, ["operator", "verkehrsunternehmen", "rnv"]): + return "operator" + if candidate.country: + return "unknown_country" + return "unknown" + + +def _infer_overlap_role(authority_level: str, coverage_scope: str, candidate: FeedCandidate, text: str) -> str: + if authority_level == "mirror": + return "bootstrap_mirror" + if authority_level == "secondary_discovery": + return "discovery_evidence" + if candidate.status == "registration_required": + return "review_baseline" if coverage_scope == "national" else "review_candidate" + if authority_level == "national_official" and coverage_scope == "national": + return "canonical_baseline" + if authority_level == "regional_authority": + return "regional_override_candidate" + if authority_level == "operator_feed": + return "operator_detail_candidate" + if authority_level == "aggregator_catalog": + return "catalog_metadata" + if _contains_any(text, ["active", "gtfs"]): + return "gap_fill_candidate" + return "review_candidate" + + +def _infer_import_policy(authority_level: str, overlap_role: str, candidate: FeedCandidate) -> str: + if candidate.status == "registration_required" or not candidate.selected_url: + return "review_before_import" + if overlap_role == "canonical_baseline": + return "canonical_candidate" + if overlap_role in {"regional_override_candidate", "operator_detail_candidate"}: + return "qa_or_override_not_additive" + if overlap_role == "bootstrap_mirror": + return "bootstrap_only" + if authority_level in {"secondary_discovery", "aggregator_catalog"}: + return "metadata_only" + return "review_before_import" + + +def _infer_overlap_group(candidate: FeedCandidate, authority_level: str, coverage_scope: str, text: str) -> str: + country = candidate.country or "unknown" + if coverage_scope == "national": + return f"{country}:national" + identity_text = _hierarchy_identity_text(candidate) + for token in [ + "vbb", + "vrr", + "vvs", + "vrs", + "mvv", + "hvv", + "rmv", + "nvv", + "nwl", + "avv", + "nah-sh", + "nah.sh", + "rnv", + ]: + if token in identity_text: + normalized = token.replace(".", "-") + return f"{country}:{coverage_scope}:{normalized}" + key = _source_key(candidate.provider or candidate.feed_name) + if key: + return f"{country}:{coverage_scope}:{key}" + return f"{country}:{coverage_scope}:{authority_level}" + + +def _is_mirror_candidate(candidate: FeedCandidate, text: str) -> bool: + if any(_url_is_mirror_url(url) for url in [candidate.selected_url, candidate.direct_download_url, candidate.latest_url if not candidate.selected_url else ""]): + return True + mirror_notes = " ".join([candidate.source_basis, candidate.notes]).lower() + return _contains_any(mirror_notes, ["european transport feeds mirror", "bootstrap mirror"]) + + +def _source_key(value: str) -> str: + cleaned = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") + return cleaned[:80] + + +def _contains_any(text: str, tokens: Iterable[str]) -> bool: + return any(token in text for token in tokens) + + +def _hierarchy_identity_text(candidate: FeedCandidate) -> str: + return " ".join( + [ + candidate.provider, + candidate.feed_name, + candidate.stable_id, + candidate.ptna_feed_id, + _url_hierarchy_text(candidate.selected_url), + _url_hierarchy_text(candidate.direct_download_url), + _url_hierarchy_text(candidate.original_release_url), + _url_hierarchy_text(candidate.details_url), + ] + ).lower() + + +def _url_hierarchy_text(url: str) -> str: + parsed = urlparse(url or "") + return " ".join(part for part in [parsed.netloc, parsed.path] if part) + + +def _url_is_mirror_url(url: str) -> bool: + parsed = urlparse(url or "") + host_path = f"{parsed.netloc.lower()}{parsed.path.lower()}" + return _contains_any(host_path, ["files.mobilitydatabase.org", "openmobilitydata-data.s3", "scraped.data.public-transport.earth"]) @dataclass @@ -732,6 +1163,211 @@ def _looks_like_download_url(url: str) -> bool: return False +def _opendata_oepnv_dataset_slug_from_url(url: str) -> str: + query = parse_qs(urlparse(unescape(url)).query) + return _clean_text((query.get("tx_vrrkit_view[dataset_name]") or [""])[0]) + + +def _opendata_oepnv_link_score(url: str) -> int: + parsed = urlparse(url) + score = 0 + if "/organisation/" in parsed.path: + score += 2 + if parsed.path.rstrip("/").endswith("startseite"): + score += 1 + return score + + +def _opendata_oepnv_title(html: str) -> str: + matches = re.findall(r"<div[^>]*class=[\"'][^\"']*element-header[^\"']*[\"'][^>]*>(.*?)</div>", html, re.IGNORECASE | re.DOTALL) + for raw in reversed(matches): + title = _html_text(raw) + if title and title.lower() not in {"hinweise", "spielregeln", "lizenzen"}: + return title + match = re.search(r"<title>(.*?)", html, re.IGNORECASE | re.DOTALL) + return _html_text(match.group(1)) if match else "" + + +def _opendata_oepnv_download_urls(html: str, detail_url: str) -> list[str]: + urls: list[str] = [] + for link in _all_links(html, detail_url): + normalized = _normalize_feed_url(link) + if normalized and _looks_like_download_url(normalized) and normalized not in urls: + urls.append(normalized) + return urls + + +def _choose_opendata_oepnv_download_url(urls: list[str], data_type: str) -> str: + if data_type == "gtfs_rt": + for url in urls: + lower = url.lower() + if "realtime" in lower or "gtfs-rt" in lower or "gtfs_rt" in lower: + return url + return "" + if data_type != "gtfs": + return "" + if not urls: + return "" + static_urls = [url for url in urls if "realtime" not in url.lower() and "gtfs-rt" not in url.lower() and "gtfs_rt" not in url.lower()] + if not static_urls: + return "" + for url in static_urls: + lower = url.lower() + if "latest" in lower or "aktueller" in lower or lower.rstrip("/").endswith("/gtfs.zip"): + return url + return static_urls[0] + + +def _opendata_oepnv_license(html: str, detail_url: str) -> tuple[str, str]: + match = re.search( + r"id=[\"']dodp_license[\"'][^>]*>.*?]*href=[\"'](?P[^\"']+)[\"'][^>]*>(?P.*?)", + html, + re.IGNORECASE | re.DOTALL, + ) + if not match: + return "", "" + return urljoin(detail_url, unescape(match.group("href"))), _html_text(match.group("text")) + + +def _opendata_oepnv_requires_login(text: str) -> bool: + lower = text.lower() + return ( + "available for registered users" in lower + or "download is only available for registered users" in lower + or "this download is only available for registered users" in lower + or "anmeldung erforderlich" in lower + or "nur für registrierte nutzer" in lower + or "nur fuer registrierte nutzer" in lower + ) + + +def _opendata_oepnv_data_type(title: str, text: str) -> str: + title_lower = title.lower() + lower = f"{title} {text}".lower() + if "gtfs-rt" in lower or "gtfs realtime" in lower or "gtfs-realtime" in lower: + return "gtfs_rt" + if "gtfs" in title_lower: + return "gtfs" + if "haltestell" in title_lower or "zhv" in title_lower: + return "stops" + if "netex" in title_lower: + return "netex" + if "trias" in lower: + return "api" + if "netex" in lower: + return "netex" + if "gtfs" in lower: + return "gtfs" + if "haltestell" in lower or "zhv" in lower: + return "stops" + if "linien" in lower: + return "line_metadata" + return "dataset" + + +def _opendata_oepnv_features(data_type: str, title: str, text: str) -> str: + features = ["OpenData ÖPNV"] + if data_type == "gtfs": + features.extend(["GTFS", "Schedules"]) + elif data_type == "netex": + features.extend(["NeTEx", "Schedules"]) + elif data_type == "gtfs_rt": + features.extend(["GTFS-Realtime"]) + elif data_type == "stops": + features.append("Stops") + lower = f"{title} {text}".lower() + if "regionalbahn" in lower or "s-bahn" in lower or "bahn" in lower: + features.append("Rail") + if "u-bahn" in lower or "metro" in lower: + features.append("Metro") + if "tram" in lower or "straßenbahn" in lower: + features.append("Tram") + if "bus" in lower: + features.append("Bus") + return "|".join(dict.fromkeys(features)) + + +def _opendata_oepnv_priority(provider: str, data_type: str, selected_url: str) -> str: + if provider == "DELFI e.V." and data_type in {"gtfs", "netex", "stops"}: + return "P0" + if data_type == "gtfs" and selected_url: + return "P1" + if data_type in {"netex", "gtfs_rt", "stops"}: + return "P2" + return "P3" + + +def _opendata_oepnv_provider(detail_url: str, slug: str) -> str: + path = urlparse(detail_url).path.lower() + haystack = f"{path}/{slug.lower()}/" + providers = { + "delfi": "DELFI e.V.", + "nah-sh": "NAH.SH", + "avv": "Aachener Verkehrsverbund", + "baden-wuerttemberg": "Baden-Württemberg", + "bawue": "Baden-Württemberg", + "hvv": "Hamburger Verkehrsverbund", + "mvv": "Münchner Verkehrs- und Tarifverbund", + "nvv": "Nordhessischer VerkehrsVerbund", + "nwl": "Nahverkehr Westfalen-Lippe", + "nordrhein-westfalen": "Nordrhein-Westfalen", + "nrw": "Nordrhein-Westfalen", + "rmv": "Rhein-Main-Verkehrsverbund", + "rnv": "Rhein-Neckar-Verkehr", + "thueringen": "Thüringen", + "vbb": "Verkehrsverbund Berlin-Brandenburg", + "vrr": "Verkehrsverbund Rhein-Ruhr", + "vrs": "Verkehrsverbund Rhein-Sieg", + "vvs": "Verkehrs- und Tarifverbund Stuttgart", + } + for token, provider in providers.items(): + if f"/{token}/" in haystack or f"-{token}" in haystack or f"_{token}" in haystack: + return provider + return "OpenData ÖPNV" + + +def _opendata_oepnv_subdivision(detail_url: str, slug: str) -> str: + text = f"{urlparse(detail_url).path} {slug}".lower() + subdivisions = { + "baden-wuerttemberg": "Baden-Württemberg", + "bawue": "Baden-Württemberg", + "thueringen": "Thüringen", + "nrw": "Nordrhein-Westfalen", + "nordrhein-westfalen": "Nordrhein-Westfalen", + "vbb": "Berlin-Brandenburg", + "hvv": "Hamburg", + "mvv": "Bayern", + "vrr": "Nordrhein-Westfalen", + "vrs": "Nordrhein-Westfalen", + "nwl": "Nordrhein-Westfalen", + "avv": "Nordrhein-Westfalen", + "rmv": "Hessen", + "nvv": "Hessen", + "rnv": "Baden-Württemberg/Rheinland-Pfalz/Hessen", + "vvs": "Baden-Württemberg", + } + for token, subdivision in subdivisions.items(): + if token in text: + return subdivision + return "" + + +def _opendata_oepnv_download_is_registration_gated(candidate: FeedCandidate) -> bool: + url_text = " ".join([candidate.selected_url, candidate.direct_download_url, candidate.original_release_url]).lower() + if "opendata-oepnv.de" not in url_text: + return False + descriptive_text = " ".join([candidate.provider, candidate.feed_name, candidate.status, candidate.notes]).lower() + return "registration required" in descriptive_text or "user registration required" in descriptive_text + + +def _html_text(value: str) -> str: + return _clean_text(re.sub(r"<[^>]+>", " ", value or "")) + + +def _title_from_slug(slug: str) -> str: + return _clean_text(slug.replace("_", " ").replace("-", " ").title()) + + def _normalize_feed_url(url: str) -> str: cleaned = _clean_text(url) if not cleaned: @@ -800,6 +1436,10 @@ def _merge_candidate(existing: FeedCandidate, incoming: FeedCandidate) -> None: new_value = getattr(incoming, field_name, "") if new_value: setattr(existing, field_name, new_value) + if existing.selected_url and incoming.selected_url and _url_is_mirror_url(existing.selected_url) and not _url_is_mirror_url(incoming.selected_url): + existing.selected_url = incoming.selected_url + existing.direct_download_url = incoming.direct_download_url or incoming.selected_url + existing.notes = _join_notes(existing.notes, "Selected non-mirror publisher URL from merged source evidence.") existing.discovery_source = _join_unique(existing.discovery_source, incoming.discovery_source) for field_name in CANONICAL_HEADERS: if field_name == "candidate_id": diff --git a/app/gtfs_diff.py b/app/gtfs_diff.py new file mode 100644 index 0000000..db2c13d --- /dev/null +++ b/app/gtfs_diff.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from sqlalchemy import func, select +from sqlalchemy.orm import Session, joinedload + +from app.models import Dataset, GtfsFeedDiffItem, GtfsFeedDiffRun, GtfsRoute, GtfsStop, GtfsTrip, Source + +MAX_DIFF_ITEMS_PER_TYPE = 200 + + +def create_gtfs_update_diff_from_stage( + session: Session, + *, + source: Source, + previous_dataset: Dataset | None, + new_dataset: Dataset, + stage_path: Path, +) -> dict[str, Any] | None: + if previous_dataset is None or previous_dataset.kind != "gtfs" or not stage_path.exists(): + return None + old_routes = _main_routes(session, previous_dataset.id) + old_stops = _main_stops(session, previous_dataset.id) + old_counts = { + "routes": len(old_routes), + "stops": len(old_stops), + "trips": _main_count(session, GtfsTrip, previous_dataset.id), + } + with sqlite3.connect(stage_path) as connection: + connection.row_factory = sqlite3.Row + new_routes = _stage_routes(connection) + new_stops = _stage_stops(connection) + new_counts = { + "routes": len(new_routes), + "stops": len(new_stops), + "trips": _stage_count(connection, "gtfs_trips"), + } + + route_summary, route_items = _diff_objects( + item_type="route", + old=old_routes, + new=new_routes, + identity_label="route", + ) + stop_summary, stop_items = _diff_objects( + item_type="stop", + old=old_stops, + new=new_stops, + identity_label="stop", + ) + summary = { + "source_id": source.id, + "previous_dataset_id": previous_dataset.id, + "new_dataset_id": new_dataset.id, + "counts": {"old": old_counts, "new": new_counts}, + "routes": route_summary, + "stops": stop_summary, + "items_stored": { + "route": min(len(route_items), MAX_DIFF_ITEMS_PER_TYPE), + "stop": min(len(stop_items), MAX_DIFF_ITEMS_PER_TYPE), + }, + } + diff_key = _diff_key(source.id, previous_dataset.id, new_dataset.id, summary) + existing = session.scalar(select(GtfsFeedDiffRun).where(GtfsFeedDiffRun.diff_key == diff_key)) + if existing is not None: + return diff_run_payload(existing) + + now = datetime.now(timezone.utc) + diff_run = GtfsFeedDiffRun( + source_id=source.id, + previous_dataset_id=previous_dataset.id, + new_dataset_id=new_dataset.id, + status="completed", + diff_key=diff_key, + summary_json=_json_dumps(summary), + metadata_json=_json_dumps({"stage_path": str(stage_path), "created_from": "gtfs_activation"}), + created_at=now, + ) + session.add(diff_run) + session.flush() + for item in [*route_items[:MAX_DIFF_ITEMS_PER_TYPE], *stop_items[:MAX_DIFF_ITEMS_PER_TYPE]]: + session.add( + GtfsFeedDiffItem( + diff_run_id=diff_run.id, + source_id=source.id, + previous_dataset_id=previous_dataset.id, + new_dataset_id=new_dataset.id, + item_type=item["item_type"], + object_key=item["object_key"], + change_type=item["change_type"], + severity=item["severity"], + status="open", + title=item["title"], + old_payload_json=_json_dumps(item["old"]) if item.get("old") else None, + new_payload_json=_json_dumps(item["new"]) if item.get("new") else None, + diff_json=_json_dumps(item["diff"]) if item.get("diff") else None, + created_at=now, + ) + ) + session.flush() + return diff_run_payload(diff_run) + + +def latest_gtfs_update_diffs(session: Session, *, source_id: int | None = None, limit: int = 20) -> list[GtfsFeedDiffRun]: + stmt = ( + select(GtfsFeedDiffRun) + .options(joinedload(GtfsFeedDiffRun.source)) + .order_by(GtfsFeedDiffRun.created_at.desc(), GtfsFeedDiffRun.id.desc()) + .limit(max(1, min(int(limit), 100))) + ) + if source_id is not None: + stmt = stmt.where(GtfsFeedDiffRun.source_id == int(source_id)) + return session.scalars(stmt).all() + + +def gtfs_update_diff_detail(session: Session, diff_id: int, *, item_limit: int = 200) -> dict[str, Any] | None: + diff_run = session.get(GtfsFeedDiffRun, diff_id) + if diff_run is None: + return None + items = session.scalars( + select(GtfsFeedDiffItem) + .where(GtfsFeedDiffItem.diff_run_id == diff_run.id) + .order_by(_change_type_order(), GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.object_key) + .limit(max(1, min(int(item_limit), 1000))) + ).all() + return {**diff_run_payload(diff_run), "items": [diff_item_payload(item) for item in items]} + + +def gtfs_update_diff_summary(session: Session) -> dict[str, Any]: + runs = session.scalar(select(func.count()).select_from(GtfsFeedDiffRun)) or 0 + rows = session.execute( + select(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.change_type, GtfsFeedDiffItem.severity, func.count()) + .select_from(GtfsFeedDiffItem) + .group_by(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.change_type, GtfsFeedDiffItem.severity) + ).all() + by_type: dict[str, int] = {} + by_change: dict[str, int] = {} + by_severity: dict[str, int] = {} + for item_type, change_type, severity, count in rows: + value = int(count or 0) + by_type[str(item_type)] = by_type.get(str(item_type), 0) + value + by_change[str(change_type)] = by_change.get(str(change_type), 0) + value + by_severity[str(severity)] = by_severity.get(str(severity), 0) + value + return {"runs": int(runs), "by_type": by_type, "by_change": by_change, "by_severity": by_severity} + + +def diff_run_payload(diff_run: GtfsFeedDiffRun) -> dict[str, Any]: + summary = _json_object(diff_run.summary_json) + return { + "id": diff_run.id, + "source_id": diff_run.source_id, + "source_name": None if diff_run.source is None else diff_run.source.name, + "previous_dataset_id": diff_run.previous_dataset_id, + "new_dataset_id": diff_run.new_dataset_id, + "status": diff_run.status, + "diff_key": diff_run.diff_key, + "summary": summary, + "metadata": _json_object(diff_run.metadata_json), + "created_at": diff_run.created_at.isoformat(), + } + + +def diff_item_payload(item: GtfsFeedDiffItem) -> dict[str, Any]: + return { + "id": item.id, + "diff_run_id": item.diff_run_id, + "source_id": item.source_id, + "previous_dataset_id": item.previous_dataset_id, + "new_dataset_id": item.new_dataset_id, + "item_type": item.item_type, + "object_key": item.object_key, + "change_type": item.change_type, + "severity": item.severity, + "status": item.status, + "title": item.title, + "old": _json_object(item.old_payload_json), + "new": _json_object(item.new_payload_json), + "diff": _json_object(item.diff_json), + "created_at": item.created_at.isoformat(), + "resolved_at": None if item.resolved_at is None else item.resolved_at.isoformat(), + } + + +def _main_routes(session: Session, dataset_id: int) -> dict[str, dict[str, Any]]: + trip_counts = { + str(route_id): int(count or 0) + for route_id, count in session.execute( + select(GtfsTrip.route_id, func.count()).where(GtfsTrip.dataset_id == dataset_id).group_by(GtfsTrip.route_id) + ).all() + } + routes = session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == dataset_id)).all() + return { + str(route.route_id): _normalized_route_payload( + { + "route_id": route.route_id, + "short_name": route.short_name, + "long_name": route.long_name, + "mode": route.mode, + "route_scope": route.route_scope, + "operator_name": route.operator_name, + "route_key": route.route_key, + "operator_key": route.operator_key, + "has_geometry": bool(route.geometry_geojson), + "trip_count": trip_counts.get(str(route.route_id), 0), + } + ) + for route in routes + } + + +def _stage_routes(connection: sqlite3.Connection) -> dict[str, dict[str, Any]]: + trip_counts = { + str(row["route_id"]): int(row["count"] or 0) + for row in connection.execute("SELECT route_id, COUNT(*) AS count FROM gtfs_trips GROUP BY route_id").fetchall() + } + rows = connection.execute( + """ + SELECT route_id, short_name, long_name, mode, route_scope, operator_name, route_key, operator_key, geometry_geojson + FROM gtfs_routes + """ + ).fetchall() + return { + str(row["route_id"]): _normalized_route_payload( + { + "route_id": row["route_id"], + "short_name": row["short_name"], + "long_name": row["long_name"], + "mode": row["mode"], + "route_scope": row["route_scope"], + "operator_name": row["operator_name"], + "route_key": row["route_key"], + "operator_key": row["operator_key"], + "has_geometry": bool(row["geometry_geojson"]), + "trip_count": trip_counts.get(str(row["route_id"]), 0), + } + ) + for row in rows + } + + +def _main_stops(session: Session, dataset_id: int) -> dict[str, dict[str, Any]]: + stops = session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == dataset_id)).all() + return { + str(stop.stop_id): _normalized_stop_payload( + { + "stop_id": stop.stop_id, + "name": stop.name, + "lat": stop.lat, + "lon": stop.lon, + "parent_station": stop.parent_station, + } + ) + for stop in stops + } + + +def _stage_stops(connection: sqlite3.Connection) -> dict[str, dict[str, Any]]: + rows = connection.execute("SELECT stop_id, name, lat, lon, parent_station FROM gtfs_stops").fetchall() + return { + str(row["stop_id"]): _normalized_stop_payload( + { + "stop_id": row["stop_id"], + "name": row["name"], + "lat": row["lat"], + "lon": row["lon"], + "parent_station": row["parent_station"], + } + ) + for row in rows + } + + +def _diff_objects( + *, + item_type: str, + old: dict[str, dict[str, Any]], + new: dict[str, dict[str, Any]], + identity_label: str, +) -> tuple[dict[str, int], list[dict[str, Any]]]: + old_keys = set(old) + new_keys = set(new) + added = sorted(new_keys - old_keys) + removed = sorted(old_keys - new_keys) + common = sorted(old_keys & new_keys) + items: list[dict[str, Any]] = [] + unchanged = 0 + changed = 0 + for key in common: + differences = _payload_differences(old[key], new[key]) + if not differences: + unchanged += 1 + continue + changed += 1 + items.append( + { + "item_type": item_type, + "object_key": key, + "change_type": "changed", + "severity": "warn", + "title": f"{identity_label.title()} changed: {key}", + "old": old[key], + "new": new[key], + "diff": differences, + } + ) + for key in added: + items.append( + { + "item_type": item_type, + "object_key": key, + "change_type": "added", + "severity": "info", + "title": f"{identity_label.title()} added: {key}", + "new": new[key], + "diff": {"added": True}, + } + ) + for key in removed: + items.append( + { + "item_type": item_type, + "object_key": key, + "change_type": "removed", + "severity": "warn", + "title": f"{identity_label.title()} removed: {key}", + "old": old[key], + "diff": {"removed": True}, + } + ) + summary = { + "old": len(old), + "new": len(new), + "added": len(added), + "removed": len(removed), + "changed": changed, + "unchanged": unchanged, + } + return summary, sorted(items, key=lambda item: (_change_type_rank(item["change_type"]), item["item_type"], item["object_key"])) + + +def _payload_differences(old: dict[str, Any], new: dict[str, Any]) -> dict[str, dict[str, Any]]: + differences = {} + for key in sorted(set(old) | set(new)): + if old.get(key) != new.get(key): + differences[key] = {"old": old.get(key), "new": new.get(key)} + return differences + + +def _normalized_route_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "route_id": str(payload.get("route_id") or ""), + "short_name": _clean_text(payload.get("short_name")), + "long_name": _clean_text(payload.get("long_name")), + "mode": _clean_text(payload.get("mode")), + "route_scope": _clean_text(payload.get("route_scope")), + "operator_name": _clean_text(payload.get("operator_name")), + "route_key": _clean_text(payload.get("route_key")), + "operator_key": _clean_text(payload.get("operator_key")), + "has_geometry": bool(payload.get("has_geometry")), + "trip_count": int(payload.get("trip_count") or 0), + } + + +def _normalized_stop_payload(payload: dict[str, Any]) -> dict[str, Any]: + return { + "stop_id": str(payload.get("stop_id") or ""), + "name": _clean_text(payload.get("name")), + "lat": _rounded_coord(payload.get("lat")), + "lon": _rounded_coord(payload.get("lon")), + "parent_station": _clean_text(payload.get("parent_station")), + } + + +def _main_count(session: Session, model, dataset_id: int) -> int: + return int(session.scalar(select(func.count()).select_from(model).where(model.dataset_id == dataset_id)) or 0) + + +def _stage_count(connection: sqlite3.Connection, table_name: str) -> int: + return int(connection.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0] or 0) + + +def _diff_key(source_id: int, previous_dataset_id: int, new_dataset_id: int, summary: dict[str, Any]) -> str: + payload = { + "source_id": source_id, + "previous_dataset_id": previous_dataset_id, + "new_dataset_id": new_dataset_id, + "routes": summary.get("routes"), + "stops": summary.get("stops"), + } + digest = hashlib.sha256(_json_dumps(payload).encode("utf-8")).hexdigest() + return f"gtfs-diff-{digest[:32]}" + + +def _change_type_order(): + from sqlalchemy import case + + return case( + (GtfsFeedDiffItem.change_type == "changed", 0), + (GtfsFeedDiffItem.change_type == "removed", 1), + (GtfsFeedDiffItem.change_type == "added", 2), + else_=9, + ) + + +def _change_type_rank(value: str) -> int: + return {"changed": 0, "removed": 1, "added": 2}.get(value, 9) + + +def _clean_text(value: object) -> str | None: + if value is None: + return None + text = " ".join(str(value).strip().split()) + return text or None + + +def _rounded_coord(value: object) -> float | None: + try: + return round(float(value), 7) + except (TypeError, ValueError): + return None + + +def _json_object(value: str | None) -> dict[str, Any]: + if not value: + return {} + try: + payload = json.loads(value) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def _json_dumps(value: dict[str, Any]) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str) diff --git a/app/harmonization.py b/app/harmonization.py index 701425c..d79b58f 100644 --- a/app/harmonization.py +++ b/app/harmonization.py @@ -1,9 +1,14 @@ from __future__ import annotations +import copy +import hashlib +import json +import re +import threading from datetime import date, datetime, timezone from typing import Any -from sqlalchemy import and_, func, select +from sqlalchemy import and_, func, or_, select from sqlalchemy.orm import Session, aliased from app.data_management import dataset_row_counts @@ -12,6 +17,9 @@ from app.models import ( Dataset, GtfsCalendar, GtfsCalendarDate, + GtfsHarmonizedSnapshot, + GtfsHarmonizedSnapshotDataset, + GtfsHarmonizedSnapshotRoute, GtfsRoute, GtfsStop, GtfsStopTime, @@ -22,20 +30,387 @@ from app.models import ( GTFS_QA_NOTE_PREFIX = "[GTFS QA]" +GTFS_LICENSE_FLAG_KEYS = ( + "can_import", + "can_derive", + "can_redistribute", + "requires_attribution", + "commercial_restrictions", +) +GTFS_AUTHORITY_LEVELS = { + "unknown", + "national_official", + "regional_authority", + "operator", + "aggregator", + "mirror", + "secondary_discovery", +} +GTFS_AUTHORITY_SCORES = { + "national_official": 600, + "regional_authority": 500, + "operator": 420, + "unknown": 320, + "aggregator": 260, + "mirror": 120, + "secondary_discovery": 80, +} +GTFS_PRIORITY_SCORES = { + "P0": 60, + "P1": 40, + "P2": 20, + "P3": 10, +} +GTFS_SHADOW_ROUTE_RATIO = 0.65 +GTFS_SHADOW_ROUTE_MIN_OVERLAP = 10 +GTFS_SHADOW_SMALL_ROUTE_RATIO = 0.80 +GTFS_SHADOW_SMALL_ROUTE_MIN_OVERLAP = 2 +GTFS_ROUTE_SHADOW_BBOX_MARGIN_DEG = 0.02 +_SNAPSHOT_CACHE_LOCK = threading.RLock() +_SNAPSHOT_CACHE_REVISION: tuple[object, ...] | None = None +_SNAPSHOT_CACHE: dict[str, Any] | None = None +_ROUTE_SHADOW_CACHE_LOCK = threading.RLock() +_ROUTE_SHADOW_CACHE_REVISION: tuple[object, ...] | None = None +_ROUTE_SHADOW_CACHE: set[int] | None = None + + +def active_harmonized_gtfs_dataset_ids(session: Session, source_ids: list[int] | None = None) -> list[int]: + if source_ids: + return _raw_active_gtfs_dataset_ids(session, source_ids=source_ids) + persisted_ids = _active_persisted_gtfs_dataset_ids(session) + if persisted_ids: + return persisted_ids + snapshot = computed_gtfs_harmonized_snapshot(session) + return [int(item["dataset_id"]) for item in snapshot["datasets"] if item["role"] == "included"] + + +def active_harmonized_gtfs_shadowed_route_ids(session: Session, source_ids: list[int] | None = None) -> set[int]: + if source_ids: + return set() + persisted = _active_persisted_gtfs_route_shadow_map(session) + if persisted is not None: + return set(persisted) + global _ROUTE_SHADOW_CACHE, _ROUTE_SHADOW_CACHE_REVISION + revision = _snapshot_revision(session) + with _ROUTE_SHADOW_CACHE_LOCK: + if _ROUTE_SHADOW_CACHE is not None and _ROUTE_SHADOW_CACHE_REVISION == revision: + return set(_ROUTE_SHADOW_CACHE) + items = _active_gtfs_snapshot_items(session) + _apply_dataset_shadowing(items) + result = _route_shadow_result(session, items) + shadowed_route_ids = set(result["shadowed_route_ids"]) + with _ROUTE_SHADOW_CACHE_LOCK: + _ROUTE_SHADOW_CACHE_REVISION = revision + _ROUTE_SHADOW_CACHE = set(shadowed_route_ids) + return shadowed_route_ids + + +def active_harmonized_gtfs_route_shadow_map(session: Session, source_ids: list[int] | None = None) -> dict[int, int]: + if source_ids: + return {} + persisted = _active_persisted_gtfs_route_shadow_map(session) + if persisted is not None: + return persisted + return _computed_gtfs_route_shadow_map(session) + + +def _computed_gtfs_route_shadow_map(session: Session) -> dict[int, int]: + items = _active_gtfs_snapshot_items(session) + _apply_dataset_shadowing(items) + result = _route_shadow_result(session, items) + return dict(result["shadowed_route_map"]) + + +def gtfs_harmonized_snapshot(session: Session) -> dict[str, Any]: + persisted = active_persisted_gtfs_snapshot(session) + if persisted is not None: + return persisted + return computed_gtfs_harmonized_snapshot(session) + + +def gtfs_harmonized_snapshot_diagnostics(session: Session) -> dict[str, Any]: + active = active_persisted_gtfs_snapshot(session) + computed = computed_gtfs_harmonized_snapshot(session) + computed_summary = computed.get("summary") or {} + active_summary = active.get("summary") if active is not None else {} + if not isinstance(active_summary, dict): + active_summary = {} + computed_key = _materialized_snapshot_key(computed) + active_key = str(active.get("snapshot_key") or "") if active is not None else "" + active_route_rows = _summary_int(active_summary, "snapshot_route_rows") + active_route_shadows = _summary_int(active_summary, "route_shadowed_routes") + computed_included_datasets = _summary_int(computed_summary, "included_datasets") + computed_route_shadows = _summary_int(computed_summary, "route_shadowed_routes") + reasons: list[str] = [] + if active is None: + reasons.append("no_active_snapshot") + elif active_key != computed_key: + reasons.append("snapshot_inputs_changed") + if active is not None and computed_included_datasets > 0 and active_route_rows == 0: + reasons.append("route_ownership_not_materialized") + if active is not None and active_route_rows > 0 and active_route_shadows != computed_route_shadows: + reasons.append("route_shadow_count_changed") + return { + "needs_rebuild": bool(reasons), + "reasons": reasons, + "active_snapshot_id": active.get("id") if active is not None else None, + "active_snapshot_key": active_key or None, + "computed_snapshot_key": computed_key, + "route_rows_materialized": active_route_rows > 0, + "active": { + "persisted": active is not None, + "included_datasets": _summary_int(active_summary, "included_datasets"), + "shadowed_datasets": _summary_int(active_summary, "shadowed_datasets"), + "route_rows": active_route_rows, + "route_shadowed_routes": active_route_shadows, + }, + "computed": { + "raw_active_datasets": _summary_int(computed_summary, "raw_active_datasets"), + "included_datasets": computed_included_datasets, + "shadowed_datasets": _summary_int(computed_summary, "shadowed_datasets"), + "route_shadowed_routes": computed_route_shadows, + }, + } + + +def list_active_harmonized_snapshot_routes( + session: Session, + *, + role: str | None = None, + dataset_id: int | None = None, + source_id: int | None = None, + shadowed_by_dataset_id: int | None = None, + mode: str | None = None, + q: str | None = None, + limit: int = 200, + offset: int = 0, + max_limit: int = 1000, +) -> dict[str, Any]: + snapshot = session.scalar( + select(GtfsHarmonizedSnapshot) + .where(GtfsHarmonizedSnapshot.status == "active") + .order_by(GtfsHarmonizedSnapshot.activated_at.desc(), GtfsHarmonizedSnapshot.id.desc()) + .limit(1) + ) + if snapshot is None: + return {"snapshot": None, "summary": {}, "routes": []} + conditions = [GtfsHarmonizedSnapshotRoute.snapshot_id == snapshot.id] + if role: + conditions.append(GtfsHarmonizedSnapshotRoute.role == role) + if dataset_id is not None: + conditions.append(GtfsHarmonizedSnapshotRoute.dataset_id == int(dataset_id)) + if source_id is not None: + conditions.append(GtfsHarmonizedSnapshotRoute.source_id == int(source_id)) + if shadowed_by_dataset_id is not None: + conditions.append(GtfsHarmonizedSnapshotRoute.shadowed_by_dataset_id == int(shadowed_by_dataset_id)) + mode_text = str(mode or "").strip() + if mode_text: + conditions.append(GtfsRoute.mode == mode_text) + query_text = " ".join(str(q or "").strip().split()) + if query_text: + pattern = f"%{query_text}%" + conditions.append( + or_( + GtfsHarmonizedSnapshotRoute.route_id.ilike(pattern), + GtfsHarmonizedSnapshotRoute.overlap_key.ilike(pattern), + GtfsRoute.short_name.ilike(pattern), + GtfsRoute.long_name.ilike(pattern), + GtfsRoute.operator_name.ilike(pattern), + ) + ) + stmt = ( + select(GtfsHarmonizedSnapshotRoute) + .join(GtfsRoute, GtfsRoute.id == GtfsHarmonizedSnapshotRoute.gtfs_route_id) + .where(*conditions) + ) + count_stmt = ( + select(func.count()) + .select_from(GtfsHarmonizedSnapshotRoute) + .join(GtfsRoute, GtfsRoute.id == GtfsHarmonizedSnapshotRoute.gtfs_route_id) + .where(*conditions) + ) + selected_limit = max(1, min(int(limit), max(1, int(max_limit)))) + selected_offset = max(0, int(offset)) + rows = session.scalars( + stmt.order_by( + GtfsHarmonizedSnapshotRoute.role, + GtfsHarmonizedSnapshotRoute.dataset_id, + GtfsHarmonizedSnapshotRoute.overlap_key, + GtfsHarmonizedSnapshotRoute.id, + ) + .offset(selected_offset) + .limit(selected_limit) + ).all() + return { + "snapshot": { + "id": snapshot.id, + "snapshot_key": snapshot.snapshot_key, + "status": snapshot.status, + "activated_at": _iso(snapshot.activated_at), + }, + "summary": { + **_snapshot_route_row_counts(session, snapshot.id), + "filtered_total": int(session.scalar(count_stmt) or 0), + "limit": selected_limit, + "offset": selected_offset, + }, + "routes": [_snapshot_route_payload(row) for row in rows], + } + + +def computed_gtfs_harmonized_snapshot(session: Session) -> dict[str, Any]: + global _SNAPSHOT_CACHE, _SNAPSHOT_CACHE_REVISION + revision = _snapshot_revision(session) + with _SNAPSHOT_CACHE_LOCK: + if _SNAPSHOT_CACHE is not None and _SNAPSHOT_CACHE_REVISION == revision: + return copy.deepcopy(_SNAPSHOT_CACHE) + snapshot = _compute_gtfs_harmonized_snapshot(session) + with _SNAPSHOT_CACHE_LOCK: + _SNAPSHOT_CACHE_REVISION = revision + _SNAPSHOT_CACHE = copy.deepcopy(snapshot) + return snapshot + + +def active_persisted_gtfs_snapshot(session: Session) -> dict[str, Any] | None: + snapshot = session.scalar( + select(GtfsHarmonizedSnapshot) + .where(GtfsHarmonizedSnapshot.status == "active") + .order_by(GtfsHarmonizedSnapshot.activated_at.desc(), GtfsHarmonizedSnapshot.id.desc()) + .limit(1) + ) + if snapshot is None: + return None + return _persisted_snapshot_payload(session, snapshot) + + +def create_gtfs_harmonized_snapshot(session: Session, *, activate: bool = True, note: str | None = None) -> dict[str, Any]: + computed = computed_gtfs_harmonized_snapshot(session) + snapshot_key = _materialized_snapshot_key(computed) + snapshot = session.scalar(select(GtfsHarmonizedSnapshot).where(GtfsHarmonizedSnapshot.snapshot_key == snapshot_key)) + now = datetime.now(timezone.utc) + dataset_items_by_id = {int(item["dataset_id"]): item for item in computed["datasets"]} + if snapshot is None: + snapshot = GtfsHarmonizedSnapshot( + snapshot_key=snapshot_key, + status="draft", + source="computed", + note=note, + summary_json=json.dumps(computed["summary"], sort_keys=True, separators=(",", ":"), default=str), + metadata_json=json.dumps( + { + "computed_at": now.isoformat(), + "revision": _snapshot_revision(session), + "dataset_ids": computed["dataset_ids"], + }, + sort_keys=True, + separators=(",", ":"), + default=str, + ), + created_at=now, + ) + session.add(snapshot) + session.flush() + for item in computed["datasets"]: + session.add( + GtfsHarmonizedSnapshotDataset( + snapshot_id=snapshot.id, + dataset_id=int(item["dataset_id"]), + source_id=int(item["source_id"]), + source_name=item.get("source_name"), + role=str(item["role"]), + reason=item.get("reason"), + shadowed_by_dataset_id=_optional_int(item.get("shadowed_by_dataset_id")), + shadowed_by_source_id=_optional_int(item.get("shadowed_by_source_id")), + authority_level=item.get("authority_level"), + review_authority_level=item.get("review_authority_level"), + route_count=int(item.get("route_count") or 0), + route_key_count=int(item.get("route_key_count") or 0), + route_key_overlap=int(item.get("route_key_overlap") or 0), + route_key_overlap_ratio=float(item.get("route_key_overlap_ratio") or 0), + metadata_json=json.dumps(item, sort_keys=True, separators=(",", ":"), default=str), + created_at=now, + ) + ) + _persist_harmonized_snapshot_routes(session, snapshot, dataset_items_by_id=dataset_items_by_id, now=now) + elif note is not None and note != snapshot.note: + snapshot.note = note + if snapshot.id is not None and not _snapshot_has_route_rows(session, int(snapshot.id)): + _persist_harmonized_snapshot_routes(session, snapshot, dataset_items_by_id=dataset_items_by_id, now=now) + if activate: + for older in session.scalars( + select(GtfsHarmonizedSnapshot).where(GtfsHarmonizedSnapshot.status == "active", GtfsHarmonizedSnapshot.id != snapshot.id) + ).all(): + older.status = "archived" + snapshot.status = "active" + if snapshot.activated_at is None: + snapshot.activated_at = now + session.flush() + return _persisted_snapshot_payload(session, snapshot) + + +def _compute_gtfs_harmonized_snapshot(session: Session) -> dict[str, Any]: + items = _active_gtfs_snapshot_items(session) + _apply_dataset_shadowing(items) + route_shadow = _route_shadow_result(session, items) + for item in items: + dataset_shadow = route_shadow["by_dataset"].get(int(item["dataset_id"]), {}) + shadow_count = int(dataset_shadow.get("route_shadow_count") or 0) + item["route_shadow_count"] = shadow_count + item["route_shadow_ratio"] = round(shadow_count / max(1, int(item.get("route_count") or 0)), 3) + item["route_shadowed_by_dataset_ids"] = dataset_shadow.get("route_shadowed_by_dataset_ids", []) + + public_items = [_snapshot_public_item(item) for item in items] + included = [item for item in public_items if item["role"] == "included"] + shadowed = [item for item in public_items if item["role"] == "shadowed"] + excluded = [item for item in public_items if item["role"] == "excluded"] + return { + "summary": { + "raw_active_datasets": len(items), + "included_datasets": len(included), + "shadowed_datasets": len(shadowed), + "excluded_datasets": len(excluded), + "included_sources": len({item["source_id"] for item in included}), + "shadowed_sources": len({item["source_id"] for item in shadowed}), + "route_shadowed_routes": route_shadow["route_shadowed_routes"], + "route_shadow_digest": _route_shadow_digest(route_shadow["shadowed_route_map"]), + }, + "dataset_ids": [item["dataset_id"] for item in included], + "datasets": public_items, + "included": included, + "shadowed": shadowed, + "excluded": excluded, + "persisted": False, + "status": "computed", + } def gtfs_harmonization_inventory(session: Session) -> dict[str, Any]: feeds = [_feed_inventory_item(session, source) for source in _gtfs_sources(session)] + snapshot = gtfs_harmonized_snapshot(session) + snapshot_diagnostics = gtfs_harmonized_snapshot_diagnostics(session) + snapshot_by_dataset_id = {int(item["dataset_id"]): item for item in snapshot["datasets"]} + for feed in feeds: + active_dataset = feed.get("active_dataset") + feed["snapshot"] = ( + snapshot_by_dataset_id.get(int(active_dataset["id"])) + if active_dataset is not None + else None + ) summary = { "sources": len(feeds), "active_sources": sum(1 for feed in feeds if feed["active_dataset"] is not None), "datasets": sum(len(feed["datasets"]) for feed in feeds), + "snapshot_included_datasets": snapshot["summary"]["included_datasets"], + "snapshot_shadowed_datasets": snapshot["summary"]["shadowed_datasets"], + "snapshot_excluded_datasets": snapshot["summary"]["excluded_datasets"], "ready": sum(1 for feed in feeds if feed["qa_status"] == "ready"), "needs_review": sum(1 for feed in feeds if feed["qa_status"] == "needs_review"), "blocked": sum(1 for feed in feeds if feed["qa_status"] == "blocked"), } return { "summary": summary, + "snapshot": snapshot, + "snapshot_diagnostics": snapshot_diagnostics, "feeds": feeds, } @@ -45,16 +420,778 @@ def gtfs_harmonization_feed_detail(session: Session, source_id: int) -> dict[str if source is None or source.kind != "gtfs": return None feed = _feed_inventory_item(session, source) + snapshot = gtfs_harmonized_snapshot(session) + if feed["active_dataset"] is not None: + snapshot_by_dataset_id = {int(item["dataset_id"]): item for item in snapshot["datasets"]} + feed["snapshot"] = snapshot_by_dataset_id.get(int(feed["active_dataset"]["id"])) + else: + feed["snapshot"] = None return { **feed, + "snapshot_summary": snapshot["summary"], "sections": _feed_sections(feed), } +def gtfs_qa_review_payload(notes: str | None) -> dict[str, Any]: + return _qa_review_payload(notes) + + +def gtfs_route_overlap_key(route: GtfsRoute) -> str | None: + return _route_overlap_key( + route_id=route.route_id, + route_key=route.route_key, + mode=route.mode, + short_name=route.short_name, + long_name=route.long_name, + operator_key=route.operator_key, + operator_name=route.operator_name, + ) + + def _gtfs_sources(session: Session) -> list[Source]: return session.scalars(select(Source).where(Source.kind == "gtfs").order_by(Source.country, Source.priority, Source.name, Source.id)).all() +def _active_persisted_gtfs_dataset_ids(session: Session) -> list[int]: + snapshot = session.scalar( + select(GtfsHarmonizedSnapshot.id) + .where(GtfsHarmonizedSnapshot.status == "active") + .order_by(GtfsHarmonizedSnapshot.activated_at.desc(), GtfsHarmonizedSnapshot.id.desc()) + .limit(1) + ) + if snapshot is None: + return [] + rows = session.execute( + select(GtfsHarmonizedSnapshotDataset.dataset_id) + .join(Dataset, Dataset.id == GtfsHarmonizedSnapshotDataset.dataset_id) + .where( + GtfsHarmonizedSnapshotDataset.snapshot_id == int(snapshot), + GtfsHarmonizedSnapshotDataset.role == "included", + Dataset.kind == "gtfs", + Dataset.is_active.is_(True), + Dataset.status == "imported", + ) + .order_by(GtfsHarmonizedSnapshotDataset.id) + ).all() + return [int(row[0]) for row in rows] + + +def _active_persisted_gtfs_route_shadow_map(session: Session) -> dict[int, int] | None: + snapshot_id = session.scalar( + select(GtfsHarmonizedSnapshot.id) + .where(GtfsHarmonizedSnapshot.status == "active") + .order_by(GtfsHarmonizedSnapshot.activated_at.desc(), GtfsHarmonizedSnapshot.id.desc()) + .limit(1) + ) + if snapshot_id is None: + return None + rows = session.execute( + select(GtfsHarmonizedSnapshotRoute.gtfs_route_id, GtfsHarmonizedSnapshotRoute.shadowed_by_gtfs_route_id) + .where( + GtfsHarmonizedSnapshotRoute.snapshot_id == int(snapshot_id), + GtfsHarmonizedSnapshotRoute.role == "shadowed", + GtfsHarmonizedSnapshotRoute.shadowed_by_gtfs_route_id.is_not(None), + ) + ).all() + if not rows: + has_route_rows = bool( + session.scalar( + select(GtfsHarmonizedSnapshotRoute.id) + .where(GtfsHarmonizedSnapshotRoute.snapshot_id == int(snapshot_id)) + .limit(1) + ) + ) + return {} if has_route_rows else None + return {int(route_id): int(shadowed_by_route_id) for route_id, shadowed_by_route_id in rows if shadowed_by_route_id is not None} + + +def _persisted_snapshot_payload(session: Session, snapshot: GtfsHarmonizedSnapshot) -> dict[str, Any]: + rows = session.scalars( + select(GtfsHarmonizedSnapshotDataset) + .where(GtfsHarmonizedSnapshotDataset.snapshot_id == snapshot.id) + .order_by(GtfsHarmonizedSnapshotDataset.id) + ).all() + active_dataset_ids = set(_active_dataset_ids_for_snapshot_rows(session, rows)) + datasets = [_persisted_snapshot_dataset_payload(row, dataset_active=int(row.dataset_id) in active_dataset_ids) for row in rows] + included = [item for item in datasets if item["role"] == "included" and item.get("dataset_active")] + shadowed = [item for item in datasets if item["role"] == "shadowed"] + excluded = [item for item in datasets if item["role"] == "excluded"] + try: + summary = json.loads(snapshot.summary_json) + except json.JSONDecodeError: + summary = {} + if not isinstance(summary, dict): + summary = {} + route_rows = _snapshot_route_row_counts(session, snapshot.id) + summary = { + **summary, + "included_datasets": len(included), + "shadowed_datasets": len(shadowed), + "excluded_datasets": len(excluded), + "inactive_included_datasets": sum(1 for item in datasets if item["role"] == "included" and not item.get("dataset_active")), + "snapshot_route_rows": route_rows["total"], + "snapshot_route_included_rows": route_rows["included"], + "snapshot_route_shadowed_rows": route_rows["shadowed"], + "snapshot_route_excluded_rows": route_rows["excluded"], + "route_shadowed_routes": route_rows["shadowed"] if route_rows["total"] else summary.get("route_shadowed_routes", 0), + } + return { + "id": snapshot.id, + "snapshot_key": snapshot.snapshot_key, + "status": snapshot.status, + "persisted": True, + "source": snapshot.source, + "note": snapshot.note, + "created_at": _iso(snapshot.created_at), + "activated_at": _iso(snapshot.activated_at), + "summary": summary, + "dataset_ids": [item["dataset_id"] for item in included], + "datasets": datasets, + "included": included, + "shadowed": shadowed, + "excluded": excluded, + } + + +def _snapshot_route_row_counts(session: Session, snapshot_id: int) -> dict[str, int]: + rows = session.execute( + select(GtfsHarmonizedSnapshotRoute.role, func.count()) + .where(GtfsHarmonizedSnapshotRoute.snapshot_id == int(snapshot_id)) + .group_by(GtfsHarmonizedSnapshotRoute.role) + ).all() + by_role = {str(role): int(count or 0) for role, count in rows} + return { + "total": sum(by_role.values()), + "included": by_role.get("included", 0), + "shadowed": by_role.get("shadowed", 0), + "excluded": by_role.get("excluded", 0), + } + + +def _snapshot_has_route_rows(session: Session, snapshot_id: int) -> bool: + return bool( + session.scalar( + select(GtfsHarmonizedSnapshotRoute.id) + .where(GtfsHarmonizedSnapshotRoute.snapshot_id == int(snapshot_id)) + .limit(1) + ) + ) + + +def _summary_int(summary: dict[str, Any], key: str) -> int: + try: + return int(summary.get(key) or 0) + except (TypeError, ValueError): + return 0 + + +def _snapshot_route_payload(row: GtfsHarmonizedSnapshotRoute) -> dict[str, Any]: + metadata = _json_object(row.metadata_json) + return { + "id": row.id, + "snapshot_id": row.snapshot_id, + "gtfs_route_id": row.gtfs_route_id, + "dataset_id": row.dataset_id, + "source_id": row.source_id, + "source_name": metadata.get("source_name"), + "route_id": row.route_id, + "route_ref": metadata.get("route_ref"), + "route_name": metadata.get("route_name"), + "mode": metadata.get("mode"), + "operator": metadata.get("operator"), + "overlap_key": row.overlap_key, + "role": row.role, + "reason": row.reason, + "shadowed_by_gtfs_route_id": row.shadowed_by_gtfs_route_id, + "shadowed_by_dataset_id": row.shadowed_by_dataset_id, + "shadowed_by_source_id": row.shadowed_by_source_id, + "metadata": metadata, + "created_at": _iso(row.created_at), + } + + +def _persist_harmonized_snapshot_routes( + session: Session, + snapshot: GtfsHarmonizedSnapshot, + *, + dataset_items_by_id: dict[int, dict[str, Any]], + now: datetime, +) -> None: + if not dataset_items_by_id: + return + route_shadow_map = _computed_gtfs_route_shadow_map(session) + shadow_winners = { + int(route.id): route + for route in session.scalars( + select(GtfsRoute).where(GtfsRoute.id.in_(sorted(set(route_shadow_map.values())))) + ).all() + } if route_shadow_map else {} + routes = session.scalars( + select(GtfsRoute) + .where(GtfsRoute.dataset_id.in_(sorted(dataset_items_by_id))) + .order_by(GtfsRoute.dataset_id, GtfsRoute.id) + ).all() + rows: list[GtfsHarmonizedSnapshotRoute] = [] + for route in routes: + dataset_item = dataset_items_by_id.get(int(route.dataset_id)) + if dataset_item is None: + continue + winner = shadow_winners.get(int(route_shadow_map.get(int(route.id), 0))) + if winner is not None: + role = "shadowed" + reason = "route_covered_by_higher_precedence_dataset" + shadowed_by_route_id = int(winner.id) + shadowed_by_dataset_id = int(winner.dataset_id) + shadowed_by_source_id = _optional_int(dataset_items_by_id.get(int(winner.dataset_id), {}).get("source_id")) + else: + role = str(dataset_item.get("role") or "included") + reason = dataset_item.get("reason") + shadowed_by_route_id = None + shadowed_by_dataset_id = _optional_int(dataset_item.get("shadowed_by_dataset_id")) + shadowed_by_source_id = _optional_int(dataset_item.get("shadowed_by_source_id")) + metadata = { + "route_ref": route.short_name, + "route_name": route.long_name, + "mode": route.mode, + "operator": route.operator_name, + "route_key": route.route_key, + "operator_key": route.operator_key, + "source_name": dataset_item.get("source_name"), + "authority_level": dataset_item.get("authority_level"), + "review_authority_level": dataset_item.get("review_authority_level"), + } + rows.append( + GtfsHarmonizedSnapshotRoute( + snapshot_id=snapshot.id, + gtfs_route_id=int(route.id), + dataset_id=int(route.dataset_id), + source_id=int(dataset_item["source_id"]), + route_id=str(route.route_id), + overlap_key=gtfs_route_overlap_key(route), + role=role, + reason=reason, + shadowed_by_gtfs_route_id=shadowed_by_route_id, + shadowed_by_dataset_id=shadowed_by_dataset_id, + shadowed_by_source_id=shadowed_by_source_id, + metadata_json=json.dumps(metadata, sort_keys=True, separators=(",", ":"), default=str), + created_at=now, + ) + ) + if len(rows) >= 5000: + session.add_all(rows) + session.flush() + rows.clear() + if rows: + session.add_all(rows) + session.flush() + + +def _active_dataset_ids_for_snapshot_rows(session: Session, rows: list[GtfsHarmonizedSnapshotDataset]) -> list[int]: + dataset_ids = [int(row.dataset_id) for row in rows] + if not dataset_ids: + return [] + return [ + int(row[0]) + for row in session.execute( + select(Dataset.id).where( + Dataset.id.in_(dataset_ids), + Dataset.kind == "gtfs", + Dataset.status == "imported", + Dataset.is_active.is_(True), + ) + ).all() + ] + + +def _persisted_snapshot_dataset_payload(row: GtfsHarmonizedSnapshotDataset, *, dataset_active: bool) -> dict[str, Any]: + return { + "dataset_id": row.dataset_id, + "source_id": row.source_id, + "source_name": row.source_name, + "country": _snapshot_dataset_metadata(row).get("country"), + "priority": _snapshot_dataset_metadata(row).get("priority"), + "authority_level": row.authority_level, + "review_authority_level": row.review_authority_level, + "review_status": _snapshot_dataset_metadata(row).get("review_status"), + "route_count": row.route_count, + "route_key_count": row.route_key_count, + "role": row.role, + "reason": row.reason, + "shadowed_by_dataset_id": row.shadowed_by_dataset_id, + "shadowed_by_source_id": row.shadowed_by_source_id, + "route_key_overlap": row.route_key_overlap, + "route_key_overlap_ratio": row.route_key_overlap_ratio, + "route_shadow_count": int(_snapshot_dataset_metadata(row).get("route_shadow_count") or 0), + "route_shadow_ratio": float(_snapshot_dataset_metadata(row).get("route_shadow_ratio") or 0), + "route_shadowed_by_dataset_ids": _snapshot_dataset_metadata(row).get("route_shadowed_by_dataset_ids") or [], + "dataset_active": dataset_active, + } + + +def _snapshot_dataset_metadata(row: GtfsHarmonizedSnapshotDataset) -> dict[str, Any]: + if not row.metadata_json: + return {} + try: + payload = json.loads(row.metadata_json) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def _materialized_snapshot_key(snapshot: dict[str, Any]) -> str: + payload = { + "datasets": [ + { + "dataset_id": item.get("dataset_id"), + "source_id": item.get("source_id"), + "role": item.get("role"), + "reason": item.get("reason"), + "shadowed_by_dataset_id": item.get("shadowed_by_dataset_id"), + "route_count": item.get("route_count"), + "route_key_count": item.get("route_key_count"), + "route_key_overlap": item.get("route_key_overlap"), + "route_shadow_count": item.get("route_shadow_count"), + } + for item in snapshot.get("datasets", []) + ], + "dataset_ids": snapshot.get("dataset_ids", []), + "route_shadow_digest": (snapshot.get("summary") or {}).get("route_shadow_digest"), + } + digest = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")).hexdigest() + return f"gtfs-snapshot-{digest[:32]}" + + +def _route_shadow_digest(route_shadow_map: dict[int, int]) -> str: + if not route_shadow_map: + return "" + pairs = sorted((int(route_id), int(shadowed_by_route_id)) for route_id, shadowed_by_route_id in route_shadow_map.items()) + encoded = json.dumps(pairs, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:32] + + +def _snapshot_revision(session: Session) -> tuple[object, ...]: + rows = session.execute( + select( + Dataset.id, + Dataset.sha256, + Dataset.status, + Source.id, + Source.name, + Source.country, + Source.priority, + Source.enabled, + Source.status, + Source.last_run_at, + Source.source_basis, + Source.notes, + func.count(GtfsRoute.id), + func.max(GtfsRoute.id), + ) + .join(Source, Source.id == Dataset.source_id) + .outerjoin(GtfsRoute, GtfsRoute.dataset_id == Dataset.id) + .where(Dataset.kind == "gtfs", Dataset.is_active.is_(True)) + .group_by( + Dataset.id, + Dataset.sha256, + Dataset.status, + Source.id, + Source.name, + Source.country, + Source.priority, + Source.enabled, + Source.status, + Source.last_run_at, + Source.source_basis, + Source.notes, + ) + .order_by(Dataset.id) + ).all() + return tuple( + ( + int(row[0]), + row[1], + row[2], + int(row[3]), + row[4], + row[5], + row[6], + bool(row[7]), + row[8], + _iso(row[9]), + row[10], + row[11], + int(row[12] or 0), + int(row[13] or 0), + ) + for row in rows + ) + + +def _raw_active_gtfs_dataset_ids(session: Session, source_ids: list[int] | None = None) -> list[int]: + stmt = select(Dataset.id).where(Dataset.kind == "gtfs", Dataset.is_active.is_(True)).order_by(Dataset.id) + if source_ids: + stmt = stmt.where(Dataset.source_id.in_(source_ids)) + return [int(row[0]) for row in session.execute(stmt).all()] + + +def _active_gtfs_snapshot_items(session: Session) -> list[dict[str, Any]]: + active_rows = session.execute( + select(Dataset, Source) + .join(Source, Source.id == Dataset.source_id) + .where(Dataset.kind == "gtfs", Dataset.is_active.is_(True)) + .order_by(Source.country, Source.priority, Source.name, Dataset.id) + ).all() + dataset_ids = [int(dataset.id) for dataset, _source in active_rows] + route_profiles = _route_profiles_by_dataset(session, dataset_ids) + bboxes = _route_bboxes_by_dataset(session, dataset_ids) + items: list[dict[str, Any]] = [] + for dataset, source in active_rows: + profile = route_profiles.get(int(dataset.id), {"route_count": 0, "route_keys": set()}) + route_keys = profile["route_keys"] + review = _qa_review_payload(source.notes) + authority_level = _effective_authority_level(source, review) + exclusion = _source_snapshot_exclusion(source, review) + items.append( + { + "dataset_id": int(dataset.id), + "source_id": int(source.id), + "source_name": source.name, + "country": source.country, + "priority": source.priority, + "authority_level": authority_level, + "review_authority_level": review["authority_level"], + "review_status": review["status"], + "route_count": int(profile["route_count"]), + "route_key_count": len(route_keys), + "route_keys": route_keys, + "bbox": bboxes.get(int(dataset.id)), + "score": _source_precedence_score(source, authority_level, len(route_keys)), + "role": "excluded" if exclusion else "included", + "reason": exclusion, + "shadowed_by_dataset_id": None, + "shadowed_by_source_id": None, + "route_key_overlap": 0, + "route_key_overlap_ratio": 0.0, + "route_shadow_count": 0, + "route_shadow_ratio": 0.0, + "route_shadowed_by_dataset_ids": [], + } + ) + return items + + +def _apply_dataset_shadowing(items: list[dict[str, Any]]) -> None: + eligible = [item for item in items if item["role"] == "included"] + for item in eligible: + shadow = _best_shadowing_dataset(item, eligible) + if shadow is None: + continue + other, overlap, ratio = shadow + item["role"] = "shadowed" + item["reason"] = "covered_by_higher_precedence_dataset" + item["shadowed_by_dataset_id"] = other["dataset_id"] + item["shadowed_by_source_id"] = other["source_id"] + item["route_key_overlap"] = overlap + item["route_key_overlap_ratio"] = round(ratio, 3) + + +def _route_profiles_by_dataset(session: Session, dataset_ids: list[int]) -> dict[int, dict[str, Any]]: + profiles: dict[int, dict[str, Any]] = { + int(dataset_id): {"route_count": 0, "route_keys": set()} + for dataset_id in dataset_ids + } + if not dataset_ids: + return profiles + rows = session.execute( + select( + GtfsRoute.dataset_id, + GtfsRoute.route_id, + GtfsRoute.route_key, + GtfsRoute.mode, + GtfsRoute.short_name, + GtfsRoute.long_name, + GtfsRoute.operator_key, + GtfsRoute.operator_name, + ).where(GtfsRoute.dataset_id.in_(dataset_ids)) + ).all() + for row in rows: + dataset_id = int(row.dataset_id) + profile = profiles.setdefault(dataset_id, {"route_count": 0, "route_keys": set()}) + profile["route_count"] += 1 + key = _route_overlap_key( + route_id=row.route_id, + route_key=row.route_key, + mode=row.mode, + short_name=row.short_name, + long_name=row.long_name, + operator_key=row.operator_key, + operator_name=row.operator_name, + ) + if key: + profile["route_keys"].add(key) + return profiles + + +def _route_bboxes_by_dataset(session: Session, dataset_ids: list[int]) -> dict[int, tuple[float, float, float, float]]: + if not dataset_ids: + return {} + rows = session.execute( + select( + GtfsRoute.dataset_id, + func.min(GtfsRoute.min_lon), + func.min(GtfsRoute.min_lat), + func.max(GtfsRoute.max_lon), + func.max(GtfsRoute.max_lat), + ) + .where(GtfsRoute.dataset_id.in_(dataset_ids)) + .group_by(GtfsRoute.dataset_id) + ).all() + bboxes: dict[int, tuple[float, float, float, float]] = {} + for dataset_id, min_lon, min_lat, max_lon, max_lat in rows: + if None in {min_lon, min_lat, max_lon, max_lat}: + continue + bboxes[int(dataset_id)] = (float(min_lon), float(min_lat), float(max_lon), float(max_lat)) + return bboxes + + +def _route_shadow_result(session: Session, items: list[dict[str, Any]]) -> dict[str, Any]: + candidate_items = {int(item["dataset_id"]): item for item in items if item["role"] == "included"} + if len(candidate_items) < 2: + return {"shadowed_route_ids": set(), "shadowed_route_map": {}, "route_shadowed_routes": 0, "by_dataset": {}} + routes_by_key: dict[str, list[GtfsRoute]] = {} + for route in session.scalars( + select(GtfsRoute) + .where(GtfsRoute.dataset_id.in_(list(candidate_items))) + .order_by(GtfsRoute.dataset_id, GtfsRoute.route_id, GtfsRoute.id) + ).all(): + key = gtfs_route_overlap_key(route) + if not key: + continue + routes_by_key.setdefault(key, []).append(route) + + shadowed_route_ids: set[int] = set() + shadowed_route_map: dict[int, int] = {} + by_dataset: dict[int, dict[str, Any]] = {} + for routes in routes_by_key.values(): + if len({int(route.dataset_id) for route in routes}) < 2: + continue + for route in routes: + loser_item = candidate_items.get(int(route.dataset_id)) + if loser_item is None: + continue + winner = _best_route_shadow(route, loser_item, routes, candidate_items) + if winner is None: + continue + shadowed_route_ids.add(int(route.id)) + shadowed_route_map[int(route.id)] = int(winner.id) + dataset_shadow = by_dataset.setdefault( + int(route.dataset_id), + {"route_shadow_count": 0, "route_shadowed_by_dataset_ids": set()}, + ) + dataset_shadow["route_shadow_count"] += 1 + dataset_shadow["route_shadowed_by_dataset_ids"].add(int(winner.dataset_id)) + public_by_dataset = { + dataset_id: { + "route_shadow_count": int(payload["route_shadow_count"]), + "route_shadowed_by_dataset_ids": sorted(payload["route_shadowed_by_dataset_ids"]), + } + for dataset_id, payload in by_dataset.items() + } + return { + "shadowed_route_ids": shadowed_route_ids, + "shadowed_route_map": shadowed_route_map, + "route_shadowed_routes": len(shadowed_route_ids), + "by_dataset": public_by_dataset, + } + + +def _best_route_shadow( + route: GtfsRoute, + loser_item: dict[str, Any], + candidates: list[GtfsRoute], + candidate_items: dict[int, dict[str, Any]], +) -> GtfsRoute | None: + route_bbox = _route_bbox(route) + if route_bbox is None: + return None + matches: list[tuple[tuple[int, int, int, int], GtfsRoute]] = [] + for candidate in candidates: + if int(candidate.id) == int(route.id) or int(candidate.dataset_id) == int(route.dataset_id): + continue + winner_item = candidate_items.get(int(candidate.dataset_id)) + if winner_item is None or not _can_shadow_route(winner_item, loser_item): + continue + candidate_bbox = _route_bbox(candidate) + if candidate_bbox is None: + continue + if not _snapshot_bboxes_compatible(candidate_bbox, route_bbox, margin=GTFS_ROUTE_SHADOW_BBOX_MARGIN_DEG): + continue + matches.append((winner_item["score"], candidate)) + if not matches: + return None + return max(matches, key=lambda item: item[0])[1] + + +def _can_shadow_route(other: dict[str, Any], item: dict[str, Any]) -> bool: + if other["country"] and item["country"] and str(other["country"]).upper() != str(item["country"]).upper(): + return False + return tuple(other["score"]) > tuple(item["score"]) + + +def _route_bbox(route: GtfsRoute) -> tuple[float, float, float, float] | None: + values = (route.min_lon, route.min_lat, route.max_lon, route.max_lat) + if any(value is None for value in values): + return None + return (float(route.min_lon), float(route.min_lat), float(route.max_lon), float(route.max_lat)) + + +def _route_overlap_key( + *, + route_id: str | None, + route_key: str | None, + mode: str | None, + short_name: str | None, + long_name: str | None, + operator_key: str | None, + operator_name: str | None, +) -> str | None: + _ = operator_key, operator_name + route_part = route_key or short_name or long_name or route_id + normalized_route = _snapshot_key_text(route_part) + if not normalized_route: + return None + mode_part = _snapshot_key_text(mode) + return "|".join(part for part in [mode_part, normalized_route] if part) + + +def _snapshot_key_text(value: str | None) -> str: + text = str(value or "").casefold().strip() + text = text.replace("ß", "ss") + text = re.sub(r"[^a-z0-9]+", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def _source_snapshot_exclusion(source: Source, review: dict[str, Any]) -> str | None: + if source.enabled is False: + return "source_disabled" + if review["status"] in {"blocked", "rejected"}: + return f"review_{review['status']}" + if any(review.get(key) == "no" for key in ("can_import", "can_derive", "can_redistribute")): + return "license_blocks_publication" + if review.get("commercial_restrictions") == "yes": + return "commercial_restrictions" + return None + + +def _effective_authority_level(source: Source, review: dict[str, Any]) -> str: + reviewed = str(review.get("authority_level") or "unknown") + if reviewed != "unknown": + return reviewed + text = " ".join(str(value or "") for value in [source.name, source.source_basis, source.notes]).casefold() + if "delfi" in text or "gtfs.de" in text or "national gtfs" in text or "national official" in text: + return "national_official" + if "regional authority" in text: + return "regional_authority" + if "operator feed" in text: + return "operator" + if "mobility database mirror" in text or "transitland" in text: + return "secondary_discovery" + return "unknown" + + +def _source_precedence_score(source: Source, authority_level: str, route_key_count: int) -> tuple[int, int, int, int]: + authority = GTFS_AUTHORITY_SCORES.get(str(authority_level or "unknown"), GTFS_AUTHORITY_SCORES["unknown"]) + priority = GTFS_PRIORITY_SCORES.get(str(source.priority or "").upper(), 0) + coverage = min(max(int(route_key_count), 0), 100_000) + source_id_tiebreak = -int(source.id or 0) + return (authority, priority, coverage, source_id_tiebreak) + + +def _best_shadowing_dataset(item: dict[str, Any], candidates: list[dict[str, Any]]) -> tuple[dict[str, Any], int, float] | None: + item_keys = item["route_keys"] + if not item_keys: + return None + matches: list[tuple[tuple[int, int, int, int], float, int, dict[str, Any]]] = [] + for other in candidates: + if other["dataset_id"] == item["dataset_id"]: + continue + if not _can_shadow_dataset(other, item): + continue + if not _snapshot_bboxes_compatible(other.get("bbox"), item.get("bbox")): + continue + overlap = len(item_keys & other["route_keys"]) + if overlap <= 0: + continue + ratio = overlap / max(1, len(item_keys)) + if not _route_overlap_is_shadowing(item, overlap, ratio): + continue + matches.append((other["score"], ratio, overlap, other)) + if not matches: + return None + _score, ratio, overlap, other = max(matches, key=lambda row: (row[0], row[1], row[2])) + return other, overlap, ratio + + +def _can_shadow_dataset(other: dict[str, Any], item: dict[str, Any]) -> bool: + if other["country"] and item["country"] and str(other["country"]).upper() != str(item["country"]).upper(): + return False + if tuple(other["score"]) > tuple(item["score"]): + return True + if tuple(other["score"]) == tuple(item["score"]): + return int(other["route_key_count"]) > int(item["route_key_count"]) + return False + + +def _snapshot_bboxes_compatible( + left: tuple[float, float, float, float] | None, + right: tuple[float, float, float, float] | None, + *, + margin: float = 0.0, +) -> bool: + if left is None or right is None: + return True + left_min_lon, left_min_lat, left_max_lon, left_max_lat = left + right_min_lon, right_min_lat, right_max_lon, right_max_lat = right + return not ( + left_max_lon + margin < right_min_lon + or left_min_lon - margin > right_max_lon + or left_max_lat + margin < right_min_lat + or left_min_lat - margin > right_max_lat + ) + + +def _route_overlap_is_shadowing(item: dict[str, Any], overlap: int, ratio: float) -> bool: + route_key_count = int(item["route_key_count"]) + if route_key_count < GTFS_SHADOW_ROUTE_MIN_OVERLAP: + return overlap >= GTFS_SHADOW_SMALL_ROUTE_MIN_OVERLAP and ratio >= GTFS_SHADOW_SMALL_ROUTE_RATIO + return overlap >= GTFS_SHADOW_ROUTE_MIN_OVERLAP and ratio >= GTFS_SHADOW_ROUTE_RATIO + + +def _snapshot_public_item(item: dict[str, Any]) -> dict[str, Any]: + return { + "dataset_id": item["dataset_id"], + "source_id": item["source_id"], + "source_name": item["source_name"], + "country": item["country"], + "priority": item["priority"], + "authority_level": item["authority_level"], + "review_authority_level": item["review_authority_level"], + "review_status": item["review_status"], + "route_count": item["route_count"], + "route_key_count": item["route_key_count"], + "role": item["role"], + "reason": item["reason"], + "shadowed_by_dataset_id": item["shadowed_by_dataset_id"], + "shadowed_by_source_id": item["shadowed_by_source_id"], + "route_key_overlap": item["route_key_overlap"], + "route_key_overlap_ratio": item["route_key_overlap_ratio"], + "route_shadow_count": item["route_shadow_count"], + "route_shadow_ratio": item["route_shadow_ratio"], + "route_shadowed_by_dataset_ids": item["route_shadowed_by_dataset_ids"], + } + + def _feed_inventory_item(session: Session, source: Source) -> dict[str, Any]: datasets = sorted([dataset for dataset in source.datasets if dataset.kind == "gtfs"], key=lambda item: (not item.is_active, item.created_at, item.id)) active_dataset = next((dataset for dataset in datasets if dataset.is_active), None) @@ -240,38 +1377,121 @@ def _overlap_summary(session: Session, dataset: Dataset | None) -> dict[str, Any def _license_payload(source: Source) -> dict[str, Any]: text = (source.license or "").strip() + review = _qa_review_payload(source.notes) + flags = {key: review.get(key, "unknown") for key in GTFS_LICENSE_FLAG_KEYS} + missing_required = any(flags[key] != "yes" for key in ("can_import", "can_derive", "can_redistribute")) + blocked = ( + any(flags[key] == "no" for key in ("can_import", "can_derive", "can_redistribute")) + or flags["commercial_restrictions"] == "yes" + ) unknown = not text or "unknown" in text.lower() + if blocked: + redistribution_status = "blocked" + tone = "bad" + elif not missing_required and not unknown: + redistribution_status = "allowed" + tone = "good" + elif unknown: + redistribution_status = "unknown" + tone = "warn" + else: + redistribution_status = "review_required" + tone = "warn" return { "label": text or "unknown", - "redistribution_status": "unknown" if unknown else "review_required", - "tone": "warn" if unknown else "info", + "redistribution_status": redistribution_status, + "tone": tone, + "flags": flags, + "authority_level": review.get("authority_level", "unknown"), } def _license_issues(source: Source) -> list[dict[str, str]]: - if _license_payload(source)["redistribution_status"] == "unknown": - return [_issue("license_unknown", "warn", "License/redistribution status is unknown", "Publication needs explicit import, derivation, redistribution, and attribution flags.")] - return [] + payload = _license_payload(source) + review = _qa_review_payload(source.notes) + flags = payload["flags"] + issues: list[dict[str, str]] = [] + if review["status"] in {"blocked", "rejected"}: + issues.append(_issue("manual_review_blocked", "bad", "Feed review blocks publication", f"Reviewer decision is {review['status']}.")) + elif review["status"] == "needs_review": + issues.append(_issue("manual_review_needed", "warn", "Feed needs manual review", review["note"] or "Review decision is marked as needs review.")) + if payload["redistribution_status"] == "blocked": + issues.append(_issue("license_blocked", "bad", "License blocks publication", "Import, derivation, redistribution, or commercial-use review explicitly blocks publication.")) + elif payload["redistribution_status"] == "unknown": + issues.append(_issue("license_unknown", "warn", "License/redistribution status is unknown", "Publication needs explicit import, derivation, redistribution, and attribution flags.")) + elif payload["redistribution_status"] == "review_required": + missing = [ + label + for key, label in [ + ("can_import", "import"), + ("can_derive", "derivation"), + ("can_redistribute", "redistribution"), + ] + if flags.get(key) != "yes" + ] + issues.append(_issue("license_flags_incomplete", "warn", "License decision is incomplete", f"Missing explicit approval for: {', '.join(missing)}.")) + if payload.get("authority_level") == "unknown": + issues.append(_issue("authority_unknown", "warn", "Source authority ranking is unknown", "Rank the source as national official, regional authority, operator, aggregator, mirror, or secondary discovery.")) + return issues def _qa_review_payload(notes: str | None) -> dict[str, Any]: + default = _default_qa_review_payload() if not notes: - return {"status": "unreviewed", "note": "", "updated_at": None} + return default for line in str(notes).splitlines(): if not line.startswith(GTFS_QA_NOTE_PREFIX): continue + raw = line[len(GTFS_QA_NOTE_PREFIX) :].strip() + if raw.startswith("{"): + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = {} + if isinstance(parsed, dict): + return _normalise_qa_review_payload({**default, **parsed}) payload: dict[str, str] = {} - for part in line[len(GTFS_QA_NOTE_PREFIX) :].strip().split(";"): + for part in raw.split(";"): if "=" not in part: continue key, value = part.split("=", 1) payload[key.strip()] = value.strip() - return { - "status": payload.get("status") or "unreviewed", - "note": payload.get("note") or "", + return _normalise_qa_review_payload({**default, **payload}) + return default + + +def _default_qa_review_payload() -> dict[str, Any]: + return { + "status": "unreviewed", + "note": "", + "updated_at": None, + "can_import": "unknown", + "can_derive": "unknown", + "can_redistribute": "unknown", + "requires_attribution": "unknown", + "commercial_restrictions": "unknown", + "authority_level": "unknown", + } + + +def _normalise_qa_review_payload(payload: dict[str, Any]) -> dict[str, Any]: + result = _default_qa_review_payload() + result.update( + { + "status": _choice(payload.get("status"), {"unreviewed", "approved", "needs_review", "blocked", "rejected"}, "unreviewed"), + "note": str(payload.get("note") or ""), "updated_at": payload.get("updated_at"), + "authority_level": _choice(payload.get("authority_level"), GTFS_AUTHORITY_LEVELS, "unknown"), } - return {"status": "unreviewed", "note": "", "updated_at": None} + ) + for key in GTFS_LICENSE_FLAG_KEYS: + result[key] = _choice(payload.get(key), {"unknown", "yes", "no"}, "unknown") + return result + + +def _choice(value: Any, allowed: set[str], default: str) -> str: + text = str(value or "").strip() + return text if text in allowed else default def _routes_without_trips(session: Session, dataset_id: int) -> int: @@ -363,11 +1583,26 @@ def _qa_status(issues: list[dict[str, str]], dataset: Dataset | None) -> str: def _feed_sections(feed: dict[str, Any]) -> list[dict[str, Any]]: + license_payload = feed["license"] + flags = license_payload.get("flags", {}) return [ {"id": "validation", "title": "GTFS Validation", "items": feed["validation"]["items"]}, {"id": "service", "title": "Service Horizon", "items": feed["service"]["items"]}, {"id": "overlap", "title": "Overlap and Deduplication", "items": feed["overlap"]["items"]}, - {"id": "license", "title": "License", "items": [_metric("Redistribution", feed["license"]["redistribution_status"], feed["license"]["tone"]), _metric("License", feed["license"]["label"], feed["license"]["tone"])]}, + { + "id": "license", + "title": "License and Authority", + "items": [ + _metric("Redistribution", license_payload["redistribution_status"], license_payload["tone"]), + _metric("License", license_payload["label"], license_payload["tone"]), + _metric("Can import", flags.get("can_import", "unknown"), "good" if flags.get("can_import") == "yes" else "warn"), + _metric("Can derive", flags.get("can_derive", "unknown"), "good" if flags.get("can_derive") == "yes" else "warn"), + _metric("Can redistribute", flags.get("can_redistribute", "unknown"), "good" if flags.get("can_redistribute") == "yes" else "warn"), + _metric("Attribution", flags.get("requires_attribution", "unknown"), "info"), + _metric("Commercial restrictions", flags.get("commercial_restrictions", "unknown"), "bad" if flags.get("commercial_restrictions") == "yes" else "info"), + _metric("Authority", license_payload.get("authority_level", "unknown"), "warn" if license_payload.get("authority_level") == "unknown" else "good"), + ], + }, ] @@ -390,5 +1625,22 @@ def _max_int(*values: int | None) -> int | None: return max(clean) if clean else None +def _optional_int(value: object) -> int | None: + try: + return None if value is None else int(value) + except (TypeError, ValueError): + return None + + +def _json_object(value: str | None) -> dict[str, Any]: + if not value: + return {} + try: + payload = json.loads(value) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + def _iso(value: datetime | None) -> str | None: return None if value is None else value.isoformat() diff --git a/app/jobs.py b/app/jobs.py index 09cb1bb..d53948d 100644 --- a/app/jobs.py +++ b/app/jobs.py @@ -23,6 +23,7 @@ from app.data_management import ( from app.db import SessionLocal, engine, init_db from app.db_lock import DatabaseWriteBusy, database_write_lock from app.gtfs_storage import missing_sidecar_paths as gtfs_missing_sidecar_paths +from app.harmonization import create_gtfs_harmonized_snapshot from app.models import Dataset, Job, JobEvent, Source, SourceUpdateCheck from app.osm_storage import missing_sidecar_paths as osm_missing_sidecar_paths from app.pipeline.gtfs import backfill_gtfs_shapes @@ -34,6 +35,7 @@ from app.pipeline.route_layer import rebuild_route_layer from app.pipeline.run import run_source from app.pipeline.sample_data import clear_project_data, load_sample_project from app.source_catalog import import_ingestable_sources, import_source_catalog, source_catalog_summary +from app.workbench import generate_map_gtfs_review_items ROUTE_MATCHING_JOB_KIND = "route_matching" @@ -44,6 +46,8 @@ SOURCE_IMPORT_JOB_KIND = "source_import" SOURCE_DELETE_JOB_KIND = "source_delete" DATASET_DELETE_JOB_KIND = "dataset_delete" MAINTENANCE_JOB_KIND = "maintenance" +MAP_GTFS_REVIEW_JOB_KIND = "map_gtfs_review" +GTFS_HARMONIZED_SNAPSHOT_JOB_KIND = "gtfs_harmonized_snapshot" TERMINAL_JOB_STATUSES = {"completed", "failed", "cancelled"} ACTIVE_JOB_STATUSES = {"queued", "running", "paused"} LEASE_SECONDS = max(300, int(settings.queue_job_lease_seconds)) @@ -124,6 +128,70 @@ def create_route_matching_job(session: Session, *, priority: int = 0) -> Job: return job +def create_map_gtfs_review_job( + session: Session, + *, + source_id: int | None = None, + dataset_id: int | None = None, + limit: int = 1000, + priority: int = 0, +) -> Job: + result = { + "source_id": source_id, + "dataset_id": dataset_id, + "limit": max(1, min(int(limit), 50_000)), + } + job = Job( + kind=MAP_GTFS_REVIEW_JOB_KIND, + status="queued", + description="Generate Map/GTFS workbench review queue", + progress_current=0, + progress_total=2, + priority=int(priority), + result_json=json.dumps(result, separators=(",", ":")), + ) + session.add(job) + session.flush() + add_job_event( + session, + job, + event_type="queued", + message="Map/GTFS review generation queued.", + progress_current=0, + progress_total=job.progress_total, + metadata=result, + ) + return job + + +def create_gtfs_harmonized_snapshot_job(session: Session, *, activate: bool = True, priority: int = 0) -> Job: + result = { + "activate": bool(activate), + "queued_pid": os.getpid(), + } + job = Job( + kind=GTFS_HARMONIZED_SNAPSHOT_JOB_KIND, + status="queued", + description="Build harmonized GTFS snapshot", + progress_current=0, + progress_total=2, + priority=int(priority), + result_json=json.dumps(result, separators=(",", ":")), + ) + session.add(job) + session.flush() + add_job_event( + session, + job, + event_type="queued", + message="Harmonized GTFS snapshot build queued.", + progress_current=0, + progress_total=job.progress_total, + metadata=result, + ) + return job + + def create_osm_relabel_job( session: Session, *, @@ -855,12 +923,14 @@ def job_queue_revision(session: Session, *, include_dismissed: bool = False) -> def job_events(session: Session, job_id: int, *, limit: int = 100) -> list[JobEvent]: - return session.scalars( + event_limit = max(1, min(limit, 500)) + events = session.scalars( select(JobEvent) .where(JobEvent.job_id == job_id) - .order_by(JobEvent.created_at, JobEvent.id) - .limit(max(1, min(limit, 500))) + .order_by(JobEvent.created_at.desc(), JobEvent.id.desc()) + .limit(event_limit) ).all() + return list(reversed(events)) def request_job_control(job_id: int, action: str) -> dict[str, Any]: @@ -990,6 +1060,10 @@ def run_claimed_job(job_id: int, worker_id: str) -> None: _run_dataset_delete_job(job_id, worker_id) elif job.kind == MAINTENANCE_JOB_KIND: _run_maintenance_job(job_id, worker_id) + elif job.kind == MAP_GTFS_REVIEW_JOB_KIND: + _run_map_gtfs_review_job(job_id, worker_id) + elif job.kind == GTFS_HARMONIZED_SNAPSHOT_JOB_KIND: + _run_gtfs_harmonized_snapshot_job(job_id, worker_id) else: raise ValueError(f"unsupported job kind: {job.kind}") except JobPaused: @@ -1292,6 +1366,46 @@ def _run_osm_relabel_job(job_id: int, worker_id: str) -> None: session.commit() +def _run_map_gtfs_review_job(job_id: int, worker_id: str) -> None: + init_db() + with SessionLocal() as session: + job = _job_for_worker(session, job_id, worker_id) + options = _json_object(job.result_json) + _job_running(session, job, worker_id, "started", "Generating Map/GTFS workbench review queue.", 1) + _check_job_control(session, job) + progress_callback = _job_progress_callback(session, job, worker_id, update_job_progress=False) + result = generate_map_gtfs_review_items( + session, + source_id=_optional_int(options.get("source_id")), + dataset_id=_optional_int(options.get("dataset_id")), + limit=int(options.get("limit") or 1000), + progress_callback=progress_callback, + ) + job = _job_for_worker(session, job_id, worker_id) + _complete_job(session, job, "Map/GTFS workbench review queue generated.", {**options, "review_result": result}) + session.commit() + + +def _run_gtfs_harmonized_snapshot_job(job_id: int, worker_id: str) -> None: + init_db() + with SessionLocal() as session: + job = _job_for_worker(session, job_id, worker_id) + options = _json_object(job.result_json) + options["worker_pid"] = os.getpid() + options["worker_id"] = worker_id + job.result_json = json.dumps(options, separators=(",", ":")) + _job_running(session, job, worker_id, "started", "Building harmonized GTFS snapshot.", 1) + _check_job_control(session, job) + snapshot = create_gtfs_harmonized_snapshot( + session, + activate=bool(options.get("activate", True)), + note=f"job #{job.id}", + ) + job = _job_for_worker(session, job_id, worker_id) + _complete_job(session, job, "Harmonized GTFS snapshot built.", {**options, "snapshot": snapshot}) + session.commit() + + def _run_source_import_job(job_id: int, worker_id: str) -> None: init_db() with SessionLocal() as session: @@ -1335,7 +1449,16 @@ def _run_source_import_job(job_id: int, worker_id: str) -> None: if options.get("run_match"): _job_running(session, job, worker_id, "matching", "Running route matcher after import.", 3) progress_callback = _job_progress_callback(session, job, worker_id) - match_result = run_route_matching(session, progress_callback=progress_callback) + imported_gtfs_dataset_id = ( + int(options["dataset_id"]) + if options.get("dataset_kind") == "gtfs" and options.get("dataset_id") + else None + ) + match_result = run_route_matching( + session, + progress_callback=progress_callback, + gtfs_dataset_ids=[imported_gtfs_dataset_id] if imported_gtfs_dataset_id is not None else None, + ) job = _job_for_worker(session, job_id, worker_id) options = _json_object(job.result_json) options["match_result"] = match_result diff --git a/app/journey.py b/app/journey.py index 446fbe7..fa161bc 100644 --- a/app/journey.py +++ b/app/journey.py @@ -11,7 +11,8 @@ from typing import Iterator, Optional from shapely.geometry import LineString, MultiLineString, Point, mapping, shape from shapely.ops import linemerge, substring -from sqlalchemy import and_, bindparam, case, exists, func, or_, select, text +from sqlalchemy import String, and_, bindparam, case, exists, func, or_, select, text +from sqlalchemy.dialects.postgresql import ARRAY from sqlalchemy.orm import Session, aliased from app.address_search import ( @@ -38,6 +39,7 @@ from app.gtfs_storage import ( stop_times_for_trip_range, uses_sidecar_stop_times, ) +from app.harmonization import active_harmonized_gtfs_dataset_ids, active_harmonized_gtfs_shadowed_route_ids from app.models import ( CanonicalStop, CanonicalStopLink, @@ -77,17 +79,26 @@ MAX_GROUP_STOP_IDS = 120 MAX_ROUTER_BOARDING_CANDIDATES = 2200 MAX_ROUTER_TRANSIT_LEGS = 6 MAX_JOURNEY_DATASET_PAIRS = 40 +LOW_WALK_MIN_SAVINGS_M = 250 +LOW_WALK_MIN_RELATIVE_SAVING = 0.35 +LOW_WALK_MAX_EXTRA_SECONDS = 45 * 60 +LOW_WALK_GOOD_ENOUGH_M = 350 WALKING_TRANSFER_RADIUS_M = 450 WALKING_TRANSFER_RADIUS_DEG = WALKING_TRANSFER_RADIUS_M / 111_320 WALKING_TRANSFER_SPEED_MPS = 1.25 MAX_WALKING_TRANSFER_SOURCE_STOPS = 80 MAX_WALKING_TRANSFER_NEIGHBORS_PER_STOP = 8 +STOP_ACCESS_RADIUS_M = 450 +STOP_ACCESS_CANDIDATES = 4 +STOP_ACCESS_MAX_PAIR_CANDIDATES = 8 +STOP_ACCESS_MAX_SECONDS = 15 * 60 ACCESS_TRANSFER_MAX_SECONDS = 45 * 60 MAX_ACCESS_TRANSFER_CANDIDATES = 4 PUBLIC_TRANSPORT_WALK_OPTION_MAX_SECONDS = 45 * 60 ADDRESS_ACCESS_RADIUS_M = 1800 ADDRESS_ACCESS_MAX_SECONDS = 30 * 60 ADDRESS_ACCESS_STOP_CANDIDATES = 4 +ADDRESS_ACCESS_DIRECT_PAIR_CANDIDATES = 4 ADDRESS_ACCESS_MAX_PAIR_CANDIDATES = 8 ADDRESS_ACCESS_MAX_DEEP_PAIR_CANDIDATES = 4 ADDRESS_ACCESS_SHORT_DIRECT_WALK_SECONDS = 20 * 60 @@ -324,11 +335,21 @@ def nearest_scheduled_stops( source_ids: list[int] | None = None, limit: int = 3, radius_m: float = 900, + grouped: bool = True, ) -> list[dict]: active_dataset_ids = _active_gtfs_dataset_ids(db, source_ids=source_ids) if not active_dataset_ids: return [] selected_limit = max(1, min(int(limit), 25)) + if not grouped: + return _nearest_exact_scheduled_stops( + db, + lat=lat, + lon=lon, + dataset_ids=active_dataset_ids, + limit=selected_limit, + radius_m=radius_m, + ) if settings.is_postgresql_database: rows = _nearest_canonical_stop_rows_postgresql( db, @@ -399,6 +420,261 @@ def nearest_scheduled_stops( return candidates[:selected_limit] +def _nearest_exact_scheduled_stops( + db: Session, + *, + lat: float, + lon: float, + dataset_ids: list[int], + limit: int, + radius_m: float, +) -> list[dict]: + selected_limit = max(1, min(int(limit), 25)) + if settings.is_postgresql_database: + rows = _nearest_gtfs_stop_rows_postgresql( + db, + lat=lat, + lon=lon, + dataset_ids=dataset_ids, + limit=selected_limit * 24, + radius_m=radius_m, + ) + visual_rows = _nearest_visual_stop_rows_postgresql( + db, + lat=lat, + lon=lon, + dataset_ids=dataset_ids, + limit=selected_limit * 8, + radius_m=radius_m, + ) + results: list[dict] = [] + seen: set[tuple[int, str]] = set() + for stop, source_id, source_name, distance_m in rows: + if not _has_scheduled_stop(db, stop): + continue + key = (stop.dataset_id, stop.stop_id) + if key in seen: + continue + seen.add(key) + results.append(_exact_stop_result_payload(db, stop, source_id, source_name, float(distance_m or 0))) + if len(results) >= selected_limit: + return results + for canonical_stop_id, preferred_dataset_id, distance_m in visual_rows: + stop = _nearest_linked_scheduled_stop_for_canonical( + db, + canonical_stop_id=int(canonical_stop_id), + dataset_ids=dataset_ids, + preferred_dataset_id=int(preferred_dataset_id), + lat=lat, + lon=lon, + ) + if stop is None: + continue + key = (stop.dataset_id, stop.stop_id) + if key in seen: + continue + seen.add(key) + source = _source_payload_for_dataset_id(db, stop.dataset_id) or {} + results.append(_exact_stop_result_payload(db, stop, source.get("id"), source.get("name"), float(distance_m or 0))) + if len(results) >= selected_limit: + break + results.sort(key=lambda item: (float(item.get("distance_m") or 0), item.get("display_name") or item.get("name") or "")) + return results[:selected_limit] + + radius_deg = float(radius_m) / 111_320 + rows = db.execute( + select(GtfsStop, Source.id, Source.name) + .join(Dataset, Dataset.id == GtfsStop.dataset_id) + .join(Source, Source.id == Dataset.source_id) + .where( + GtfsStop.dataset_id.in_(dataset_ids), + GtfsStop.lat.is_not(None), + GtfsStop.lon.is_not(None), + GtfsStop.lat >= float(lat) - radius_deg, + GtfsStop.lat <= float(lat) + radius_deg, + GtfsStop.lon >= float(lon) - radius_deg, + GtfsStop.lon <= float(lon) + radius_deg, + ) + .order_by(GtfsStop.name, GtfsStop.id) + .limit(selected_limit * 48) + ).all() + candidates: list[tuple[float, GtfsStop, int, str]] = [] + for stop, source_id, source_name in rows: + if stop.lat is None or stop.lon is None or not _has_scheduled_stop(db, stop): + continue + distance_m = _distance_m(float(lat), float(lon), float(stop.lat), float(stop.lon)) + if distance_m <= radius_m: + candidates.append((distance_m, stop, int(source_id), str(source_name))) + candidates.sort(key=lambda item: (item[0], item[1].name or "", item[1].stop_id)) + return [ + _exact_stop_result_payload(db, stop, source_id, source_name, distance_m) + for distance_m, stop, source_id, source_name in candidates[:selected_limit] + ] + + +def _nearest_gtfs_stop_rows_postgresql( + db: Session, + *, + lat: float, + lon: float, + dataset_ids: list[int], + limit: int, + radius_m: float, +) -> list[tuple[GtfsStop, int, str, float]]: + radius_deg = float(radius_m) / 111_320 + stmt = text( + """ + WITH point AS ( + SELECT ST_SetSRID(ST_MakePoint(:lon, :lat), 4326) AS geom + ) + SELECT + gtfs_stops.id AS stop_pk, + sources.id AS source_id, + sources.name AS source_name, + ST_DistanceSphere(gtfs_stops.geom, point.geom) AS distance_m + FROM gtfs_stops + JOIN datasets + ON datasets.id = gtfs_stops.dataset_id + AND datasets.kind = 'gtfs' + AND datasets.is_active IS TRUE + JOIN sources ON sources.id = datasets.source_id + CROSS JOIN point + WHERE gtfs_stops.dataset_id IN :dataset_ids + AND gtfs_stops.geom IS NOT NULL + AND gtfs_stops.geom && ST_Expand(point.geom, :radius_deg) + AND ST_DWithin(gtfs_stops.geom::geography, point.geom::geography, :radius_m) + ORDER BY gtfs_stops.geom <-> point.geom, gtfs_stops.id + LIMIT :limit + """ + ).bindparams(bindparam("dataset_ids", expanding=True)) + rows = db.execute( + stmt, + { + "lat": float(lat), + "lon": float(lon), + "dataset_ids": tuple(dataset_ids), + "radius_deg": radius_deg, + "radius_m": float(radius_m), + "limit": max(1, int(limit)), + }, + ).all() + stop_ids = [int(row.stop_pk) for row in rows] + stops = {stop.id: stop for stop in db.scalars(select(GtfsStop).where(GtfsStop.id.in_(stop_ids))).all()} if stop_ids else {} + return [ + (stops[int(row.stop_pk)], int(row.source_id), str(row.source_name), float(row.distance_m or 0)) + for row in rows + if int(row.stop_pk) in stops + ] + + +def _nearest_linked_scheduled_stop_for_canonical( + db: Session, + *, + canonical_stop_id: int, + dataset_ids: list[int], + preferred_dataset_id: int | None, + lat: float, + lon: float, +) -> GtfsStop | None: + if settings.is_postgresql_database: + stmt = text( + """ + WITH point AS ( + SELECT ST_SetSRID(ST_MakePoint(:lon, :lat), 4326) AS geom + ) + SELECT gtfs_stops.id AS stop_pk + FROM canonical_stop_links + JOIN gtfs_stops + ON gtfs_stops.dataset_id = canonical_stop_links.dataset_id + AND gtfs_stops.id = canonical_stop_links.object_id + CROSS JOIN point + WHERE canonical_stop_links.canonical_stop_id = :canonical_stop_id + AND canonical_stop_links.object_type = 'gtfs_stop' + AND canonical_stop_links.dataset_id IN :dataset_ids + ORDER BY + CASE WHEN canonical_stop_links.dataset_id = :preferred_dataset_id THEN 0 ELSE 1 END, + CASE WHEN gtfs_stops.geom IS NULL THEN 1 ELSE 0 END, + gtfs_stops.geom <-> point.geom, + canonical_stop_links.role, + canonical_stop_links.id + LIMIT 64 + """ + ).bindparams(bindparam("dataset_ids", expanding=True)) + rows = db.execute( + stmt, + { + "canonical_stop_id": int(canonical_stop_id), + "dataset_ids": tuple(dataset_ids), + "preferred_dataset_id": preferred_dataset_id, + "lat": float(lat), + "lon": float(lon), + }, + ).all() + stops = [db.get(GtfsStop, int(row.stop_pk)) for row in rows] + else: + rows = db.scalars( + select(GtfsStop) + .join( + CanonicalStopLink, + and_( + CanonicalStopLink.dataset_id == GtfsStop.dataset_id, + CanonicalStopLink.object_id == GtfsStop.id, + CanonicalStopLink.object_type == "gtfs_stop", + ), + ) + .where( + CanonicalStopLink.canonical_stop_id == canonical_stop_id, + CanonicalStopLink.dataset_id.in_(dataset_ids), + ) + .order_by( + case((CanonicalStopLink.dataset_id == preferred_dataset_id, 0), else_=1), + CanonicalStopLink.role, + CanonicalStopLink.id, + ) + .limit(64) + ).all() + stops = list(rows) + best: tuple[float, GtfsStop] | None = None + for stop in stops: + if stop is None or not _has_scheduled_stop(db, stop): + continue + distance_m = ( + _distance_m(float(lat), float(lon), float(stop.lat), float(stop.lon)) + if stop.lat is not None and stop.lon is not None + else float("inf") + ) + if best is None or distance_m < best[0]: + best = (distance_m, stop) + return None if best is None else best[1] + + +def _exact_stop_result_payload( + db: Session, + stop: GtfsStop, + source_id: int | None, + source_name: str | None, + distance_m: float, +) -> dict: + canonical = canonical_stop_for_gtfs_stop(db, stop) + payload = _stop_payload(_stop_summary(stop)) + payload.update( + { + "id": _stop_exact_token(stop.dataset_id, stop.stop_id), + "kind": "stop", + "canonical_stop_id": None if canonical is None else canonical.id, + "display_name": stop.name, + "source_id": source_id, + "source_name": source_name, + "scheduled": True, + "grouped": False, + "grouped_stop_count": 1, + "sample_stop_ids": [stop.stop_id], + "distance_m": float(distance_m or 0), + } + ) + return payload + + def _nearest_canonical_stop_rows_postgresql( db: Session, *, @@ -798,6 +1074,10 @@ def _stop_group_token(dataset_id: int, group_id: str) -> str: return f"{STOP_GROUP_PREFIX}{dataset_id}:{group_id}" +def _stop_exact_token(dataset_id: int, stop_id: str) -> str: + return f"{STOP_EXACT_PREFIX}{dataset_id}:{stop_id}" + + def _stop_place_token(canonical_stop_id: int, dataset_id: int) -> str: return f"{STOP_PLACE_PREFIX}{canonical_stop_id}:{dataset_id}" @@ -1004,13 +1284,17 @@ def _resolve_stop_selection(db: Session, value: int | str, source_ids: list[int] token = token[len(STOP_EXACT_PREFIX) :] exact_external_stop_id = True - stop = _active_stop_by_external_stop_id(db, token, active_dataset_ids) if token else None + stop = _stop_by_exact_token(db, token, active_dataset_ids) if exact_external_stop_id else None + if stop is None: + stop = _active_stop_by_external_stop_id(db, token, active_dataset_ids) if token else None if stop is None and not exact_external_stop_id and token.isdigit(): candidate = db.get(GtfsStop, int(token)) if candidate is not None and (not active_dataset_ids or candidate.dataset_id in active_dataset_ids): stop = candidate if stop is None: raise ValueError("from_stop_id and to_stop_id must reference existing GTFS stops") + if exact_external_stop_id: + return _selection_for_exact_stop(db, stop, active_dataset_ids) return _selection_for_stop(db, stop, active_dataset_ids) @@ -1105,6 +1389,29 @@ def _active_stop_by_external_stop_id(db: Session, stop_id: str, active_dataset_i return db.scalar(stmt) +def _stop_by_exact_token(db: Session, token: str, active_dataset_ids: list[int]) -> GtfsStop | None: + if ":" not in token: + return None + dataset_text, stop_id = token.split(":", 1) + if not dataset_text.isdigit() or not stop_id: + return None + dataset_id = int(dataset_text) + if active_dataset_ids and dataset_id not in active_dataset_ids: + return None + return db.scalar(select(GtfsStop).where(GtfsStop.dataset_id == dataset_id, GtfsStop.stop_id == stop_id)) + + +def _selection_for_exact_stop(db: Session, stop: GtfsStop, active_dataset_ids: list[int]) -> StopSelection: + if not _has_scheduled_stop(db, stop): + return _selection_for_group(db, stop.dataset_id, stop.parent_station or stop.stop_id) + canonical = canonical_stop_for_gtfs_stop(db, stop) + return StopSelection( + display=_stop_summary(stop), + stop_ids_by_dataset={stop.dataset_id: (stop.stop_id,)}, + canonical_stop_id=None if canonical is None else canonical.id, + ) + + def _selection_for_stop(db: Session, stop: GtfsStop, active_dataset_ids: list[int]) -> StopSelection: if _has_scheduled_stop(db, stop): canonical = canonical_stop_for_gtfs_stop(db, stop) @@ -1256,6 +1563,8 @@ def find_journeys( service_date: str | date | None = None, _allow_access_transfer: bool = True, _allow_address_access: bool = True, + _allow_stop_access: bool = True, + _allow_destination_walk: bool = True, ) -> dict: if via_stop_id is not None and str(via_stop_id).strip(): return _find_journeys_via( @@ -1308,12 +1617,13 @@ def find_journeys( if service_ids == set(): continue direct.extend( - _find_direct_journeys( + _find_direct_journeys_with_final_walks( db=db, dataset_id=dataset_id, service_ids=service_ids, from_stop_ids=from_selection.stop_ids_by_dataset[dataset_id], to_stop_ids=to_selection.stop_ids_by_dataset[dataset_id], + target_canonical_stop_id=to_selection.canonical_stop_id if _allow_destination_walk else None, earliest_departure=departure_seconds, limit=max_journeys, stop_cache=stop_cache, @@ -1322,7 +1632,7 @@ def find_journeys( ) direct = sorted(direct, key=_journey_sort_key)[:max_journeys] if max_transfers > 0: - direct_arrival = direct[0]["arrival_seconds"] if direct else None + direct_arrival = _best_walk_free_public_transport_arrival(direct) transfer_journeys: list[dict] = [] for first_dataset_id, second_dataset_id in _journey_dataset_pairs(from_selection, to_selection): first_service_ids = service_ids_by_dataset.get(first_dataset_id) @@ -1339,7 +1649,7 @@ def find_journeys( from_stop_ids=from_selection.stop_ids_by_dataset[first_dataset_id], to_stop_ids=to_selection.stop_ids_by_dataset[second_dataset_id], origin_canonical_stop_id=from_selection.canonical_stop_id, - target_canonical_stop_id=to_selection.canonical_stop_id, + target_canonical_stop_id=to_selection.canonical_stop_id if _allow_destination_walk else None, earliest_departure=departure_seconds, latest_arrival=direct_arrival, transfer_seconds=max(0, transfer_seconds), @@ -1353,14 +1663,7 @@ def find_journeys( key=_journey_sort_key, )[: max_journeys * 3] if max_transfers > 1: - best_known_arrival = min( - ( - int(journey["arrival_seconds"]) - for journey in [*direct, *transfer_journeys] - if journey.get("arrival_seconds") is not None - ), - default=None, - ) + best_known_arrival = _best_walk_free_public_transport_arrival([*direct, *transfer_journeys]) round_journeys: list[dict] = [] for dataset_id in common_dataset_ids: service_ids = service_ids_by_dataset.get(dataset_id) @@ -1395,11 +1698,32 @@ def find_journeys( departure_seconds=departure_seconds, ) walk_journeys = [] if walk_journey is None else [walk_journey] - journeys = _filter_reasonable_journeys([*walk_journeys, *transfer_journeys, *direct]) + stop_access_journeys = ( + _find_exact_stop_access_journeys( + db=db, + from_stop_id=from_stop_id, + to_stop_id=to_stop_id, + from_selection=from_selection, + to_selection=to_selection, + departure_seconds=departure_seconds, + max_transfers=max_transfers, + transfer_seconds=transfer_seconds, + limit=max_journeys, + source_ids=source_ids, + service_date=parsed_service_date, + ) + if _allow_stop_access + else [] + ) + journeys = deduplicate_equivalent_journeys( + _filter_dominated_journeys( + _filter_reasonable_journeys([*walk_journeys, *stop_access_journeys, *transfer_journeys, *direct]) + ) + ) unique: dict[tuple[str, ...], dict] = {} for journey in sorted(journeys, key=_journey_sort_key): - key = tuple(_journey_leg_signature(leg) for leg in journey["legs"]) + key = journey_equivalence_key(journey) unique.setdefault(key, journey) selected = _select_diverse_journeys(unique.values(), limit=max_journeys) @@ -1419,8 +1743,8 @@ def find_journeys( ) selected = list( { - tuple(_journey_leg_signature(leg) for leg in journey["legs"]): journey - for journey in sorted(access_journeys, key=_journey_sort_key) + journey_equivalence_key(journey): journey + for journey in deduplicate_equivalent_journeys(access_journeys) }.values() )[:max_journeys] selected_dataset_ids = sorted( @@ -1521,12 +1845,19 @@ def _find_journeys_with_address_access( destination_candidates = [] candidate_pairs = [] else: + max_candidate_pairs = ( + ADDRESS_ACCESS_DIRECT_PAIR_CANDIDATES + if max_transfers <= 0 + else ADDRESS_ACCESS_MAX_DEEP_PAIR_CANDIDATES + if max_transfers > 1 + else ADDRESS_ACCESS_MAX_PAIR_CANDIDATES + ) candidate_pairs = _address_access_candidate_pairs( origin_candidates, destination_candidates, origin_is_address=origin_is_address, destination_is_address=destination_is_address, - max_pairs=ADDRESS_ACCESS_MAX_DEEP_PAIR_CANDIDATES if max_transfers > 1 else ADDRESS_ACCESS_MAX_PAIR_CANDIDATES, + max_pairs=max_candidate_pairs, ) access_leg_cache: dict[str, dict | None] = {} transit_departure_cache: dict[str, int | None] = {} @@ -1570,10 +1901,13 @@ def _find_journeys_with_address_access( service_date=parsed_service_date, _allow_access_transfer=include_major_hubs, _allow_address_access=False, + _allow_destination_walk=not destination_is_address, ) except ValueError: continue for transit_journey in transit.get("journeys", [])[: max_journeys * 2]: + if (access_leg is not None or destination_is_address) and _journey_is_walk_only(transit_journey): + continue egress_leg = None if destination_is_address: arrival_seconds = transit_journey.get("arrival_seconds") @@ -1602,10 +1936,12 @@ def _find_journeys_with_address_access( if include_major_hubs and len(journeys) >= max_journeys: break - unique: dict[tuple[str, ...], dict] = {} - for journey in sorted(_filter_reasonable_journeys(journeys), key=_journey_sort_key): - key = tuple(_journey_leg_signature(leg) for leg in journey["legs"]) - unique.setdefault(key, journey) + unique = { + journey_equivalence_key(journey): journey + for journey in deduplicate_equivalent_journeys( + _filter_dominated_journeys(_filter_reasonable_journeys(journeys)) + ) + } selected = _select_diverse_journeys(unique.values(), limit=max_journeys) selected_dataset_ids = sorted( { @@ -1622,7 +1958,13 @@ def _find_journeys_with_address_access( "origin_candidates": len(origin_candidates), "destination_candidates": len(destination_candidates), "searched_pairs": len(candidate_pairs), - "max_pairs": ADDRESS_ACCESS_MAX_DEEP_PAIR_CANDIDATES if max_transfers > 1 else ADDRESS_ACCESS_MAX_PAIR_CANDIDATES, + "max_pairs": ( + ADDRESS_ACCESS_DIRECT_PAIR_CANDIDATES + if max_transfers <= 0 + else ADDRESS_ACCESS_MAX_DEEP_PAIR_CANDIDATES + if max_transfers > 1 + else ADDRESS_ACCESS_MAX_PAIR_CANDIDATES + ), "major_hubs": include_major_hubs, } } @@ -1642,6 +1984,206 @@ def _find_journeys_with_address_access( } +def _find_exact_stop_access_journeys( + db: Session, + *, + from_stop_id: int | str, + to_stop_id: int | str, + from_selection: StopSelection, + to_selection: StopSelection, + departure_seconds: int, + max_transfers: int, + transfer_seconds: int, + limit: int, + source_ids: list[int] | None, + service_date: date | None, +) -> list[dict]: + origin_base = _AccessStopCandidate(str(from_stop_id), from_selection, 0) + destination_base = _AccessStopCandidate(str(to_stop_id), to_selection, 0) + origin_candidates = [origin_base] + destination_candidates = [destination_base] + if _selection_allows_stop_access(from_selection): + origin_candidates.extend(_nearby_exact_stop_candidates(db, from_selection, source_ids=source_ids)) + if _selection_allows_stop_access(to_selection): + destination_candidates.extend(_nearby_exact_stop_candidates(db, to_selection, source_ids=source_ids)) + pairs = _stop_access_candidate_pairs( + origin_candidates, + destination_candidates, + original_origin_token=origin_base.token, + original_destination_token=destination_base.token, + max_pairs=STOP_ACCESS_MAX_PAIR_CANDIDATES, + ) + if not pairs: + return [] + + journeys: list[dict] = [] + access_leg_cache: dict[str, dict | None] = {} + egress_leg_cache: dict[tuple[str, int], dict | None] = {} + for origin, destination in pairs: + access_leg = None + transit_departure_seconds = departure_seconds + if origin.token != origin_base.token: + access_leg = access_leg_cache.get(origin.token) + if origin.token not in access_leg_cache: + access_leg = _walk_leg_between_summaries( + db, + from_stop=from_selection.display, + to_stop=origin.selection.display, + departure_seconds=departure_seconds, + dataset_id=origin.selection.dataset_id, + max_duration_seconds=STOP_ACCESS_MAX_SECONDS, + route_geometry=True, + ) + access_leg_cache[origin.token] = access_leg + if access_leg is None: + continue + transit_departure_seconds = int(access_leg["arrival_seconds"]) + transit_departure = format_gtfs_time(transit_departure_seconds) + if transit_departure is None: + continue + try: + transit = find_journeys( + db=db, + from_stop_id=origin.token, + to_stop_id=destination.token, + departure=transit_departure, + max_transfers=max_transfers, + transfer_seconds=transfer_seconds, + limit=max(limit, 4), + source_ids=source_ids, + service_date=service_date, + _allow_stop_access=False, + _allow_destination_walk=destination.token == destination_base.token, + ) + except ValueError: + continue + for transit_journey in transit.get("journeys", [])[: max(limit * 2, limit)]: + if (access_leg is not None or destination.token != destination_base.token) and _journey_is_walk_only(transit_journey): + continue + egress_leg = None + if destination.token != destination_base.token: + arrival_seconds = transit_journey.get("arrival_seconds") + if arrival_seconds is None: + continue + egress_key = (destination.token, int(arrival_seconds)) + egress_leg = egress_leg_cache.get(egress_key) + if egress_key not in egress_leg_cache: + egress_leg = _walk_leg_between_summaries( + db, + from_stop=destination.selection.display, + to_stop=to_selection.display, + departure_seconds=int(arrival_seconds), + dataset_id=destination.selection.dataset_id, + max_duration_seconds=STOP_ACCESS_MAX_SECONDS, + route_geometry=True, + ) + egress_leg_cache[egress_key] = egress_leg + if egress_leg is None: + continue + combined = _compose_address_access_journey(transit_journey, access_leg=access_leg, egress_leg=egress_leg) + if combined is not None: + combined["stop_access_composed"] = True + journeys.append(combined) + if len(journeys) >= limit * 3: + break + if len(journeys) >= limit * 3: + break + return sorted(journeys, key=_journey_sort_key)[:limit] + + +def _selection_allows_stop_access(selection: StopSelection) -> bool: + return ( + _concrete_stop_key(selection) is not None + and selection.display.lat is not None + and selection.display.lon is not None + ) + + +def _nearby_exact_stop_candidates( + db: Session, + selection: StopSelection, + *, + source_ids: list[int] | None, +) -> list[_AccessStopCandidate]: + if selection.display.lat is None or selection.display.lon is None: + return [] + own_key = _concrete_stop_key(selection) + rows = nearest_scheduled_stops( + db, + lat=float(selection.display.lat), + lon=float(selection.display.lon), + source_ids=source_ids, + limit=STOP_ACCESS_CANDIDATES + 2, + radius_m=STOP_ACCESS_RADIUS_M, + grouped=False, + ) + candidates: list[_AccessStopCandidate] = [] + seen: set[tuple[int, str]] = set() + for row in rows: + token = str(row.get("id") or "") + if not token: + continue + try: + candidate_selection = _resolve_stop_selection(db, token, source_ids=source_ids) + except ValueError: + continue + key = _concrete_stop_key(candidate_selection) + if key is None or key == own_key or key in seen: + continue + seen.add(key) + candidates.append( + _AccessStopCandidate( + token=token, + selection=candidate_selection, + distance_m=float(row.get("distance_m") or 0), + ) + ) + if len(candidates) >= STOP_ACCESS_CANDIDATES: + break + return candidates + + +def _stop_access_candidate_pairs( + origins: list[_AccessStopCandidate], + destinations: list[_AccessStopCandidate], + *, + original_origin_token: str, + original_destination_token: str, + max_pairs: int, +) -> list[tuple[_AccessStopCandidate, _AccessStopCandidate]]: + pairs: list[tuple[float, _AccessStopCandidate, _AccessStopCandidate]] = [] + seen: set[tuple[str, str]] = set() + for origin in origins: + for destination in destinations: + if origin.token == original_origin_token and destination.token == original_destination_token: + continue + if origin.token == destination.token: + continue + key = (origin.token, destination.token) + if key in seen: + continue + seen.add(key) + distance = ( + (origin.distance_m if origin.token != original_origin_token else 0) + + (destination.distance_m if destination.token != original_destination_token else 0) + ) + pairs.append((distance, origin, destination)) + pairs.sort(key=lambda item: (item[0], item[1].distance_m, item[2].distance_m, item[1].token, item[2].token)) + return [(origin, destination) for _, origin, destination in pairs[: max(1, max_pairs)]] + + +def _concrete_stop_key(selection: StopSelection) -> tuple[int, str] | None: + if len(selection.stop_ids_by_dataset) != 1: + return None + dataset_id, stop_ids = next(iter(selection.stop_ids_by_dataset.items())) + if len(stop_ids) != 1: + return None + stop_id = str(stop_ids[0]) + if selection.display.stop_id != stop_id: + return None + return int(dataset_id), stop_id + + def _address_access_candidate_pairs( origins: list[_AccessStopCandidate], destinations: list[_AccessStopCandidate], @@ -2123,12 +2665,51 @@ def _walk_leg_between_summaries( return leg +def _retime_access_walk_to_onward_departure(access_leg: dict | None, onward_journey: dict) -> dict | None: + if access_leg is None: + return None + onward_departure = _journey_departure_seconds(onward_journey) + if onward_departure is None: + return access_leg + try: + duration_seconds = max(0, int(math.ceil(float(access_leg.get("duration_seconds") or 0)))) + except (TypeError, ValueError): + return access_leg + departure_seconds = max(0, int(onward_departure) - duration_seconds) + retimed = dict(access_leg) + retimed["departure_seconds"] = departure_seconds + retimed["arrival_seconds"] = int(onward_departure) + retimed["departure_time"] = format_gtfs_time(departure_seconds) + retimed["arrival_time"] = format_gtfs_time(int(onward_departure)) + retimed["departure_time_label"] = format_gtfs_time_label(departure_seconds) + retimed["arrival_time_label"] = format_gtfs_time_label(int(onward_departure)) + return retimed + + +def _journey_departure_seconds(journey: dict) -> int | None: + value = journey.get("departure_seconds") + if value is not None: + try: + return int(value) + except (TypeError, ValueError): + pass + parsed = parse_gtfs_time(str(journey.get("departure_time") or "")) + if parsed is not None: + return parsed + for leg in journey.get("legs") or []: + parsed = parse_gtfs_time(str(leg.get("departure_time") or "")) + if parsed is not None: + return parsed + return None + + def _compose_address_access_journey( journey: dict, *, access_leg: dict | None, egress_leg: dict | None, ) -> dict | None: + access_leg = _retime_access_walk_to_onward_departure(access_leg, journey) public_legs: list[dict] = [] features: list[dict] = [] leg_offset = 0 @@ -2186,32 +2767,148 @@ def _offset_feature_legs(features: list[dict], offset: int) -> list[dict]: return copied +def deduplicate_equivalent_journeys(journeys) -> list[dict]: + unique: dict[tuple[object, ...], dict] = {} + for journey in sorted((dict(journey) for journey in journeys), key=_journey_duplicate_sort_key): + unique.setdefault(journey_equivalence_key(journey), journey) + return list(unique.values()) + + +def journey_equivalence_key(journey: dict) -> tuple[object, ...]: + legs = journey.get("legs") or [] + return tuple(_journey_leg_equivalence_key(leg) for leg in legs) + + +def journey_duplicate_preference_key(journey: dict) -> tuple[float, int, str]: + dataset_ids = [ + int(leg["dataset_id"]) + for leg in journey.get("legs") or [] + if leg.get("mode") != "walk" and _is_int_like(leg.get("dataset_id")) + ] + min_dataset_id = min(dataset_ids, default=10**9) + source_signature = "|".join( + sorted( + { + str(leg.get("source_name") or "") + for leg in journey.get("legs") or [] + if leg.get("mode") != "walk" and leg.get("source_name") + } + ) + ) + return (-_journey_detail_score(journey), min_dataset_id, source_signature) + + +def _journey_duplicate_sort_key(journey: dict) -> tuple[tuple[float, float, float, int, int], tuple[float, int, str]]: + return (_journey_sort_key(journey), journey_duplicate_preference_key(journey)) + + +def _journey_detail_score(journey: dict) -> float: + score = 0.0 + for leg in journey.get("legs") or []: + if leg.get("mode") == "walk": + continue + score += float(leg.get("stop_count") or 0) + score += float(leg.get("intermediate_stop_count") or 0) + stops = leg.get("stops") + if isinstance(stops, list): + score += len(stops) + if leg.get("geometry_source") and leg.get("geometry_source") != "straight_line": + score += 1 + return score + + +def _journey_leg_equivalence_key(leg: dict) -> tuple[object, ...]: + mode = _equivalence_text(leg.get("mode")) + route = "walk" if mode == "walk" else _equivalence_text(leg.get("route_ref") or leg.get("route_name") or leg.get("route_id")) + return ( + mode, + route, + _leg_endpoint_equivalence(leg, "from"), + _leg_endpoint_equivalence(leg, "to"), + leg.get("departure_time") or leg.get("departure_time_label"), + leg.get("arrival_time") or leg.get("arrival_time_label"), + ) + + +def _leg_endpoint_equivalence(leg: dict, side: str) -> str: + stops = leg.get("stops") + stop = None + if isinstance(stops, list) and stops: + stop = stops[0] if side == "from" else stops[-1] + if isinstance(stop, dict): + canonical = stop.get("canonical_stop") + if isinstance(canonical, dict) and canonical.get("id") is not None: + return f"canonical:{canonical['id']}" + name = _equivalence_text(stop.get("name")) + if name: + return name + payload = leg.get(side) + if isinstance(payload, dict): + name = _equivalence_text(payload.get("name")) + if name: + return name + stop_id = _equivalence_text(payload.get("stop_id")) + if stop_id: + return stop_id + return "" + + +def _equivalence_text(value: object) -> str: + text = str(value or "").casefold() + return re.sub(r"[^a-z0-9]+", " ", text).strip() + + +def _is_int_like(value: object) -> bool: + try: + int(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return False + return True + + def _select_diverse_journeys(journeys, *, limit: int) -> list[dict]: ranked = sorted((dict(journey) for journey in journeys), key=_journey_sort_key) selected: list[dict] = [] - seen_exact: set[str] = set() + seen_exact: set[tuple[object, ...]] = set() seen_diversity: set[tuple[object, ...]] = set() - for journey in ranked: - exact = "||".join(_journey_leg_signature(leg) for leg in journey.get("legs") or []) + + def append(journey: dict | None, *, force: bool = False) -> bool: + if journey is None: + return False + exact = journey_equivalence_key(journey) if exact in seen_exact: - continue + return False seen_exact.add(exact) diversity_key = _journey_diversity_key(journey) - if diversity_key in seen_diversity and len(selected) >= max(3, limit // 2): - continue + if not force and diversity_key in seen_diversity and len(selected) >= max(3, limit // 2): + return False seen_diversity.add(diversity_key) selected.append(journey) + return True + + if ranked: + append(ranked[0], force=True) + append(_best_public_transport_option(ranked), force=True) + append(_least_walking_public_transport_option(ranked), force=True) + append(_fewest_transfer_public_transport_option(ranked), force=True) + append(_walk_only_option(ranked), force=True) + + for journey in ranked: + append(journey) if len(selected) >= limit: break + selected = selected[:limit] if len(selected) < min(limit, 3): + selected_exact = {journey_equivalence_key(existing) for existing in selected} for journey in ranked: - exact = "||".join(_journey_leg_signature(leg) for leg in journey.get("legs") or []) - if exact in {"||".join(_journey_leg_signature(leg) for leg in existing.get("legs") or []) for existing in selected}: + exact = journey_equivalence_key(journey) + if exact in selected_exact: continue selected.append(journey) + selected_exact.add(exact) if len(selected) >= min(limit, 3): break - return _ensure_walk_only_option(selected, ranked, limit=limit) + return _ensure_walk_only_option(selected[:limit], ranked, limit=limit) def _ensure_walk_only_option(selected: list[dict], ranked: list[dict], *, limit: int) -> list[dict]: @@ -2232,6 +2929,72 @@ def _journey_is_walk_only(journey: dict) -> bool: return bool(legs) and all(leg.get("mode") == "walk" for leg in legs) +def _walk_only_option(journeys: list[dict]) -> dict | None: + return next((journey for journey in journeys if _journey_is_walk_only(journey)), None) + + +def _journey_has_public_transport(journey: dict) -> bool: + return any(leg.get("mode") != "walk" for leg in journey.get("legs") or []) + + +def _best_public_transport_option(journeys: list[dict]) -> dict | None: + return next((journey for journey in journeys if _journey_has_public_transport(journey)), None) + + +def _least_walking_public_transport_option(journeys: list[dict]) -> dict | None: + public = [journey for journey in journeys if _journey_has_public_transport(journey)] + if not public: + return None + fastest = public[0] + lowest = min(public, key=lambda journey: (_journey_walking_m(journey), _journey_arrival_seconds(journey), _journey_duration_seconds(journey))) + if lowest is fastest: + return lowest + fastest_walking = _journey_walking_m(fastest) + lowest_walking = _journey_walking_m(lowest) + walking_saved = fastest_walking - lowest_walking + if lowest_walking <= LOW_WALK_GOOD_ENOUGH_M: + return lowest + if walking_saved < LOW_WALK_MIN_SAVINGS_M: + return None + if fastest_walking > 0 and walking_saved / fastest_walking < LOW_WALK_MIN_RELATIVE_SAVING: + return None + extra_seconds = _journey_arrival_seconds(lowest) - _journey_arrival_seconds(fastest) + if extra_seconds > LOW_WALK_MAX_EXTRA_SECONDS: + return None + return lowest + + +def _fewest_transfer_public_transport_option(journeys: list[dict]) -> dict | None: + public = [journey for journey in journeys if _journey_has_public_transport(journey)] + if not public: + return None + return min( + public, + key=lambda journey: ( + int(journey.get("transfers") or 0), + _journey_walking_m(journey), + _journey_arrival_seconds(journey), + ), + ) + + +def _journey_walking_m(journey: dict) -> float: + return sum(float(leg.get("distance_m") or 0) for leg in journey.get("legs") or [] if leg.get("mode") == "walk") + + +def _journey_arrival_seconds(journey: dict) -> float: + value = journey.get("arrival_seconds") + return float("inf") if value is None else float(value) + + +def _journey_duration_seconds(journey: dict) -> float: + value = journey.get("duration_seconds") + if value is not None: + return float(value) + minutes = journey.get("duration_minutes") + return float("inf") if minutes is None else float(minutes) * 60 + + def _journey_diversity_key(journey: dict) -> tuple[object, ...]: route_signature = tuple( str(leg.get("route_ref") or leg.get("route_id") or leg.get("mode") or "") @@ -2289,9 +3052,9 @@ def _find_journeys_via( for second in second_result.get("journeys", [])[:max_journeys]: combined.append(_combine_via_journey(first, second)) - unique: dict[tuple[str, ...], dict] = {} - for journey in sorted(combined, key=_journey_sort_key): - key = tuple(_journey_leg_signature(leg) for leg in journey["legs"]) + unique: dict[tuple[object, ...], dict] = {} + for journey in deduplicate_equivalent_journeys(combined): + key = journey_equivalence_key(journey) unique.setdefault(key, journey) selected = list(unique.values())[:max_journeys] dataset_ids = sorted( @@ -2415,10 +3178,98 @@ def _journey_sort_key(journey: dict) -> tuple[float, float, float, int, int]: ) +def _best_walk_free_public_transport_arrival(journeys: list[dict]) -> int | None: + arrivals = [ + int(journey["arrival_seconds"]) + for journey in journeys + if journey.get("arrival_seconds") is not None + and any(leg.get("mode") != "walk" for leg in journey.get("legs") or []) + and not any(leg.get("mode") == "walk" for leg in journey.get("legs") or []) + ] + return min(arrivals, default=None) + + def _filter_reasonable_journeys(journeys: list[dict]) -> list[dict]: return [journey for journey in journeys if _journey_is_reasonable(journey)] +def _filter_dominated_journeys(journeys: list[dict]) -> list[dict]: + kept: list[dict] = [] + for candidate in sorted(journeys, key=_journey_sort_key): + if any(_journey_dominates(existing, candidate) for existing in kept): + continue + kept.append(candidate) + return kept + + +def _journey_dominates(left: dict, right: dict) -> bool: + left_endpoint = _journey_last_transit_endpoint_key(left) + if left_endpoint is None or left_endpoint != _journey_last_transit_endpoint_key(right): + return False + + left_arrival = _journey_arrival_seconds(left) + right_arrival = _journey_arrival_seconds(right) + left_transfers = int(left.get("transfers") or 0) + right_transfers = int(right.get("transfers") or 0) + left_transit_legs = _journey_transit_leg_count(left) + right_transit_legs = _journey_transit_leg_count(right) + left_walking = _journey_walking_m(left) + right_walking = _journey_walking_m(right) + + if left_arrival > right_arrival: + return False + if left_transfers > right_transfers: + return False + if left_transit_legs > right_transit_legs: + return False + if left_walking > right_walking + 50: + return False + + return ( + left_arrival < right_arrival + or left_transfers < right_transfers + or left_transit_legs < right_transit_legs + or left_walking + 50 < right_walking + ) + + +def _journey_last_transit_endpoint_key(journey: dict) -> tuple[object, ...] | None: + for leg in reversed(journey.get("legs") or []): + if leg.get("mode") == "walk": + continue + to_stop = leg.get("to") or {} + canonical_key = _stop_payload_canonical_key(to_stop) + if canonical_key is not None: + return canonical_key + stops = leg.get("stops") or [] + if stops: + canonical_key = _stop_payload_canonical_key(stops[-1]) + if canonical_key is not None: + return canonical_key + dataset_id = leg.get("dataset_id") + stop_id = to_stop.get("stop_id") + if dataset_id is not None and stop_id: + try: + return ("stop", int(dataset_id), str(stop_id)) + except (TypeError, ValueError): + return ("stop", str(dataset_id), str(stop_id)) + return None + + +def _stop_payload_canonical_key(stop: dict) -> tuple[str, int | str] | None: + canonical_id = (stop.get("canonical_stop") or {}).get("id") or stop.get("canonical_stop_id") + if canonical_id is None: + return None + try: + return ("canonical", int(canonical_id)) + except (TypeError, ValueError): + return ("canonical", str(canonical_id)) + + +def _journey_transit_leg_count(journey: dict) -> int: + return sum(1 for leg in journey.get("legs") or [] if leg.get("mode") != "walk") + + def _journey_is_reasonable(journey: dict) -> bool: path: list[int] = [] for leg in journey.get("legs") or []: @@ -2510,6 +3361,18 @@ def _active_service_ids(db: Session, dataset_id: int, service_date: date) -> set def _where_trip_service_active(stmt, trip_model, service_ids: set[str] | None): if service_ids is None: return stmt + if settings.is_postgresql_database: + return stmt.where( + trip_model.service_id + == func.any( + bindparam( + "active_service_ids", + sorted(str(service_id) for service_id in service_ids), + type_=ARRAY(String), + unique=True, + ) + ) + ) return stmt.where(trip_model.service_id.in_(service_ids)) @@ -2559,6 +3422,8 @@ def _trip_route_lookup( for trip, route in db.execute(stmt).all(): if service_filter is not None and str(trip.service_id) not in service_filter: continue + if _route_is_shadowed(db, route): + continue lookup.setdefault(trip.trip_id, (trip, route)) return lookup @@ -2883,6 +3748,8 @@ def _find_round_journeys( return [] origin_id = from_selection.canonical_stop_id target_id = to_selection.canonical_stop_id + origin_stop_ids = from_selection.stop_ids_by_dataset.get(dataset_id) + target_stop_ids = set(to_selection.stop_ids_by_dataset.get(dataset_id, ())) best: dict[int, _RouterLabel] = {origin_id: _RouterLabel(origin_id, earliest_departure)} marked = {origin_id} solutions: list[_RouterLabel] = [] @@ -2926,6 +3793,9 @@ def _find_round_journeys( service_ids=service_ids, board_ready=board_ready, latest_arrival=latest_arrival, + stop_ids_override_by_canonical={origin_id: origin_stop_ids} + if round_index == 0 and origin_stop_ids + else None, ) if not boardings: break @@ -2948,6 +3818,12 @@ def _find_round_journeys( continue if latest_arrival is not None and arrival >= latest_arrival: continue + if ( + canonical_stop_id == target_id + and target_stop_ids + and call.stop_id not in target_stop_ids + ): + continue current = best.get(canonical_stop_id) if current is not None and current.arrival_seconds <= arrival: continue @@ -3170,10 +4046,15 @@ def _router_boardings_for_marked_stops( service_ids: set[str] | None, board_ready: dict[int, int], latest_arrival: int | None = None, + stop_ids_override_by_canonical: dict[int, tuple[str, ...]] | None = None, ) -> list[_RouterBoarding]: if not board_ready: return [] stop_ids_by_canonical = _gtfs_stop_ids_for_canonical_ids(db, dataset_id, set(board_ready)) + if stop_ids_override_by_canonical: + for canonical_stop_id, stop_ids in stop_ids_override_by_canonical.items(): + if canonical_stop_id in board_ready and stop_ids: + stop_ids_by_canonical[int(canonical_stop_id)] = tuple(str(stop_id) for stop_id in stop_ids) stop_to_canonical = { stop_id: canonical_stop_id for canonical_stop_id, stop_ids in stop_ids_by_canonical.items() @@ -3245,7 +4126,11 @@ def _router_boarding_rows( stmt = _where_trip_service_active(stmt, GtfsTrip, service_ids) if latest_departure is not None: stmt = stmt.where(or_(GtfsStopTime.departure_seconds.is_(None), GtfsStopTime.departure_seconds < latest_departure)) - return db.execute(stmt).all() + return [ + (call, trip, route) + for call, trip, route in db.execute(stmt).all() + if not _route_is_shadowed(db, route) + ] def _gtfs_stop_ids_for_canonical_ids( @@ -3357,6 +4242,9 @@ def _find_walk_only_journey( to_lat=float(to_selection.display.lat), mode="walk", max_visited=80_000, + allow_python_fallback=not settings.is_postgresql_database, + pgrouting_timeout_ms=700, + pgrouting_padding_km=(0.5, 1.5, 4), ) except Exception: # noqa: BLE001 - walking comparison is optional return None @@ -3378,9 +4266,21 @@ def _find_walk_only_journey( arrival_seconds=arrival_seconds, ), from_selection.display.dataset_id, + route_geometry=False, ) + routed_geometry, routed_distance, routed_duration_seconds = _walk_geometry_from_route_payload(route) + if routed_geometry is not None: + leg["geometry"] = routed_geometry + leg["geometry_source"] = "routing_layer:walk" + leg["distance_m"] = routed_distance + if routed_duration_seconds is not None: + routed_duration = max(0, int(math.ceil(routed_duration_seconds))) + routed_arrival = departure_seconds + routed_duration + leg["duration_seconds"] = routed_duration + leg["arrival_seconds"] = routed_arrival + leg["arrival_time"] = format_gtfs_time(routed_arrival) + leg["arrival_time_label"] = format_gtfs_time_label(routed_arrival) leg["route_name"] = "Walk only" - leg["duration_seconds"] = duration_seconds return _journey_payload([leg]) @@ -3412,6 +4312,84 @@ def _find_direct_journeys( return sorted(candidates, key=_journey_sort_key)[:limit] +def _find_direct_journeys_with_final_walks( + db: Session, + dataset_id: int, + service_ids: set[str] | None, + from_stop_ids: tuple[str, ...], + to_stop_ids: tuple[str, ...], + target_canonical_stop_id: int | None, + earliest_departure: int, + limit: int, + stop_cache: dict[tuple[int, str], StopSummary], + osm_stop_cache: dict[tuple[int, str], dict], +) -> list[dict]: + if service_ids == set(): + return [] + + candidates: list[dict] = [] + seen: set[tuple[str, ...]] = set() + destination_groups = _destination_stop_groups_with_final_walks( + db, + dataset_id=dataset_id, + to_stop_ids=to_stop_ids, + target_canonical_stop_id=target_canonical_stop_id, + ) + for destination_stop_ids, final_walk_by_canonical in destination_groups: + direct_legs = _find_direct_legs( + db, + dataset_id, + service_ids, + from_stop_ids, + destination_stop_ids, + earliest_departure, + stop_cache, + osm_stop_cache, + max_legs=max(limit * 4, limit), + ) + if not final_walk_by_canonical: + for leg in direct_legs: + journey = _journey_payload([leg]) + key = tuple(_journey_leg_signature(payload_leg) for payload_leg in journey["legs"]) + if key in seen: + continue + seen.add(key) + candidates.append(journey) + continue + + canonical_by_stop_id = _canonical_ids_for_stop_ids( + db, + dataset_id, + {str((leg.get("to") or {}).get("stop_id")) for leg in direct_legs if (leg.get("to") or {}).get("stop_id")}, + ) + for leg in direct_legs: + arrival_seconds = leg.get("arrival_seconds") + if arrival_seconds is None: + continue + stop_id = (leg.get("to") or {}).get("stop_id") + canonical_stop_id = canonical_by_stop_id.get(str(stop_id)) + final_walk_template = final_walk_by_canonical.get(canonical_stop_id) + if final_walk_template is None or canonical_stop_id is None: + continue + final_arrival_seconds = int(arrival_seconds) + _walking_transfer_seconds(final_walk_template.distance_m) + final_walk = _RouterWalkBacklink( + previous_label=_RouterLabel(canonical_stop_id, int(arrival_seconds)), + from_stop=final_walk_template.from_stop, + to_stop=final_walk_template.to_stop, + distance_m=final_walk_template.distance_m, + departure_seconds=int(arrival_seconds), + arrival_seconds=final_arrival_seconds, + ) + journey = _journey_payload([leg, _walk_leg_payload(db, final_walk, dataset_id)]) + key = tuple(_journey_leg_signature(payload_leg) for payload_leg in journey["legs"]) + if key in seen: + continue + seen.add(key) + candidates.append(journey) + + return sorted(candidates, key=_journey_sort_key)[:limit] + + def _find_direct_legs( db: Session, dataset_id: int, @@ -3478,6 +4456,8 @@ def _find_direct_legs( candidates: list[dict] = [] seen: set[tuple[object, ...]] = set() for origin, dest, trip, route in db.execute(stmt).all(): + if _route_is_shadowed(db, route): + continue dep_seconds = _departure_seconds(origin) arr_seconds = _arrival_seconds(dest) if dep_seconds is None or arr_seconds is None: @@ -3614,11 +4594,11 @@ def _find_one_transfer_journeys( latest_arrival=latest_first_arrival_limit, ) searched_second_legs = 0 - best_candidate_arrival: int | None = None + best_exact_candidate_arrival: int | None = None for canonical_stop_id, second in second_leg_options: if searched_second_legs >= MAX_BACKWARD_SECOND_LEG_OPTIONS and candidates: break - if best_candidate_arrival is not None and candidates and second.arrival_seconds > best_candidate_arrival: + if best_exact_candidate_arrival is not None and candidates and second.arrival_seconds > best_exact_candidate_arrival: break searched_second_legs += 1 transfer_stop_ids = transfer_stop_ids_by_canonical.get(canonical_stop_id) @@ -3668,7 +4648,12 @@ def _find_one_transfer_journeys( if key in seen: continue seen.add(key) - best_candidate_arrival = candidate_arrival if best_candidate_arrival is None else min(best_candidate_arrival, candidate_arrival) + if final_walk is None: + best_exact_candidate_arrival = ( + candidate_arrival + if best_exact_candidate_arrival is None + else min(best_exact_candidate_arrival, candidate_arrival) + ) candidates.append( _OneTransferCandidate( arrival_seconds=candidate_arrival, @@ -4128,6 +5113,8 @@ def _latest_direct_leg_to_stops( if excluded_trip_id: stmt = stmt.where(GtfsTrip.trip_id != excluded_trip_id) for origin, dest, trip, route in db.execute(stmt).all(): + if _route_is_shadowed(db, route): + continue departure = _departure_seconds(origin) arrival = _arrival_seconds(dest) if departure is None or arrival is None: @@ -4249,6 +5236,8 @@ def _destination_arrivals( stmt = stmt.where(or_(GtfsStopTime.arrival_seconds.is_(None), GtfsStopTime.arrival_seconds <= latest_arrival)) rows = [] for stop_time, trip, route in db.execute(stmt).all(): + if _route_is_shadowed(db, route): + continue arrival = _arrival_seconds(stop_time) if arrival is None or arrival < earliest_departure: continue @@ -4312,6 +5301,8 @@ def _origin_boardings( stmt = stmt.where(or_(GtfsStopTime.departure_seconds.is_(None), GtfsStopTime.departure_seconds < latest_departure)) boardings: list[_Boarding] = [] for call, trip, route in db.execute(stmt).all(): + if _route_is_shadowed(db, route): + continue departure = _departure_seconds(call) if departure is None or departure < earliest_departure: continue @@ -4489,8 +5480,8 @@ def _walk_leg_payload(db: Session, backlink: _RouterWalkBacklink, dataset_id: in "geometry": geometry, "geometry_source": geometry_source, "stops": [ - _canonical_walk_stop_payload(backlink.from_stop, 1), - _canonical_walk_stop_payload(backlink.to_stop, 2), + _walk_stop_payload(db, backlink.from_stop, 1), + _walk_stop_payload(db, backlink.to_stop, 2), ], } @@ -4516,9 +5507,19 @@ def _walk_geometry_from_routing(db: Session, from_stop: StopSummary, to_stop: St to_lat=float(to_stop.lat), mode="walk", max_visited=5_000, + allow_python_fallback=not settings.is_postgresql_database, + pgrouting_timeout_ms=500, + pgrouting_padding_km=(0.5, 1.5), ) except Exception: # noqa: BLE001 - routing graph may be unavailable during import return None, 0.0, None + result = _walk_geometry_from_route_payload(route) + if result[0] is not None: + _walk_geometry_cache_put(cache_key, result) + return _copy_walk_geometry_cache_value(result) + + +def _walk_geometry_from_route_payload(route: dict[str, object]) -> tuple[dict | None, float, float | None]: features = (route.get("features") or {}).get("features") if isinstance(route, dict) else None if not isinstance(features, list): return None, 0.0, None @@ -4540,7 +5541,6 @@ def _walk_geometry_from_routing(db: Session, from_stop: StopSummary, to_stop: St result = ({"type": "LineString", "coordinates": geometry}, float(route.get("distance_m") or 0), duration_seconds) else: result = ({"type": "MultiLineString", "coordinates": coordinates}, float(route.get("distance_m") or 0), duration_seconds) - _walk_geometry_cache_put(cache_key, result) return _copy_walk_geometry_cache_value(result) @@ -4575,15 +5575,30 @@ def _copy_walk_geometry_cache_value(value: tuple[dict | None, float, float | Non return copied_geometry, distance_m, duration_seconds -def _canonical_walk_stop_payload(stop: StopSummary, sequence: int) -> dict: +def _walk_stop_payload(db: Session, stop: StopSummary, sequence: int) -> dict: payload = _stop_payload(stop) payload["stop_sequence"] = sequence is_external_location = is_location_token(stop.stop_id) - payload["visual_source"] = "address" if is_external_location else "canonical_stop" + canonical_id = None + canonical_name = None + visual_source = "address" if is_external_location else "gtfs_stop" + if is_external_location: + pass + elif stop.stop_id.startswith("canonical:"): + canonical_id = stop.id + canonical_name = stop.name + visual_source = "canonical_stop" + elif stop.id: + gtfs_stop = db.get(GtfsStop, stop.id) + canonical = canonical_stop_for_gtfs_stop(db, gtfs_stop) if gtfs_stop is not None else None + if canonical is not None: + canonical_id = canonical.id + canonical_name = canonical.name payload["visual_lon"] = stop.lon payload["visual_lat"] = stop.lat payload["osm"] = None - payload["canonical_stop"] = None if is_external_location else {"id": stop.id, "name": stop.name} + payload["visual_source"] = visual_source + payload["canonical_stop"] = None if canonical_id is None else {"id": canonical_id, "name": canonical_name} return payload @@ -5364,12 +6379,17 @@ def _stop_payload(stop: StopSummary) -> dict: def _active_gtfs_dataset_ids(db: Session, source_ids: Optional[list[int]] = None) -> list[int]: + if not source_ids: + return active_harmonized_gtfs_dataset_ids(db) stmt = select(Dataset.id).where(Dataset.is_active.is_(True), Dataset.kind == "gtfs") - if source_ids: - stmt = stmt.where(Dataset.source_id.in_(source_ids)) + stmt = stmt.where(Dataset.source_id.in_(source_ids)) return [row[0] for row in db.execute(stmt).all()] +def _route_is_shadowed(db: Session, route: GtfsRoute) -> bool: + return int(route.id) in active_harmonized_gtfs_shadowed_route_ids(db) + + def _journey_leg_signature(leg: dict) -> str: return "|".join( str(part or "") diff --git a/app/journey_search.py b/app/journey_search.py index 8dee844..090b7a3 100644 --- a/app/journey_search.py +++ b/app/journey_search.py @@ -13,8 +13,16 @@ from typing import Any from sqlalchemy import select from app.address_search import is_location_token +from app.config import settings from app.db import SessionLocal -from app.journey import find_journeys, parse_service_date, resolve_location_summary +from app.harmonization import active_harmonized_gtfs_dataset_ids +from app.journey import ( + find_journeys, + journey_duplicate_preference_key, + journey_equivalence_key, + parse_service_date, + resolve_location_summary, +) from app.models import JourneySearchCache from app.routing import direct_route_between_points, route_between_points @@ -24,8 +32,15 @@ TRANSIT_STAGE_CACHE_TTL_SECONDS = 5 * 60 TRANSIT_STAGE_CACHE_MAX_ENTRIES = 256 PROGRESSIVE_SEARCH_CACHE_TTL_SECONDS = 10 * 60 PROGRESSIVE_SEARCH_CACHE_MAX_ENTRIES = 128 -JOURNEY_SEARCH_CACHE_VERSION = "journey-search-v7" -_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix="journey-search") +JOURNEY_SEARCH_CACHE_VERSION = "journey-search-v13-route-shadow" +LOW_WALK_MIN_SAVINGS_M = 250 +LOW_WALK_MIN_RELATIVE_SAVING = 0.35 +LOW_WALK_MAX_EXTRA_SECONDS = 45 * 60 +LOW_WALK_GOOD_ENOUGH_M = 350 +_executor = ThreadPoolExecutor( + max_workers=max(1, min(int(settings.journey_search_worker_count), 16)), + thread_name_prefix="journey-search", +) _lock = threading.RLock() _searches: dict[str, "_SearchState"] = {} _progressive_search_inflight: dict[tuple[object, ...], str] = {} @@ -106,9 +121,9 @@ def _run_search(search_id: str) -> None: state.updated_at = time.time() request = dict(state.request) try: - mode = str(request.get("mode") or "transit") - if mode in {"walk", "drive", "car"}: - _run_point_route_search(search_id, "drive" if mode == "car" else mode, request) + mode = _canonical_search_mode(request.get("mode")) + if mode in {"walk", "bike", "drive"}: + _run_point_route_search(search_id, mode, request) else: _run_transit_search(search_id, request) except Exception as exc: # noqa: BLE001 - report progressive-search failure to client @@ -129,25 +144,53 @@ def _run_transit_search(search_id: str, request: dict[str, Any]) -> None: diagnostics: dict[str, Any] = {"stages": []} best_count = 0 stale_stages = 0 + ranked: list[dict] = [] + stopped_early_reason = "" for transfers in stages: if _is_cancelled(search_id): return label = "direct" if transfers == 0 else f"up to {transfers} transfer{'s' if transfers != 1 else ''}" _publish_status(search_id, "running", f"Searching {label}...", f"transfers_{transfers}") stage_started_at = time.monotonic() - with SessionLocal() as db: - result = _cached_find_journeys( - db, - from_stop_id=str(request.get("from_stop_id") or ""), - to_stop_id=str(request.get("to_stop_id") or ""), - via_stop_id=request.get("via_stop_id") or None, - source_ids=source_ids, - departure=str(request.get("departure") or "08:00"), - service_date=service_date, - max_transfers=transfers, - transfer_seconds=transfer_seconds, - limit=stage_limit, + try: + with SessionLocal() as db: + result = _cached_find_journeys( + db, + from_stop_id=str(request.get("from_stop_id") or ""), + to_stop_id=str(request.get("to_stop_id") or ""), + via_stop_id=request.get("via_stop_id") or None, + source_ids=source_ids, + departure=str(request.get("departure") or "08:00"), + service_date=service_date, + max_transfers=transfers, + transfer_seconds=transfer_seconds, + limit=stage_limit, + ) + except Exception as exc: # noqa: BLE001 - preserve already-published progressive results on DB timeouts + if not _is_statement_timeout_error(exc) or not ranked: + raise + elapsed_ms = int((time.monotonic() - stage_started_at) * 1000) + diagnostics["stages"].append( + { + "transfers": transfers, + "cache": "error", + "elapsed_ms": elapsed_ms, + "journeys": 0, + "timed_out": True, + "error": _short_error_message(exc), + } ) + context["diagnostics"] = diagnostics + stopped_early_reason = f"Stopped after {label} timed out; showing the best options found so far." + _publish_results( + search_id, + journeys=ranked, + context=context, + status="running", + stage=f"transfers_{transfers}", + message=stopped_early_reason, + ) + break cache_status = str(result.pop("_cache_status", "miss")) elapsed_ms = int((time.monotonic() - stage_started_at) * 1000) stage_diagnostics = { @@ -179,11 +222,19 @@ def _run_transit_search(search_id: str, request: dict[str, Any]) -> None: else: stale_stages = 0 best_count = max(best_count, len(ranked)) + if _has_sufficient_progressive_options(ranked, transfers=transfers, limit=limit): + stopped_early_reason = "Stopped after finding a balanced set of fast and low-walk options." + break if ranked and stale_stages >= 2 and transfers >= 2: + stopped_early_reason = "Stopped after deeper transfer stages stopped improving the options." break if _major_hub_address_stage_is_complete(result_diagnostics, ranked, transfers=transfers, limit=limit): + stopped_early_reason = "Stopped after finding major-hub address access options." break complete_message = ( + f"{stopped_early_reason} Found {best_count} option{'s' if best_count != 1 else ''}." + if stopped_early_reason and best_count + else f"Search complete. Found {best_count} option{'s' if best_count != 1 else ''}." if best_count else "Search complete. No route found in the imported timetable." @@ -270,6 +321,7 @@ def _cached_find_journeys( to_stop_id, str(via_stop_id or ""), tuple(sorted(int(source_id) for source_id in source_ids or [])), + tuple(active_harmonized_gtfs_dataset_ids(db, source_ids=source_ids)), departure, str(service_date or ""), int(max_transfers), @@ -406,12 +458,14 @@ def _as_utc(value: datetime | None) -> datetime | None: def _progressive_cache_key(request: dict[str, Any]) -> tuple[object, ...]: source_ids = _csv_ints(request.get("source_id")) + mode = _canonical_search_mode(request.get("mode")) return ( - str(request.get("mode") or "transit"), + mode, str(request.get("from_stop_id") or ""), str(request.get("to_stop_id") or ""), str(request.get("via_stop_id") or ""), tuple(sorted(int(source_id) for source_id in source_ids or [])), + _active_dataset_cache_fragment(source_ids) if mode == "transit" else (), str(request.get("departure") or "08:00"), str(request.get("service_date") or ""), bool(request.get("direct_only")), @@ -421,6 +475,23 @@ def _progressive_cache_key(request: dict[str, Any]) -> tuple[object, ...]: ) +def _canonical_search_mode(mode: object) -> str: + value = str(mode or "transit").lower() + if value in {"bike", "bicycle", "cycle", "cycling"}: + return "bike" + if value == "car": + return "drive" + return value if value in {"transit", "walk", "drive"} else "transit" + + +def _active_dataset_cache_fragment(source_ids: list[int] | None) -> tuple[int, ...]: + try: + with SessionLocal() as session: + return tuple(active_harmonized_gtfs_dataset_ids(session, source_ids=source_ids)) + except Exception: # noqa: BLE001 - cache key generation must not prevent search + return () + + def _progressive_cache_get(key: tuple[object, ...]) -> dict[str, Any] | None: now = time.monotonic() with _lock: @@ -567,24 +638,7 @@ def _context_from_result(result: dict[str, Any]) -> dict[str, Any]: def _journey_key(journey: dict[str, Any]) -> str: - parts = [] - for leg in journey.get("legs") or []: - parts.append( - "|".join( - str(part or "") - for part in [ - leg.get("dataset_id"), - leg.get("mode"), - leg.get("route_id"), - leg.get("trip_id"), - (leg.get("from") or {}).get("stop_id") or (leg.get("from") or {}).get("name"), - (leg.get("to") or {}).get("stop_id") or (leg.get("to") or {}).get("name"), - leg.get("departure_time"), - leg.get("arrival_time"), - ] - ) - ) - return "||".join(parts) + return json.dumps(journey_equivalence_key(journey), ensure_ascii=False, separators=(",", ":")) def _rank_journeys(journeys, ranking: str) -> list[dict]: @@ -623,36 +677,47 @@ def _rank_journeys(journeys, ranking: str) -> list[dict]: walking, ) - return sorted((dict(journey) for journey in journeys), key=key) + return sorted((dict(journey) for journey in journeys), key=lambda journey: (key(journey), journey_duplicate_preference_key(journey))) def _select_diverse_journeys(journeys: list[dict], *, limit: int) -> list[dict]: + ranked = list(journeys) selected: list[dict] = [] selected_exact: set[str] = set() selected_diversity: set[tuple[object, ...]] = set() - for journey in journeys: + + def append(journey: dict | None, *, force: bool = False) -> bool: + if journey is None: + return False exact_key = _journey_key(journey) if exact_key in selected_exact: - continue + return False diversity_key = _journey_diversity_key(journey) - if diversity_key in selected_diversity and len(selected) >= 3: - continue + if not force and diversity_key in selected_diversity and len(selected) >= 3: + return False selected.append(journey) selected_exact.add(exact_key) selected_diversity.add(diversity_key) + return True + + if ranked: + append(ranked[0], force=True) + append(_best_public_transport_option(ranked), force=True) + append(_least_walking_public_transport_option(ranked), force=True) + append(_fewest_transfer_public_transport_option(ranked), force=True) + append(_walk_only_option(ranked), force=True) + + for journey in ranked: + append(journey) if len(selected) >= limit: - return selected + return _ensure_walk_only_option(selected[:limit], ranked, limit=limit) if len(selected) >= min(3, limit): return selected - for journey in journeys: - exact_key = _journey_key(journey) - if exact_key in selected_exact: - continue - selected.append(journey) - selected_exact.add(exact_key) + for journey in ranked: + append(journey, force=True) if len(selected) >= min(3, limit): break - return _ensure_walk_only_option(selected, journeys, limit=limit) + return _ensure_walk_only_option(selected[:limit], ranked, limit=limit) def _ensure_walk_only_option(selected: list[dict], ranked: list[dict], *, limit: int) -> list[dict]: @@ -673,6 +738,95 @@ def _journey_is_walk_only(journey: dict) -> bool: return bool(legs) and all(leg.get("mode") == "walk" for leg in legs) +def _walk_only_option(journeys: list[dict]) -> dict | None: + return next((journey for journey in journeys if _journey_is_walk_only(journey)), None) + + +def _journey_has_public_transport(journey: dict) -> bool: + return any(leg.get("mode") != "walk" for leg in journey.get("legs") or []) + + +def _best_public_transport_option(journeys: list[dict]) -> dict | None: + return next((journey for journey in journeys if _journey_has_public_transport(journey)), None) + + +def _least_walking_public_transport_option(journeys: list[dict]) -> dict | None: + public = [journey for journey in journeys if _journey_has_public_transport(journey)] + if not public: + return None + fastest = public[0] + lowest = min(public, key=lambda journey: (_journey_walking_m(journey), _journey_arrival_seconds(journey), _journey_duration_seconds(journey))) + if lowest is fastest: + return lowest + fastest_walking = _journey_walking_m(fastest) + lowest_walking = _journey_walking_m(lowest) + walking_saved = fastest_walking - lowest_walking + if lowest_walking <= LOW_WALK_GOOD_ENOUGH_M: + return lowest + if walking_saved < LOW_WALK_MIN_SAVINGS_M: + return None + if fastest_walking > 0 and walking_saved / fastest_walking < LOW_WALK_MIN_RELATIVE_SAVING: + return None + extra_seconds = _journey_arrival_seconds(lowest) - _journey_arrival_seconds(fastest) + if extra_seconds > LOW_WALK_MAX_EXTRA_SECONDS: + return None + return lowest + + +def _fewest_transfer_public_transport_option(journeys: list[dict]) -> dict | None: + public = [journey for journey in journeys if _journey_has_public_transport(journey)] + if not public: + return None + return min( + public, + key=lambda journey: ( + int(journey.get("transfers") or 0), + _journey_walking_m(journey), + _journey_arrival_seconds(journey), + ), + ) + + +def _journey_walking_m(journey: dict) -> float: + return sum(float(leg.get("distance_m") or 0) for leg in journey.get("legs") or [] if leg.get("mode") == "walk") + + +def _journey_arrival_seconds(journey: dict) -> float: + value = journey.get("arrival_seconds") + return float("inf") if value is None else float(value) + + +def _journey_duration_seconds(journey: dict) -> float: + value = journey.get("duration_seconds") + if value is not None: + return float(value) + minutes = journey.get("duration_minutes") + return float("inf") if minutes is None else float(minutes) * 60 + + +def _has_sufficient_progressive_options(journeys: list[dict], *, transfers: int, limit: int) -> bool: + if transfers < 2 or len(journeys) < min(3, limit): + return False + public = [journey for journey in journeys if _journey_has_public_transport(journey)] + if not public: + return False + low_walk = _least_walking_public_transport_option(public) + if low_walk is None: + return False + return _journey_walking_m(low_walk) <= max(LOW_WALK_GOOD_ENOUGH_M * 2, _journey_walking_m(public[0]) * 0.75) + + +def _is_statement_timeout_error(exc: Exception) -> bool: + text = " ".join(str(part).lower() for part in [exc, getattr(exc, "orig", "")]) + return "statement timeout" in text or "canceling statement due to statement timeout" in text or "querycanceled" in text + + +def _short_error_message(exc: Exception) -> str: + text = " ".join(str(part) for part in [getattr(exc, "orig", ""), exc] if part) + text = " ".join(text.split()) + return text[:240] if text else exc.__class__.__name__ + + def _journey_diversity_key(journey: dict[str, Any]) -> tuple[object, ...]: route_signature = tuple( str(leg.get("route_ref") or leg.get("route_id") or leg.get("mode") or "") diff --git a/app/main.py b/app/main.py index 8a1eb93..0335d62 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,7 @@ from __future__ import annotations +import csv +import io import json from contextlib import asynccontextmanager from datetime import datetime, timezone @@ -24,8 +26,17 @@ from app.dataset_search import search_datasets from app.db import engine, get_db, init_db from app.db_lock import DatabaseWriteBusy, database_write_lock, database_write_status from app.geofabrik import create_geofabrik_source, geofabrik_catalog +from app.gtfs_diff import diff_run_payload, gtfs_update_diff_detail, gtfs_update_diff_summary, latest_gtfs_update_diffs from app.gtfs_storage import scheduled_stop_ids -from app.harmonization import GTFS_QA_NOTE_PREFIX, gtfs_harmonization_feed_detail, gtfs_harmonization_inventory +from app.harmonization import ( + GTFS_QA_NOTE_PREFIX, + gtfs_harmonization_feed_detail, + gtfs_harmonization_inventory, + gtfs_harmonized_snapshot, + gtfs_harmonized_snapshot_diagnostics, + gtfs_qa_review_payload, + list_active_harmonized_snapshot_routes, +) from app.itineraries import generate_itineraries, itinerary_payload, recent_itineraries, set_itinerary_saved, set_leg_locked from app.journey import find_journeys, nearest_scheduled_stops, search_scheduled_stops from app.journey_search import cancel_journey_search, journey_search_payload, start_journey_search @@ -37,6 +48,8 @@ from app.jobs import ( cancel_job, create_dataset_delete_job, create_address_index_rebuild_job, + create_gtfs_harmonized_snapshot_job, + create_map_gtfs_review_job, create_maintenance_job, create_osm_relabel_job, create_route_matching_job, @@ -69,6 +82,7 @@ from app.models import ( ItineraryLeg, Job, MatchRule, + MapGtfsReviewItem, OsmFeature, PipelineRun, RouteMatch, @@ -106,7 +120,21 @@ from app.source_catalog import ( source_catalog_rows, source_catalog_summary, ) -from app.worker_supervisor import queue_worker_status, start_queue_workers, stop_queue_workers +from app.worker_supervisor import ( + queue_worker_status, + restart_queue_workers, + start_queue_workers, + stop_configured_queue_workers, + stop_queue_workers, +) +from app.workbench import ( + generate_map_gtfs_review_items, + list_map_gtfs_review_items, + map_gtfs_review_summary, + record_route_match_decision, + review_item_payload, + update_map_gtfs_review_item, +) @asynccontextmanager async def lifespan(_app: FastAPI): @@ -125,6 +153,7 @@ async def lifespan(_app: FastAPI): app = FastAPI(title="Mobility Workbench", version="0.1.0", lifespan=lifespan) app.mount("/static", StaticFiles(directory=Path(__file__).parent / "static"), name="static") templates = Jinja2Templates(directory=str(Path(__file__).parent / "templates")) +STATIC_ASSET_VERSION = "20260706-bike-mode" MAP_FEATURE_LIMIT = 5000 MAP_FEATURE_LIMIT_MAX = 20000 @@ -146,6 +175,12 @@ class GtfsFeedReviewUpdate(BaseModel): license: Optional[str] = None review_status: Optional[str] = None review_note: Optional[str] = None + can_import: Optional[str] = None + can_derive: Optional[str] = None + can_redistribute: Optional[str] = None + requires_attribution: Optional[str] = None + commercial_restrictions: Optional[str] = None + authority_level: Optional[str] = None enabled: Optional[bool] = None @@ -221,6 +256,11 @@ class JobPriorityRequest(BaseModel): priority: int +class WorkbenchReviewUpdate(BaseModel): + status: Optional[str] = None + note: Optional[str] = None + + def write_endpoint(operation: str): def decorator(fn): @wraps(fn) @@ -249,7 +289,7 @@ async def database_operational_error_handler(_request: Request, exc: Operational @app.get("/", response_class=HTMLResponse) def index(request: Request) -> HTMLResponse: - return templates.TemplateResponse(request=request, name="index.html") + return templates.TemplateResponse(request=request, name="index.html", context={"static_version": STATIC_ASSET_VERSION}) @app.get("/api/sources") @@ -623,6 +663,21 @@ def queue_route_matching(priority: int = 0, db: Session = Depends(get_db)) -> di return job_payload(job) +@app.post("/api/jobs/map-gtfs-review") +@write_endpoint("queue Map/GTFS review generation") +def queue_map_gtfs_review( + source_id: Optional[int] = None, + dataset_id: Optional[int] = None, + limit: int = 1000, + priority: int = 0, + db: Session = Depends(get_db), +) -> dict: + job = create_map_gtfs_review_job(db, source_id=source_id, dataset_id=dataset_id, limit=limit, priority=priority) + db.commit() + db.refresh(job) + return job_payload(job) + + @app.post("/api/jobs/osm-relabel") @write_endpoint("queue OSM relabeling") def queue_osm_relabel( @@ -658,6 +713,28 @@ def list_jobs( } +@app.post("/api/workers/start") +def start_workers_endpoint() -> dict: + started = start_queue_workers(force=True) + return _worker_control_payload("start", started=started) + + +@app.post("/api/workers/stop") +def stop_workers_endpoint() -> dict: + stopped = stop_configured_queue_workers() + return _worker_control_payload("stop", stopped=stopped) + + +@app.post("/api/workers/restart") +def restart_workers_endpoint() -> dict: + result = restart_queue_workers() + return _worker_control_payload( + "restart", + stopped=result.get("stopped", []), + started=result.get("started", []), + ) + + @app.get("/api/jobs/revision") def get_jobs_revision( response: Response, @@ -870,6 +947,152 @@ def gtfs_harmonization_inventory_endpoint(db: Session = Depends(get_db)) -> dict return gtfs_harmonization_inventory(db) +@app.get("/api/harmonization/gtfs/snapshot") +def gtfs_harmonized_snapshot_endpoint(db: Session = Depends(get_db)) -> dict: + return gtfs_harmonized_snapshot(db) + + +@app.get("/api/harmonization/gtfs/snapshot/routes") +def gtfs_harmonized_snapshot_routes_endpoint( + role: Optional[str] = None, + dataset_id: Optional[int] = None, + source_id: Optional[int] = None, + shadowed_by_dataset_id: Optional[int] = None, + mode: Optional[str] = None, + q: Optional[str] = None, + limit: int = 200, + offset: int = 0, + db: Session = Depends(get_db), +) -> dict: + if role is not None and role not in {"included", "shadowed", "excluded"}: + raise HTTPException(status_code=400, detail="role must be included, shadowed, or excluded") + return list_active_harmonized_snapshot_routes( + db, + role=role, + dataset_id=dataset_id, + source_id=source_id, + shadowed_by_dataset_id=shadowed_by_dataset_id, + mode=mode, + q=q, + limit=limit, + offset=offset, + ) + + +@app.get("/api/harmonization/gtfs/snapshot/routes.csv") +def gtfs_harmonized_snapshot_routes_csv_endpoint( + role: Optional[str] = None, + dataset_id: Optional[int] = None, + source_id: Optional[int] = None, + shadowed_by_dataset_id: Optional[int] = None, + mode: Optional[str] = None, + q: Optional[str] = None, + limit: int = 10000, + offset: int = 0, + db: Session = Depends(get_db), +) -> Response: + if role is not None and role not in {"included", "shadowed", "excluded"}: + raise HTTPException(status_code=400, detail="role must be included, shadowed, or excluded") + payload = list_active_harmonized_snapshot_routes( + db, + role=role, + dataset_id=dataset_id, + source_id=source_id, + shadowed_by_dataset_id=shadowed_by_dataset_id, + mode=mode, + q=q, + limit=limit, + offset=offset, + max_limit=100_000, + ) + output = io.StringIO() + fieldnames = [ + "snapshot_id", + "gtfs_route_id", + "dataset_id", + "source_id", + "source_name", + "route_id", + "route_ref", + "route_name", + "mode", + "operator", + "overlap_key", + "role", + "reason", + "shadowed_by_gtfs_route_id", + "shadowed_by_dataset_id", + "shadowed_by_source_id", + ] + writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + for route in payload.get("routes", []): + writer.writerow(route) + snapshot = payload.get("snapshot") or {} + filename = f"gtfs_harmonized_snapshot_routes_{snapshot.get('id') or 'none'}.csv" + return Response( + content=output.getvalue(), + media_type="text/csv; charset=utf-8", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +@app.get("/api/harmonization/gtfs/diffs") +def list_gtfs_update_diffs( + source_id: Optional[int] = None, + limit: int = 20, + db: Session = Depends(get_db), +) -> dict: + return { + "summary": gtfs_update_diff_summary(db), + "diffs": [diff_run_payload(diff) for diff in latest_gtfs_update_diffs(db, source_id=source_id, limit=limit)], + } + + +@app.get("/api/harmonization/gtfs/diffs/{diff_id}") +def get_gtfs_update_diff(diff_id: int, item_limit: int = 200, db: Session = Depends(get_db)) -> dict: + detail = gtfs_update_diff_detail(db, diff_id, item_limit=item_limit) + if detail is None: + raise HTTPException(status_code=404, detail="GTFS update diff not found") + return detail + + +@app.get("/api/workbench/gtfs-workflow") +def gtfs_workflow_status_endpoint(db: Session = Depends(get_db)) -> dict: + inventory = gtfs_harmonization_inventory(db) + snapshot_diagnostics = inventory.get("snapshot_diagnostics") or gtfs_harmonized_snapshot_diagnostics(db) + snapshot = inventory["snapshot"] + snapshot_summary = snapshot.get("summary") or {} + snapshot_route_preview = ( + list_active_harmonized_snapshot_routes(db, role="shadowed", limit=8) + if snapshot.get("persisted") and int(snapshot_summary.get("snapshot_route_rows") or 0) > 0 + else {"snapshot": None, "summary": {}, "routes": []} + ) + review = map_gtfs_review_summary(db) + diff_summary = gtfs_update_diff_summary(db) + latest_diffs = [diff_run_payload(diff) for diff in latest_gtfs_update_diffs(db, limit=5)] + jobs = latest_jobs(db, limit=8) + return { + "inventory_summary": inventory["summary"], + "snapshot": snapshot, + "snapshot_diagnostics": snapshot_diagnostics, + "snapshot_route_preview": snapshot_route_preview, + "review_summary": review, + "diff_summary": diff_summary, + "latest_diffs": latest_diffs, + "jobs": [job_payload(job) for job in jobs], + } + + +@app.post("/api/jobs/gtfs-harmonized-snapshot") +@write_endpoint("queue harmonized GTFS snapshot build") +def queue_gtfs_harmonized_snapshot(activate: bool = True, priority: int = 0, db: Session = Depends(get_db)) -> dict: + job = create_gtfs_harmonized_snapshot_job(db, activate=activate, priority=priority) + db.commit() + db.refresh(job) + return job_payload(job) + + @app.get("/api/harmonization/gtfs/sources/{source_id}") def gtfs_harmonization_feed_detail_endpoint(source_id: int, db: Session = Depends(get_db)) -> dict: detail = gtfs_harmonization_feed_detail(db, source_id) @@ -886,15 +1109,48 @@ def update_gtfs_harmonization_feed_review(source_id: int, payload: GtfsFeedRevie raise HTTPException(status_code=404, detail="GTFS source not found") if payload.review_status is not None and payload.review_status not in {"unreviewed", "approved", "needs_review", "blocked", "rejected"}: raise HTTPException(status_code=400, detail="review_status must be unreviewed, approved, needs_review, blocked, or rejected") + license_flags = { + "can_import": payload.can_import, + "can_derive": payload.can_derive, + "can_redistribute": payload.can_redistribute, + "requires_attribution": payload.requires_attribution, + "commercial_restrictions": payload.commercial_restrictions, + } + for field, value in license_flags.items(): + if value is not None and value not in {"unknown", "yes", "no"}: + raise HTTPException(status_code=400, detail=f"{field} must be unknown, yes, or no") + if payload.authority_level is not None and payload.authority_level not in { + "unknown", + "national_official", + "regional_authority", + "operator", + "aggregator", + "mirror", + "secondary_discovery", + }: + raise HTTPException(status_code=400, detail="authority_level is not valid") if payload.license is not None: source.license = _truncate(payload.license, 255) if payload.enabled is not None: source.enabled = bool(payload.enabled) - if payload.review_status is not None or payload.review_note is not None: + existing_review = gtfs_qa_review_payload(source.notes) + review_fields_present = ( + payload.review_status is not None + or payload.review_note is not None + or any(value is not None for value in license_flags.values()) + or payload.authority_level is not None + ) + if review_fields_present: source.notes = _upsert_gtfs_qa_note( source.notes, - status=payload.review_status or "unreviewed", - note=payload.review_note or "", + status=payload.review_status or existing_review["status"], + note=payload.review_note if payload.review_note is not None else existing_review["note"], + can_import=payload.can_import or existing_review["can_import"], + can_derive=payload.can_derive or existing_review["can_derive"], + can_redistribute=payload.can_redistribute or existing_review["can_redistribute"], + requires_attribution=payload.requires_attribution or existing_review["requires_attribution"], + commercial_restrictions=payload.commercial_restrictions or existing_review["commercial_restrictions"], + authority_level=payload.authority_level or existing_review["authority_level"], ) db.commit() detail = gtfs_harmonization_feed_detail(db, source_id) @@ -903,6 +1159,84 @@ def update_gtfs_harmonization_feed_review(source_id: int, payload: GtfsFeedRevie return detail +@app.get("/api/workbench/map-gtfs/summary") +def map_gtfs_workbench_summary_endpoint(db: Session = Depends(get_db)) -> dict: + return map_gtfs_review_summary(db) + + +@app.get("/api/workbench/map-gtfs/review-items") +def list_map_gtfs_workbench_review_items( + queue: Optional[str] = None, + status: Optional[str] = "open", + severity: Optional[str] = None, + mode: Optional[str] = None, + q: Optional[str] = None, + source_id: Optional[int] = None, + dataset_id: Optional[int] = None, + limit: int = 100, + db: Session = Depends(get_db), +) -> dict: + items = list_map_gtfs_review_items( + db, + queue=queue, + status=status, + severity=severity, + mode=mode, + q=q, + source_id=source_id, + dataset_id=dataset_id, + limit=limit, + ) + return { + "summary": map_gtfs_review_summary(db), + "filters": { + "queue": queue, + "status": status, + "severity": severity, + "mode": mode, + "q": q, + "source_id": source_id, + "dataset_id": dataset_id, + }, + "items": [review_item_payload(item) for item in items], + } + + +@app.get("/api/workbench/map-gtfs/review-items/{item_id}") +def get_map_gtfs_workbench_review_item(item_id: int, db: Session = Depends(get_db)) -> dict: + item = db.get(MapGtfsReviewItem, item_id) + if item is None: + raise HTTPException(status_code=404, detail="review item not found") + return review_item_payload(item) + + +@app.post("/api/workbench/map-gtfs/review-items/generate") +@write_endpoint("generate Map/GTFS review queue") +def generate_map_gtfs_workbench_review_items( + source_id: Optional[int] = None, + dataset_id: Optional[int] = None, + limit: int = 1000, + db: Session = Depends(get_db), +) -> dict: + result = generate_map_gtfs_review_items(db, source_id=source_id, dataset_id=dataset_id, limit=limit) + db.commit() + return {"summary": map_gtfs_review_summary(db), "result": result} + + +@app.patch("/api/workbench/map-gtfs/review-items/{item_id}") +@write_endpoint("update Map/GTFS review item") +def update_map_gtfs_workbench_review_item(item_id: int, payload: WorkbenchReviewUpdate, db: Session = Depends(get_db)) -> dict: + try: + item = update_map_gtfs_review_item(db, item_id, status=payload.status, note=payload.note) + except ValueError as exc: + detail = str(exc) + status_code = 404 if detail == "review item not found" else 400 + raise HTTPException(status_code=status_code, detail=detail) from exc + db.commit() + db.refresh(item) + return review_item_payload(item) + + @app.get("/api/matches") def list_matches( status: Optional[str] = None, @@ -933,6 +1267,7 @@ def accept_match(match_id: int, db: Session = Depends(get_db)) -> dict: match.rule_source = "manual" match.updated_at = datetime.now(timezone.utc) _persist_match_rule(db, match, "accept_match") + record_route_match_decision(db, match, note="Accepted from route match review.") db.commit() return {"id": match.id, "status": match.status} @@ -947,6 +1282,7 @@ def reject_match(match_id: int, db: Session = Depends(get_db)) -> dict: match.rule_source = "manual" match.updated_at = datetime.now(timezone.utc) _persist_match_rule(db, match, "reject_match") + record_route_match_decision(db, match, note="Rejected from route match review.") db.commit() return {"id": match.id, "status": match.status} @@ -1030,6 +1366,7 @@ def accept_match_candidate(match_id: int, osm_feature_id: str, db: Session = Dep match.reasons_json = json.dumps(reasons, separators=(",", ":")) match.updated_at = datetime.now(timezone.utc) _persist_match_rule(db, match, "accept_match") + record_route_match_decision(db, match, note="Accepted from route candidate review.") db.commit() db.refresh(match) return {"id": match.id, "status": match.status, "confidence": match.confidence, "match": match_row(match)} @@ -1423,6 +1760,7 @@ def journey_nearest_location( source_ids=_csv_ints(source_id, "source_id"), limit=1, radius_m=max(5, min(float(stop_radius_m), 120)), + grouped=False, ) location = stops[0] if stops else None if location is not None: @@ -1509,7 +1847,8 @@ def journey_search( @app.post("/api/journey/searches") def start_progressive_journey_search(payload: JourneySearchRequest) -> dict: - mode = payload.mode if payload.mode in {"transit", "walk", "drive", "car"} else "transit" + requested_mode = str(payload.mode or "transit").lower() + mode = "bike" if requested_mode in {"bike", "bicycle", "cycle", "cycling"} else requested_mode if requested_mode in {"transit", "walk", "drive", "car"} else "transit" ranking = payload.ranking if payload.ranking in {"recommended", "earliest_arrival", "duration", "fewest_transfers"} else "recommended" try: return start_journey_search( @@ -1978,6 +2317,29 @@ def _json_object_value(value: object, key: str) -> object: return value.get(key) if isinstance(value, dict) else None +def _worker_control_payload(action: str, **groups: list[object]) -> dict: + return { + "action": action, + "operations": { + name: [_worker_handle_payload(handle) for handle in handles] + for name, handles in groups.items() + }, + "workers": queue_worker_status(), + } + + +def _worker_handle_payload(handle: object) -> dict: + return { + "index": getattr(handle, "index", None), + "worker_id": getattr(handle, "worker_id", None), + "pid": getattr(handle, "pid", None), + "status": getattr(handle, "status", None), + "pid_file": str(getattr(handle, "pid_file", "")), + "log_file": str(getattr(handle, "log_file", "")), + "started_by_server": bool(getattr(handle, "started_by_server", False)), + } + + def _job_queue_revision_payload( db: Session, *, @@ -2018,19 +2380,51 @@ def _set_etag(response: Response, revision: str) -> None: response.headers["ETag"] = f'W/"{revision}"' -def _upsert_gtfs_qa_note(notes: str | None, *, status: str, note: str) -> str | None: +def _upsert_gtfs_qa_note( + notes: str | None, + *, + status: str, + note: str, + can_import: str | None = None, + can_derive: str | None = None, + can_redistribute: str | None = None, + requires_attribution: str | None = None, + commercial_restrictions: str | None = None, + authority_level: str | None = None, +) -> str | None: status_text = (status or "unreviewed").strip() or "unreviewed" note_text = " ".join((note or "").strip().split()) updated_at = datetime.now(timezone.utc).isoformat() - marker = f"{GTFS_QA_NOTE_PREFIX} status={status_text}; updated_at={updated_at}" - if note_text: - marker = f"{marker}; note={note_text}" + payload = { + "status": status_text, + "updated_at": updated_at, + "note": note_text, + "can_import": can_import or "unknown", + "can_derive": can_derive or "unknown", + "can_redistribute": can_redistribute or "unknown", + "requires_attribution": requires_attribution or "unknown", + "commercial_restrictions": commercial_restrictions or "unknown", + "authority_level": authority_level or "unknown", + } + marker = f"{GTFS_QA_NOTE_PREFIX} {json.dumps(payload, sort_keys=True, separators=(',', ':'))}" preserved = [ line for line in str(notes or "").splitlines() if line.strip() and not line.startswith(GTFS_QA_NOTE_PREFIX) ] - if status_text != "unreviewed" or note_text: + has_decision = ( + status_text != "unreviewed" + or note_text + or any(payload[key] != "unknown" for key in [ + "can_import", + "can_derive", + "can_redistribute", + "requires_attribution", + "commercial_restrictions", + "authority_level", + ]) + ) + if has_decision: preserved.insert(0, marker) return "\n".join(preserved) or None diff --git a/app/models.py b/app/models.py index 823374d..b5ad45e 100644 --- a/app/models.py +++ b/app/models.py @@ -81,6 +81,115 @@ class Dataset(Base): source: Mapped[Source] = relationship(back_populates="datasets") +class GtfsHarmonizedSnapshot(Base): + __tablename__ = "gtfs_harmonized_snapshots" + __table_args__ = (UniqueConstraint("snapshot_key", name="uq_gtfs_harmonized_snapshot_key"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + snapshot_key: Mapped[str] = mapped_column(String(128), nullable=False, index=True) + status: Mapped[str] = mapped_column(String(64), nullable=False, default="draft", index=True) + source: Mapped[str] = mapped_column(String(64), nullable=False, default="computed") + note: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + summary_json: Mapped[str] = mapped_column(Text, nullable=False) + metadata_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now_utc, nullable=False, index=True) + activated_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True, index=True) + + datasets: Mapped[list["GtfsHarmonizedSnapshotDataset"]] = relationship(back_populates="snapshot", cascade="all, delete-orphan") + routes: Mapped[list["GtfsHarmonizedSnapshotRoute"]] = relationship(back_populates="snapshot", cascade="all, delete-orphan") + + +class GtfsHarmonizedSnapshotDataset(Base): + __tablename__ = "gtfs_harmonized_snapshot_datasets" + __table_args__ = (UniqueConstraint("snapshot_id", "dataset_id", name="uq_gtfs_harmonized_snapshot_dataset"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + snapshot_id: Mapped[int] = mapped_column(ForeignKey("gtfs_harmonized_snapshots.id"), nullable=False, index=True) + dataset_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + source_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + source_name: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + role: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + reason: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + shadowed_by_dataset_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, index=True) + shadowed_by_source_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, index=True) + authority_level: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, index=True) + review_authority_level: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + route_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + route_key_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + route_key_overlap: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + route_key_overlap_ratio: Mapped[float] = mapped_column(Float, nullable=False, default=0.0) + metadata_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now_utc, nullable=False, index=True) + + snapshot: Mapped[GtfsHarmonizedSnapshot] = relationship(back_populates="datasets") + + +class GtfsHarmonizedSnapshotRoute(Base): + __tablename__ = "gtfs_harmonized_snapshot_routes" + __table_args__ = (UniqueConstraint("snapshot_id", "gtfs_route_id", name="uq_gtfs_harmonized_snapshot_route"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + snapshot_id: Mapped[int] = mapped_column(ForeignKey("gtfs_harmonized_snapshots.id"), nullable=False, index=True) + gtfs_route_id: Mapped[int] = mapped_column(ForeignKey("gtfs_routes.id"), nullable=False, index=True) + dataset_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + source_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + route_id: Mapped[str] = mapped_column(Text, nullable=False) + overlap_key: Mapped[Optional[str]] = mapped_column(String(255), nullable=True, index=True) + role: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + reason: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + shadowed_by_gtfs_route_id: Mapped[Optional[int]] = mapped_column(ForeignKey("gtfs_routes.id"), nullable=True, index=True) + shadowed_by_dataset_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, index=True) + shadowed_by_source_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, index=True) + metadata_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now_utc, nullable=False, index=True) + + snapshot: Mapped[GtfsHarmonizedSnapshot] = relationship(back_populates="routes") + gtfs_route: Mapped["GtfsRoute"] = relationship(foreign_keys=[gtfs_route_id]) + shadowed_by_gtfs_route: Mapped[Optional["GtfsRoute"]] = relationship(foreign_keys=[shadowed_by_gtfs_route_id]) + + +class GtfsFeedDiffRun(Base): + __tablename__ = "gtfs_feed_diff_runs" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source_id: Mapped[int] = mapped_column(ForeignKey("sources.id"), nullable=False, index=True) + previous_dataset_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, index=True) + new_dataset_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + status: Mapped[str] = mapped_column(String(64), nullable=False, default="completed", index=True) + diff_key: Mapped[str] = mapped_column(String(128), nullable=False, index=True) + summary_json: Mapped[str] = mapped_column(Text, nullable=False) + metadata_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now_utc, nullable=False, index=True) + + source: Mapped[Source] = relationship() + items: Mapped[list["GtfsFeedDiffItem"]] = relationship(back_populates="diff_run", cascade="all, delete-orphan") + + +class GtfsFeedDiffItem(Base): + __tablename__ = "gtfs_feed_diff_items" + __table_args__ = (UniqueConstraint("diff_run_id", "item_type", "object_key", "change_type", name="uq_gtfs_feed_diff_item"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + diff_run_id: Mapped[int] = mapped_column(ForeignKey("gtfs_feed_diff_runs.id"), nullable=False, index=True) + source_id: Mapped[int] = mapped_column(ForeignKey("sources.id"), nullable=False, index=True) + previous_dataset_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True, index=True) + new_dataset_id: Mapped[int] = mapped_column(Integer, nullable=False, index=True) + item_type: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + object_key: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + change_type: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + severity: Mapped[str] = mapped_column(String(32), nullable=False, default="info", index=True) + status: Mapped[str] = mapped_column(String(64), nullable=False, default="open", index=True) + title: Mapped[str] = mapped_column(Text, nullable=False) + old_payload_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + new_payload_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + diff_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now_utc, nullable=False, index=True) + resolved_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True, index=True) + + diff_run: Mapped[GtfsFeedDiffRun] = relationship(back_populates="items") + source: Mapped[Source] = relationship() + + class SourceUpdateCheck(Base): __tablename__ = "source_update_checks" @@ -543,6 +652,83 @@ class MatchRule(Base): created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now_utc, nullable=False) +class MapGtfsDecision(Base): + __tablename__ = "map_gtfs_decisions" + __table_args__ = (UniqueConstraint("decision_key", name="uq_map_gtfs_decision_key"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + decision_key: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + decision_type: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + status: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + active: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False, index=True) + source_id: Mapped[Optional[int]] = mapped_column(ForeignKey("sources.id"), nullable=True, index=True) + gtfs_dataset_id: Mapped[Optional[int]] = mapped_column(ForeignKey("datasets.id"), nullable=True, index=True) + gtfs_route_id: Mapped[Optional[int]] = mapped_column(ForeignKey("gtfs_routes.id"), nullable=True, index=True) + gtfs_stop_id: Mapped[Optional[int]] = mapped_column(ForeignKey("gtfs_stops.id"), nullable=True, index=True) + osm_dataset_id: Mapped[Optional[int]] = mapped_column(ForeignKey("datasets.id"), nullable=True, index=True) + osm_feature_id: Mapped[Optional[int]] = mapped_column(ForeignKey("osm_features.id"), nullable=True, index=True) + route_match_id: Mapped[Optional[int]] = mapped_column(ForeignKey("route_matches.id"), nullable=True, index=True) + canonical_stop_id: Mapped[Optional[int]] = mapped_column(ForeignKey("canonical_stops.id"), nullable=True, index=True) + canonical_stop_link_id: Mapped[Optional[int]] = mapped_column(ForeignKey("canonical_stop_links.id"), nullable=True, index=True) + confidence: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + evidence_hash: Mapped[Optional[str]] = mapped_column(String(128), nullable=True, index=True) + selector_json: Mapped[str] = mapped_column(Text, nullable=False) + action_json: Mapped[str] = mapped_column(Text, nullable=False) + note: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + reviewer: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + metadata_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now_utc, nullable=False, index=True) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now_utc, nullable=False, index=True) + + source: Mapped[Optional[Source]] = relationship(foreign_keys=[source_id]) + gtfs_dataset: Mapped[Optional[Dataset]] = relationship(foreign_keys=[gtfs_dataset_id]) + gtfs_route: Mapped[Optional[GtfsRoute]] = relationship() + gtfs_stop: Mapped[Optional[GtfsStop]] = relationship() + osm_dataset: Mapped[Optional[Dataset]] = relationship(foreign_keys=[osm_dataset_id]) + osm_feature: Mapped[Optional[OsmFeature]] = relationship() + route_match: Mapped[Optional[RouteMatch]] = relationship() + canonical_stop: Mapped[Optional[CanonicalStop]] = relationship() + canonical_stop_link: Mapped[Optional[CanonicalStopLink]] = relationship() + + +class MapGtfsReviewItem(Base): + __tablename__ = "map_gtfs_review_items" + __table_args__ = (UniqueConstraint("fingerprint", name="uq_map_gtfs_review_item_fingerprint"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + fingerprint: Mapped[str] = mapped_column(String(255), nullable=False, index=True) + queue: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + item_type: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + status: Mapped[str] = mapped_column(String(64), nullable=False, default="open", index=True) + severity: Mapped[str] = mapped_column(String(32), nullable=False, default="info", index=True) + title: Mapped[str] = mapped_column(Text, nullable=False) + description: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + source_id: Mapped[Optional[int]] = mapped_column(ForeignKey("sources.id"), nullable=True, index=True) + dataset_id: Mapped[Optional[int]] = mapped_column(ForeignKey("datasets.id"), nullable=True, index=True) + gtfs_route_id: Mapped[Optional[int]] = mapped_column(ForeignKey("gtfs_routes.id"), nullable=True, index=True) + gtfs_stop_id: Mapped[Optional[int]] = mapped_column(ForeignKey("gtfs_stops.id"), nullable=True, index=True) + osm_feature_id: Mapped[Optional[int]] = mapped_column(ForeignKey("osm_features.id"), nullable=True, index=True) + route_match_id: Mapped[Optional[int]] = mapped_column(ForeignKey("route_matches.id"), nullable=True, index=True) + canonical_stop_id: Mapped[Optional[int]] = mapped_column(ForeignKey("canonical_stops.id"), nullable=True, index=True) + canonical_stop_link_id: Mapped[Optional[int]] = mapped_column(ForeignKey("canonical_stop_links.id"), nullable=True, index=True) + decision_id: Mapped[Optional[int]] = mapped_column(ForeignKey("map_gtfs_decisions.id"), nullable=True, index=True) + evidence_hash: Mapped[Optional[str]] = mapped_column(String(128), nullable=True, index=True) + metadata_json: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now_utc, nullable=False, index=True) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=now_utc, nullable=False, index=True) + resolved_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True, index=True) + + source: Mapped[Optional[Source]] = relationship(foreign_keys=[source_id]) + dataset: Mapped[Optional[Dataset]] = relationship(foreign_keys=[dataset_id]) + gtfs_route: Mapped[Optional[GtfsRoute]] = relationship() + gtfs_stop: Mapped[Optional[GtfsStop]] = relationship() + osm_feature: Mapped[Optional[OsmFeature]] = relationship() + route_match: Mapped[Optional[RouteMatch]] = relationship() + canonical_stop: Mapped[Optional[CanonicalStop]] = relationship() + canonical_stop_link: Mapped[Optional[CanonicalStopLink]] = relationship() + decision: Mapped[Optional[MapGtfsDecision]] = relationship() + + class JourneySearchCache(Base): __tablename__ = "journey_search_cache" diff --git a/app/osm_storage.py b/app/osm_storage.py index a6c9928..5570ac2 100644 --- a/app/osm_storage.py +++ b/app/osm_storage.py @@ -293,7 +293,12 @@ def query_osm_features( dataset.id: dataset for dataset in session.scalars(select(Dataset).where(Dataset.id.in_([int(value) for value in dataset_ids]))).all() } - materialized_ids = _materialized_ids_by_identity(session, list(datasets)) if prefer_materialized_ids else {} + sidecar_dataset_ids = [ + dataset_id + for dataset_id, dataset in datasets.items() + if features_are_sidecar(dataset) + ] + materialized_ids = _materialized_ids_by_identity(session, sidecar_dataset_ids) if prefer_materialized_ids else {} rows: list[OsmFeature] = [] main_dataset_ids = [dataset_id for dataset_id, dataset in datasets.items() if not features_are_sidecar(dataset)] if main_dataset_ids: diff --git a/app/pipeline/gtfs.py b/app/pipeline/gtfs.py index 09f5979..e091911 100644 --- a/app/pipeline/gtfs.py +++ b/app/pipeline/gtfs.py @@ -15,6 +15,8 @@ from sqlalchemy import func, select, text from sqlalchemy.orm import Session from app.config import settings +from app.decision_replay import apply_gtfs_decision_replay, capture_gtfs_decision_replay +from app.gtfs_diff import create_gtfs_update_diff_from_stage from app.gtfs_storage import GTFS_STORAGE_MAIN, GTFS_STORAGE_METADATA_KEY, GTFS_STORAGE_SIDECAR_STOP_TIMES, effective_gtfs_timetable_storage from app.models import ( Dataset, @@ -64,6 +66,7 @@ GTFS_EXTENDED_MODE_RANGES = [ GTFS_IMPORTER_VERSION = "gtfs_import_v6_sidecar_stop_times" REQUIRED_FILES = {"agency.txt", "stops.txt", "routes.txt", "trips.txt", "stop_times.txt"} +GTFS_CALENDAR_WEEKDAY_COLUMNS = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"] GTFS_STAGE_BATCH_SIZE = 50_000 ProgressCallback = Callable[[str, str, int | None, int | None, dict[str, Any] | None], None] @@ -82,7 +85,7 @@ def run_gtfs_source(session: Session, source: Source, progress_callback: Progres ) .order_by(Dataset.id.desc()) ) - if existing is not None and _dataset_importer_version(existing) == GTFS_IMPORTER_VERSION: + if existing is not None and _dataset_import_can_reuse(existing): return existing return import_gtfs_zip(session=session, source=source, zip_path=local_path, source_hash=source_hash, progress_callback=progress_callback) @@ -395,15 +398,96 @@ def _activate_staged_gtfs( dataset = session.get(Dataset, dataset.id) or dataset source = session.get(Source, source.id) or source replaced_datasets = [existing for existing in list(source.datasets) if existing.id != dataset.id and existing.kind == "gtfs"] + previous_dataset = next( + ( + existing + for existing in sorted(replaced_datasets, key=lambda item: (not item.is_active, item.created_at, item.id)) + if existing.status == "imported" + ), + None, + ) for existing in source.datasets: if existing.id != dataset.id: existing.is_active = False copy_stop_times = _copy_stop_times_to_main(summary) heavy_index_drop = copy_stop_times and _should_drop_indexes_for_activation(stage_path) + diff_result: dict[str, Any] | None = None + decision_replay_plan: dict[str, Any] | None = None if heavy_index_drop: _emit_progress(progress_callback, "gtfs_activation_indexes_dropped", "Dropping heavy GTFS lookup indexes before bulk activation.", None, None, None) _drop_gtfs_bulk_indexes(session.connection()) try: + if previous_dataset is not None: + _emit_progress( + progress_callback, + "gtfs_update_diff_started", + f"Comparing staged GTFS dataset #{dataset.id} with previous dataset #{previous_dataset.id}.", + None, + None, + {"previous_dataset_id": previous_dataset.id, "new_dataset_id": dataset.id}, + ) + try: + diff_result = create_gtfs_update_diff_from_stage( + session, + source=source, + previous_dataset=previous_dataset, + new_dataset=dataset, + stage_path=stage_path, + ) + except Exception as exc: # noqa: BLE001 - update diff must not block activation + diff_result = None + summary["gtfs_update_diff_error"] = str(exc) + _emit_progress( + progress_callback, + "gtfs_update_diff_failed", + "GTFS update diff failed; continuing dataset activation.", + None, + None, + {"error": str(exc), "previous_dataset_id": previous_dataset.id, "new_dataset_id": dataset.id}, + ) + if diff_result is not None: + summary["gtfs_update_diff"] = { + "id": diff_result["id"], + "summary": diff_result["summary"], + } + _emit_progress( + progress_callback, + "gtfs_update_diff_completed", + "Stored GTFS update diff before replacing the previous dataset.", + None, + None, + diff_result, + ) + try: + decision_replay_plan = capture_gtfs_decision_replay( + session, + previous_dataset, + diff_run_id=None if diff_result is None else int(diff_result["id"]), + ) + summary["gtfs_decision_replay"] = { + "captured": len(decision_replay_plan.get("decisions", [])), + "diff_run_id": decision_replay_plan.get("diff_run_id"), + "previous_dataset_id": previous_dataset.id, + } + _emit_progress( + progress_callback, + "gtfs_decision_replay_captured", + "Captured existing GTFS-to-map decisions for update replay.", + None, + None, + summary["gtfs_decision_replay"], + ) + except Exception as exc: # noqa: BLE001 - replay capture must not block activation + decision_replay_plan = None + summary["gtfs_decision_replay_error"] = str(exc) + _emit_progress( + progress_callback, + "gtfs_decision_replay_capture_failed", + "GTFS decision replay capture failed; continuing dataset activation.", + None, + None, + {"error": str(exc), "previous_dataset_id": previous_dataset.id, "new_dataset_id": dataset.id}, + ) if replaced_datasets: _emit_progress( progress_callback, @@ -511,6 +595,33 @@ def _activate_staged_gtfs( ], progress_callback, ) + if decision_replay_plan is not None: + try: + replay_result = apply_gtfs_decision_replay( + session, + decision_replay_plan, + new_dataset=dataset, + diff_run_id=None if diff_result is None else int(diff_result["id"]), + ) + summary["gtfs_decision_replay"] = replay_result + _emit_progress( + progress_callback, + "gtfs_decision_replay_completed", + "Replayed unchanged GTFS-to-map decisions and queued changed decisions for review.", + None, + None, + replay_result, + ) + except Exception as exc: # noqa: BLE001 - replay must not block activation + summary["gtfs_decision_replay_error"] = str(exc) + _emit_progress( + progress_callback, + "gtfs_decision_replay_failed", + "GTFS decision replay failed; continuing dataset activation.", + None, + None, + {"error": str(exc), "new_dataset_id": dataset.id}, + ) finally: if heavy_index_drop: _emit_progress(progress_callback, "gtfs_activation_indexes_rebuilding", "Rebuilding GTFS lookup indexes after bulk activation.", None, None, None) @@ -557,10 +668,7 @@ def _copy_stage_table( rows = cursor.fetchmany(GTFS_STAGE_BATCH_SIZE) if not rows: break - payload = [ - {"dataset_id": dataset_id, **{column: row[index] for index, column in enumerate(columns)}} - for row in rows - ] + payload = [_copy_stage_row_payload(table, dataset_id, columns, row) for row in rows] session.execute(text(insert_sql), payload) copied += len(rows) _emit_progress( @@ -573,6 +681,15 @@ def _copy_stage_table( ) +def _copy_stage_row_payload(table: str, dataset_id: int, columns: list[str], row: sqlite3.Row | tuple[Any, ...]) -> dict[str, Any]: + payload = {"dataset_id": dataset_id, **{column: row[index] for index, column in enumerate(columns)}} + if table == "gtfs_calendars": + for column in GTFS_CALENDAR_WEEKDAY_COLUMNS: + if column in payload: + payload[column] = _bool_flag(payload[column]) + return payload + + def _should_drop_indexes_for_activation(stage_path: Path) -> bool: if settings.is_postgresql_database: return False @@ -1325,3 +1442,13 @@ def _dataset_importer_version(dataset: Dataset) -> str: return str(json.loads(dataset.metadata_json or "{}").get("importer") or "") except json.JSONDecodeError: return "" + + +def _dataset_import_can_reuse(dataset: Dataset) -> bool: + try: + metadata = json.loads(dataset.metadata_json or "{}") + except json.JSONDecodeError: + return False + if str(metadata.get("importer") or "") != GTFS_IMPORTER_VERSION: + return False + return int(metadata.get("stop_times_import_limit") or 0) == int(settings.gtfs_stop_times_import_limit or 0) diff --git a/app/pipeline/matcher.py b/app/pipeline/matcher.py index b9d1fe3..f50fa2a 100644 --- a/app/pipeline/matcher.py +++ b/app/pipeline/matcher.py @@ -2,13 +2,15 @@ from __future__ import annotations from dataclasses import dataclass from datetime import datetime, timezone +import hashlib import json -from typing import Callable, Optional +from typing import Callable, Optional, Sequence from shapely.geometry import LineString, MultiLineString, Point, shape from sqlalchemy import delete, select from sqlalchemy.orm import Session +from app.harmonization import active_harmonized_gtfs_shadowed_route_ids from app.config import settings from app.models import Dataset, GtfsRoute, MatchRule, OsmFeature, RouteMatch from app.osm_storage import ensure_main_osm_feature, osm_feature_bbox, query_osm_features @@ -34,7 +36,7 @@ MAX_EXACT_REF_CANDIDATES = 120 OSM_SCOPE_NEAR_DISTANCE_DEG = 0.15 GEOMETRY_PROXIMITY_DEG = 0.0035 GEOMETRY_SAMPLE_POINTS = 24 -MATCHER_VERSION = "matcher_v4_scope_spatial_manual_rules" +MATCHER_VERSION = "matcher_v5_spatial_candidate_gate" ProgressCallback = Callable[[str, str, int | None, int | None, dict[str, object] | None], None] @@ -78,6 +80,7 @@ def run_route_matching( *, progress_callback: ProgressCallback | None = None, batch_size: int | None = None, + gtfs_dataset_ids: Sequence[int] | None = None, ) -> dict[str, object]: """Match active GTFS routes against active OSM route features.""" active_datasets = session.execute( @@ -86,22 +89,37 @@ def run_route_matching( if not active_datasets: return {"routes": 0, "matches": 0, "missing": 0} dataset_source_ids = {int(dataset_id): int(source_id) for dataset_id, _, source_id in active_datasets} - gtfs_dataset_ids = [int(dataset_id) for dataset_id, kind, _ in active_datasets if kind == "gtfs"] + active_gtfs_dataset_ids = [int(dataset_id) for dataset_id, kind, _ in active_datasets if kind == "gtfs"] osm_dataset_ids = [int(dataset_id) for dataset_id, kind, _ in active_datasets if kind == "osm_geojson"] - if not gtfs_dataset_ids: + if gtfs_dataset_ids is None: + selected_gtfs_dataset_ids = active_gtfs_dataset_ids + else: + requested_gtfs_dataset_ids = {int(dataset_id) for dataset_id in gtfs_dataset_ids} + selected_gtfs_dataset_ids = [ + dataset_id + for dataset_id in active_gtfs_dataset_ids + if dataset_id in requested_gtfs_dataset_ids + ] + if not selected_gtfs_dataset_ids: return {"routes": 0, "matches": 0, "missing": 0} - route_row_ids = session.scalars( - select(GtfsRoute.id) - .where(GtfsRoute.dataset_id.in_(gtfs_dataset_ids)) - .order_by(GtfsRoute.dataset_id, GtfsRoute.route_id, GtfsRoute.id) - ).all() + all_route_row_ids = [ + int(route_id) + for route_id in session.scalars( + select(GtfsRoute.id) + .where(GtfsRoute.dataset_id.in_(selected_gtfs_dataset_ids)) + .order_by(GtfsRoute.dataset_id, GtfsRoute.route_id, GtfsRoute.id) + ).all() + ] + shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session) + route_row_ids = [route_id for route_id in all_route_row_ids if route_id not in shadowed_route_ids] + shadowed_skipped = len(all_route_row_ids) - len(route_row_ids) # Reconcile current match rows from auto scoring plus durable manual rules. total_routes = len(route_row_ids) if total_routes == 0: - return {"routes": 0, "matches": 0, "missing": 0} + return {"routes": 0, "matches": 0, "missing": 0, "shadowed_skipped": shadowed_skipped} - dependency = _route_matching_dependency(session, active_datasets) + dependency = _route_matching_dependency(session, active_datasets, shadowed_route_ids=shadowed_route_ids) run = start_pipeline_run( session, stage=STAGE_MATCH_ROUTES, @@ -117,11 +135,49 @@ def run_route_matching( f"Matching {total_routes} GTFS routes in batches of {effective_batch_size}.", 0, total_routes, - {"gtfs_datasets": gtfs_dataset_ids, "osm_datasets": osm_dataset_ids, "batch_size": effective_batch_size}, + { + "gtfs_datasets": selected_gtfs_dataset_ids, + "osm_datasets": osm_dataset_ids, + "batch_size": effective_batch_size, + "shadowed_skipped": shadowed_skipped, + }, ) manual_rules = _manual_match_rules(session) osm_scope_bbox = osm_feature_bbox(session, osm_dataset_ids, kinds=["route"]) - counts = {"routes": total_routes, "matches": 0, "missing": 0, "manual": 0, "created": 0, "updated": 0, "unchanged": 0} + _emit_progress( + progress_callback, + "route_matching_osm_index_started", + "Loading OSM route candidates for matcher.", + 0, + None, + {"osm_datasets": osm_dataset_ids}, + ) + osm_routes = query_osm_features( + session, + osm_dataset_ids, + kinds=["route"], + prefer_materialized_ids=False, + ) + osm_route_index = _build_osm_route_index(osm_routes) + feature_profile_cache: dict[tuple[int, str, str], _GeometryProfile | None] = {} + _emit_progress( + progress_callback, + "route_matching_osm_index_completed", + f"Loaded {len(osm_routes)} OSM route candidates.", + len(osm_routes), + None, + {"osm_routes": len(osm_routes)}, + ) + counts = { + "routes": total_routes, + "matches": 0, + "missing": 0, + "manual": 0, + "created": 0, + "updated": 0, + "unchanged": 0, + "shadowed_skipped": shadowed_skipped, + } scoped_counts = {"in_osm_scope": 0, "near_osm_scope": 0, "outside_osm_scope": 0, "unknown_scope": 0} processed = 0 for chunk in _chunks_int(route_row_ids, effective_batch_size): @@ -137,6 +193,8 @@ def run_route_matching( dataset_source_ids=dataset_source_ids, manual_rules=manual_rules, osm_scope_bbox=osm_scope_bbox, + osm_route_index=osm_route_index, + feature_profile_cache=feature_profile_cache, scoped_counts=scoped_counts, ) counts["matches"] += batch_counts["matches"] @@ -178,7 +236,12 @@ def run_route_matching( return result -def _route_matching_dependency(session: Session, active_datasets) -> dict[str, object]: +def _route_matching_dependency( + session: Session, + active_datasets, + *, + shadowed_route_ids: set[int] | None = None, +) -> dict[str, object]: datasets = [ {"id": int(dataset_id), "kind": str(kind), "source_id": int(source_id), "sha256": _dataset_sha(session, int(dataset_id))} for dataset_id, kind, source_id in active_datasets @@ -193,7 +256,15 @@ def _route_matching_dependency(session: Session, active_datasets) -> dict[str, o } for rule in session.scalars(select(MatchRule).order_by(MatchRule.id)).all() ] - return {"version": MATCHER_VERSION, "active_datasets": datasets, "manual_rules": rules} + return { + "version": MATCHER_VERSION, + "active_datasets": datasets, + "manual_rules": rules, + "shadowed_routes": { + "count": len(shadowed_route_ids or set()), + "digest": _int_set_digest(shadowed_route_ids or set()), + }, + } def _dataset_sha(session: Session, dataset_id: int) -> str | None: @@ -201,6 +272,13 @@ def _dataset_sha(session: Session, dataset_id: int) -> str | None: return None if dataset is None else dataset.sha256 +def _int_set_digest(values: set[int]) -> str: + if not values: + return "" + payload = ",".join(str(value) for value in sorted(values)) + return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32] + + def _match_route_batch( *, session: Session, @@ -209,6 +287,8 @@ def _match_route_batch( dataset_source_ids: dict[int, int], manual_rules: list[_ManualMatchRule], osm_scope_bbox: tuple[float | None, float | None, float | None, float | None], + osm_route_index: _OsmRouteIndex, + feature_profile_cache: dict[tuple[int, str, str], _GeometryProfile | None], scoped_counts: dict[str, int], ) -> dict[str, int]: matches = 0 @@ -265,10 +345,13 @@ def _match_route_batch( best_score = 0.0 best_reasons: dict[str, object] = {} route_geometry_profile = _geometry_profile(route.geometry_geojson) - for feature in candidate_osm_routes_for_route(session, route, osm_dataset_ids): + for feature in _candidate_osm_routes(route, osm_route_index): if _is_rejected_pair(manual_rules, route, route_source_id, feature, dataset_source_ids.get(feature.dataset_id)): continue - feature_geometry_profile = _geometry_profile(feature.geometry_geojson) + feature_key = _feature_identity_key(feature) + if feature_key not in feature_profile_cache: + feature_profile_cache[feature_key] = _geometry_profile(feature.geometry_geojson) + feature_geometry_profile = feature_profile_cache[feature_key] score, reasons = score_route_pair( route, feature, @@ -393,22 +476,26 @@ def _build_osm_route_index(osm_routes: list[OsmFeature]) -> _OsmRouteIndex: def _candidate_osm_routes(route: GtfsRoute, index: _OsmRouteIndex) -> list[OsmFeature]: selected: list[OsmFeature] = [] - seen: set[int] = set() + seen: set[tuple[int, str, str]] = set() - def add(features: list[OsmFeature], *, require_compatible_mode: bool = True) -> None: + def add(features: list[OsmFeature], *, require_compatible_mode: bool = True, require_spatial: bool = False) -> None: for feature in features: - if feature.id in seen: + key = (feature.dataset_id, feature.osm_type, feature.osm_id) + if key in seen: continue if require_compatible_mode and not _mode_compatible(route.mode or "", feature.mode or ""): continue - seen.add(feature.id) + if require_spatial and not _candidate_is_spatially_plausible(route, feature): + continue + seen.add(key) selected.append(feature) route_ref = norm_ref(route.short_name or route.route_id) + route_has_bbox = _has_known_bbox((route.min_lon, route.min_lat, route.max_lon, route.max_lat)) if route_ref: - add(index.by_ref.get(route_ref, [])) + add(index.by_ref.get(route_ref, []), require_spatial=route_has_bbox) if route.route_key: - add(index.by_route_key.get(route.route_key, [])) + add(index.by_route_key.get(route.route_key, []), require_spatial=route_has_bbox) if selected: return _spatially_ranked_candidates(route, selected, MAX_EXACT_REF_CANDIDATES) @@ -431,7 +518,7 @@ def _candidate_osm_routes(route: GtfsRoute, index: _OsmRouteIndex) -> list[OsmFe near_candidates.append((distance, feature)) fallback_limit = MAX_FALLBACK_CANDIDATES_WITH_REF if route_ref else MAX_FALLBACK_CANDIDATES_WITHOUT_REF fallback = [feature for _, feature in sorted(near_candidates, key=lambda item: item[0])[:fallback_limit]] - if not fallback: + if not fallback and not route_has_bbox: fallback = mode_candidates[:fallback_limit] add(fallback) return _spatially_ranked_candidates(route, selected, fallback_limit) @@ -443,17 +530,22 @@ def candidate_osm_routes_for_route(session: Session, route: GtfsRoute, osm_datas selected: list[OsmFeature] = [] seen: set[tuple[int, str, str]] = set() - def add(features: list[OsmFeature], *, require_compatible_mode: bool = True) -> None: + def add(features: list[OsmFeature], *, require_compatible_mode: bool = True, require_spatial: bool = False) -> None: for feature in features: key = (feature.dataset_id, feature.osm_type, feature.osm_id) if key in seen: continue if require_compatible_mode and not _mode_compatible(route.mode or "", feature.mode or ""): continue + if require_spatial and not _candidate_is_spatially_plausible(route, feature): + continue seen.add(key) selected.append(feature) route_ref = norm_ref(route.short_name or route.route_id) + gtfs_bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat) + route_has_bbox = _has_known_bbox(gtfs_bbox) + exact_ref_bbox = _expanded_bbox(gtfs_bbox, OSM_SCOPE_NEAR_DISTANCE_DEG) if route_has_bbox else None route_keys = [key for key in [route.route_key, route_ref] if key] for route_key in dict.fromkeys(route_keys): add( @@ -462,14 +554,15 @@ def candidate_osm_routes_for_route(session: Session, route: GtfsRoute, osm_datas osm_dataset_ids, kinds=["route"], route_key=route_key, - ) + bbox=exact_ref_bbox, + ), + require_spatial=route_has_bbox, ) if selected: return _spatially_ranked_candidates(route, selected, MAX_EXACT_REF_CANDIDATES) - gtfs_bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat) compatible_modes = sorted(MODE_GROUPS.get(route.mode or "", {route.mode or ""}) - {""}) - if not any(value is None for value in gtfs_bbox): + if route_has_bbox: bbox = _expanded_bbox(gtfs_bbox, 0.10) add( query_osm_features( @@ -482,7 +575,7 @@ def candidate_osm_routes_for_route(session: Session, route: GtfsRoute, osm_datas ), require_compatible_mode=False, ) - if not selected: + if not selected and not route_has_bbox: add( query_osm_features( session, @@ -634,6 +727,27 @@ def route_match_scope(route: GtfsRoute, osm_scope_bbox: tuple[float | None, floa return "outside_osm_scope" +def _candidate_is_spatially_plausible(route: GtfsRoute, feature: OsmFeature) -> bool: + route_bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat) + feature_bbox = (feature.min_lon, feature.min_lat, feature.max_lon, feature.max_lat) + if not _has_known_bbox(route_bbox): + return True + if not _has_known_bbox(feature_bbox): + return False + if bbox_overlap(route_bbox, feature_bbox): + return True + distance = approx_bbox_center_distance_deg(route_bbox, feature_bbox) + return distance is not None and distance < OSM_SCOPE_NEAR_DISTANCE_DEG + + +def _has_known_bbox(bbox: tuple[float | None, float | None, float | None, float | None]) -> bool: + return not any(value is None for value in bbox) + + +def _feature_identity_key(feature: OsmFeature) -> tuple[int, str, str]: + return (int(feature.dataset_id), str(feature.osm_type), str(feature.osm_id)) + + def _combined_bbox(features: list[OsmFeature]) -> tuple[float | None, float | None, float | None, float | None]: boxes = [ (feature.min_lon, feature.min_lat, feature.max_lon, feature.max_lat) diff --git a/app/pipeline/route_layer.py b/app/pipeline/route_layer.py index 7e3b3d9..aa16d78 100644 --- a/app/pipeline/route_layer.py +++ b/app/pipeline/route_layer.py @@ -11,6 +11,7 @@ from sqlalchemy.orm import Session from app.config import settings from app.gtfs_storage import all_scheduled_stop_ids, stop_times_by_trip as storage_stop_times_by_trip +from app.harmonization import active_harmonized_gtfs_shadowed_route_ids from app.models import ( CanonicalStop, CanonicalStopLink, @@ -36,7 +37,7 @@ from app.pipeline.utils import bbox_overlap, geometry_json_and_bbox, norm_ref, n from app.spatial import analyze_postgresql_tables, refresh_postgis_geometries, using_postgresql -ROUTE_LAYER_VERSION = "route_layer_v3_stop_alias_matching" +ROUTE_LAYER_VERSION = "route_layer_v4_harmonized_route_ownership" GTFS_ROUTE_PATTERN_NULL_SHAPE = "__route__" OSM_STOP_LINK_RADIUS_DEG = 0.0018 OSM_STOP_NAME_LINK_RADIUS_DEG = 0.0032 @@ -144,6 +145,7 @@ def rebuild_route_layer( "route_patterns_updated": pattern_result.get("route_patterns_updated", 0), "route_patterns_reused": pattern_result.get("route_patterns_reused", 0), "route_patterns_removed": pattern_result.get("route_patterns_removed", 0), + "shadowed_routes_skipped": pattern_result.get("shadowed_routes_skipped", 0), "route_pattern_links": pattern_result["route_pattern_links"], "trip_pattern_links": pattern_result["trip_pattern_links"], "route_pattern_stops": pattern_result["route_pattern_stops"], @@ -161,6 +163,7 @@ def _route_layer_dependency(session: Session) -> dict[str, object]: for dataset in session.scalars(select(Dataset).where(Dataset.is_active.is_(True)).order_by(Dataset.kind, Dataset.id)).all() ] match_rows = session.execute(select(RouteMatch.id, RouteMatch.gtfs_route_id, RouteMatch.osm_feature_id, RouteMatch.status, RouteMatch.updated_at).order_by(RouteMatch.id)).all() + shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session) match_signature = dependency_hash( [ [ @@ -177,6 +180,10 @@ def _route_layer_dependency(session: Session) -> dict[str, object]: "version": ROUTE_LAYER_VERSION, "active_datasets": active_datasets, "route_matches": {"count": len(match_rows), "signature": match_signature}, + "shadowed_routes": { + "count": len(shadowed_route_ids), + "signature": dependency_hash(sorted(shadowed_route_ids)) if shadowed_route_ids else "", + }, } @@ -945,14 +952,15 @@ def _build_route_patterns( ) -> dict[str, int]: osm_candidates = _osm_route_candidates(session, progress_callback=progress_callback) overrides = _route_layer_overrides(session) - seeds = _gtfs_pattern_seeds(session) + shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session) + seeds, shadowed_routes_skipped = _gtfs_pattern_seeds(session, shadowed_route_ids=shadowed_route_ids) _emit_progress( progress_callback, "route_layer_pattern_seeds", f"Loaded {len(seeds)} GTFS route-pattern seeds.", 0, len(seeds), - {"seeds": len(seeds)}, + {"seeds": len(seeds), "shadowed_routes_skipped": shadowed_routes_skipped}, ) link_count = 0 stop_count = 0 @@ -1256,6 +1264,7 @@ def _build_route_patterns( "trip_pattern_links": trip_link_count, "route_pattern_stops": stop_count, "gtfs_proposed_patterns": proposed_count, + "shadowed_routes_skipped": shadowed_routes_skipped, } _emit_progress( progress_callback, @@ -1494,10 +1503,11 @@ def _route_layer_overrides(session: Session) -> _RouteLayerOverrides: return _RouteLayerOverrides(accepted_by_gtfs_route_id=accepted, rejected_by_gtfs_route_id=rejected) -def _gtfs_pattern_seeds(session: Session) -> list[_GtfsPatternSeed]: +def _gtfs_pattern_seeds(session: Session, *, shadowed_route_ids: set[int] | None = None) -> tuple[list[_GtfsPatternSeed], int]: active_gtfs_dataset_ids = _active_dataset_ids(session, "gtfs") if not active_gtfs_dataset_ids: - return [] + return [], 0 + shadowed_route_ids = shadowed_route_ids or set() rows = session.execute( select(GtfsRoute, GtfsTrip.shape_id, func.min(GtfsTrip.trip_id)) .join(GtfsTrip, and_(GtfsTrip.dataset_id == GtfsRoute.dataset_id, GtfsTrip.route_id == GtfsRoute.route_id)) @@ -1525,7 +1535,11 @@ def _gtfs_pattern_seeds(session: Session) -> list[_GtfsPatternSeed]: for dataset_id, shape_id, geometry, min_lon, min_lat, max_lon, max_lat in shape_rows } seeds = [] + skipped = 0 for route, shape_id, trip_id in rows: + if int(route.id) in shadowed_route_ids: + skipped += 1 + continue geometry_text = None geometry_source = "none" bbox = (route.min_lon, route.min_lat, route.max_lon, route.max_lat) @@ -1556,7 +1570,7 @@ def _gtfs_pattern_seeds(session: Session) -> list[_GtfsPatternSeed]: center_point=center_point, ) ) - return seeds + return seeds, skipped def _choose_osm_candidate( diff --git a/app/pipeline/sample_data.py b/app/pipeline/sample_data.py index 2f8dbe4..756f06d 100644 --- a/app/pipeline/sample_data.py +++ b/app/pipeline/sample_data.py @@ -19,6 +19,11 @@ from app.models import ( GtfsAgency, GtfsCalendar, GtfsCalendarDate, + GtfsFeedDiffItem, + GtfsFeedDiffRun, + GtfsHarmonizedSnapshot, + GtfsHarmonizedSnapshotDataset, + GtfsHarmonizedSnapshotRoute, GtfsRoute, GtfsRoutePatternLink, GtfsShape, @@ -30,6 +35,8 @@ from app.models import ( ItineraryLeg, Job, JobEvent, + MapGtfsDecision, + MapGtfsReviewItem, MatchRule, OsmDiffState, OsmFeature, @@ -96,8 +103,15 @@ def clear_project_data( ItineraryLeg, Itinerary, TravelRequest, + GtfsFeedDiffItem, + GtfsFeedDiffRun, + GtfsHarmonizedSnapshotRoute, + GtfsHarmonizedSnapshotDataset, + GtfsHarmonizedSnapshot, SourceUpdateCheck, OsmDiffState, + MapGtfsReviewItem, + MapGtfsDecision, MatchRule, RouteMatch, GtfsTripRoutePatternLink, diff --git a/app/routing.py b/app/routing.py index 331a9da..cad5c87 100644 --- a/app/routing.py +++ b/app/routing.py @@ -20,12 +20,18 @@ from app.serializers import feature_collection WALK_HEURISTIC_MPS = 1.6 +BIKE_HEURISTIC_MPS = 6.0 DRIVE_HEURISTIC_MPS = 36.0 +BIKE_SPEED_MPS = 4.8 DEFAULT_MAX_VISITED = 160_000 PGR_WALK_BBOX_PADDING_KM = [0.5, 1.5, 4, 10, 25] +PGR_BIKE_BBOX_PADDING_KM = [1, 3, 8, 20, 60] PGR_DRIVE_BBOX_PADDING_KM = [2, 8, 25, 75, 200] PGR_WALK_STATEMENT_TIMEOUT_MS = 2_500 +PGR_BIKE_STATEMENT_TIMEOUT_MS = 4_000 PGR_DRIVE_STATEMENT_TIMEOUT_MS = 7_500 +ROUTING_MODES = {"walk", "bike", "drive"} +ROUTING_MODE_ALIASES = {"bicycle": "bike", "cycle": "bike", "cycling": "bike", "car": "drive"} ROUTE_CACHE_TTL_SECONDS = 15 * 60 ROUTE_CACHE_MAX_ENTRIES = 512 _route_cache_lock = threading.RLock() @@ -123,6 +129,43 @@ def _metadata(dataset: Dataset) -> dict[str, object]: return value if isinstance(value, dict) else {} +def _normalize_routing_mode(mode: str) -> str: + normalized = ROUTING_MODE_ALIASES.get(str(mode or "").lower(), str(mode or "").lower()) + if normalized not in ROUTING_MODES: + raise ValueError("mode must be walk, bike, or drive") + return normalized + + +def _cost_columns(mode: str) -> tuple[str, str]: + if mode == "drive": + return "drive_cost_s", "reverse_drive_cost_s" + return "walk_cost_s", "reverse_walk_cost_s" + + +def _heuristic_mps(mode: str) -> float: + if mode == "walk": + return WALK_HEURISTIC_MPS + if mode == "bike": + return BIKE_HEURISTIC_MPS + return DRIVE_HEURISTIC_MPS + + +def _pgrouting_padding_km(mode: str) -> list[float]: + if mode == "walk": + return PGR_WALK_BBOX_PADDING_KM + if mode == "bike": + return PGR_BIKE_BBOX_PADDING_KM + return PGR_DRIVE_BBOX_PADDING_KM + + +def _pgrouting_timeout_ms(mode: str) -> int: + if mode == "walk": + return PGR_WALK_STATEMENT_TIMEOUT_MS + if mode == "bike": + return PGR_BIKE_STATEMENT_TIMEOUT_MS + return PGR_DRIVE_STATEMENT_TIMEOUT_MS + + def route_between_points( db: Session, *, @@ -133,9 +176,11 @@ def route_between_points( mode: str = "walk", dataset_id: int | None = None, max_visited: int = DEFAULT_MAX_VISITED, + allow_python_fallback: bool = True, + pgrouting_timeout_ms: int | None = None, + pgrouting_padding_km: tuple[float, ...] | list[float] | None = None, ) -> dict[str, object]: - if mode not in {"walk", "drive"}: - raise ValueError("mode must be walk or drive") + mode = _normalize_routing_mode(mode) dataset = db.get(Dataset, dataset_id) if dataset_id is not None else active_routing_dataset(db) if dataset is None: raise ValueError("No routing dataset is available.") @@ -152,7 +197,8 @@ def route_between_points( payload = _single_point_route(start, from_lon, from_lat, to_lon, to_lat, mode, dataset_id) _route_cache_put(cache_key, payload) return payload - if settings.is_postgresql_database and _pgrouting_installed(db): + pgrouting_available = settings.is_postgresql_database and _pgrouting_installed(db) + if pgrouting_available: try: payload = _route_with_pgrouting( db, @@ -164,15 +210,23 @@ def route_between_points( from_lat=from_lat, to_lon=to_lon, to_lat=to_lat, + timeout_ms=pgrouting_timeout_ms, + padding_km=pgrouting_padding_km, ) _route_cache_put(cache_key, payload) return payload except ValueError: - pass + if not allow_python_fallback: + raise except SQLAlchemyError: db.rollback() + if not allow_python_fallback: + raise - heuristic_mps = WALK_HEURISTIC_MPS if mode == "walk" else DRIVE_HEURISTIC_MPS + if not allow_python_fallback: + raise ValueError("pgRouting did not find a bounded route and Python fallback is disabled.") + + heuristic_mps = _heuristic_mps(mode) queue: list[tuple[float, float, int]] = [] heapq.heappush(queue, (0.0, 0.0, start.osm_node_id)) costs: dict[int, float] = {start.osm_node_id: 0.0} @@ -226,8 +280,7 @@ def direct_route_between_points( dataset_id: int | None = None, reason: str | None = None, ) -> dict[str, object]: - if mode not in {"walk", "drive"}: - raise ValueError("mode must be walk or drive") + mode = _normalize_routing_mode(mode) dataset = db.get(Dataset, dataset_id) if dataset_id is not None else active_routing_dataset(db) payload = _direct_route_payload( dataset_id=0 if dataset is None else int(dataset.id), @@ -251,8 +304,7 @@ def snap_point_to_routing_graph( dataset_id: int | None = None, max_distance_m: float = 250, ) -> dict[str, object] | None: - if mode not in {"walk", "drive"}: - raise ValueError("mode must be walk or drive") + mode = _normalize_routing_mode(mode) dataset = db.get(Dataset, dataset_id) if dataset_id is not None else active_routing_dataset(db) if dataset is None: return None @@ -288,8 +340,8 @@ def _snap_point_to_routing_edge_postgresql( mode: str, max_distance_m: float, ) -> dict[str, object] | None: - cost_column = "walk_cost_s" if mode == "walk" else "drive_cost_s" - reverse_cost_column = "reverse_walk_cost_s" if mode == "walk" else "reverse_drive_cost_s" + cost_column, reverse_cost_column = _cost_columns(mode) + access_filter = f"AND {_bike_access_sql('edge')}" if mode == "bike" else "" radius_deg = max_distance_m / 111_320 row = db.execute( text( @@ -319,6 +371,7 @@ def _snap_point_to_routing_edge_postgresql( CROSS JOIN point WHERE edge.dataset_id = :dataset_id AND (edge.{cost_column} IS NOT NULL OR edge.{reverse_cost_column} IS NOT NULL) + {access_filter} AND box(point(edge.max_lon, edge.max_lat), point(edge.min_lon, edge.min_lat)) && box( point(:lon + :radius_deg, :lat + :radius_deg), @@ -337,13 +390,16 @@ def _snap_point_to_routing_edge_postgresql( ST_DistanceSphere(edges.edge_geom, point.geom) AS distance_m, CASE WHEN edges.highway IN ('footway', 'pedestrian', 'steps') THEN 0 - WHEN edges.highway IN ('path', 'cycleway', 'bridleway') THEN 1 + WHEN edges.highway = 'cycleway' THEN 0 + WHEN edges.highway IN ('path', 'bridleway') THEN 1 WHEN edges.highway IN ('living_street', 'residential') THEN 2 WHEN edges.highway = 'service' THEN 3 ELSE 4 END AS highway_rank, CASE - WHEN :mode != 'walk' THEN 0 + WHEN :mode NOT IN ('walk', 'bike') THEN 0 + WHEN :mode = 'bike' AND edges.highway = 'cycleway' THEN -12 + WHEN :mode = 'bike' AND edges.highway IN ('path', 'living_street', 'residential') THEN -4 WHEN edges.highway = 'service' THEN 20 WHEN edges.highway IN ('primary', 'primary_link', 'secondary', 'secondary_link') THEN 10 WHEN edges.highway IN ('tertiary', 'tertiary_link', 'unclassified', 'road') THEN 5 @@ -359,7 +415,9 @@ def _snap_point_to_routing_edge_postgresql( ) ORDER BY ST_DistanceSphere(edges.edge_geom, point.geom) + CASE - WHEN :mode != 'walk' THEN 0 + WHEN :mode NOT IN ('walk', 'bike') THEN 0 + WHEN :mode = 'bike' AND edges.highway = 'cycleway' THEN -12 + WHEN :mode = 'bike' AND edges.highway IN ('path', 'living_street', 'residential') THEN -4 WHEN edges.highway = 'service' THEN 20 WHEN edges.highway IN ('primary', 'primary_link', 'secondary', 'secondary_link') THEN 10 WHEN edges.highway IN ('tertiary', 'tertiary_link', 'unclassified', 'road') THEN 5 @@ -455,15 +513,26 @@ def _route_with_pgrouting( from_lat: float, to_lon: float, to_lat: float, + timeout_ms: int | None = None, + padding_km: tuple[float, ...] | list[float] | None = None, ) -> dict[str, object]: - cost_column = "walk_cost_s" if mode == "walk" else "drive_cost_s" - reverse_cost_column = "reverse_walk_cost_s" if mode == "walk" else "reverse_drive_cost_s" + cost_column, reverse_cost_column = _cost_columns(mode) routing_cost = _routing_cost_expression(cost_column, mode) - reverse_routing_cost = _routing_cost_expression(reverse_cost_column, mode) - for padding_km in PGR_WALK_BBOX_PADDING_KM if mode == "walk" else PGR_DRIVE_BBOX_PADDING_KM: + reverse_routing_cost = _routing_cost_expression(reverse_cost_column, mode, reverse=True) + actual_cost_sql = "steps.cost" if mode == "bike" else f""" + CASE + WHEN steps.from_node = edge.source_osm_node_id THEN edge.{cost_column} + ELSE edge.{reverse_cost_column} + END + """ + paddings = list(padding_km or _pgrouting_padding_km(mode)) + statement_timeout_ms = int( + timeout_ms if timeout_ms is not None else _pgrouting_timeout_ms(mode) + ) + for padding_km in paddings: _set_local_statement_timeout( db, - PGR_WALK_STATEMENT_TIMEOUT_MS if mode == "walk" else PGR_DRIVE_STATEMENT_TIMEOUT_MS, + statement_timeout_ms, ) bbox = _expanded_bbox( min(from_lon, to_lon, start.lon, target.lon), @@ -517,10 +586,7 @@ def _route_with_pgrouting( edge.highway, edge.name, edge.geometry_geojson, - CASE - WHEN steps.from_node = edge.source_osm_node_id THEN edge.{cost_column} - ELSE edge.{reverse_cost_column} - END AS actual_cost_s + {actual_cost_sql} AS actual_cost_s FROM steps JOIN routing_edges AS edge ON edge.id = steps.edge WHERE steps.edge <> -1 @@ -616,33 +682,109 @@ def _pgrouting_payload( return payload -def _routing_cost_expression(column: str, mode: str) -> str: - if mode != "walk": +def _routing_cost_expression(column: str, mode: str, *, reverse: bool = False) -> str: + if mode == "drive": return column + if mode == "bike": + direction_filter = f"WHEN {_bike_direction_forbidden_sql(reverse=reverse)} THEN NULL" + return f""" + CASE + WHEN {column} IS NULL THEN NULL + WHEN NOT ({_bike_access_sql()}) THEN NULL + {direction_filter} + ELSE (length_m / {BIKE_SPEED_MPS:.4f}) * CASE + WHEN highway = 'cycleway' THEN 0.82 + WHEN highway = 'path' THEN 0.95 + WHEN highway IN ('living_street', 'residential') THEN 1.00 + WHEN highway = 'service' THEN 1.08 + WHEN highway = 'track' THEN 1.10 + WHEN highway IN ('unclassified', 'road', 'tertiary', 'tertiary_link') THEN 1.12 + WHEN highway IN ('secondary', 'secondary_link') THEN 1.22 + WHEN highway IN ('primary', 'primary_link') THEN 1.35 + WHEN highway IN ('footway', 'pedestrian', 'bridleway') THEN 1.40 + ELSE 1.05 + END + END + """ return f""" CASE WHEN {column} IS NULL THEN NULL ELSE {column} * CASE - WHEN highway IN ('footway', 'pedestrian') THEN 0.70 - WHEN highway = 'path' THEN 0.78 - WHEN highway = 'steps' THEN 0.95 - WHEN highway = 'cycleway' THEN 1.05 - WHEN highway = 'bridleway' THEN 1.10 - WHEN highway IN ('living_street', 'track') THEN 1.15 - WHEN highway IN ('residential', 'service') THEN 1.35 - WHEN highway IN ('unclassified', 'road') THEN 1.55 - WHEN highway IN ('tertiary', 'tertiary_link') THEN 1.80 - WHEN highway IN ('secondary', 'secondary_link') THEN 2.15 - WHEN highway IN ('primary', 'primary_link') THEN 2.50 - ELSE 1.30 + WHEN highway IN ('footway', 'pedestrian') THEN 0.92 + WHEN highway = 'path' THEN 0.94 + WHEN highway = 'steps' THEN 1.05 + WHEN highway = 'cycleway' THEN 1.02 + WHEN highway = 'bridleway' THEN 1.05 + WHEN highway = 'living_street' THEN 1.00 + WHEN highway IN ('residential', 'service') THEN 1.05 + WHEN highway = 'track' THEN 1.10 + WHEN highway IN ('unclassified', 'road') THEN 1.12 + WHEN highway IN ('tertiary', 'tertiary_link') THEN 1.18 + WHEN highway IN ('secondary', 'secondary_link') THEN 1.28 + WHEN highway IN ('primary', 'primary_link') THEN 1.40 + ELSE 1.08 END END """ +def _routing_tags_sql(alias: str | None = None) -> str: + prefix = f"{alias}." if alias else "" + return f"COALESCE(NULLIF({prefix}tags_json, '')::jsonb, '{{}}'::jsonb)" + + +def _bike_access_sql(alias: str | None = None) -> str: + tags = _routing_tags_sql(alias) + highway = f"{alias}.highway" if alias else "highway" + return f""" + ( + {highway} NOT IN ('motorway', 'motorway_link', 'steps', 'platform') + AND NOT ( + COALESCE({tags} ->> 'bicycle', '') IN ('no', 'private') + OR ( + COALESCE({tags} ->> 'access', '') IN ('no', 'private') + AND COALESCE({tags} ->> 'bicycle', '') NOT IN ('yes', 'designated', 'permissive', 'destination') + AND COALESCE({tags} ->> 'vehicle', '') NOT IN ('yes', 'designated', 'permissive', 'destination') + ) + ) + AND NOT ( + {highway} IN ('footway', 'pedestrian', 'bridleway') + AND COALESCE({tags} ->> 'bicycle', '') NOT IN ('yes', 'designated', 'permissive', 'destination') + ) + AND NOT ( + {highway} IN ('trunk', 'trunk_link') + AND COALESCE({tags} ->> 'bicycle', '') NOT IN ('yes', 'designated', 'permissive', 'destination') + ) + ) + """ + + +def _bike_direction_forbidden_sql(alias: str | None = None, *, reverse: bool) -> str: + tags = _routing_tags_sql(alias) + highway = f"{alias}.highway" if alias else "highway" + bicycle_direction_key = "bicycle:backward" if reverse else "bicycle:forward" + one_way_condition = ( + f"COALESCE({tags} ->> 'oneway', '') IN ('yes', 'true', '1') " + f"OR COALESCE({tags} ->> 'junction', '') = 'roundabout' " + f"OR {highway} = 'motorway'" + if reverse + else f"COALESCE({tags} ->> 'oneway', '') = '-1'" + ) + return f""" + ( + ( + {one_way_condition} + ) + AND COALESCE({tags} ->> 'oneway:bicycle', '') != 'no' + AND COALESCE({tags} ->> 'bicycle:oneway', '') != 'no' + AND COALESCE({tags} ->> '{bicycle_direction_key}', '') NOT IN ('yes', 'designated', 'permissive') + ) + """ + + def _nearest_node(db: Session, dataset_id: int, lon: float, lat: float, mode: str) -> _GraphNode | None: - cost_column = "walk_cost_s" if mode == "walk" else "drive_cost_s" - reverse_cost_column = "reverse_walk_cost_s" if mode == "walk" else "reverse_drive_cost_s" + cost_column, reverse_cost_column = _cost_columns(mode) + access_filter = f"AND {_bike_access_sql('edge')}" if mode == "bike" else "" row = None for candidate_limit in (64, 512, 4096): row = db.execute( @@ -663,6 +805,7 @@ def _nearest_node(db: Session, dataset_id: int, lon: float, lat: float, mode: st SELECT 1 FROM routing_edges AS edge WHERE edge.dataset_id = :dataset_id + {access_filter} AND ( (edge.source_osm_node_id = nearest.osm_node_id AND edge.{cost_column} IS NOT NULL) OR (edge.target_osm_node_id = nearest.osm_node_id AND edge.{reverse_cost_column} IS NOT NULL) @@ -686,8 +829,7 @@ def _nearest_node(db: Session, dataset_id: int, lon: float, lat: float, mode: st def _outgoing_edges(db: Session, dataset_id: int, node_id: int, mode: str) -> list[_Traversal]: - cost_column = "walk_cost_s" if mode == "walk" else "drive_cost_s" - reverse_cost_column = "reverse_walk_cost_s" if mode == "walk" else "reverse_drive_cost_s" + cost_column, reverse_cost_column = _cost_columns(mode) rows = db.execute( text( f""" @@ -695,6 +837,7 @@ def _outgoing_edges(db: Session, dataset_id: int, node_id: int, mode: str) -> li id, source_osm_node_id, target_osm_node_id, source_lon, source_lat, target_lon, target_lat, length_m, highway, name, geometry_geojson, + tags_json, CASE WHEN source_osm_node_id = :node_id THEN {cost_column} ELSE {reverse_cost_column} @@ -713,6 +856,14 @@ def _outgoing_edges(db: Session, dataset_id: int, node_id: int, mode: str) -> li edges = [] for row in rows: forward = bool(row.forward) + cost_s = float(row.cost_s) + if mode == "bike": + tags = _json_object(str(row.tags_json or "")) + if not _bike_access_python(row.highway, tags): + continue + if _bike_direction_forbidden_python(row.highway, tags, reverse=not forward): + continue + cost_s = _bike_edge_seconds(float(row.length_m), row.highway, tags) if forward: to_node = int(row.target_osm_node_id) from_lon, from_lat = float(row.source_lon), float(row.source_lat) @@ -730,7 +881,7 @@ def _outgoing_edges(db: Session, dataset_id: int, node_id: int, mode: str) -> li from_lat=from_lat, to_lon=to_lon, to_lat=to_lat, - cost_s=float(row.cost_s), + cost_s=cost_s, length_m=float(row.length_m), highway=row.highway, name=row.name, @@ -869,10 +1020,82 @@ def _connector_feature(kind: str, mode: str, coordinates: list[list[float]], dis def _connector_seconds(distance_m: float, mode: str) -> float: - speed = 1.35 if mode == "walk" else 8.0 + speed = 1.35 if mode == "walk" else BIKE_SPEED_MPS if mode == "bike" else 8.0 return float(distance_m) / speed +def _bike_edge_seconds(length_m: float, highway: str | None, tags: dict[str, object]) -> float: + factor = { + "cycleway": 0.82, + "path": 0.95, + "living_street": 1.0, + "residential": 1.0, + "service": 1.08, + "track": 1.10, + "unclassified": 1.12, + "road": 1.12, + "tertiary": 1.12, + "tertiary_link": 1.12, + "secondary": 1.22, + "secondary_link": 1.22, + "primary": 1.35, + "primary_link": 1.35, + "footway": 1.40, + "pedestrian": 1.40, + "bridleway": 1.40, + }.get(str(highway or ""), 1.05) + return float(length_m) / BIKE_SPEED_MPS * factor + + +def _bike_access_python(highway: str | None, tags: dict[str, object]) -> bool: + highway_text = str(highway or "") + bicycle = _tag_text(tags, "bicycle") + access = _tag_text(tags, "access") + vehicle = _tag_text(tags, "vehicle") + if highway_text in {"motorway", "motorway_link", "steps", "platform"}: + return False + if bicycle in {"no", "private"}: + return False + if access in {"no", "private"} and bicycle not in {"yes", "designated", "permissive", "destination"} and vehicle not in {"yes", "designated", "permissive", "destination"}: + return False + if highway_text in {"footway", "pedestrian", "bridleway"} and bicycle not in {"yes", "designated", "permissive", "destination"}: + return False + if highway_text in {"trunk", "trunk_link"} and bicycle not in {"yes", "designated", "permissive", "destination"}: + return False + return True + + +def _bike_direction_forbidden_python(highway: str | None, tags: dict[str, object], *, reverse: bool) -> bool: + oneway = _tag_text(tags, "oneway") + junction = _tag_text(tags, "junction") + if reverse: + one_way = oneway in {"yes", "true", "1"} or junction == "roundabout" or str(highway or "") == "motorway" + else: + one_way = oneway == "-1" + if not one_way: + return False + bicycle_direction_key = "bicycle:backward" if reverse else "bicycle:forward" + return ( + _tag_text(tags, "oneway:bicycle") != "no" + and _tag_text(tags, "bicycle:oneway") != "no" + and _tag_text(tags, bicycle_direction_key) not in {"yes", "designated", "permissive"} + ) + + +def _tag_text(tags: dict[str, object], key: str) -> str: + return str(tags.get(key) or "").strip().lower() + + +def _json_object(value: str | None) -> dict[str, object]: + if not value: + return {} + try: + parsed = json.loads(value) + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + def _duration_minutes_ceil(seconds: int | float | None) -> int | None: if seconds is None: return None diff --git a/app/static/app.js b/app/static/app.js index 0d92783..0843d61 100644 --- a/app/static/app.js +++ b/app/static/app.js @@ -33,13 +33,17 @@ let jobListRefreshTimer; let jobListRefreshInFlight = false; let jobListRefreshFailureShown = false; let layerLoadAbortController; +let mapGtfsReviewSearchTimer; let allSources = []; let sourceCatalogEntries = []; let sourceCatalogSummary = {}; const JOB_DETAILS_POLL_MS = 4000; +const JOB_DETAILS_EVENT_FETCH_LIMIT = 500; +const JOB_DETAILS_VISIBLE_EVENTS = 200; const JOB_LIST_REFRESH_MS = 5000; const JOB_LIST_REFRESH_HIDDEN_MS = 15000; const JOURNEY_STOP_SEARCH_DEBOUNCE_MS = 400; +const MAP_GTFS_REVIEW_SEARCH_DEBOUNCE_MS = 350; const SIDEBAR_COLLAPSED_STORAGE_KEY = 'mobilitySidebarCollapsed'; const MAP_VIEW_STORAGE_KEY = 'mobilityMapView'; const DEFAULT_MAP_VIEW = { center: [52.52, 13.405], zoom: 11 }; @@ -971,6 +975,281 @@ function qaDecisionLabel(value) { return value || 'Architecture undecided'; } +async function loadGtfsWorkflow() { + const container = document.getElementById('gtfsWorkflow'); + if (!container) return; + try { + const data = await api('/api/workbench/gtfs-workflow'); + renderGtfsWorkflow(data); + } catch (err) { + container.classList.remove('muted'); + container.innerHTML = `

${escapeHtml(err.message)}

`; + } +} + +function renderGtfsWorkflow(data) { + const container = document.getElementById('gtfsWorkflow'); + if (!container) return; + const inventory = data.inventory_summary || {}; + const snapshot = data.snapshot || {}; + const review = data.review_summary || {}; + const diffSummary = data.diff_summary || {}; + const latestDiffs = data.latest_diffs || []; + const snapshotSummary = snapshot.summary || {}; + const snapshotDiagnostics = data.snapshot_diagnostics || {}; + const snapshotRoutePreview = data.snapshot_route_preview || {}; + const activeSnapshotState = snapshotDiagnostics.active || {}; + const openReviews = Number(review.by_status?.open || 0); + const resolvedReviews = Number(review.by_status?.resolved || 0); + const dedupOpen = Number(review.by_queue?.gtfs_deduplication?.open || 0); + const updateDiffOpen = Number(review.by_queue?.gtfs_update_diff?.open || 0); + const routeOpen = Number(review.by_queue?.route_matching?.open || 0); + const stopOpen = Number(review.by_queue?.stop_matching?.open || 0); + const snapshotNeedsRebuild = Boolean(snapshotDiagnostics.needs_rebuild); + const snapshotRouteRows = Number(snapshotSummary.snapshot_route_rows || activeSnapshotState.route_rows || 0); + const snapshotRouteShadows = Number(snapshotSummary.route_shadowed_routes || activeSnapshotState.route_shadowed_routes || 0); + const snapshotRouteIncludedRows = Number(snapshotSummary.snapshot_route_included_rows || 0); + const snapshotRouteShadowedRows = Number(snapshotSummary.snapshot_route_shadowed_rows || snapshotRouteShadows || 0); + const snapshotBody = [ + `${formatCount(snapshotSummary.included_datasets || activeSnapshotState.included_datasets || 0)} contributing source datasets`, + snapshotRouteRows + ? `${formatCount(snapshotRouteIncludedRows)} included route rows · ${formatCount(snapshotRouteShadowedRows)} shadowed route rows` + : `${formatCount(snapshotRouteRows)} route rows`, + formatSnapshotDiagnostics(snapshotDiagnostics), + ].filter(Boolean).join(' · '); + const steps = [ + { + title: 'Update feeds', + state: inventory.active_sources ? 'done' : 'current', + body: `${formatCount(inventory.active_sources || 0)} active feeds · ${formatCount(diffSummary.runs || 0)} stored update diffs`, + actions: [{ label: 'Refresh feeds', action: 'refresh-feeds' }], + }, + { + title: 'Build snapshot', + state: snapshotNeedsRebuild ? 'current' : snapshot.persisted ? 'done' : 'current', + body: snapshotBody, + actions: [ + { label: snapshot.persisted ? 'Rebuild snapshot' : 'Build snapshot', action: 'build-snapshot' }, + ...(snapshot.persisted && snapshotRouteRows ? [{ label: 'Route CSV', action: 'download-route-csv' }] : []), + ], + }, + { + title: 'Generate review queue', + state: openReviews || resolvedReviews ? 'done' : 'current', + body: `${formatCount(openReviews)} open · ${formatCount(resolvedReviews)} resolved`, + actions: [{ label: 'Generate review queue', action: 'generate-review' }], + }, + { + title: 'Review changed and conflicted data', + state: openReviews ? 'current' : resolvedReviews ? 'done' : 'pending', + body: `${formatCount(dedupOpen)} GTFS dedup · ${formatCount(updateDiffOpen)} update diff · ${formatCount(routeOpen)} route matching · ${formatCount(stopOpen)} stop matching`, + actions: [ + { label: 'Show open queue', action: 'show-open-review' }, + ...(dedupOpen ? [{ label: 'GTFS dedup', action: 'show-dedup-review' }] : []), + ...(updateDiffOpen ? [{ label: 'Update diff', action: 'show-update-review' }] : []), + ...(routeOpen ? [{ label: 'Route matches', action: 'show-route-review' }] : []), + ...(stopOpen ? [{ label: 'Stop matches', action: 'show-stop-review' }] : []), + ], + }, + { + title: 'Publish route layer', + state: 'pending', + body: 'Run matcher and route-layer build after snapshot/review changes.', + actions: [ + { label: 'Run matcher', action: 'run-matcher' }, + { label: 'Build route layer', action: 'build-route-layer' }, + ], + }, + ]; + container.classList.remove('muted'); + container.innerHTML = ` +
+ ${steps.map((step, index) => renderGtfsWorkflowStep(step, index + 1)).join('')} +
+ ${renderGtfsSnapshotRouteAudit(snapshotRoutePreview, snapshotDiagnostics)} +
+

Latest feed diffs

+ ${latestDiffs.length ? latestDiffs.map(renderGtfsWorkflowDiff).join('') : '

No GTFS update diff stored yet.

'} +
+ `; +} + +function formatSnapshotDiagnostics(diagnostics) { + if (!diagnostics?.needs_rebuild) return ''; + const computed = diagnostics.computed || {}; + const reasons = new Set(diagnostics.reasons || []); + const parts = []; + if (Number(computed.route_shadowed_routes || 0)) { + parts.push(`${formatCount(computed.route_shadowed_routes)} computed route shadows`); + } + if (reasons.has('route_ownership_not_materialized')) { + parts.push('route ownership not materialized'); + } else if (reasons.has('snapshot_inputs_changed')) { + parts.push('snapshot inputs changed'); + } else if (reasons.has('no_active_snapshot')) { + parts.push('no active snapshot'); + } + return parts.length ? `${parts.join(' · ')}; rebuild needed` : 'rebuild needed'; +} + +function renderGtfsSnapshotRouteAudit(preview, diagnostics) { + const routes = preview?.routes || []; + const summary = preview?.summary || {}; + const needsMaterialization = new Set(diagnostics?.reasons || []).has('route_ownership_not_materialized'); + if (needsMaterialization) { + return ` +
+

Snapshot route ownership

+

Route ownership rows are not materialized yet. Rebuild the snapshot to persist route-level inclusion and shadowing.

+
+ `; + } + if (!routes.length) { + return ` +
+

Snapshot route ownership

+

No shadowed route rows in the active snapshot preview.

+
+ `; + } + return ` +
+
+ Snapshot route ownership + +
+

${formatCount(summary.filtered_total || routes.length)} shadowed route rows in active snapshot.

+
+ ${routes.map(renderSnapshotRoutePreviewRow).join('')} +
+
+ `; +} + +function renderSnapshotRoutePreviewRow(route) { + const label = [route.route_ref || route.route_id, route.route_name, route.mode].filter(Boolean).join(' · '); + const loser = [route.source_name || `source #${route.source_id}`, `dataset #${route.dataset_id}`].filter(Boolean).join(' · '); + const winner = route.shadowed_by_dataset_id ? `dataset #${route.shadowed_by_dataset_id}` : 'higher-precedence route'; + return ` +
+ ${escapeHtml(label || `route #${route.gtfs_route_id}`)} + ${escapeHtml(loser)} shadowed by ${escapeHtml(winner)} +
+ `; +} + +function renderGtfsWorkflowStep(step, index) { + return ` +
+ ${escapeHtml(String(index))} +
+ ${escapeHtml(step.title)} +
${escapeHtml(step.body || '')}
+
+ ${(step.actions || []).map(action => ``).join('')} +
+
+
+ `; +} + +function renderGtfsWorkflowDiff(diff) { + const summary = diff.summary || {}; + const routes = summary.routes || {}; + const stops = summary.stops || {}; + return ` + + `; +} + +async function handleGtfsWorkflowClick(event) { + const diffButton = event.target.closest('[data-gtfs-diff-detail]'); + if (diffButton) { + showGtfsUpdateDiffDetail(diffButton.dataset.gtfsDiffDetail).catch(err => alert(err.message)); + return; + } + const actionButton = event.target.closest('[data-gtfs-workflow-action]'); + if (!actionButton) return; + const action = actionButton.dataset.gtfsWorkflowAction; + if (action === 'download-route-csv') { + window.open('/api/harmonization/gtfs/snapshot/routes.csv?limit=100000', '_blank', 'noopener'); + return; + } + const originalText = actionButton.textContent; + actionButton.disabled = true; + actionButton.textContent = 'Working...'; + try { + if (action === 'refresh-feeds') { + await Promise.all([loadSources(), loadGtfsHarmonizationInventory()]); + } else if (action === 'build-snapshot') { + await queueGtfsSnapshotBuild(actionButton); + } else if (action === 'generate-review') { + await queueMapGtfsReviewGeneration(actionButton); + } else if (action === 'show-open-review') { + await showMapGtfsReviewQueue(''); + } else if (action === 'show-dedup-review') { + await showMapGtfsReviewQueue('gtfs_deduplication'); + } else if (action === 'show-update-review') { + await showMapGtfsReviewQueue('gtfs_update_diff'); + } else if (action === 'show-route-review') { + await showMapGtfsReviewQueue('route_matching'); + } else if (action === 'show-stop-review') { + await showMapGtfsReviewQueue('stop_matching'); + } else if (action === 'run-matcher') { + await queueMatcherRun(actionButton); + } else if (action === 'build-route-layer') { + await queueRouteLayerBuild(actionButton); + } + await loadGtfsWorkflow(); + } catch (err) { + alert(err.message); + } finally { + actionButton.disabled = false; + actionButton.textContent = originalText; + } +} + +async function showMapGtfsReviewQueue(queueName) { + const queue = document.getElementById('mapGtfsReviewQueueFilter'); + const status = document.getElementById('mapGtfsReviewStatus'); + const severity = document.getElementById('mapGtfsReviewSeverityFilter'); + const mode = document.getElementById('mapGtfsReviewModeFilter'); + const search = document.getElementById('mapGtfsReviewSearch'); + if (queue) queue.value = queueName || ''; + if (status) status.value = 'open'; + if (severity) severity.value = ''; + if (mode) mode.value = ''; + if (search) search.value = ''; + openSidebarSection('map-gtfs-workbench'); + await loadMapGtfsWorkbenchReview(); + focusWorkbenchReviewQueue(); +} + +function openSidebarSection(sectionKey) { + const section = document.querySelector(`[data-sidebar-section="${sectionKey}"]`); + if (!section) return null; + section.open = true; + let parent = section.parentElement?.closest?.('details[data-sidebar-section]'); + while (parent) { + parent.open = true; + parent = parent.parentElement?.closest?.('details[data-sidebar-section]'); + } + return section; +} + +function focusWorkbenchReviewQueue() { + const section = document.querySelector('[data-sidebar-section="map-gtfs-workbench"]'); + const queue = document.getElementById('mapGtfsReviewQueue'); + if (!queue) return; + queue.setAttribute('tabindex', '-1'); + (section || queue).scrollIntoView({ behavior: 'smooth', block: 'start' }); + queue.focus({ preventScroll: true }); +} + async function loadGtfsHarmonizationInventory() { const container = document.getElementById('gtfsHarmonizationInventory'); if (!container) return; @@ -987,18 +1266,24 @@ function renderGtfsHarmonizationInventory(data) { const container = document.getElementById('gtfsHarmonizationInventory'); if (!container) return; const summary = data.summary || {}; + const snapshot = data.snapshot || {}; const feeds = data.feeds || []; + setJourneyTransitSnapshotFromGtfsSnapshot(snapshot); container.classList.remove('muted'); if (!feeds.length) { container.classList.add('muted'); - container.innerHTML = 'No GTFS sources registered yet.'; + container.innerHTML = 'No registered GTFS feeds yet.'; return; } container.innerHTML = `
${[ - ['Sources', summary.sources || 0], - ['Active', summary.active_sources || 0], + ['Feeds', summary.sources || 0], + ['Active feeds', summary.active_sources || 0], + ['Raw datasets', summary.datasets || 0], + ['Snapshot', summary.snapshot_included_datasets || 0], + ['Shadowed', summary.snapshot_shadowed_datasets || 0], + [snapshot.persisted ? `Snapshot #${snapshot.id}` : 'Computed', snapshot.persisted ? 'active' : 'live'], ['Ready', summary.ready || 0], ['Review', summary.needs_review || 0], ['Blocked', summary.blocked || 0], @@ -1016,11 +1301,15 @@ function renderGtfsFeedQaCard(feed) { const counts = feed.counts || {}; const service = feed.service || {}; const issues = feed.issues || []; + const snapshot = feed.snapshot || {}; return `
- ${escapeHtml(source.name || `Source #${source.id}`)} - ${escapeHtml(gtfsQaStatusLabel(feed.qa_status))} + ${escapeHtml(source.name || `Feed #${source.id}`)} + + ${escapeHtml(gtfsQaStatusLabel(feed.qa_status))} + ${snapshot.role ? `${escapeHtml(gtfsSnapshotRoleLabel(snapshot.role))}` : ''} +
${escapeHtml([source.country || 'n/a', source.priority, source.license || 'unknown license'].filter(Boolean).join(' · '))}
${[ @@ -1028,8 +1317,9 @@ function renderGtfsFeedQaCard(feed) { `${formatCount(counts.routes || 0)} routes`, `${formatCount(counts.stops || 0)} stops`, `${formatCount(counts.stop_times || 0)} stop_times`, + snapshot.shadowed_by_dataset_id ? `shadowed by #${snapshot.shadowed_by_dataset_id}` : null, service.end_date ? `service to ${service.end_date}` : 'no service horizon', - ].map(metric).join('')}
+ ].filter(Boolean).map(metric).join('')}
${issues.length ? `
${issues.slice(0, 3).map(renderCompactIssue).join('')}
` : '
No blocking feed QA issue detected.
'}
@@ -1042,6 +1332,344 @@ function renderCompactIssue(issue) { return `${escapeHtml(issue.title || issue.id || '')}`; } +function gtfsSnapshotRoleClass(role) { + if (role === 'included') return 'ok'; + if (role === 'shadowed') return 'warn'; + if (role === 'excluded') return 'error'; + return 'weak'; +} + +function gtfsSnapshotRoleLabel(role) { + if (role === 'included') return 'snapshot'; + if (role === 'shadowed') return 'shadowed'; + if (role === 'excluded') return 'excluded'; + return role || 'unknown'; +} + +async function loadMapGtfsWorkbenchReview() { + const container = document.getElementById('mapGtfsReviewQueue'); + if (!container) return; + const params = new URLSearchParams({ limit: '80' }); + const queue = document.getElementById('mapGtfsReviewQueueFilter')?.value || ''; + const status = document.getElementById('mapGtfsReviewStatus')?.value || ''; + const severity = document.getElementById('mapGtfsReviewSeverityFilter')?.value || ''; + const mode = document.getElementById('mapGtfsReviewModeFilter')?.value || ''; + const query = (document.getElementById('mapGtfsReviewSearch')?.value || '').trim(); + if (queue) params.set('queue', queue); + if (status) params.set('status', status); + if (severity) params.set('severity', severity); + if (mode) params.set('mode', mode); + if (query) params.set('q', query); + try { + const data = await api(`/api/workbench/map-gtfs/review-items?${params.toString()}`); + renderMapGtfsWorkbenchReview(data); + } catch (err) { + container.classList.remove('muted'); + container.innerHTML = `

${escapeHtml(err.message)}

`; + } +} + +function renderMapGtfsWorkbenchReview(data) { + const container = document.getElementById('mapGtfsReviewQueue'); + if (!container) return; + const summary = data.summary || {}; + const items = data.items || []; + container.classList.remove('muted'); + container.innerHTML = ` +
+ ${[ + ['Open', summary.by_status?.open || 0], + ['Reviewing', summary.by_status?.in_review || 0], + ['Resolved', summary.by_status?.resolved || 0], + ['Decisions', summary.active_decisions || 0], + ].map(([label, value]) => `
${escapeHtml(formatCount(value))}${escapeHtml(label)}
`).join('')} +
+ ${items.length ? items.map(renderMapGtfsReviewItem).join('') : '

No review items for this filter.

'} + `; +} + +function renderMapGtfsReviewItem(item) { + const route = item.gtfs_route; + const stop = item.gtfs_stop; + const label = route + ? [route.ref || route.route_id, route.name, route.mode].filter(Boolean).join(' · ') + : stop + ? [stop.name || stop.stop_id, stop.stop_id].filter(Boolean).join(' · ') + : item.item_type; + return ` +
+
+ ${escapeHtml(item.title || label)} + ${escapeHtml(item.status || '')} +
+
${escapeHtml(label || '')}
+
${escapeHtml(item.description || '')}
+
${[ + item.queue, + item.item_type, + item.source?.name, + item.dataset ? `dataset #${item.dataset.id}` : null, + ].filter(Boolean).map(metric).join('')}
+
+ + ${item.route_match_id ? `` : ''} + ${item.canonical_stop_id ? `` : ''} + ${item.status !== 'resolved' ? `` : ''} + ${item.status !== 'dismissed' ? `` : ''} +
+
+ `; +} + +async function showMapGtfsReviewItemDetail(itemId) { + openOverlay('Review item', '

Loading review evidence...

', { mapReview: true }); + try { + const item = await api(`/api/workbench/map-gtfs/review-items/${encodeURIComponent(itemId)}`); + openOverlay(item.title || 'Review item', renderMapGtfsReviewItemDetail(item), { mapReview: true }); + } catch (err) { + openOverlay('Review item', `

${escapeHtml(err.message)}

`, { mapReview: true }); + } +} + +function renderMapGtfsReviewItemDetail(item) { + const metadata = item.metadata || {}; + return ` +
+
+
+ ${escapeHtml(item.severity || 'info')} + ${escapeHtml(item.status || '')} + ${escapeHtml(item.queue || '')} +
+

${escapeHtml(item.description || '')}

+
${[ + item.source?.name, + item.dataset ? `dataset #${item.dataset.id}` : null, + item.evidence_hash ? `evidence ${String(item.evidence_hash).slice(0, 10)}` : null, + ].filter(Boolean).map(metric).join('')}
+
+ ${renderReviewEntitySummary(item)} + ${renderReviewEvidenceByType(item, metadata)} +
+

Raw evidence

+
+ Show JSON +
${escapeHtml(JSON.stringify(metadata, null, 2))}
+
+
+
+ `; +} + +function renderReviewEntitySummary(item) { + return ` +
+

Referenced data

+
+ ${renderReviewEntityCard('GTFS route', item.gtfs_route, route => [ + ['Ref', route.ref || route.route_id], + ['Name', route.name], + ['Mode', route.mode], + ['Operator', route.operator], + ['Route ID', route.route_id], + ['Dataset', route.dataset_id ? `#${route.dataset_id}` : null], + ])} + ${renderReviewEntityCard('GTFS stop', item.gtfs_stop, stop => [ + ['Name', stop.name], + ['Stop ID', stop.stop_id], + ['Dataset', stop.dataset_id ? `#${stop.dataset_id}` : null], + ['Coordinate', stop.lat !== null && stop.lon !== null ? `${stop.lat}, ${stop.lon}` : null], + ])} + ${renderReviewEntityCard('OSM feature', item.osm_feature, feature => [ + ['Name', feature.name], + ['Ref', feature.ref], + ['Mode', feature.mode], + ['Kind', feature.kind], + ['OSM', [feature.osm_type, feature.osm_id].filter(Boolean).join(' ')], + ['Dataset', feature.dataset_id ? `#${feature.dataset_id}` : null], + ])} +
+
+ `; +} + +function renderReviewEntityCard(title, entity, rowsFactory) { + if (!entity) return ''; + return ` +
+

${escapeHtml(title)}

+ ${rowsFactory(entity) + .filter(([, value]) => value !== null && value !== undefined && value !== '') + .map(([label, value]) => `
${escapeHtml(label)}${escapeHtml(value)}
`) + .join('')} +
+ `; +} + +function renderReviewEvidenceByType(item, metadata) { + if (item.queue === 'gtfs_deduplication') return renderGtfsDedupEvidence(metadata); + if (item.queue === 'gtfs_update_diff') return renderGtfsUpdateDiffEvidence(metadata); + if (item.queue === 'route_matching') return renderRouteMatchingEvidence(metadata); + if (item.queue === 'stop_matching') return renderStopMatchingEvidence(metadata); + return renderGenericReviewEvidence(metadata); +} + +function renderGtfsDedupEvidence(metadata) { + const shadowed = metadata.shadowed_dataset || {}; + const included = metadata.included_dataset || {}; + const differences = metadata.differences || []; + return ` +
+

Deduplication decision

+

${metadata.data_differs ? 'The shadowed route differs from the selected active route.' : 'The shadowed route looks equivalent on first-pass evidence.'}

+
+ ${renderReviewObjectPanel('Shadowed route', shadowed)} + ${renderReviewObjectPanel('Active route', included)} +
+
${[ + metadata.overlap_key ? `overlap ${metadata.overlap_key}` : null, + differences.length ? `differs: ${differences.join(', ')}` : 'no stored field differences', + metadata.snapshot?.reason, + ].filter(Boolean).map(metric).join('')}
+
+ `; +} + +function renderGtfsUpdateDiffEvidence(metadata) { + const diff = metadata.diff || {}; + const rows = Object.entries(diff); + return ` +
+

Feed update diff

+
${[ + metadata.item_type, + metadata.object_key, + metadata.change_type, + Array.isArray(metadata.previous_decisions) && metadata.previous_decisions.length ? `${metadata.previous_decisions.length} previous decisions` : null, + ].filter(Boolean).map(metric).join('')}
+ ${rows.length ? ` +
+ ${rows.map(([field, values]) => ` +
${escapeHtml(field)}${escapeHtml(values?.old)}${escapeHtml(values?.new)}
+ `).join('')} +
+ ` : '

No field-level diff stored for this item.

'} +
+ `; +} + +function renderRouteMatchingEvidence(metadata) { + return ` +
+

Route matching evidence

+
${[ + metadata.route_match_status, + metadata.confidence !== undefined ? `confidence ${metadata.confidence}` : null, + metadata.osm_feature_id ? `OSM feature #${metadata.osm_feature_id}` : null, + ].filter(Boolean).map(metric).join('')}
+ ${metadata.reasons ? `
${escapeHtml(JSON.stringify(metadata.reasons, null, 2))}
` : '

No candidate scoring reasons stored.

'} +
+ `; +} + +function renderStopMatchingEvidence(metadata) { + const gtfs = metadata.gtfs || {}; + return ` +
+

Stop matching evidence

+
${[ + metadata.missing_canonical_stop ? 'missing canonical stop' : null, + metadata.missing_visual_stop ? 'missing OSM stop/platform link' : null, + metadata.canonical_stop_id ? `canonical stop #${metadata.canonical_stop_id}` : null, + ].filter(Boolean).map(metric).join('')}
+ ${renderReviewObjectPanel('GTFS stop evidence', gtfs)} +
+ `; +} + +function renderGenericReviewEvidence(metadata) { + const entries = Object.entries(metadata || {}).filter(([, value]) => value !== null && typeof value !== 'object'); + if (!entries.length) return ''; + return ` +
+

Evidence

+
+ ${entries.map(([key, value]) => `
${escapeHtml(key)}${escapeHtml(value)}
`).join('')} +
+
+ `; +} + +function renderReviewObjectPanel(title, data) { + const entries = Object.entries(data || {}).filter(([, value]) => value !== null && value !== undefined && typeof value !== 'object'); + return ` +
+

${escapeHtml(title)}

+ ${entries.length + ? entries.map(([key, value]) => `
${escapeHtml(key)}${escapeHtml(value)}
`).join('') + : '

No structured values.

'} +
+ `; +} + +async function queueMapGtfsReviewGeneration(button) { + const originalText = button?.textContent; + if (button) { + button.disabled = true; + button.textContent = 'Queuing...'; + } + try { + const job = await api('/api/jobs/map-gtfs-review?limit=5000', { method: 'POST' }); + updateMapStatus(`Queued Map/GTFS review generation as job #${job.id}.`); + await Promise.all([loadJobs(), loadGtfsWorkflow()]); + pollJob(job.id); + } finally { + if (button) { + button.disabled = false; + button.textContent = originalText; + } + } +} + +async function queueGtfsSnapshotBuild(button) { + const originalText = button?.textContent; + if (button) { + button.disabled = true; + button.textContent = 'Queuing...'; + } + try { + const job = await api('/api/jobs/gtfs-harmonized-snapshot?activate=true', { method: 'POST' }); + updateMapStatus(`Queued harmonized GTFS snapshot build as job #${job.id}.`); + await Promise.all([loadJobs(), loadGtfsWorkflow()]); + pollJob(job.id); + } finally { + if (button) { + button.disabled = false; + button.textContent = originalText; + } + } +} + +async function updateMapGtfsReviewItemStatus(itemId, status, button) { + const originalText = button?.textContent; + if (button) { + button.disabled = true; + button.textContent = 'Saving...'; + } + try { + await api(`/api/workbench/map-gtfs/review-items/${encodeURIComponent(itemId)}`, { + method: 'PATCH', + body: JSON.stringify({ status }) + }); + await loadMapGtfsWorkbenchReview(); + } finally { + if (button) { + button.disabled = false; + button.textContent = originalText; + } + } +} + async function showGtfsHarmonizationDetail(sourceId) { openOverlay('GTFS feed QA', '

Loading feed QA...

'); try { @@ -1053,11 +1681,83 @@ async function showGtfsHarmonizationDetail(sourceId) { } } +async function showGtfsUpdateDiffDetail(diffId) { + openOverlay('GTFS update diff', '

Loading GTFS update diff...

'); + try { + const data = await api(`/api/harmonization/gtfs/diffs/${encodeURIComponent(diffId)}?item_limit=300`); + document.getElementById('overlayTitle').textContent = `GTFS diff: ${data.source_name || `source #${data.source_id}`}`; + document.getElementById('overlayContent').innerHTML = renderGtfsUpdateDiffDetail(data); + } catch (err) { + document.getElementById('overlayContent').innerHTML = `

${escapeHtml(err.message)}

`; + } +} + +function renderGtfsUpdateDiffDetail(data) { + const summary = data.summary || {}; + const routes = summary.routes || {}; + const stops = summary.stops || {}; + const items = data.items || []; + return ` +
+
+
+
+ ${escapeHtml(data.source_name || `Source #${data.source_id}`)} +
dataset #${escapeHtml(String(data.previous_dataset_id || 'n/a'))} -> #${escapeHtml(String(data.new_dataset_id || 'n/a'))} · ${escapeHtml(formatDateTime(data.created_at))}
+
+ ${escapeHtml(data.status || '')} +
+
+ ${[ + ['Routes added', routes.added || 0], + ['Routes changed', routes.changed || 0], + ['Routes removed', routes.removed || 0], + ['Stops added', stops.added || 0], + ['Stops changed', stops.changed || 0], + ['Stops removed', stops.removed || 0], + ].map(([label, value]) => ` +
+ ${escapeHtml(formatCount(value))} + ${escapeHtml(label)} +
+ `).join('')} +
+
+
+

Changed Items

+
+ ${items.length ? items.map(renderGtfsUpdateDiffItem).join('') : '

No stored diff items.

'} +
+
+
+ `; +} + +function renderGtfsUpdateDiffItem(item) { + const diff = item.diff || {}; + const fields = Object.keys(diff).filter(key => !['added', 'removed'].includes(key)); + return ` +
+
+ ${escapeHtml(item.title || item.object_key)} + ${escapeHtml(item.change_type || '')} +
+
${[ + item.item_type, + item.object_key, + fields.length ? `fields ${fields.join(', ')}` : null, + ].filter(Boolean).map(metric).join('')}
+ ${fields.length ? `
${escapeHtml(fields.map(field => `${field}: ${diff[field]?.old ?? 'n/a'} -> ${diff[field]?.new ?? 'n/a'}`).join(' · '))}
` : ''} +
+ `; +} + function renderGtfsHarmonizationDetail(feed) { const source = feed.source || {}; const dataset = feed.active_dataset; const issues = feed.issues || []; const review = source.qa_review || {}; + const snapshot = feed.snapshot || {}; return `
@@ -1075,6 +1775,19 @@ function renderGtfsHarmonizationDetail(feed) { ].map(metric).join('')}
${escapeHtml(shorten(source.url || '', 140))}
+ ${snapshot.role ? ` +
+

Snapshot Role

+
${[ + `role ${gtfsSnapshotRoleLabel(snapshot.role)}`, + snapshot.reason ? `reason ${snapshot.reason}` : null, + snapshot.authority_level ? `authority ${snapshot.authority_level}` : null, + snapshot.shadowed_by_dataset_id ? `shadowed by dataset #${snapshot.shadowed_by_dataset_id}` : null, + snapshot.route_key_overlap ? `${formatCount(snapshot.route_key_overlap)} overlapping route keys` : null, + snapshot.route_key_overlap_ratio ? `${Math.round(Number(snapshot.route_key_overlap_ratio) * 100)}% overlap` : null, + ].filter(Boolean).map(metric).join('')}
+
+ ` : ''}

Review Decision

@@ -1092,6 +1805,16 @@ function renderGtfsHarmonizationDetail(feed) { ].map(([value, label]) => ``).join('')} + +
+ ${gtfsLicenseFlagSelect('can_import', 'Can import', review.can_import)} + ${gtfsLicenseFlagSelect('can_derive', 'Can derive', review.can_derive)} + ${gtfsLicenseFlagSelect('can_redistribute', 'Can redistribute', review.can_redistribute)} + ${gtfsLicenseFlagSelect('requires_attribution', 'Attribution required', review.requires_attribution)} + ${gtfsLicenseFlagSelect('commercial_restrictions', 'Commercial restrictions', review.commercial_restrictions)} +
@@ -1140,6 +1863,38 @@ function renderGtfsHarmonizationDetail(feed) { `; } +function gtfsLicenseFlagSelect(name, label, value) { + const selected = String(value || 'unknown'); + return ` + + `; +} + +function gtfsAuthoritySelect(value) { + const selected = String(value || 'unknown'); + return ` + + `; +} + function renderGtfsHarmonizationDataset(dataset) { const counts = dataset.counts || {}; return ` @@ -1174,7 +1929,7 @@ async function saveGtfsFeedReview(sourceId, payload, button) { document.getElementById('overlayTitle').textContent = `GTFS QA: ${data.source?.name || `source #${sourceId}`}`; document.getElementById('overlayContent').innerHTML = renderGtfsHarmonizationDetail(data); updateMapStatus(`Saved GTFS QA review for ${data.source?.name || `source #${sourceId}`}.`); - await Promise.all([loadSources(), loadGtfsHarmonizationInventory()]); + await Promise.all([loadSources(), loadGtfsWorkflow(), loadGtfsHarmonizationInventory()]); } catch (err) { alert(err.message); } finally { @@ -1191,6 +1946,12 @@ function gtfsReviewPayloadFromForm(form) { license: String(formData.get('license') || '').trim(), review_status: String(formData.get('review_status') || 'unreviewed'), review_note: String(formData.get('review_note') || '').trim(), + can_import: String(formData.get('can_import') || 'unknown'), + can_derive: String(formData.get('can_derive') || 'unknown'), + can_redistribute: String(formData.get('can_redistribute') || 'unknown'), + requires_attribution: String(formData.get('requires_attribution') || 'unknown'), + commercial_restrictions: String(formData.get('commercial_restrictions') || 'unknown'), + authority_level: String(formData.get('authority_level') || 'unknown'), enabled: formData.has('enabled') }; } @@ -1309,30 +2070,38 @@ function renderJobs(jobs, workers = []) { ? '
' : ''; if (!jobs.length) { - container.classList.add('muted'); - container.innerHTML = `${workerHtml}
No jobs yet.
`; + container.classList.remove('muted'); + container.innerHTML = ` +
${workerHtml}
+
No jobs yet.
+ `; return; } container.classList.remove('muted'); - container.innerHTML = `${workerHtml}${toolbarHtml}${jobs.map(job => ` -
-
- ${escapeHtml(job.description || job.kind)} - ${escapeHtml(job.status)} -
-
- - priority ${Number(job.priority || 0)} · ${escapeHtml(formatDateTime(job.updated_at || job.created_at))} -
- ${job.requested_action ? `
Requested: ${escapeHtml(job.requested_action)}
` : ''} - ${job.lease_owner ? `
Worker: ${escapeHtml(job.lease_owner)}
` : ''} - ${job.error ? `
${escapeHtml(job.error)}
` : ''} - ${job.result && Object.keys(job.result).length ? `
${escapeHtml(shorten(JSON.stringify(job.result), 120))}
` : ''} -
- ${jobActionButtons(job)} + container.innerHTML = ` +
${workerHtml}
+
+ ${toolbarHtml}${jobs.map(job => ` +
+
+ ${escapeHtml(job.description || job.kind)} + ${escapeHtml(job.status)} +
+
+ + priority ${Number(job.priority || 0)} · ${escapeHtml(formatDateTime(job.updated_at || job.created_at))} +
+ ${job.requested_action ? `
Requested: ${escapeHtml(job.requested_action)}
` : ''} + ${job.lease_owner ? `
Worker: ${escapeHtml(job.lease_owner)}
` : ''} + ${job.error ? `
${escapeHtml(job.error)}
` : ''} + ${job.result && Object.keys(job.result).length ? `
${escapeHtml(shorten(JSON.stringify(job.result), 120))}
` : ''} +
+ ${jobActionButtons(job)} +
+ `).join('')}
- `).join('')}`; + `; } function jobStepDefinitions(job) { @@ -1467,6 +2236,8 @@ function renderJobEventMetadata(event) { function renderJobDetails(data, queueData = {}) { const job = data.job || {}; const events = data.events || []; + const visibleEvents = events.slice(-JOB_DETAILS_VISIBLE_EVENTS); + const hiddenFetchedEvents = Math.max(0, events.length - visibleEvents.length); const queueJobs = queueData.jobs || []; const latestEvent = events.length ? events[events.length - 1] : null; const progressMax = Number(job.progress_total || 1); @@ -1519,11 +2290,12 @@ function renderJobDetails(data, queueData = {}) { ` : '

No queue rows returned.

'}
-

Events ${events.length}

+

Events latest ${visibleEvents.length}

+ ${hiddenFetchedEvents ? `

Showing the latest ${visibleEvents.length} of ${events.length} fetched events.

` : ''}
- ${events.length ? events.map((event, index) => ` + ${visibleEvents.length ? visibleEvents.map((event, index) => `
- ${index + 1} + ${hiddenFetchedEvents + index + 1}
${escapeHtml(event.event_type || 'event')} @@ -1555,7 +2327,7 @@ async function loadJobDetails(jobId) { } try { const [details, queue] = await Promise.all([ - api(`/api/jobs/${encodeURIComponent(jobId)}/events?limit=200`), + api(`/api/jobs/${encodeURIComponent(jobId)}/events?limit=${JOB_DETAILS_EVENT_FETCH_LIMIT}`), api('/api/jobs?limit=20') ]); if (activeJobDetailsId !== String(jobId)) return; @@ -1584,13 +2356,30 @@ function jobKindLabel(job) { function renderWorkerStatus(workers) { if (!workers.length) { - return '
worker disabledSet QUEUE_WORKER_AUTOSTART=true to start workers with the server.
'; + return ` +
+
+ worker not configured + Set QUEUE_WORKER_COUNT above 0 to enable queue workers. +
+
+ `; } + const runningCount = workers.filter(worker => worker.running).length; + const workerControls = runningCount > 0 + ? ` + + + ` + : ''; return `
+
+ ${workerControls} +
${workers.map(worker => `
- ${escapeHtml(worker.worker_id)} ${worker.running ? 'running' : 'stopped'} + ${escapeHtml(worker.worker_id)} ${escapeHtml(worker.status || (worker.running ? 'running' : 'stopped'))} ${worker.pid ? `pid ${escapeHtml(String(worker.pid))}` : 'no pid'} · ${escapeHtml(shorten(worker.log_file || '', 80))}
`).join('')} @@ -1620,6 +2409,11 @@ function jobActionButtons(job) { } async function handleJobAction(event) { + const workerButton = event.target.closest('[data-worker-action]'); + if (workerButton) { + await handleWorkerAction(workerButton); + return; + } const clearButton = event.target.closest('[data-jobs-clear-terminal]'); if (clearButton) { const originalText = clearButton.textContent; @@ -1674,8 +2468,31 @@ async function handleJobAction(event) { } } -async function queueRouteLayerBuild() { - const button = document.getElementById('buildRouteLayerBtn'); +async function handleWorkerAction(button) { + const action = button.dataset.workerAction; + if (!['start', 'stop', 'restart'].includes(action)) return; + if ((action === 'stop' || action === 'restart') && !confirm(`${action === 'stop' ? 'Stop' : 'Restart'} the queue worker?`)) { + return; + } + const originalText = button.textContent; + button.disabled = true; + button.textContent = 'Working...'; + try { + const result = await api(`/api/workers/${action}`, { method: 'POST' }); + const running = (result.workers || []).filter(worker => worker.running).length; + const total = (result.workers || []).length; + updateMapStatus(`Worker ${action} complete: ${running} / ${total} running.`); + await loadJobs(); + } catch (err) { + alert(err.message); + } finally { + button.disabled = false; + button.textContent = originalText; + } +} + +async function queueRouteLayerBuild(buttonOverride = null) { + const button = buttonOverride || document.getElementById('buildRouteLayerBtn'); const originalText = button?.textContent; if (button) { button.disabled = true; @@ -1684,7 +2501,7 @@ async function queueRouteLayerBuild() { try { const job = await api('/api/jobs/route-layer-build', { method: 'POST' }); updateMapStatus(`Route-layer rebuild queued as job #${job.id}.`); - await loadJobs(); + await Promise.all([loadJobs(), loadGtfsWorkflow()]); pollJob(job.id); } finally { if (button) { @@ -1694,8 +2511,8 @@ async function queueRouteLayerBuild() { } } -async function queueMatcherRun() { - const button = document.getElementById('runMatchBtn'); +async function queueMatcherRun(buttonOverride = null) { + const button = buttonOverride || document.getElementById('runMatchBtn'); const originalText = button?.textContent; if (button) { button.disabled = true; @@ -1704,7 +2521,7 @@ async function queueMatcherRun() { try { const job = await api('/api/jobs/match-run', { method: 'POST' }); updateMapStatus(`Route matching queued as job #${job.id}.`); - await loadJobs(); + await Promise.all([loadJobs(), loadGtfsWorkflow()]); pollJob(job.id); } finally { if (button) { @@ -1801,7 +2618,7 @@ function renderSources() { container, queryId: 'sourceSearch', kinds: ['gtfs'], - emptyMessage: 'No matching GTFS sources.' + emptyMessage: 'No matching registered GTFS feeds.' }); } @@ -1906,7 +2723,7 @@ function sourceReadinessWarning(source) { return '
No active extracted OSM visual dataset yet. Import this source before matching or building the route layer.
'; } if (source.kind === 'gtfs' && !hasActiveGtfsDataset(source)) { - return '
No active GTFS timetable dataset yet. Import this source before routing or matching.
'; + return '
No active GTFS timetable dataset yet. Import this feed before routing or matching.
'; } if (source.kind === 'osm_geojson' && !hasActiveOsmDataset(source)) { return '
No active OSM visual dataset yet. Import this source before matching or displaying routes.
'; @@ -2151,7 +2968,7 @@ function renderSourceCatalog() { .sort(([left], [right]) => left.localeCompare(right)) .map(([priority, count]) => `${priority}: ${count}`) .join(' · '); - summary.textContent = `${sourceCatalogSummary.catalog_entries || 0} backlog entries${priorities ? ` · ${priorities}` : ''}`; + summary.textContent = `${sourceCatalogSummary.catalog_entries || 0} discovery entries${priorities ? ` · ${priorities}` : ''}`; } const container = document.getElementById('sourceCatalog'); if (!container) return; @@ -2164,7 +2981,7 @@ function renderSourceCatalog() { function sourceCatalogEntryCard(entry) { const kind = inferCatalogSourceKind(entry); - const actionLabel = kind && kind.startsWith('osm_') ? 'Use as map source' : 'Use as GTFS source'; + const actionLabel = kind && kind.startsWith('osm_') ? 'Register map source' : 'Register GTFS feed'; return `
@@ -2174,7 +2991,7 @@ function sourceCatalogEntryCard(entry) {
${[ entry.source_category, entry.formats_apis, - entry.linked_source_count ? `${entry.linked_source_count} source links` : null + entry.linked_source_count ? `${entry.linked_source_count} registered links` : null ].filter(Boolean).map(value => metric(shorten(value, 34))).join('')}
${escapeHtml(shorten(entry.coverage_notes || '', 150))}
${entry.geometry_notes ? `
Geometry QA: ${escapeHtml(shorten(entry.geometry_notes, 150))}
` : ''} @@ -2202,7 +3019,7 @@ function fillSourceFormFromCatalog(entryId) { if (form.elements.kind) form.elements.kind.value = isMapSource ? kind : 'gtfs'; form.elements.url.focus(); form.elements.url.select(); - updateMapStatus(`Prepared source from catalog: ${entry.source_name || entry.id}`); + updateMapStatus(`Prepared feed/source from discovery catalog: ${entry.source_name || entry.id}`); } function inferCatalogSourceKind(entry) { @@ -2391,7 +3208,25 @@ function setJourneySources(sources) { snapshot.classList.remove('muted'); snapshot.innerHTML = ` Harmonized transit snapshot - ${formatCount(gtfsSources.length)} active GTFS source${gtfsSources.length === 1 ? '' : 's'} · ${formatCount(activeDatasetCount)} active timetable dataset${activeDatasetCount === 1 ? '' : 's'} + ${formatCount(gtfsSources.length)} active raw GTFS source${gtfsSources.length === 1 ? '' : 's'} · ${formatCount(activeDatasetCount)} active raw timetable input${activeDatasetCount === 1 ? '' : 's'} + `; +} + +function setJourneyTransitSnapshotFromGtfsSnapshot(data) { + const snapshot = document.getElementById('journeyTransitSnapshot'); + if (!snapshot || !data || !data.summary) return; + const summary = data.summary || {}; + const included = summary.included_datasets || 0; + const raw = summary.raw_active_datasets || 0; + if (!raw) { + snapshot.classList.add('muted'); + snapshot.innerHTML = 'Harmonized transit snapshotNo active GTFS feed imported.'; + return; + } + snapshot.classList.remove('muted'); + snapshot.innerHTML = ` + Harmonized transit snapshot + ${formatCount(included)} routing dataset${included === 1 ? '' : 's'} from ${formatCount(raw)} active raw input${raw === 1 ? '' : 's'}${summary.shadowed_datasets ? ` · ${formatCount(summary.shadowed_datasets)} shadowed` : ''} `; } @@ -3153,7 +3988,7 @@ function durationText(row) { function legMetaText(leg) { if (!leg) return ''; - if (normalizeMode(leg.mode) === 'walk') { + if (['walk', 'bike', 'drive', 'car'].includes(normalizeMode(leg.mode))) { return distanceText(leg.distance_m); } const intermediate = Number(leg.intermediate_stop_count); @@ -3212,6 +4047,8 @@ function modeIcon(mode) { coach: ['Coach', 'C'], ferry: ['Ferry', 'F'], walk: ['Walk', 'W'], + bike: ['Bike', 'BI'], + bicycle: ['Bike', 'BI'], drive: ['Car', 'C'], car: ['Car', 'C'], monorail: ['Monorail', 'M'], @@ -3229,6 +4066,8 @@ function transportModeColor(mode) { const key = normalizeMode(mode); return { walk: '#16a34a', + bike: '#0d9488', + bicycle: '#0d9488', tram: '#dc2626', light_rail: '#dc2626', subway: '#ef4444', @@ -3484,7 +4323,7 @@ async function submitSourceForm(event) { } await api('/api/sources', { method: 'POST', body: JSON.stringify(payload) }); form.reset(); - await Promise.all([loadSources(), loadStats(), loadSourceCatalog(), loadGtfsHarmonizationInventory()]); + await Promise.all([loadSources(), loadStats(), loadSourceCatalog(), loadGtfsWorkflow(), loadGtfsHarmonizationInventory()]); } async function showCandidates(matchId) { @@ -3542,7 +4381,7 @@ async function acceptCandidate(matchId, osmFeatureId, button) { } try { await api(`/api/matches/${encodeURIComponent(matchId)}/candidates/${encodeURIComponent(osmFeatureId)}/accept`, { method: 'POST' }); - await Promise.all([loadMatches(), loadStats(), loadMapLayers()]); + await Promise.all([loadMatches(), loadStats(), loadMapLayers(), loadMapGtfsWorkbenchReview()]); await showCandidates(matchId); } catch (err) { alert(err.message); @@ -3764,7 +4603,7 @@ function formatTimeInput(totalMinutes) { async function updateMatch(id, action) { await api(`/api/matches/${id}/${action}`, { method: 'POST' }); - await Promise.all([loadMatches(), loadStats(), loadMapLayers()]); + await Promise.all([loadMatches(), loadStats(), loadMapLayers(), loadMapGtfsWorkbenchReview()]); } function shorten(str, len) { @@ -3774,7 +4613,18 @@ function shorten(str, len) { async function refreshAll() { await loadSources(); - await Promise.all([loadStats(), loadQaSummary(), loadGtfsHarmonizationInventory(), loadJobs(), loadMatches(), loadMapLayers(), loadSourceCatalog(), loadItineraries()]); + await Promise.all([ + loadStats(), + loadQaSummary(), + loadGtfsWorkflow(), + loadGtfsHarmonizationInventory(), + loadMapGtfsWorkbenchReview(), + loadJobs(), + loadMatches(), + loadMapLayers(), + loadSourceCatalog(), + loadItineraries() + ]); } function setupSidebarSections() { @@ -3879,7 +4729,11 @@ function setupEvents() { }); bindClick('refreshBtn', refreshAll); bindClick('refreshQaBtn', loadQaSummary); + bindClick('refreshGtfsWorkflowBtn', loadGtfsWorkflow); bindClick('refreshGtfsHarmonizationBtn', loadGtfsHarmonizationInventory); + bindClick('buildGtfsSnapshotBtn', event => queueGtfsSnapshotBuild(event.currentTarget).catch(err => alert(err.message))); + bindClick('refreshMapGtfsWorkbenchBtn', loadMapGtfsWorkbenchReview); + bindClick('queueMapGtfsReviewBtn', event => queueMapGtfsReviewGeneration(event.currentTarget).catch(err => alert(err.message))); document.querySelectorAll('[data-admin-action]').forEach(button => { button.addEventListener('click', () => runAdminAction(button.dataset.adminAction, button)); }); @@ -3931,11 +4785,45 @@ function setupEvents() { showDatasetSearchFeature(row.dataset.searchFeatureType, row.dataset.searchFeatureId, row); }); document.getElementById('sources')?.addEventListener('click', handleSourceAction); + document.getElementById('gtfsWorkflow')?.addEventListener('click', handleGtfsWorkflowClick); document.getElementById('gtfsHarmonizationInventory')?.addEventListener('click', event => { const button = event.target.closest('[data-gtfs-feed-detail]'); if (!button) return; showGtfsHarmonizationDetail(button.dataset.gtfsFeedDetail); }); + document.getElementById('mapGtfsReviewQueueFilter')?.addEventListener('change', () => loadMapGtfsWorkbenchReview()); + document.getElementById('mapGtfsReviewStatus')?.addEventListener('change', () => loadMapGtfsWorkbenchReview()); + document.getElementById('mapGtfsReviewSeverityFilter')?.addEventListener('change', () => loadMapGtfsWorkbenchReview()); + document.getElementById('mapGtfsReviewModeFilter')?.addEventListener('change', () => loadMapGtfsWorkbenchReview()); + document.getElementById('mapGtfsReviewSearch')?.addEventListener('input', () => { + window.clearTimeout(mapGtfsReviewSearchTimer); + mapGtfsReviewSearchTimer = window.setTimeout(loadMapGtfsWorkbenchReview, MAP_GTFS_REVIEW_SEARCH_DEBOUNCE_MS); + }); + document.getElementById('mapGtfsReviewQueue')?.addEventListener('click', event => { + const detailButton = event.target.closest('[data-map-gtfs-review-detail]'); + if (detailButton) { + showMapGtfsReviewItemDetail(detailButton.dataset.mapGtfsReviewDetail); + return; + } + const statusButton = event.target.closest('[data-map-gtfs-review-status]'); + if (statusButton) { + updateMapGtfsReviewItemStatus( + statusButton.dataset.reviewItemId, + statusButton.dataset.mapGtfsReviewStatus, + statusButton + ).catch(err => alert(err.message)); + return; + } + const candidateButton = event.target.closest('[data-candidates]'); + if (candidateButton) { + showCandidates(candidateButton.dataset.candidates); + return; + } + const canonicalStopButton = event.target.closest('[data-canonical-stop]'); + if (canonicalStopButton) { + showCanonicalStop(canonicalStopButton.dataset.canonicalStop); + } + }); document.getElementById('mappingSources')?.addEventListener('click', handleSourceAction); document.getElementById('importSourceCatalogBtn').addEventListener('click', () => importSourceCatalog().catch(err => alert(err.message))); document.getElementById('importIngestableSourcesBtn').addEventListener('click', () => importIngestableSources().catch(err => alert(err.message))); diff --git a/app/static/style.css b/app/static/style.css index 9eb4e6a..51dd592 100644 --- a/app/static/style.css +++ b/app/static/style.css @@ -383,10 +383,16 @@ input.journey-selected-stop { gap: 8px; } .harmonization-review-form label:has(textarea), +.harmonization-review-form .harmonization-license-grid, .harmonization-review-form .source-actions, .harmonization-review-form > .muted { grid-column: 1 / -1; } +.harmonization-license-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: 8px; +} .harmonization-review-item { display: grid; gap: 2px; @@ -416,6 +422,155 @@ input.journey-selected-stop { .harmonization-review-item span { color: #52606d; } +.map-gtfs-review-queue { + display: grid; + gap: 8px; + max-height: 360px; + overflow-x: hidden; + overflow-y: auto; + padding-right: 4px; + min-width: 0; +} +.map-gtfs-review-summary { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: 5px; +} +.map-gtfs-review-summary div { + border: 1px solid #e2e8f0; + border-radius: 6px; + background: #f8fafc; + padding: 6px; + min-width: 0; +} +.map-gtfs-review-summary strong, +.map-gtfs-review-summary span { + display: block; + min-width: 0; + overflow-wrap: anywhere; +} +.map-gtfs-review-summary strong { + font-size: 13px; + color: #17212b; +} +.map-gtfs-review-summary span { + font-size: 10px; + color: #5a6875; +} +.map-gtfs-review-item { + display: grid; + gap: 5px; + border-left: 4px solid #64748b; + border-radius: 6px; + background: #f8fafc; + padding: 8px; + font-size: 12px; + min-width: 0; +} +.map-gtfs-review-item.error { + border-left-color: #c24141; + background: #fff7f7; +} +.map-gtfs-review-item.warn { + border-left-color: #c47a12; + background: #fffaf0; +} +.map-gtfs-review-item.info { + border-left-color: #2563eb; + background: #f6f9ff; +} +.map-gtfs-review-title { + display: flex; + justify-content: space-between; + gap: 8px; + min-width: 0; +} +.map-gtfs-review-title > * { + min-width: 0; + overflow-wrap: anywhere; +} +.map-gtfs-review-actions { + display: flex; + flex-wrap: wrap; + gap: 6px; +} +.gtfs-workflow, +.gtfs-workflow-steps, +.gtfs-workflow-diffs, +.gtfs-diff-item-list { + display: grid; + gap: 8px; + min-width: 0; +} +.gtfs-workflow-step { + grid-template-columns: auto minmax(0, 1fr); +} +.gtfs-workflow-step.pending { + opacity: 0.82; +} +.gtfs-workflow-actions { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 6px; +} +.gtfs-workflow-diffs { + border-top: 1px solid #e2e8f0; + padding-top: 8px; +} +.gtfs-workflow-audit { + display: grid; + gap: 7px; + border-top: 1px solid #e2e8f0; + padding-top: 8px; + min-width: 0; +} +.gtfs-workflow-audit h4 { + margin: 0; + font-size: 12px; + color: #334155; +} +.snapshot-route-preview { + display: grid; + gap: 6px; + max-height: 190px; + overflow-y: auto; + padding-right: 4px; + min-width: 0; +} +.snapshot-route-row { + display: grid; + gap: 2px; + border-left: 3px solid #2563eb; + background: #f8fafc; + padding: 6px 8px; + min-width: 0; +} +.snapshot-route-row strong, +.snapshot-route-row span { + min-width: 0; + overflow-wrap: anywhere; +} +.gtfs-workflow-diffs h4 { + margin: 0; + font-size: 12px; + color: #334155; +} +.gtfs-workflow-diff { + display: grid; + gap: 2px; + width: 100%; + border: 1px solid #e2e8f0; + border-radius: 6px; + background: #f8fafc; + color: #17212b; + padding: 8px; + text-align: left; +} +.gtfs-workflow-diff span { + color: #52606d; + font-size: 11px; +} .source, .match, .catalog-entry { border-top: 1px solid #e1e7ee; padding: 8px 0; @@ -814,6 +969,14 @@ input.journey-selected-stop { gap: 8px; font-size: 12px; min-width: 0; +} +.jobs-worker-status { + min-width: 0; +} +.jobs-queue { + display: grid; + gap: 8px; + min-width: 0; max-height: 280px; overflow-x: hidden; overflow-y: auto; @@ -831,6 +994,12 @@ input.journey-selected-stop { display: grid; gap: 4px; } +.worker-controls { + display: flex; + flex-wrap: wrap; + gap: 6px; + min-width: 0; +} .worker-row { border: 1px solid #e1e7ee; border-radius: 6px; @@ -1202,6 +1371,8 @@ input.journey-selected-stop { .mode-coach { background: #fef3c7; color: #a16207; } .mode-ferry { background: #dbeafe; color: #0284c7; } .mode-walk { background: #dcfce7; color: #16a34a; } +.mode-bike, +.mode-bicycle { background: #ccfbf1; color: #0d9488; } .mode-drive, .mode-car { background: #ffedd5; color: #f97316; } .mode-monorail, @@ -1416,6 +1587,79 @@ input.journey-selected-stop { white-space: pre-wrap; overflow-wrap: anywhere; } +.review-detail { + display: grid; + gap: 14px; + font-size: 12px; +} +.review-detail section { + display: grid; + gap: 8px; +} +.review-detail h3, +.review-entity-card h4 { + margin: 0; + font-size: 13px; +} +.review-detail-title { + display: flex; + flex-wrap: wrap; + gap: 6px; +} +.review-detail-grid, +.review-comparison { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: 8px; + min-width: 0; +} +.review-entity-card { + display: grid; + gap: 5px; + border: 1px solid #e1e7ee; + border-radius: 6px; + background: #fbfdff; + padding: 8px; + min-width: 0; +} +.review-entity-card div { + display: grid; + grid-template-columns: minmax(80px, 0.42fr) minmax(0, 1fr); + gap: 8px; + min-width: 0; +} +.review-entity-card span { + color: #64748b; +} +.review-entity-card strong, +.review-diff-table span, +.review-diff-table strong { + min-width: 0; + overflow-wrap: anywhere; +} +.review-diff-table { + display: grid; + gap: 5px; +} +.review-diff-table > div { + display: grid; + grid-template-columns: minmax(90px, 0.7fr) repeat(2, minmax(0, 1fr)); + gap: 8px; + border: 1px solid #e2e8f0; + border-radius: 6px; + background: #f8fafc; + padding: 7px; +} +.review-detail pre { + margin: 5px 0 0; + padding: 8px; + border-radius: 6px; + background: #f7fafc; + white-space: pre-wrap; + overflow-wrap: anywhere; + max-height: 320px; + overflow-y: auto; +} .canonical-stop-detail { display: grid; gap: 14px; diff --git a/app/templates/index.html b/app/templates/index.html index f58f6e1..ebea250 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -5,7 +5,7 @@ Mobility Workbench - +
@@ -49,7 +49,7 @@

GTFS Harmonization