from __future__ import annotations import json import zipfile from sqlalchemy import func, select from app.data_management import delete_dataset from app.db import reset_db, session_scope from app.gtfs_storage import sidecar_path, stop_time_count, stop_times_by_trip from app.journey import find_journeys, search_scheduled_stops from app.models import ( Dataset, GtfsCalendar, GtfsFeedDiffItem, GtfsFeedDiffRun, GtfsRoute, GtfsRoutePatternLink, GtfsTripRoutePatternLink, MapGtfsDecision, MapGtfsReviewItem, RoutePattern, Source, ) from app.pipeline.gtfs import _copy_stage_row_payload from app.pipeline.run import run_source def test_gtfs_import_uses_staging_bulk_loader_and_reports_chunks(tmp_path, monkeypatch): reset_db() gtfs_path = tmp_path / "small.gtfs.zip" with zipfile.ZipFile(gtfs_path, "w") as zf: zf.writestr("agency.txt", "agency_id,agency_name,agency_url,agency_timezone\nA,Agency,https://example.invalid,Europe/Berlin\n") zf.writestr( "stops.txt", "stop_id,stop_name,stop_lat,stop_lon\nA,Alpha,52.0,13.0\nB,Beta,52.1,13.1\nC,Gamma,52.2,13.2\n", ) zf.writestr("routes.txt", "route_id,agency_id,route_short_name,route_long_name,route_type\nR,A,R1,Alpha - Gamma,3\n") zf.writestr("trips.txt", "route_id,service_id,trip_id,shape_id\nR,daily,t1,s1\nR,daily,t2,s1\n") zf.writestr("calendar.txt", "service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date\ndaily,1,1,1,1,1,1,1,20260101,20261231\n") zf.writestr( "stop_times.txt", "\n".join( [ "trip_id,arrival_time,departure_time,stop_id,stop_sequence", "t1,08:00:00,08:00:00,A,1", "t1,08:05:00,08:05:00,B,2", "t1,08:10:00,08:10:00,C,3", "t2,09:00:00,09:00:00,A,1", "t2,09:10:00,09:10:00,C,2", ] ) + "\n", ) zf.writestr("shapes.txt", "shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence\ns1,52.0,13.0,1\ns1,52.2,13.2,2\n") monkeypatch.setattr("app.pipeline.gtfs.GTFS_STAGE_BATCH_SIZE", 2) events = [] with session_scope() as session: source = Source(name="Small GTFS", kind="gtfs", url=str(gtfs_path)) session.add(source) session.flush() dataset = run_source(session, source, progress_callback=lambda *args: events.append(args)) metadata = json.loads(dataset.metadata_json or "{}") assert metadata["importer"] == "gtfs_import_v6_sidecar_stop_times" assert metadata["staging"] == "sqlite_promoted_to_sidecar" assert metadata["gtfs_storage"]["tables"]["gtfs_stop_times"] == "sidecar" assert metadata["stop_times_imported"] == 5 assert sidecar_path(dataset) is not None assert sidecar_path(dataset).exists() assert stop_time_count(session, dataset.id) == 5 assert len(stop_times_by_trip(session, dataset.id, ["t1"])["t1"]) == 3 assert session.scalar(select(func.count()).select_from(GtfsCalendar).where(GtfsCalendar.dataset_id == dataset.id)) == 1 assert session.scalar(select(func.count()).select_from(Dataset).where(Dataset.kind == "gtfs", Dataset.is_active.is_(True))) == 1 alpha = search_scheduled_stops(session, "Alpha", limit=1)[0] gamma = search_scheduled_stops(session, "Gamma", limit=1)[0] journey = find_journeys(session, alpha["id"], gamma["id"], "08:00", limit=1) assert journey["journeys"][0]["departure_time"] == "08:00:00" assert journey["journeys"][0]["arrival_time"] == "08:10:00" event_types = [event[0] for event in events] assert "gtfs_staging_started" in event_types assert "gtfs_file_chunk" in event_types assert "gtfs_activation_sidecar_stop_times" in event_types assert "gtfs_activation_completed" in event_types def test_gtfs_calendar_stage_payload_casts_weekday_flags_to_bool(): columns = ["service_id", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", "start_date", "end_date"] payload = _copy_stage_row_payload( "gtfs_calendars", 5, columns, ("Special", 0, 1, "0", "1", False, True, "", 20260401, 20261211), ) assert payload == { "dataset_id": 5, "service_id": "Special", "monday": False, "tuesday": True, "wednesday": False, "thursday": True, "friday": False, "saturday": True, "sunday": False, "start_date": 20260401, "end_date": 20261211, } def test_gtfs_reimports_same_hash_when_stop_time_limit_changes(tmp_path, monkeypatch): reset_db() gtfs_path = tmp_path / "limited.gtfs.zip" with zipfile.ZipFile(gtfs_path, "w") as zf: zf.writestr("agency.txt", "agency_id,agency_name,agency_url,agency_timezone\nA,Agency,https://example.invalid,Europe/Berlin\n") zf.writestr("stops.txt", "stop_id,stop_name,stop_lat,stop_lon\nA,Alpha,52.0,13.0\nB,Beta,52.1,13.1\nC,Gamma,52.2,13.2\n") zf.writestr("routes.txt", "route_id,agency_id,route_short_name,route_long_name,route_type\nR,A,R1,Alpha - Gamma,3\n") zf.writestr("trips.txt", "route_id,service_id,trip_id\nR,daily,t1\n") zf.writestr("calendar.txt", "service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date\ndaily,1,1,1,1,1,1,1,20260101,20261231\n") zf.writestr( "stop_times.txt", "\n".join( [ "trip_id,arrival_time,departure_time,stop_id,stop_sequence", "t1,08:00:00,08:00:00,A,1", "t1,08:05:00,08:05:00,B,2", "t1,08:10:00,08:10:00,C,3", ] ) + "\n", ) with session_scope() as session: source = Source(name="Limited GTFS", kind="gtfs", url=str(gtfs_path)) session.add(source) session.flush() monkeypatch.setattr("app.pipeline.gtfs.settings.gtfs_stop_times_import_limit", 2) first_dataset = run_source(session, source) first_id = first_dataset.id assert stop_time_count(session, first_id) == 2 monkeypatch.setattr("app.pipeline.gtfs.settings.gtfs_stop_times_import_limit", 0) second_dataset = run_source(session, source) assert second_dataset.id != first_id assert stop_time_count(session, second_dataset.id) == 3 def test_gtfs_reimport_stores_update_diff_before_replacing_old_dataset(tmp_path): reset_db() gtfs_path = tmp_path / "updating.gtfs.zip" _write_update_diff_gtfs(gtfs_path, route_long_name="Alpha - Beta", beta_name="Beta") with session_scope() as session: source = Source(name="Updating GTFS", kind="gtfs", url=str(gtfs_path)) session.add(source) session.flush() first_dataset = run_source(session, source) first_id = first_dataset.id _write_update_diff_gtfs( gtfs_path, route_long_name="Alpha - Beta revised", beta_name="Beta moved", include_extra_stop=True, include_extra_route=True, ) with session_scope() as session: source = session.scalar(select(Source).where(Source.name == "Updating GTFS")) second_dataset = run_source(session, source) second_id = second_dataset.id diff_run = session.scalar(select(GtfsFeedDiffRun).where(GtfsFeedDiffRun.new_dataset_id == second_id)) assert diff_run is not None assert diff_run.previous_dataset_id == first_id assert session.get(Dataset, first_id) is None summary = json.loads(diff_run.summary_json) assert summary["routes"]["changed"] == 1 assert summary["routes"]["added"] == 1 assert summary["stops"]["changed"] == 1 assert summary["stops"]["added"] == 1 items = { (item.item_type, item.object_key, item.change_type) for item in session.scalars(select(GtfsFeedDiffItem).where(GtfsFeedDiffItem.diff_run_id == diff_run.id)) } assert ("route", "R1", "changed") in items assert ("route", "R2", "added") in items assert ("stop", "B", "changed") in items assert ("stop", "C", "added") in items def test_gtfs_reimport_replays_unchanged_decisions_and_queues_changed_items(tmp_path): reset_db() gtfs_path = tmp_path / "decision-replay.gtfs.zip" _write_update_diff_gtfs(gtfs_path, route_long_name="Alpha - Beta", beta_name="Beta", include_stable_route=True) with session_scope() as session: source = Source(name="Decision replay GTFS", kind="gtfs", url=str(gtfs_path)) session.add(source) session.flush() first_dataset = run_source(session, source) first_id = first_dataset.id stable_route = session.scalar( select(GtfsRoute).where(GtfsRoute.dataset_id == first_id, GtfsRoute.route_id == "R_KEEP") ) assert stable_route is not None selector = { "gtfs": { "source_id": source.id, "dataset_id": first_id, "route_id": stable_route.route_id, "route_key": stable_route.route_key, "ref": stable_route.short_name, "mode": stable_route.mode, } } session.add( MapGtfsDecision( decision_key="test-stable-route-decision", decision_type="route_osm_relation", status="accepted", active=True, source_id=source.id, gtfs_dataset_id=first_id, gtfs_route_id=stable_route.id, confidence=0.91, selector_json=json.dumps(selector, sort_keys=True), action_json=json.dumps({"status": "accepted", "osm": {"osm_id": 12345}}, sort_keys=True), note="keep stable route", reviewer="test", metadata_json=json.dumps({"manual": True}, sort_keys=True), ) ) _write_update_diff_gtfs( gtfs_path, route_long_name="Alpha - Beta revised", beta_name="Beta moved", include_extra_stop=True, include_extra_route=True, include_stable_route=True, ) with session_scope() as session: source = session.scalar(select(Source).where(Source.name == "Decision replay GTFS")) second_dataset = run_source(session, source) second_id = second_dataset.id stable_route = session.scalar( select(GtfsRoute).where(GtfsRoute.dataset_id == second_id, GtfsRoute.route_id == "R_KEEP") ) assert stable_route is not None replayed = session.scalar( select(MapGtfsDecision).where(MapGtfsDecision.gtfs_dataset_id == second_id, MapGtfsDecision.gtfs_route_id == stable_route.id) ) assert replayed is not None assert replayed.reviewer == "decision_replay" assert replayed.confidence == 0.91 replay_metadata = json.loads(replayed.metadata_json or "{}") assert replay_metadata["manual"] is True assert replay_metadata["replay"]["previous_dataset_id"] == first_id assert replay_metadata["replay"]["new_dataset_id"] == second_id assert session.scalar(select(func.count()).select_from(MapGtfsDecision).where(MapGtfsDecision.gtfs_dataset_id == first_id)) == 0 review_items = session.scalars(select(MapGtfsReviewItem).where(MapGtfsReviewItem.queue == "gtfs_update_diff")).all() review_keys = { (item.item_type, (json.loads(item.metadata_json or "{}")).get("object_key"), (json.loads(item.metadata_json or "{}")).get("change_type")) for item in review_items } assert ("gtfs_route_update", "R1", "changed") in review_keys assert ("gtfs_stop_update", "B", "changed") in review_keys def test_gtfs_dataset_delete_removes_route_pattern_links_by_pattern_id(): reset_db() with session_scope() as session: source = Source(name="Delete GTFS", kind="gtfs", url="./delete.zip") other_source = Source(name="Other GTFS", kind="gtfs", url="./other.zip") session.add_all([source, other_source]) session.flush() dataset = Dataset(source_id=source.id, kind="gtfs", local_path="./delete.zip", sha256="a" * 64, is_active=False, status="imported") other_dataset = Dataset(source_id=other_source.id, kind="gtfs", local_path="./other.zip", sha256="b" * 64, is_active=True, status="imported") session.add_all([dataset, other_dataset]) session.flush() route = GtfsRoute( dataset_id=dataset.id, route_id="R1", short_name="R1", long_name="Delete route", route_type=3, mode="bus", ) session.add(route) session.flush() pattern = RoutePattern( pattern_key="gtfs:delete:R1:S1", route_ref="R1", route_name="Delete route", mode="bus", source_kind="gtfs_proposed", status="active", gtfs_route_id=route.id, gtfs_shape_id="S1", geometry_geojson='{"type":"LineString","coordinates":[[13.0,52.0],[13.1,52.1]]}', ) session.add(pattern) session.flush() session.add( GtfsRoutePatternLink( dataset_id=other_dataset.id, gtfs_route_id=route.id, route_id="R1", shape_id="S1", route_pattern_id=pattern.id, confidence=1, status="linked", source_kind="gtfs_proposed", ) ) session.add( GtfsTripRoutePatternLink( dataset_id=other_dataset.id, trip_id="T1", route_id="R1", shape_id="S1", route_pattern_id=pattern.id, source_kind="gtfs_proposed", confidence=1, status="linked", ) ) session.flush() result = delete_dataset(session, dataset.id) assert result["deleted"] is True assert session.scalar(select(func.count()).select_from(RoutePattern).where(RoutePattern.id == pattern.id)) == 0 assert session.scalar(select(func.count()).select_from(GtfsRoutePatternLink)) == 0 assert session.scalar(select(func.count()).select_from(GtfsTripRoutePatternLink)) == 0 def _write_update_diff_gtfs( gtfs_path, *, route_long_name: str, beta_name: str, include_extra_stop: bool = False, include_extra_route: bool = False, include_stable_route: bool = False, ) -> None: stops = [ "A,Alpha,52.0,13.0", f"B,{beta_name},52.1,13.1", ] if include_extra_stop: stops.append("C,Gamma,52.2,13.2") routes = [ f"R1,A,R1,{route_long_name},3", ] if include_extra_route: routes.append("R2,A,R2,Alpha - Gamma,3") if include_stable_route: routes.append("R_KEEP,A,RK,Stable route,3") trips = [ "R1,daily,t1", ] if include_extra_route: trips.append("R2,daily,t2") if include_stable_route: trips.append("R_KEEP,daily,t_keep") stop_times = [ "t1,08:00:00,08:00:00,A,1", "t1,08:10:00,08:10:00,B,2", ] if include_extra_route: stop_times.extend( [ "t2,09:00:00,09:00:00,A,1", "t2,09:15:00,09:15:00,C,2", ] ) if include_stable_route: stop_times.extend( [ "t_keep,10:00:00,10:00:00,A,1", "t_keep,10:12:00,10:12:00,B,2", ] ) with zipfile.ZipFile(gtfs_path, "w") as zf: zf.writestr("agency.txt", "agency_id,agency_name,agency_url,agency_timezone\nA,Agency,https://example.invalid,Europe/Berlin\n") zf.writestr("stops.txt", "stop_id,stop_name,stop_lat,stop_lon\n" + "\n".join(stops) + "\n") zf.writestr("routes.txt", "route_id,agency_id,route_short_name,route_long_name,route_type\n" + "\n".join(routes) + "\n") zf.writestr("trips.txt", "route_id,service_id,trip_id\n" + "\n".join(trips) + "\n") zf.writestr("calendar.txt", "service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date\ndaily,1,1,1,1,1,1,1,20260101,20261231\n") zf.writestr("stop_times.txt", "trip_id,arrival_time,departure_time,stop_id,stop_sequence\n" + "\n".join(stop_times) + "\n")