400 lines
16 KiB
Python
400 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import zipfile
|
|
|
|
from sqlalchemy import func, select
|
|
|
|
from app.data_management import delete_dataset
|
|
from app.db import reset_db, session_scope
|
|
from app.gtfs_storage import sidecar_path, stop_time_count, stop_times_by_trip
|
|
from app.journey import find_journeys, search_scheduled_stops
|
|
from app.models import (
|
|
Dataset,
|
|
GtfsCalendar,
|
|
GtfsFeedDiffItem,
|
|
GtfsFeedDiffRun,
|
|
GtfsRoute,
|
|
GtfsRoutePatternLink,
|
|
GtfsTripRoutePatternLink,
|
|
MapGtfsDecision,
|
|
MapGtfsReviewItem,
|
|
RoutePattern,
|
|
Source,
|
|
)
|
|
from app.pipeline.gtfs import _copy_stage_row_payload
|
|
from app.pipeline.run import run_source
|
|
|
|
|
|
def test_gtfs_import_uses_staging_bulk_loader_and_reports_chunks(tmp_path, monkeypatch):
|
|
reset_db()
|
|
gtfs_path = tmp_path / "small.gtfs.zip"
|
|
with zipfile.ZipFile(gtfs_path, "w") as zf:
|
|
zf.writestr("agency.txt", "agency_id,agency_name,agency_url,agency_timezone\nA,Agency,https://example.invalid,Europe/Berlin\n")
|
|
zf.writestr(
|
|
"stops.txt",
|
|
"stop_id,stop_name,stop_lat,stop_lon\nA,Alpha,52.0,13.0\nB,Beta,52.1,13.1\nC,Gamma,52.2,13.2\n",
|
|
)
|
|
zf.writestr("routes.txt", "route_id,agency_id,route_short_name,route_long_name,route_type\nR,A,R1,Alpha - Gamma,3\n")
|
|
zf.writestr("trips.txt", "route_id,service_id,trip_id,shape_id\nR,daily,t1,s1\nR,daily,t2,s1\n")
|
|
zf.writestr("calendar.txt", "service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date\ndaily,1,1,1,1,1,1,1,20260101,20261231\n")
|
|
zf.writestr(
|
|
"stop_times.txt",
|
|
"\n".join(
|
|
[
|
|
"trip_id,arrival_time,departure_time,stop_id,stop_sequence",
|
|
"t1,08:00:00,08:00:00,A,1",
|
|
"t1,08:05:00,08:05:00,B,2",
|
|
"t1,08:10:00,08:10:00,C,3",
|
|
"t2,09:00:00,09:00:00,A,1",
|
|
"t2,09:10:00,09:10:00,C,2",
|
|
]
|
|
)
|
|
+ "\n",
|
|
)
|
|
zf.writestr("shapes.txt", "shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence\ns1,52.0,13.0,1\ns1,52.2,13.2,2\n")
|
|
|
|
monkeypatch.setattr("app.pipeline.gtfs.GTFS_STAGE_BATCH_SIZE", 2)
|
|
events = []
|
|
with session_scope() as session:
|
|
source = Source(name="Small GTFS", kind="gtfs", url=str(gtfs_path))
|
|
session.add(source)
|
|
session.flush()
|
|
dataset = run_source(session, source, progress_callback=lambda *args: events.append(args))
|
|
|
|
metadata = json.loads(dataset.metadata_json or "{}")
|
|
assert metadata["importer"] == "gtfs_import_v6_sidecar_stop_times"
|
|
assert metadata["staging"] == "sqlite_promoted_to_sidecar"
|
|
assert metadata["gtfs_storage"]["tables"]["gtfs_stop_times"] == "sidecar"
|
|
assert metadata["stop_times_imported"] == 5
|
|
assert sidecar_path(dataset) is not None
|
|
assert sidecar_path(dataset).exists()
|
|
assert stop_time_count(session, dataset.id) == 5
|
|
assert len(stop_times_by_trip(session, dataset.id, ["t1"])["t1"]) == 3
|
|
assert session.scalar(select(func.count()).select_from(GtfsCalendar).where(GtfsCalendar.dataset_id == dataset.id)) == 1
|
|
assert session.scalar(select(func.count()).select_from(Dataset).where(Dataset.kind == "gtfs", Dataset.is_active.is_(True))) == 1
|
|
alpha = search_scheduled_stops(session, "Alpha", limit=1)[0]
|
|
gamma = search_scheduled_stops(session, "Gamma", limit=1)[0]
|
|
journey = find_journeys(session, alpha["id"], gamma["id"], "08:00", limit=1)
|
|
assert journey["journeys"][0]["departure_time"] == "08:00:00"
|
|
assert journey["journeys"][0]["arrival_time"] == "08:10:00"
|
|
|
|
event_types = [event[0] for event in events]
|
|
assert "gtfs_staging_started" in event_types
|
|
assert "gtfs_file_chunk" in event_types
|
|
assert "gtfs_activation_sidecar_stop_times" in event_types
|
|
assert "gtfs_activation_completed" in event_types
|
|
|
|
|
|
def test_gtfs_calendar_stage_payload_casts_weekday_flags_to_bool():
|
|
columns = ["service_id", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", "start_date", "end_date"]
|
|
payload = _copy_stage_row_payload(
|
|
"gtfs_calendars",
|
|
5,
|
|
columns,
|
|
("Special", 0, 1, "0", "1", False, True, "", 20260401, 20261211),
|
|
)
|
|
|
|
assert payload == {
|
|
"dataset_id": 5,
|
|
"service_id": "Special",
|
|
"monday": False,
|
|
"tuesday": True,
|
|
"wednesday": False,
|
|
"thursday": True,
|
|
"friday": False,
|
|
"saturday": True,
|
|
"sunday": False,
|
|
"start_date": 20260401,
|
|
"end_date": 20261211,
|
|
}
|
|
|
|
|
|
def test_gtfs_reimports_same_hash_when_stop_time_limit_changes(tmp_path, monkeypatch):
|
|
reset_db()
|
|
gtfs_path = tmp_path / "limited.gtfs.zip"
|
|
with zipfile.ZipFile(gtfs_path, "w") as zf:
|
|
zf.writestr("agency.txt", "agency_id,agency_name,agency_url,agency_timezone\nA,Agency,https://example.invalid,Europe/Berlin\n")
|
|
zf.writestr("stops.txt", "stop_id,stop_name,stop_lat,stop_lon\nA,Alpha,52.0,13.0\nB,Beta,52.1,13.1\nC,Gamma,52.2,13.2\n")
|
|
zf.writestr("routes.txt", "route_id,agency_id,route_short_name,route_long_name,route_type\nR,A,R1,Alpha - Gamma,3\n")
|
|
zf.writestr("trips.txt", "route_id,service_id,trip_id\nR,daily,t1\n")
|
|
zf.writestr("calendar.txt", "service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date\ndaily,1,1,1,1,1,1,1,20260101,20261231\n")
|
|
zf.writestr(
|
|
"stop_times.txt",
|
|
"\n".join(
|
|
[
|
|
"trip_id,arrival_time,departure_time,stop_id,stop_sequence",
|
|
"t1,08:00:00,08:00:00,A,1",
|
|
"t1,08:05:00,08:05:00,B,2",
|
|
"t1,08:10:00,08:10:00,C,3",
|
|
]
|
|
)
|
|
+ "\n",
|
|
)
|
|
|
|
with session_scope() as session:
|
|
source = Source(name="Limited GTFS", kind="gtfs", url=str(gtfs_path))
|
|
session.add(source)
|
|
session.flush()
|
|
|
|
monkeypatch.setattr("app.pipeline.gtfs.settings.gtfs_stop_times_import_limit", 2)
|
|
first_dataset = run_source(session, source)
|
|
first_id = first_dataset.id
|
|
assert stop_time_count(session, first_id) == 2
|
|
|
|
monkeypatch.setattr("app.pipeline.gtfs.settings.gtfs_stop_times_import_limit", 0)
|
|
second_dataset = run_source(session, source)
|
|
assert second_dataset.id != first_id
|
|
assert stop_time_count(session, second_dataset.id) == 3
|
|
|
|
|
|
def test_gtfs_reimport_stores_update_diff_before_replacing_old_dataset(tmp_path):
|
|
reset_db()
|
|
gtfs_path = tmp_path / "updating.gtfs.zip"
|
|
_write_update_diff_gtfs(gtfs_path, route_long_name="Alpha - Beta", beta_name="Beta")
|
|
|
|
with session_scope() as session:
|
|
source = Source(name="Updating GTFS", kind="gtfs", url=str(gtfs_path))
|
|
session.add(source)
|
|
session.flush()
|
|
first_dataset = run_source(session, source)
|
|
first_id = first_dataset.id
|
|
|
|
_write_update_diff_gtfs(
|
|
gtfs_path,
|
|
route_long_name="Alpha - Beta revised",
|
|
beta_name="Beta moved",
|
|
include_extra_stop=True,
|
|
include_extra_route=True,
|
|
)
|
|
|
|
with session_scope() as session:
|
|
source = session.scalar(select(Source).where(Source.name == "Updating GTFS"))
|
|
second_dataset = run_source(session, source)
|
|
second_id = second_dataset.id
|
|
|
|
diff_run = session.scalar(select(GtfsFeedDiffRun).where(GtfsFeedDiffRun.new_dataset_id == second_id))
|
|
assert diff_run is not None
|
|
assert diff_run.previous_dataset_id == first_id
|
|
assert session.get(Dataset, first_id) is None
|
|
summary = json.loads(diff_run.summary_json)
|
|
assert summary["routes"]["changed"] == 1
|
|
assert summary["routes"]["added"] == 1
|
|
assert summary["stops"]["changed"] == 1
|
|
assert summary["stops"]["added"] == 1
|
|
|
|
items = {
|
|
(item.item_type, item.object_key, item.change_type)
|
|
for item in session.scalars(select(GtfsFeedDiffItem).where(GtfsFeedDiffItem.diff_run_id == diff_run.id))
|
|
}
|
|
assert ("route", "R1", "changed") in items
|
|
assert ("route", "R2", "added") in items
|
|
assert ("stop", "B", "changed") in items
|
|
assert ("stop", "C", "added") in items
|
|
|
|
|
|
def test_gtfs_reimport_replays_unchanged_decisions_and_queues_changed_items(tmp_path):
|
|
reset_db()
|
|
gtfs_path = tmp_path / "decision-replay.gtfs.zip"
|
|
_write_update_diff_gtfs(gtfs_path, route_long_name="Alpha - Beta", beta_name="Beta", include_stable_route=True)
|
|
|
|
with session_scope() as session:
|
|
source = Source(name="Decision replay GTFS", kind="gtfs", url=str(gtfs_path))
|
|
session.add(source)
|
|
session.flush()
|
|
first_dataset = run_source(session, source)
|
|
first_id = first_dataset.id
|
|
stable_route = session.scalar(
|
|
select(GtfsRoute).where(GtfsRoute.dataset_id == first_id, GtfsRoute.route_id == "R_KEEP")
|
|
)
|
|
assert stable_route is not None
|
|
selector = {
|
|
"gtfs": {
|
|
"source_id": source.id,
|
|
"dataset_id": first_id,
|
|
"route_id": stable_route.route_id,
|
|
"route_key": stable_route.route_key,
|
|
"ref": stable_route.short_name,
|
|
"mode": stable_route.mode,
|
|
}
|
|
}
|
|
session.add(
|
|
MapGtfsDecision(
|
|
decision_key="test-stable-route-decision",
|
|
decision_type="route_osm_relation",
|
|
status="accepted",
|
|
active=True,
|
|
source_id=source.id,
|
|
gtfs_dataset_id=first_id,
|
|
gtfs_route_id=stable_route.id,
|
|
confidence=0.91,
|
|
selector_json=json.dumps(selector, sort_keys=True),
|
|
action_json=json.dumps({"status": "accepted", "osm": {"osm_id": 12345}}, sort_keys=True),
|
|
note="keep stable route",
|
|
reviewer="test",
|
|
metadata_json=json.dumps({"manual": True}, sort_keys=True),
|
|
)
|
|
)
|
|
|
|
_write_update_diff_gtfs(
|
|
gtfs_path,
|
|
route_long_name="Alpha - Beta revised",
|
|
beta_name="Beta moved",
|
|
include_extra_stop=True,
|
|
include_extra_route=True,
|
|
include_stable_route=True,
|
|
)
|
|
|
|
with session_scope() as session:
|
|
source = session.scalar(select(Source).where(Source.name == "Decision replay GTFS"))
|
|
second_dataset = run_source(session, source)
|
|
second_id = second_dataset.id
|
|
stable_route = session.scalar(
|
|
select(GtfsRoute).where(GtfsRoute.dataset_id == second_id, GtfsRoute.route_id == "R_KEEP")
|
|
)
|
|
assert stable_route is not None
|
|
|
|
replayed = session.scalar(
|
|
select(MapGtfsDecision).where(MapGtfsDecision.gtfs_dataset_id == second_id, MapGtfsDecision.gtfs_route_id == stable_route.id)
|
|
)
|
|
assert replayed is not None
|
|
assert replayed.reviewer == "decision_replay"
|
|
assert replayed.confidence == 0.91
|
|
replay_metadata = json.loads(replayed.metadata_json or "{}")
|
|
assert replay_metadata["manual"] is True
|
|
assert replay_metadata["replay"]["previous_dataset_id"] == first_id
|
|
assert replay_metadata["replay"]["new_dataset_id"] == second_id
|
|
assert session.scalar(select(func.count()).select_from(MapGtfsDecision).where(MapGtfsDecision.gtfs_dataset_id == first_id)) == 0
|
|
|
|
review_items = session.scalars(select(MapGtfsReviewItem).where(MapGtfsReviewItem.queue == "gtfs_update_diff")).all()
|
|
review_keys = {
|
|
(item.item_type, (json.loads(item.metadata_json or "{}")).get("object_key"), (json.loads(item.metadata_json or "{}")).get("change_type"))
|
|
for item in review_items
|
|
}
|
|
assert ("gtfs_route_update", "R1", "changed") in review_keys
|
|
assert ("gtfs_stop_update", "B", "changed") in review_keys
|
|
|
|
|
|
def test_gtfs_dataset_delete_removes_route_pattern_links_by_pattern_id():
|
|
reset_db()
|
|
with session_scope() as session:
|
|
source = Source(name="Delete GTFS", kind="gtfs", url="./delete.zip")
|
|
other_source = Source(name="Other GTFS", kind="gtfs", url="./other.zip")
|
|
session.add_all([source, other_source])
|
|
session.flush()
|
|
dataset = Dataset(source_id=source.id, kind="gtfs", local_path="./delete.zip", sha256="a" * 64, is_active=False, status="imported")
|
|
other_dataset = Dataset(source_id=other_source.id, kind="gtfs", local_path="./other.zip", sha256="b" * 64, is_active=True, status="imported")
|
|
session.add_all([dataset, other_dataset])
|
|
session.flush()
|
|
route = GtfsRoute(
|
|
dataset_id=dataset.id,
|
|
route_id="R1",
|
|
short_name="R1",
|
|
long_name="Delete route",
|
|
route_type=3,
|
|
mode="bus",
|
|
)
|
|
session.add(route)
|
|
session.flush()
|
|
pattern = RoutePattern(
|
|
pattern_key="gtfs:delete:R1:S1",
|
|
route_ref="R1",
|
|
route_name="Delete route",
|
|
mode="bus",
|
|
source_kind="gtfs_proposed",
|
|
status="active",
|
|
gtfs_route_id=route.id,
|
|
gtfs_shape_id="S1",
|
|
geometry_geojson='{"type":"LineString","coordinates":[[13.0,52.0],[13.1,52.1]]}',
|
|
)
|
|
session.add(pattern)
|
|
session.flush()
|
|
session.add(
|
|
GtfsRoutePatternLink(
|
|
dataset_id=other_dataset.id,
|
|
gtfs_route_id=route.id,
|
|
route_id="R1",
|
|
shape_id="S1",
|
|
route_pattern_id=pattern.id,
|
|
confidence=1,
|
|
status="linked",
|
|
source_kind="gtfs_proposed",
|
|
)
|
|
)
|
|
session.add(
|
|
GtfsTripRoutePatternLink(
|
|
dataset_id=other_dataset.id,
|
|
trip_id="T1",
|
|
route_id="R1",
|
|
shape_id="S1",
|
|
route_pattern_id=pattern.id,
|
|
source_kind="gtfs_proposed",
|
|
confidence=1,
|
|
status="linked",
|
|
)
|
|
)
|
|
session.flush()
|
|
|
|
result = delete_dataset(session, dataset.id)
|
|
|
|
assert result["deleted"] is True
|
|
assert session.scalar(select(func.count()).select_from(RoutePattern).where(RoutePattern.id == pattern.id)) == 0
|
|
assert session.scalar(select(func.count()).select_from(GtfsRoutePatternLink)) == 0
|
|
assert session.scalar(select(func.count()).select_from(GtfsTripRoutePatternLink)) == 0
|
|
|
|
|
|
def _write_update_diff_gtfs(
|
|
gtfs_path,
|
|
*,
|
|
route_long_name: str,
|
|
beta_name: str,
|
|
include_extra_stop: bool = False,
|
|
include_extra_route: bool = False,
|
|
include_stable_route: bool = False,
|
|
) -> None:
|
|
stops = [
|
|
"A,Alpha,52.0,13.0",
|
|
f"B,{beta_name},52.1,13.1",
|
|
]
|
|
if include_extra_stop:
|
|
stops.append("C,Gamma,52.2,13.2")
|
|
routes = [
|
|
f"R1,A,R1,{route_long_name},3",
|
|
]
|
|
if include_extra_route:
|
|
routes.append("R2,A,R2,Alpha - Gamma,3")
|
|
if include_stable_route:
|
|
routes.append("R_KEEP,A,RK,Stable route,3")
|
|
trips = [
|
|
"R1,daily,t1",
|
|
]
|
|
if include_extra_route:
|
|
trips.append("R2,daily,t2")
|
|
if include_stable_route:
|
|
trips.append("R_KEEP,daily,t_keep")
|
|
stop_times = [
|
|
"t1,08:00:00,08:00:00,A,1",
|
|
"t1,08:10:00,08:10:00,B,2",
|
|
]
|
|
if include_extra_route:
|
|
stop_times.extend(
|
|
[
|
|
"t2,09:00:00,09:00:00,A,1",
|
|
"t2,09:15:00,09:15:00,C,2",
|
|
]
|
|
)
|
|
if include_stable_route:
|
|
stop_times.extend(
|
|
[
|
|
"t_keep,10:00:00,10:00:00,A,1",
|
|
"t_keep,10:12:00,10:12:00,B,2",
|
|
]
|
|
)
|
|
with zipfile.ZipFile(gtfs_path, "w") as zf:
|
|
zf.writestr("agency.txt", "agency_id,agency_name,agency_url,agency_timezone\nA,Agency,https://example.invalid,Europe/Berlin\n")
|
|
zf.writestr("stops.txt", "stop_id,stop_name,stop_lat,stop_lon\n" + "\n".join(stops) + "\n")
|
|
zf.writestr("routes.txt", "route_id,agency_id,route_short_name,route_long_name,route_type\n" + "\n".join(routes) + "\n")
|
|
zf.writestr("trips.txt", "route_id,service_id,trip_id\n" + "\n".join(trips) + "\n")
|
|
zf.writestr("calendar.txt", "service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date\ndaily,1,1,1,1,1,1,1,20260101,20261231\n")
|
|
zf.writestr("stop_times.txt", "trip_id,arrival_time,departure_time,stop_id,stop_sequence\n" + "\n".join(stop_times) + "\n")
|