add-ideas/meubility-workbench
1
0
Fork 0
Code Issues 122 Pull Requests Actions Packages Projects Releases Wiki Activity

Next alpha stage commit

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-06 20:54:46 +02:00
parent e23387738b
commit 1d7ec956d2
42 changed files with 11603 additions and 2837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

329
tests/test_gtfs_import.py
View File

@@ -5,10 +5,24 @@ import zipfile
from sqlalchemy import func, select
from app.data_management import delete_dataset
from app.db import reset_db, session_scope
from app.gtfs_storage import sidecar_path, stop_time_count, stop_times_by_trip
from app.journey import find_journeys, search_scheduled_stops
from app.models import Dataset, GtfsCalendar, Source
from app.models import (
Dataset,
GtfsCalendar,
GtfsFeedDiffItem,
GtfsFeedDiffRun,
GtfsRoute,
GtfsRoutePatternLink,
GtfsTripRoutePatternLink,
MapGtfsDecision,
MapGtfsReviewItem,
RoutePattern,
Source,
)
from app.pipeline.gtfs import _copy_stage_row_payload
from app.pipeline.run import run_source
@@ -70,3 +84,316 @@ def test_gtfs_import_uses_staging_bulk_loader_and_reports_chunks(tmp_path, monke
assert "gtfs_file_chunk" in event_types
assert "gtfs_activation_sidecar_stop_times" in event_types
assert "gtfs_activation_completed" in event_types
def test_gtfs_calendar_stage_payload_casts_weekday_flags_to_bool():
columns = ["service_id", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday", "start_date", "end_date"]
payload = _copy_stage_row_payload(
"gtfs_calendars",
5,
columns,
("Special", 0, 1, "0", "1", False, True, "", 20260401, 20261211),
)
assert payload == {
"dataset_id": 5,
"service_id": "Special",
"monday": False,
"tuesday": True,
"wednesday": False,
"thursday": True,
"friday": False,
"saturday": True,
"sunday": False,
"start_date": 20260401,
"end_date": 20261211,
}
def test_gtfs_reimports_same_hash_when_stop_time_limit_changes(tmp_path, monkeypatch):
reset_db()
gtfs_path = tmp_path / "limited.gtfs.zip"
with zipfile.ZipFile(gtfs_path, "w") as zf:
zf.writestr("agency.txt", "agency_id,agency_name,agency_url,agency_timezone\nA,Agency,https://example.invalid,Europe/Berlin\n")
zf.writestr("stops.txt", "stop_id,stop_name,stop_lat,stop_lon\nA,Alpha,52.0,13.0\nB,Beta,52.1,13.1\nC,Gamma,52.2,13.2\n")
zf.writestr("routes.txt", "route_id,agency_id,route_short_name,route_long_name,route_type\nR,A,R1,Alpha - Gamma,3\n")
zf.writestr("trips.txt", "route_id,service_id,trip_id\nR,daily,t1\n")
zf.writestr("calendar.txt", "service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date\ndaily,1,1,1,1,1,1,1,20260101,20261231\n")
zf.writestr(
"stop_times.txt",
"\n".join(
[
"trip_id,arrival_time,departure_time,stop_id,stop_sequence",
"t1,08:00:00,08:00:00,A,1",
"t1,08:05:00,08:05:00,B,2",
"t1,08:10:00,08:10:00,C,3",
]
)
+ "\n",
)
with session_scope() as session:
source = Source(name="Limited GTFS", kind="gtfs", url=str(gtfs_path))
session.add(source)
session.flush()
monkeypatch.setattr("app.pipeline.gtfs.settings.gtfs_stop_times_import_limit", 2)
first_dataset = run_source(session, source)
first_id = first_dataset.id
assert stop_time_count(session, first_id) == 2
monkeypatch.setattr("app.pipeline.gtfs.settings.gtfs_stop_times_import_limit", 0)
second_dataset = run_source(session, source)
assert second_dataset.id != first_id
assert stop_time_count(session, second_dataset.id) == 3
def test_gtfs_reimport_stores_update_diff_before_replacing_old_dataset(tmp_path):
reset_db()
gtfs_path = tmp_path / "updating.gtfs.zip"
_write_update_diff_gtfs(gtfs_path, route_long_name="Alpha - Beta", beta_name="Beta")
with session_scope() as session:
source = Source(name="Updating GTFS", kind="gtfs", url=str(gtfs_path))
session.add(source)
session.flush()
first_dataset = run_source(session, source)
first_id = first_dataset.id
_write_update_diff_gtfs(
gtfs_path,
route_long_name="Alpha - Beta revised",
beta_name="Beta moved",
include_extra_stop=True,
include_extra_route=True,
)
with session_scope() as session:
source = session.scalar(select(Source).where(Source.name == "Updating GTFS"))
second_dataset = run_source(session, source)
second_id = second_dataset.id
diff_run = session.scalar(select(GtfsFeedDiffRun).where(GtfsFeedDiffRun.new_dataset_id == second_id))
assert diff_run is not None
assert diff_run.previous_dataset_id == first_id
assert session.get(Dataset, first_id) is None
summary = json.loads(diff_run.summary_json)
assert summary["routes"]["changed"] == 1
assert summary["routes"]["added"] == 1
assert summary["stops"]["changed"] == 1
assert summary["stops"]["added"] == 1
items = {
(item.item_type, item.object_key, item.change_type)
for item in session.scalars(select(GtfsFeedDiffItem).where(GtfsFeedDiffItem.diff_run_id == diff_run.id))
}
assert ("route", "R1", "changed") in items
assert ("route", "R2", "added") in items
assert ("stop", "B", "changed") in items
assert ("stop", "C", "added") in items
def test_gtfs_reimport_replays_unchanged_decisions_and_queues_changed_items(tmp_path):
reset_db()
gtfs_path = tmp_path / "decision-replay.gtfs.zip"
_write_update_diff_gtfs(gtfs_path, route_long_name="Alpha - Beta", beta_name="Beta", include_stable_route=True)
with session_scope() as session:
source = Source(name="Decision replay GTFS", kind="gtfs", url=str(gtfs_path))
session.add(source)
session.flush()
first_dataset = run_source(session, source)
first_id = first_dataset.id
stable_route = session.scalar(
select(GtfsRoute).where(GtfsRoute.dataset_id == first_id, GtfsRoute.route_id == "R_KEEP")
)
assert stable_route is not None
selector = {
"gtfs": {
"source_id": source.id,
"dataset_id": first_id,
"route_id": stable_route.route_id,
"route_key": stable_route.route_key,
"ref": stable_route.short_name,
"mode": stable_route.mode,
}
}
session.add(
MapGtfsDecision(
decision_key="test-stable-route-decision",
decision_type="route_osm_relation",
status="accepted",
active=True,
source_id=source.id,
gtfs_dataset_id=first_id,
gtfs_route_id=stable_route.id,
confidence=0.91,
selector_json=json.dumps(selector, sort_keys=True),
action_json=json.dumps({"status": "accepted", "osm": {"osm_id": 12345}}, sort_keys=True),
note="keep stable route",
reviewer="test",
metadata_json=json.dumps({"manual": True}, sort_keys=True),
)
)
_write_update_diff_gtfs(
gtfs_path,
route_long_name="Alpha - Beta revised",
beta_name="Beta moved",
include_extra_stop=True,
include_extra_route=True,
include_stable_route=True,
)
with session_scope() as session:
source = session.scalar(select(Source).where(Source.name == "Decision replay GTFS"))
second_dataset = run_source(session, source)
second_id = second_dataset.id
stable_route = session.scalar(
select(GtfsRoute).where(GtfsRoute.dataset_id == second_id, GtfsRoute.route_id == "R_KEEP")
)
assert stable_route is not None
replayed = session.scalar(
select(MapGtfsDecision).where(MapGtfsDecision.gtfs_dataset_id == second_id, MapGtfsDecision.gtfs_route_id == stable_route.id)
)
assert replayed is not None
assert replayed.reviewer == "decision_replay"
assert replayed.confidence == 0.91
replay_metadata = json.loads(replayed.metadata_json or "{}")
assert replay_metadata["manual"] is True
assert replay_metadata["replay"]["previous_dataset_id"] == first_id
assert replay_metadata["replay"]["new_dataset_id"] == second_id
assert session.scalar(select(func.count()).select_from(MapGtfsDecision).where(MapGtfsDecision.gtfs_dataset_id == first_id)) == 0
review_items = session.scalars(select(MapGtfsReviewItem).where(MapGtfsReviewItem.queue == "gtfs_update_diff")).all()
review_keys = {
(item.item_type, (json.loads(item.metadata_json or "{}")).get("object_key"), (json.loads(item.metadata_json or "{}")).get("change_type"))
for item in review_items
}
assert ("gtfs_route_update", "R1", "changed") in review_keys
assert ("gtfs_stop_update", "B", "changed") in review_keys
def test_gtfs_dataset_delete_removes_route_pattern_links_by_pattern_id():
reset_db()
with session_scope() as session:
source = Source(name="Delete GTFS", kind="gtfs", url="./delete.zip")
other_source = Source(name="Other GTFS", kind="gtfs", url="./other.zip")
session.add_all([source, other_source])
session.flush()
dataset = Dataset(source_id=source.id, kind="gtfs", local_path="./delete.zip", sha256="a" * 64, is_active=False, status="imported")
other_dataset = Dataset(source_id=other_source.id, kind="gtfs", local_path="./other.zip", sha256="b" * 64, is_active=True, status="imported")
session.add_all([dataset, other_dataset])
session.flush()
route = GtfsRoute(
dataset_id=dataset.id,
route_id="R1",
short_name="R1",
long_name="Delete route",
route_type=3,
mode="bus",
)
session.add(route)
session.flush()
pattern = RoutePattern(
pattern_key="gtfs:delete:R1:S1",
route_ref="R1",
route_name="Delete route",
mode="bus",
source_kind="gtfs_proposed",
status="active",
gtfs_route_id=route.id,
gtfs_shape_id="S1",
geometry_geojson='{"type":"LineString","coordinates":[[13.0,52.0],[13.1,52.1]]}',
)
session.add(pattern)
session.flush()
session.add(
GtfsRoutePatternLink(
dataset_id=other_dataset.id,
gtfs_route_id=route.id,
route_id="R1",
shape_id="S1",
route_pattern_id=pattern.id,
confidence=1,
status="linked",
source_kind="gtfs_proposed",
)
)
session.add(
GtfsTripRoutePatternLink(
dataset_id=other_dataset.id,
trip_id="T1",
route_id="R1",
shape_id="S1",
route_pattern_id=pattern.id,
source_kind="gtfs_proposed",
confidence=1,
status="linked",
)
)
session.flush()
result = delete_dataset(session, dataset.id)
assert result["deleted"] is True
assert session.scalar(select(func.count()).select_from(RoutePattern).where(RoutePattern.id == pattern.id)) == 0
assert session.scalar(select(func.count()).select_from(GtfsRoutePatternLink)) == 0
assert session.scalar(select(func.count()).select_from(GtfsTripRoutePatternLink)) == 0
def _write_update_diff_gtfs(
gtfs_path,
*,
route_long_name: str,
beta_name: str,
include_extra_stop: bool = False,
include_extra_route: bool = False,
include_stable_route: bool = False,
) -> None:
stops = [
"A,Alpha,52.0,13.0",
f"B,{beta_name},52.1,13.1",
]
if include_extra_stop:
stops.append("C,Gamma,52.2,13.2")
routes = [
f"R1,A,R1,{route_long_name},3",
]
if include_extra_route:
routes.append("R2,A,R2,Alpha - Gamma,3")
if include_stable_route:
routes.append("R_KEEP,A,RK,Stable route,3")
trips = [
"R1,daily,t1",
]
if include_extra_route:
trips.append("R2,daily,t2")
if include_stable_route:
trips.append("R_KEEP,daily,t_keep")
stop_times = [
"t1,08:00:00,08:00:00,A,1",
"t1,08:10:00,08:10:00,B,2",
]
if include_extra_route:
stop_times.extend(
[
"t2,09:00:00,09:00:00,A,1",
"t2,09:15:00,09:15:00,C,2",
]
)
if include_stable_route:
stop_times.extend(
[
"t_keep,10:00:00,10:00:00,A,1",
"t_keep,10:12:00,10:12:00,B,2",
]
)
with zipfile.ZipFile(gtfs_path, "w") as zf:
zf.writestr("agency.txt", "agency_id,agency_name,agency_url,agency_timezone\nA,Agency,https://example.invalid,Europe/Berlin\n")
zf.writestr("stops.txt", "stop_id,stop_name,stop_lat,stop_lon\n" + "\n".join(stops) + "\n")
zf.writestr("routes.txt", "route_id,agency_id,route_short_name,route_long_name,route_type\n" + "\n".join(routes) + "\n")
zf.writestr("trips.txt", "route_id,service_id,trip_id\n" + "\n".join(trips) + "\n")
zf.writestr("calendar.txt", "service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date\ndaily,1,1,1,1,1,1,1,20260101,20261231\n")
zf.writestr("stop_times.txt", "trip_id,arrival_time,departure_time,stop_id,stop_sequence\n" + "\n".join(stop_times) + "\n")