add-ideas/meubility-workbench
1
0
Fork 0
Code Issues 122 Pull Requests Actions Packages Projects Releases Wiki Activity

Next alpha stage commit

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-06 20:54:46 +02:00
parent e23387738b
commit 1d7ec956d2
42 changed files with 11603 additions and 2837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
app/pipeline/gtfs.py
View File

@@ -15,6 +15,8 @@ from sqlalchemy import func, select, text
from sqlalchemy.orm import Session
from app.config import settings
from app.decision_replay import apply_gtfs_decision_replay, capture_gtfs_decision_replay
from app.gtfs_diff import create_gtfs_update_diff_from_stage
from app.gtfs_storage import GTFS_STORAGE_MAIN, GTFS_STORAGE_METADATA_KEY, GTFS_STORAGE_SIDECAR_STOP_TIMES, effective_gtfs_timetable_storage
from app.models import (
Dataset,
@@ -64,6 +66,7 @@ GTFS_EXTENDED_MODE_RANGES = [
GTFS_IMPORTER_VERSION = "gtfs_import_v6_sidecar_stop_times"
REQUIRED_FILES = {"agency.txt", "stops.txt", "routes.txt", "trips.txt", "stop_times.txt"}
GTFS_CALENDAR_WEEKDAY_COLUMNS = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]
GTFS_STAGE_BATCH_SIZE = 50_000
ProgressCallback = Callable[[str, str, int | None, int | None, dict[str, Any] | None], None]
@@ -82,7 +85,7 @@ def run_gtfs_source(session: Session, source: Source, progress_callback: Progres
)
.order_by(Dataset.id.desc())
)
if existing is not None and _dataset_importer_version(existing) == GTFS_IMPORTER_VERSION:
if existing is not None and _dataset_import_can_reuse(existing):
return existing
return import_gtfs_zip(session=session, source=source, zip_path=local_path, source_hash=source_hash, progress_callback=progress_callback)
@@ -395,15 +398,96 @@ def _activate_staged_gtfs(
dataset = session.get(Dataset, dataset.id) or dataset
source = session.get(Source, source.id) or source
replaced_datasets = [existing for existing in list(source.datasets) if existing.id != dataset.id and existing.kind == "gtfs"]
previous_dataset = next(
(
existing
for existing in sorted(replaced_datasets, key=lambda item: (not item.is_active, item.created_at, item.id))
if existing.status == "imported"
),
None,
)
for existing in source.datasets:
if existing.id != dataset.id:
existing.is_active = False
copy_stop_times = _copy_stop_times_to_main(summary)
heavy_index_drop = copy_stop_times and _should_drop_indexes_for_activation(stage_path)
diff_result: dict[str, Any] | None = None
decision_replay_plan: dict[str, Any] | None = None
if heavy_index_drop:
_emit_progress(progress_callback, "gtfs_activation_indexes_dropped", "Dropping heavy GTFS lookup indexes before bulk activation.", None, None, None)
_drop_gtfs_bulk_indexes(session.connection())
try:
if previous_dataset is not None:
_emit_progress(
progress_callback,
"gtfs_update_diff_started",
f"Comparing staged GTFS dataset #{dataset.id} with previous dataset #{previous_dataset.id}.",
None,
None,
{"previous_dataset_id": previous_dataset.id, "new_dataset_id": dataset.id},
)
try:
diff_result = create_gtfs_update_diff_from_stage(
session,
source=source,
previous_dataset=previous_dataset,
new_dataset=dataset,
stage_path=stage_path,
)
except Exception as exc: # noqa: BLE001 - update diff must not block activation
diff_result = None
summary["gtfs_update_diff_error"] = str(exc)
_emit_progress(
progress_callback,
"gtfs_update_diff_failed",
"GTFS update diff failed; continuing dataset activation.",
None,
None,
{"error": str(exc), "previous_dataset_id": previous_dataset.id, "new_dataset_id": dataset.id},
)
if diff_result is not None:
summary["gtfs_update_diff"] = {
"id": diff_result["id"],
"summary": diff_result["summary"],
}
_emit_progress(
progress_callback,
"gtfs_update_diff_completed",
"Stored GTFS update diff before replacing the previous dataset.",
None,
None,
diff_result,
)
try:
decision_replay_plan = capture_gtfs_decision_replay(
session,
previous_dataset,
diff_run_id=None if diff_result is None else int(diff_result["id"]),
)
summary["gtfs_decision_replay"] = {
"captured": len(decision_replay_plan.get("decisions", [])),
"diff_run_id": decision_replay_plan.get("diff_run_id"),
"previous_dataset_id": previous_dataset.id,
}
_emit_progress(
progress_callback,
"gtfs_decision_replay_captured",
"Captured existing GTFS-to-map decisions for update replay.",
None,
None,
summary["gtfs_decision_replay"],
)
except Exception as exc: # noqa: BLE001 - replay capture must not block activation
decision_replay_plan = None
summary["gtfs_decision_replay_error"] = str(exc)
_emit_progress(
progress_callback,
"gtfs_decision_replay_capture_failed",
"GTFS decision replay capture failed; continuing dataset activation.",
None,
None,
{"error": str(exc), "previous_dataset_id": previous_dataset.id, "new_dataset_id": dataset.id},
)
if replaced_datasets:
_emit_progress(
progress_callback,
@@ -511,6 +595,33 @@ def _activate_staged_gtfs(
],
progress_callback,
)
if decision_replay_plan is not None:
try:
replay_result = apply_gtfs_decision_replay(
session,
decision_replay_plan,
new_dataset=dataset,
diff_run_id=None if diff_result is None else int(diff_result["id"]),
)
summary["gtfs_decision_replay"] = replay_result
_emit_progress(
progress_callback,
"gtfs_decision_replay_completed",
"Replayed unchanged GTFS-to-map decisions and queued changed decisions for review.",
None,
None,
replay_result,
)
except Exception as exc: # noqa: BLE001 - replay must not block activation
summary["gtfs_decision_replay_error"] = str(exc)
_emit_progress(
progress_callback,
"gtfs_decision_replay_failed",
"GTFS decision replay failed; continuing dataset activation.",
None,
None,
{"error": str(exc), "new_dataset_id": dataset.id},
)
finally:
if heavy_index_drop:
_emit_progress(progress_callback, "gtfs_activation_indexes_rebuilding", "Rebuilding GTFS lookup indexes after bulk activation.", None, None, None)
@@ -557,10 +668,7 @@ def _copy_stage_table(
rows = cursor.fetchmany(GTFS_STAGE_BATCH_SIZE)
if not rows:
break
payload = [
{"dataset_id": dataset_id, **{column: row[index] for index, column in enumerate(columns)}}
for row in rows
]
payload = [_copy_stage_row_payload(table, dataset_id, columns, row) for row in rows]
session.execute(text(insert_sql), payload)
copied += len(rows)
_emit_progress(
@@ -573,6 +681,15 @@ def _copy_stage_table(
)
def _copy_stage_row_payload(table: str, dataset_id: int, columns: list[str], row: sqlite3.Row | tuple[Any, ...]) -> dict[str, Any]:
payload = {"dataset_id": dataset_id, **{column: row[index] for index, column in enumerate(columns)}}
if table == "gtfs_calendars":
for column in GTFS_CALENDAR_WEEKDAY_COLUMNS:
if column in payload:
payload[column] = _bool_flag(payload[column])
return payload
def _should_drop_indexes_for_activation(stage_path: Path) -> bool:
if settings.is_postgresql_database:
return False
@@ -1325,3 +1442,13 @@ def _dataset_importer_version(dataset: Dataset) -> str:
return str(json.loads(dataset.metadata_json or "{}").get("importer") or "")
except json.JSONDecodeError:
return ""
def _dataset_import_can_reuse(dataset: Dataset) -> bool:
try:
metadata = json.loads(dataset.metadata_json or "{}")
except json.JSONDecodeError:
return False
if str(metadata.get("importer") or "") != GTFS_IMPORTER_VERSION:
return False
return int(metadata.get("stop_times_import_limit") or 0) == int(settings.gtfs_stop_times_import_limit or 0)