Next alpha stage commit
This commit is contained in:
440
app/gtfs_diff.py
Normal file
440
app/gtfs_diff.py
Normal file
@@ -0,0 +1,440 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import sqlite3
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.orm import Session, joinedload
|
||||
|
||||
from app.models import Dataset, GtfsFeedDiffItem, GtfsFeedDiffRun, GtfsRoute, GtfsStop, GtfsTrip, Source
|
||||
|
||||
MAX_DIFF_ITEMS_PER_TYPE = 200
|
||||
|
||||
|
||||
def create_gtfs_update_diff_from_stage(
|
||||
session: Session,
|
||||
*,
|
||||
source: Source,
|
||||
previous_dataset: Dataset | None,
|
||||
new_dataset: Dataset,
|
||||
stage_path: Path,
|
||||
) -> dict[str, Any] | None:
|
||||
if previous_dataset is None or previous_dataset.kind != "gtfs" or not stage_path.exists():
|
||||
return None
|
||||
old_routes = _main_routes(session, previous_dataset.id)
|
||||
old_stops = _main_stops(session, previous_dataset.id)
|
||||
old_counts = {
|
||||
"routes": len(old_routes),
|
||||
"stops": len(old_stops),
|
||||
"trips": _main_count(session, GtfsTrip, previous_dataset.id),
|
||||
}
|
||||
with sqlite3.connect(stage_path) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
new_routes = _stage_routes(connection)
|
||||
new_stops = _stage_stops(connection)
|
||||
new_counts = {
|
||||
"routes": len(new_routes),
|
||||
"stops": len(new_stops),
|
||||
"trips": _stage_count(connection, "gtfs_trips"),
|
||||
}
|
||||
|
||||
route_summary, route_items = _diff_objects(
|
||||
item_type="route",
|
||||
old=old_routes,
|
||||
new=new_routes,
|
||||
identity_label="route",
|
||||
)
|
||||
stop_summary, stop_items = _diff_objects(
|
||||
item_type="stop",
|
||||
old=old_stops,
|
||||
new=new_stops,
|
||||
identity_label="stop",
|
||||
)
|
||||
summary = {
|
||||
"source_id": source.id,
|
||||
"previous_dataset_id": previous_dataset.id,
|
||||
"new_dataset_id": new_dataset.id,
|
||||
"counts": {"old": old_counts, "new": new_counts},
|
||||
"routes": route_summary,
|
||||
"stops": stop_summary,
|
||||
"items_stored": {
|
||||
"route": min(len(route_items), MAX_DIFF_ITEMS_PER_TYPE),
|
||||
"stop": min(len(stop_items), MAX_DIFF_ITEMS_PER_TYPE),
|
||||
},
|
||||
}
|
||||
diff_key = _diff_key(source.id, previous_dataset.id, new_dataset.id, summary)
|
||||
existing = session.scalar(select(GtfsFeedDiffRun).where(GtfsFeedDiffRun.diff_key == diff_key))
|
||||
if existing is not None:
|
||||
return diff_run_payload(existing)
|
||||
|
||||
now = datetime.now(timezone.utc)
|
||||
diff_run = GtfsFeedDiffRun(
|
||||
source_id=source.id,
|
||||
previous_dataset_id=previous_dataset.id,
|
||||
new_dataset_id=new_dataset.id,
|
||||
status="completed",
|
||||
diff_key=diff_key,
|
||||
summary_json=_json_dumps(summary),
|
||||
metadata_json=_json_dumps({"stage_path": str(stage_path), "created_from": "gtfs_activation"}),
|
||||
created_at=now,
|
||||
)
|
||||
session.add(diff_run)
|
||||
session.flush()
|
||||
for item in [*route_items[:MAX_DIFF_ITEMS_PER_TYPE], *stop_items[:MAX_DIFF_ITEMS_PER_TYPE]]:
|
||||
session.add(
|
||||
GtfsFeedDiffItem(
|
||||
diff_run_id=diff_run.id,
|
||||
source_id=source.id,
|
||||
previous_dataset_id=previous_dataset.id,
|
||||
new_dataset_id=new_dataset.id,
|
||||
item_type=item["item_type"],
|
||||
object_key=item["object_key"],
|
||||
change_type=item["change_type"],
|
||||
severity=item["severity"],
|
||||
status="open",
|
||||
title=item["title"],
|
||||
old_payload_json=_json_dumps(item["old"]) if item.get("old") else None,
|
||||
new_payload_json=_json_dumps(item["new"]) if item.get("new") else None,
|
||||
diff_json=_json_dumps(item["diff"]) if item.get("diff") else None,
|
||||
created_at=now,
|
||||
)
|
||||
)
|
||||
session.flush()
|
||||
return diff_run_payload(diff_run)
|
||||
|
||||
|
||||
def latest_gtfs_update_diffs(session: Session, *, source_id: int | None = None, limit: int = 20) -> list[GtfsFeedDiffRun]:
|
||||
stmt = (
|
||||
select(GtfsFeedDiffRun)
|
||||
.options(joinedload(GtfsFeedDiffRun.source))
|
||||
.order_by(GtfsFeedDiffRun.created_at.desc(), GtfsFeedDiffRun.id.desc())
|
||||
.limit(max(1, min(int(limit), 100)))
|
||||
)
|
||||
if source_id is not None:
|
||||
stmt = stmt.where(GtfsFeedDiffRun.source_id == int(source_id))
|
||||
return session.scalars(stmt).all()
|
||||
|
||||
|
||||
def gtfs_update_diff_detail(session: Session, diff_id: int, *, item_limit: int = 200) -> dict[str, Any] | None:
|
||||
diff_run = session.get(GtfsFeedDiffRun, diff_id)
|
||||
if diff_run is None:
|
||||
return None
|
||||
items = session.scalars(
|
||||
select(GtfsFeedDiffItem)
|
||||
.where(GtfsFeedDiffItem.diff_run_id == diff_run.id)
|
||||
.order_by(_change_type_order(), GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.object_key)
|
||||
.limit(max(1, min(int(item_limit), 1000)))
|
||||
).all()
|
||||
return {**diff_run_payload(diff_run), "items": [diff_item_payload(item) for item in items]}
|
||||
|
||||
|
||||
def gtfs_update_diff_summary(session: Session) -> dict[str, Any]:
|
||||
runs = session.scalar(select(func.count()).select_from(GtfsFeedDiffRun)) or 0
|
||||
rows = session.execute(
|
||||
select(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.change_type, GtfsFeedDiffItem.severity, func.count())
|
||||
.select_from(GtfsFeedDiffItem)
|
||||
.group_by(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.change_type, GtfsFeedDiffItem.severity)
|
||||
).all()
|
||||
by_type: dict[str, int] = {}
|
||||
by_change: dict[str, int] = {}
|
||||
by_severity: dict[str, int] = {}
|
||||
for item_type, change_type, severity, count in rows:
|
||||
value = int(count or 0)
|
||||
by_type[str(item_type)] = by_type.get(str(item_type), 0) + value
|
||||
by_change[str(change_type)] = by_change.get(str(change_type), 0) + value
|
||||
by_severity[str(severity)] = by_severity.get(str(severity), 0) + value
|
||||
return {"runs": int(runs), "by_type": by_type, "by_change": by_change, "by_severity": by_severity}
|
||||
|
||||
|
||||
def diff_run_payload(diff_run: GtfsFeedDiffRun) -> dict[str, Any]:
|
||||
summary = _json_object(diff_run.summary_json)
|
||||
return {
|
||||
"id": diff_run.id,
|
||||
"source_id": diff_run.source_id,
|
||||
"source_name": None if diff_run.source is None else diff_run.source.name,
|
||||
"previous_dataset_id": diff_run.previous_dataset_id,
|
||||
"new_dataset_id": diff_run.new_dataset_id,
|
||||
"status": diff_run.status,
|
||||
"diff_key": diff_run.diff_key,
|
||||
"summary": summary,
|
||||
"metadata": _json_object(diff_run.metadata_json),
|
||||
"created_at": diff_run.created_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def diff_item_payload(item: GtfsFeedDiffItem) -> dict[str, Any]:
|
||||
return {
|
||||
"id": item.id,
|
||||
"diff_run_id": item.diff_run_id,
|
||||
"source_id": item.source_id,
|
||||
"previous_dataset_id": item.previous_dataset_id,
|
||||
"new_dataset_id": item.new_dataset_id,
|
||||
"item_type": item.item_type,
|
||||
"object_key": item.object_key,
|
||||
"change_type": item.change_type,
|
||||
"severity": item.severity,
|
||||
"status": item.status,
|
||||
"title": item.title,
|
||||
"old": _json_object(item.old_payload_json),
|
||||
"new": _json_object(item.new_payload_json),
|
||||
"diff": _json_object(item.diff_json),
|
||||
"created_at": item.created_at.isoformat(),
|
||||
"resolved_at": None if item.resolved_at is None else item.resolved_at.isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def _main_routes(session: Session, dataset_id: int) -> dict[str, dict[str, Any]]:
|
||||
trip_counts = {
|
||||
str(route_id): int(count or 0)
|
||||
for route_id, count in session.execute(
|
||||
select(GtfsTrip.route_id, func.count()).where(GtfsTrip.dataset_id == dataset_id).group_by(GtfsTrip.route_id)
|
||||
).all()
|
||||
}
|
||||
routes = session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == dataset_id)).all()
|
||||
return {
|
||||
str(route.route_id): _normalized_route_payload(
|
||||
{
|
||||
"route_id": route.route_id,
|
||||
"short_name": route.short_name,
|
||||
"long_name": route.long_name,
|
||||
"mode": route.mode,
|
||||
"route_scope": route.route_scope,
|
||||
"operator_name": route.operator_name,
|
||||
"route_key": route.route_key,
|
||||
"operator_key": route.operator_key,
|
||||
"has_geometry": bool(route.geometry_geojson),
|
||||
"trip_count": trip_counts.get(str(route.route_id), 0),
|
||||
}
|
||||
)
|
||||
for route in routes
|
||||
}
|
||||
|
||||
|
||||
def _stage_routes(connection: sqlite3.Connection) -> dict[str, dict[str, Any]]:
|
||||
trip_counts = {
|
||||
str(row["route_id"]): int(row["count"] or 0)
|
||||
for row in connection.execute("SELECT route_id, COUNT(*) AS count FROM gtfs_trips GROUP BY route_id").fetchall()
|
||||
}
|
||||
rows = connection.execute(
|
||||
"""
|
||||
SELECT route_id, short_name, long_name, mode, route_scope, operator_name, route_key, operator_key, geometry_geojson
|
||||
FROM gtfs_routes
|
||||
"""
|
||||
).fetchall()
|
||||
return {
|
||||
str(row["route_id"]): _normalized_route_payload(
|
||||
{
|
||||
"route_id": row["route_id"],
|
||||
"short_name": row["short_name"],
|
||||
"long_name": row["long_name"],
|
||||
"mode": row["mode"],
|
||||
"route_scope": row["route_scope"],
|
||||
"operator_name": row["operator_name"],
|
||||
"route_key": row["route_key"],
|
||||
"operator_key": row["operator_key"],
|
||||
"has_geometry": bool(row["geometry_geojson"]),
|
||||
"trip_count": trip_counts.get(str(row["route_id"]), 0),
|
||||
}
|
||||
)
|
||||
for row in rows
|
||||
}
|
||||
|
||||
|
||||
def _main_stops(session: Session, dataset_id: int) -> dict[str, dict[str, Any]]:
|
||||
stops = session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == dataset_id)).all()
|
||||
return {
|
||||
str(stop.stop_id): _normalized_stop_payload(
|
||||
{
|
||||
"stop_id": stop.stop_id,
|
||||
"name": stop.name,
|
||||
"lat": stop.lat,
|
||||
"lon": stop.lon,
|
||||
"parent_station": stop.parent_station,
|
||||
}
|
||||
)
|
||||
for stop in stops
|
||||
}
|
||||
|
||||
|
||||
def _stage_stops(connection: sqlite3.Connection) -> dict[str, dict[str, Any]]:
|
||||
rows = connection.execute("SELECT stop_id, name, lat, lon, parent_station FROM gtfs_stops").fetchall()
|
||||
return {
|
||||
str(row["stop_id"]): _normalized_stop_payload(
|
||||
{
|
||||
"stop_id": row["stop_id"],
|
||||
"name": row["name"],
|
||||
"lat": row["lat"],
|
||||
"lon": row["lon"],
|
||||
"parent_station": row["parent_station"],
|
||||
}
|
||||
)
|
||||
for row in rows
|
||||
}
|
||||
|
||||
|
||||
def _diff_objects(
|
||||
*,
|
||||
item_type: str,
|
||||
old: dict[str, dict[str, Any]],
|
||||
new: dict[str, dict[str, Any]],
|
||||
identity_label: str,
|
||||
) -> tuple[dict[str, int], list[dict[str, Any]]]:
|
||||
old_keys = set(old)
|
||||
new_keys = set(new)
|
||||
added = sorted(new_keys - old_keys)
|
||||
removed = sorted(old_keys - new_keys)
|
||||
common = sorted(old_keys & new_keys)
|
||||
items: list[dict[str, Any]] = []
|
||||
unchanged = 0
|
||||
changed = 0
|
||||
for key in common:
|
||||
differences = _payload_differences(old[key], new[key])
|
||||
if not differences:
|
||||
unchanged += 1
|
||||
continue
|
||||
changed += 1
|
||||
items.append(
|
||||
{
|
||||
"item_type": item_type,
|
||||
"object_key": key,
|
||||
"change_type": "changed",
|
||||
"severity": "warn",
|
||||
"title": f"{identity_label.title()} changed: {key}",
|
||||
"old": old[key],
|
||||
"new": new[key],
|
||||
"diff": differences,
|
||||
}
|
||||
)
|
||||
for key in added:
|
||||
items.append(
|
||||
{
|
||||
"item_type": item_type,
|
||||
"object_key": key,
|
||||
"change_type": "added",
|
||||
"severity": "info",
|
||||
"title": f"{identity_label.title()} added: {key}",
|
||||
"new": new[key],
|
||||
"diff": {"added": True},
|
||||
}
|
||||
)
|
||||
for key in removed:
|
||||
items.append(
|
||||
{
|
||||
"item_type": item_type,
|
||||
"object_key": key,
|
||||
"change_type": "removed",
|
||||
"severity": "warn",
|
||||
"title": f"{identity_label.title()} removed: {key}",
|
||||
"old": old[key],
|
||||
"diff": {"removed": True},
|
||||
}
|
||||
)
|
||||
summary = {
|
||||
"old": len(old),
|
||||
"new": len(new),
|
||||
"added": len(added),
|
||||
"removed": len(removed),
|
||||
"changed": changed,
|
||||
"unchanged": unchanged,
|
||||
}
|
||||
return summary, sorted(items, key=lambda item: (_change_type_rank(item["change_type"]), item["item_type"], item["object_key"]))
|
||||
|
||||
|
||||
def _payload_differences(old: dict[str, Any], new: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
differences = {}
|
||||
for key in sorted(set(old) | set(new)):
|
||||
if old.get(key) != new.get(key):
|
||||
differences[key] = {"old": old.get(key), "new": new.get(key)}
|
||||
return differences
|
||||
|
||||
|
||||
def _normalized_route_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"route_id": str(payload.get("route_id") or ""),
|
||||
"short_name": _clean_text(payload.get("short_name")),
|
||||
"long_name": _clean_text(payload.get("long_name")),
|
||||
"mode": _clean_text(payload.get("mode")),
|
||||
"route_scope": _clean_text(payload.get("route_scope")),
|
||||
"operator_name": _clean_text(payload.get("operator_name")),
|
||||
"route_key": _clean_text(payload.get("route_key")),
|
||||
"operator_key": _clean_text(payload.get("operator_key")),
|
||||
"has_geometry": bool(payload.get("has_geometry")),
|
||||
"trip_count": int(payload.get("trip_count") or 0),
|
||||
}
|
||||
|
||||
|
||||
def _normalized_stop_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"stop_id": str(payload.get("stop_id") or ""),
|
||||
"name": _clean_text(payload.get("name")),
|
||||
"lat": _rounded_coord(payload.get("lat")),
|
||||
"lon": _rounded_coord(payload.get("lon")),
|
||||
"parent_station": _clean_text(payload.get("parent_station")),
|
||||
}
|
||||
|
||||
|
||||
def _main_count(session: Session, model, dataset_id: int) -> int:
|
||||
return int(session.scalar(select(func.count()).select_from(model).where(model.dataset_id == dataset_id)) or 0)
|
||||
|
||||
|
||||
def _stage_count(connection: sqlite3.Connection, table_name: str) -> int:
|
||||
return int(connection.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0] or 0)
|
||||
|
||||
|
||||
def _diff_key(source_id: int, previous_dataset_id: int, new_dataset_id: int, summary: dict[str, Any]) -> str:
|
||||
payload = {
|
||||
"source_id": source_id,
|
||||
"previous_dataset_id": previous_dataset_id,
|
||||
"new_dataset_id": new_dataset_id,
|
||||
"routes": summary.get("routes"),
|
||||
"stops": summary.get("stops"),
|
||||
}
|
||||
digest = hashlib.sha256(_json_dumps(payload).encode("utf-8")).hexdigest()
|
||||
return f"gtfs-diff-{digest[:32]}"
|
||||
|
||||
|
||||
def _change_type_order():
|
||||
from sqlalchemy import case
|
||||
|
||||
return case(
|
||||
(GtfsFeedDiffItem.change_type == "changed", 0),
|
||||
(GtfsFeedDiffItem.change_type == "removed", 1),
|
||||
(GtfsFeedDiffItem.change_type == "added", 2),
|
||||
else_=9,
|
||||
)
|
||||
|
||||
|
||||
def _change_type_rank(value: str) -> int:
|
||||
return {"changed": 0, "removed": 1, "added": 2}.get(value, 9)
|
||||
|
||||
|
||||
def _clean_text(value: object) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
text = " ".join(str(value).strip().split())
|
||||
return text or None
|
||||
|
||||
|
||||
def _rounded_coord(value: object) -> float | None:
|
||||
try:
|
||||
return round(float(value), 7)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _json_object(value: str | None) -> dict[str, Any]:
|
||||
if not value:
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def _json_dumps(value: dict[str, Any]) -> str:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|
||||
Reference in New Issue
Block a user