441 lines
16 KiB
Python
441 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import sqlite3
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.orm import Session, joinedload
|
|
|
|
from app.models import Dataset, GtfsFeedDiffItem, GtfsFeedDiffRun, GtfsRoute, GtfsStop, GtfsTrip, Source
|
|
|
|
MAX_DIFF_ITEMS_PER_TYPE = 200
|
|
|
|
|
|
def create_gtfs_update_diff_from_stage(
|
|
session: Session,
|
|
*,
|
|
source: Source,
|
|
previous_dataset: Dataset | None,
|
|
new_dataset: Dataset,
|
|
stage_path: Path,
|
|
) -> dict[str, Any] | None:
|
|
if previous_dataset is None or previous_dataset.kind != "gtfs" or not stage_path.exists():
|
|
return None
|
|
old_routes = _main_routes(session, previous_dataset.id)
|
|
old_stops = _main_stops(session, previous_dataset.id)
|
|
old_counts = {
|
|
"routes": len(old_routes),
|
|
"stops": len(old_stops),
|
|
"trips": _main_count(session, GtfsTrip, previous_dataset.id),
|
|
}
|
|
with sqlite3.connect(stage_path) as connection:
|
|
connection.row_factory = sqlite3.Row
|
|
new_routes = _stage_routes(connection)
|
|
new_stops = _stage_stops(connection)
|
|
new_counts = {
|
|
"routes": len(new_routes),
|
|
"stops": len(new_stops),
|
|
"trips": _stage_count(connection, "gtfs_trips"),
|
|
}
|
|
|
|
route_summary, route_items = _diff_objects(
|
|
item_type="route",
|
|
old=old_routes,
|
|
new=new_routes,
|
|
identity_label="route",
|
|
)
|
|
stop_summary, stop_items = _diff_objects(
|
|
item_type="stop",
|
|
old=old_stops,
|
|
new=new_stops,
|
|
identity_label="stop",
|
|
)
|
|
summary = {
|
|
"source_id": source.id,
|
|
"previous_dataset_id": previous_dataset.id,
|
|
"new_dataset_id": new_dataset.id,
|
|
"counts": {"old": old_counts, "new": new_counts},
|
|
"routes": route_summary,
|
|
"stops": stop_summary,
|
|
"items_stored": {
|
|
"route": min(len(route_items), MAX_DIFF_ITEMS_PER_TYPE),
|
|
"stop": min(len(stop_items), MAX_DIFF_ITEMS_PER_TYPE),
|
|
},
|
|
}
|
|
diff_key = _diff_key(source.id, previous_dataset.id, new_dataset.id, summary)
|
|
existing = session.scalar(select(GtfsFeedDiffRun).where(GtfsFeedDiffRun.diff_key == diff_key))
|
|
if existing is not None:
|
|
return diff_run_payload(existing)
|
|
|
|
now = datetime.now(timezone.utc)
|
|
diff_run = GtfsFeedDiffRun(
|
|
source_id=source.id,
|
|
previous_dataset_id=previous_dataset.id,
|
|
new_dataset_id=new_dataset.id,
|
|
status="completed",
|
|
diff_key=diff_key,
|
|
summary_json=_json_dumps(summary),
|
|
metadata_json=_json_dumps({"stage_path": str(stage_path), "created_from": "gtfs_activation"}),
|
|
created_at=now,
|
|
)
|
|
session.add(diff_run)
|
|
session.flush()
|
|
for item in [*route_items[:MAX_DIFF_ITEMS_PER_TYPE], *stop_items[:MAX_DIFF_ITEMS_PER_TYPE]]:
|
|
session.add(
|
|
GtfsFeedDiffItem(
|
|
diff_run_id=diff_run.id,
|
|
source_id=source.id,
|
|
previous_dataset_id=previous_dataset.id,
|
|
new_dataset_id=new_dataset.id,
|
|
item_type=item["item_type"],
|
|
object_key=item["object_key"],
|
|
change_type=item["change_type"],
|
|
severity=item["severity"],
|
|
status="open",
|
|
title=item["title"],
|
|
old_payload_json=_json_dumps(item["old"]) if item.get("old") else None,
|
|
new_payload_json=_json_dumps(item["new"]) if item.get("new") else None,
|
|
diff_json=_json_dumps(item["diff"]) if item.get("diff") else None,
|
|
created_at=now,
|
|
)
|
|
)
|
|
session.flush()
|
|
return diff_run_payload(diff_run)
|
|
|
|
|
|
def latest_gtfs_update_diffs(session: Session, *, source_id: int | None = None, limit: int = 20) -> list[GtfsFeedDiffRun]:
|
|
stmt = (
|
|
select(GtfsFeedDiffRun)
|
|
.options(joinedload(GtfsFeedDiffRun.source))
|
|
.order_by(GtfsFeedDiffRun.created_at.desc(), GtfsFeedDiffRun.id.desc())
|
|
.limit(max(1, min(int(limit), 100)))
|
|
)
|
|
if source_id is not None:
|
|
stmt = stmt.where(GtfsFeedDiffRun.source_id == int(source_id))
|
|
return session.scalars(stmt).all()
|
|
|
|
|
|
def gtfs_update_diff_detail(session: Session, diff_id: int, *, item_limit: int = 200) -> dict[str, Any] | None:
|
|
diff_run = session.get(GtfsFeedDiffRun, diff_id)
|
|
if diff_run is None:
|
|
return None
|
|
items = session.scalars(
|
|
select(GtfsFeedDiffItem)
|
|
.where(GtfsFeedDiffItem.diff_run_id == diff_run.id)
|
|
.order_by(_change_type_order(), GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.object_key)
|
|
.limit(max(1, min(int(item_limit), 1000)))
|
|
).all()
|
|
return {**diff_run_payload(diff_run), "items": [diff_item_payload(item) for item in items]}
|
|
|
|
|
|
def gtfs_update_diff_summary(session: Session) -> dict[str, Any]:
|
|
runs = session.scalar(select(func.count()).select_from(GtfsFeedDiffRun)) or 0
|
|
rows = session.execute(
|
|
select(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.change_type, GtfsFeedDiffItem.severity, func.count())
|
|
.select_from(GtfsFeedDiffItem)
|
|
.group_by(GtfsFeedDiffItem.item_type, GtfsFeedDiffItem.change_type, GtfsFeedDiffItem.severity)
|
|
).all()
|
|
by_type: dict[str, int] = {}
|
|
by_change: dict[str, int] = {}
|
|
by_severity: dict[str, int] = {}
|
|
for item_type, change_type, severity, count in rows:
|
|
value = int(count or 0)
|
|
by_type[str(item_type)] = by_type.get(str(item_type), 0) + value
|
|
by_change[str(change_type)] = by_change.get(str(change_type), 0) + value
|
|
by_severity[str(severity)] = by_severity.get(str(severity), 0) + value
|
|
return {"runs": int(runs), "by_type": by_type, "by_change": by_change, "by_severity": by_severity}
|
|
|
|
|
|
def diff_run_payload(diff_run: GtfsFeedDiffRun) -> dict[str, Any]:
|
|
summary = _json_object(diff_run.summary_json)
|
|
return {
|
|
"id": diff_run.id,
|
|
"source_id": diff_run.source_id,
|
|
"source_name": None if diff_run.source is None else diff_run.source.name,
|
|
"previous_dataset_id": diff_run.previous_dataset_id,
|
|
"new_dataset_id": diff_run.new_dataset_id,
|
|
"status": diff_run.status,
|
|
"diff_key": diff_run.diff_key,
|
|
"summary": summary,
|
|
"metadata": _json_object(diff_run.metadata_json),
|
|
"created_at": diff_run.created_at.isoformat(),
|
|
}
|
|
|
|
|
|
def diff_item_payload(item: GtfsFeedDiffItem) -> dict[str, Any]:
|
|
return {
|
|
"id": item.id,
|
|
"diff_run_id": item.diff_run_id,
|
|
"source_id": item.source_id,
|
|
"previous_dataset_id": item.previous_dataset_id,
|
|
"new_dataset_id": item.new_dataset_id,
|
|
"item_type": item.item_type,
|
|
"object_key": item.object_key,
|
|
"change_type": item.change_type,
|
|
"severity": item.severity,
|
|
"status": item.status,
|
|
"title": item.title,
|
|
"old": _json_object(item.old_payload_json),
|
|
"new": _json_object(item.new_payload_json),
|
|
"diff": _json_object(item.diff_json),
|
|
"created_at": item.created_at.isoformat(),
|
|
"resolved_at": None if item.resolved_at is None else item.resolved_at.isoformat(),
|
|
}
|
|
|
|
|
|
def _main_routes(session: Session, dataset_id: int) -> dict[str, dict[str, Any]]:
|
|
trip_counts = {
|
|
str(route_id): int(count or 0)
|
|
for route_id, count in session.execute(
|
|
select(GtfsTrip.route_id, func.count()).where(GtfsTrip.dataset_id == dataset_id).group_by(GtfsTrip.route_id)
|
|
).all()
|
|
}
|
|
routes = session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == dataset_id)).all()
|
|
return {
|
|
str(route.route_id): _normalized_route_payload(
|
|
{
|
|
"route_id": route.route_id,
|
|
"short_name": route.short_name,
|
|
"long_name": route.long_name,
|
|
"mode": route.mode,
|
|
"route_scope": route.route_scope,
|
|
"operator_name": route.operator_name,
|
|
"route_key": route.route_key,
|
|
"operator_key": route.operator_key,
|
|
"has_geometry": bool(route.geometry_geojson),
|
|
"trip_count": trip_counts.get(str(route.route_id), 0),
|
|
}
|
|
)
|
|
for route in routes
|
|
}
|
|
|
|
|
|
def _stage_routes(connection: sqlite3.Connection) -> dict[str, dict[str, Any]]:
|
|
trip_counts = {
|
|
str(row["route_id"]): int(row["count"] or 0)
|
|
for row in connection.execute("SELECT route_id, COUNT(*) AS count FROM gtfs_trips GROUP BY route_id").fetchall()
|
|
}
|
|
rows = connection.execute(
|
|
"""
|
|
SELECT route_id, short_name, long_name, mode, route_scope, operator_name, route_key, operator_key, geometry_geojson
|
|
FROM gtfs_routes
|
|
"""
|
|
).fetchall()
|
|
return {
|
|
str(row["route_id"]): _normalized_route_payload(
|
|
{
|
|
"route_id": row["route_id"],
|
|
"short_name": row["short_name"],
|
|
"long_name": row["long_name"],
|
|
"mode": row["mode"],
|
|
"route_scope": row["route_scope"],
|
|
"operator_name": row["operator_name"],
|
|
"route_key": row["route_key"],
|
|
"operator_key": row["operator_key"],
|
|
"has_geometry": bool(row["geometry_geojson"]),
|
|
"trip_count": trip_counts.get(str(row["route_id"]), 0),
|
|
}
|
|
)
|
|
for row in rows
|
|
}
|
|
|
|
|
|
def _main_stops(session: Session, dataset_id: int) -> dict[str, dict[str, Any]]:
|
|
stops = session.scalars(select(GtfsStop).where(GtfsStop.dataset_id == dataset_id)).all()
|
|
return {
|
|
str(stop.stop_id): _normalized_stop_payload(
|
|
{
|
|
"stop_id": stop.stop_id,
|
|
"name": stop.name,
|
|
"lat": stop.lat,
|
|
"lon": stop.lon,
|
|
"parent_station": stop.parent_station,
|
|
}
|
|
)
|
|
for stop in stops
|
|
}
|
|
|
|
|
|
def _stage_stops(connection: sqlite3.Connection) -> dict[str, dict[str, Any]]:
|
|
rows = connection.execute("SELECT stop_id, name, lat, lon, parent_station FROM gtfs_stops").fetchall()
|
|
return {
|
|
str(row["stop_id"]): _normalized_stop_payload(
|
|
{
|
|
"stop_id": row["stop_id"],
|
|
"name": row["name"],
|
|
"lat": row["lat"],
|
|
"lon": row["lon"],
|
|
"parent_station": row["parent_station"],
|
|
}
|
|
)
|
|
for row in rows
|
|
}
|
|
|
|
|
|
def _diff_objects(
|
|
*,
|
|
item_type: str,
|
|
old: dict[str, dict[str, Any]],
|
|
new: dict[str, dict[str, Any]],
|
|
identity_label: str,
|
|
) -> tuple[dict[str, int], list[dict[str, Any]]]:
|
|
old_keys = set(old)
|
|
new_keys = set(new)
|
|
added = sorted(new_keys - old_keys)
|
|
removed = sorted(old_keys - new_keys)
|
|
common = sorted(old_keys & new_keys)
|
|
items: list[dict[str, Any]] = []
|
|
unchanged = 0
|
|
changed = 0
|
|
for key in common:
|
|
differences = _payload_differences(old[key], new[key])
|
|
if not differences:
|
|
unchanged += 1
|
|
continue
|
|
changed += 1
|
|
items.append(
|
|
{
|
|
"item_type": item_type,
|
|
"object_key": key,
|
|
"change_type": "changed",
|
|
"severity": "warn",
|
|
"title": f"{identity_label.title()} changed: {key}",
|
|
"old": old[key],
|
|
"new": new[key],
|
|
"diff": differences,
|
|
}
|
|
)
|
|
for key in added:
|
|
items.append(
|
|
{
|
|
"item_type": item_type,
|
|
"object_key": key,
|
|
"change_type": "added",
|
|
"severity": "info",
|
|
"title": f"{identity_label.title()} added: {key}",
|
|
"new": new[key],
|
|
"diff": {"added": True},
|
|
}
|
|
)
|
|
for key in removed:
|
|
items.append(
|
|
{
|
|
"item_type": item_type,
|
|
"object_key": key,
|
|
"change_type": "removed",
|
|
"severity": "warn",
|
|
"title": f"{identity_label.title()} removed: {key}",
|
|
"old": old[key],
|
|
"diff": {"removed": True},
|
|
}
|
|
)
|
|
summary = {
|
|
"old": len(old),
|
|
"new": len(new),
|
|
"added": len(added),
|
|
"removed": len(removed),
|
|
"changed": changed,
|
|
"unchanged": unchanged,
|
|
}
|
|
return summary, sorted(items, key=lambda item: (_change_type_rank(item["change_type"]), item["item_type"], item["object_key"]))
|
|
|
|
|
|
def _payload_differences(old: dict[str, Any], new: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|
differences = {}
|
|
for key in sorted(set(old) | set(new)):
|
|
if old.get(key) != new.get(key):
|
|
differences[key] = {"old": old.get(key), "new": new.get(key)}
|
|
return differences
|
|
|
|
|
|
def _normalized_route_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"route_id": str(payload.get("route_id") or ""),
|
|
"short_name": _clean_text(payload.get("short_name")),
|
|
"long_name": _clean_text(payload.get("long_name")),
|
|
"mode": _clean_text(payload.get("mode")),
|
|
"route_scope": _clean_text(payload.get("route_scope")),
|
|
"operator_name": _clean_text(payload.get("operator_name")),
|
|
"route_key": _clean_text(payload.get("route_key")),
|
|
"operator_key": _clean_text(payload.get("operator_key")),
|
|
"has_geometry": bool(payload.get("has_geometry")),
|
|
"trip_count": int(payload.get("trip_count") or 0),
|
|
}
|
|
|
|
|
|
def _normalized_stop_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
"stop_id": str(payload.get("stop_id") or ""),
|
|
"name": _clean_text(payload.get("name")),
|
|
"lat": _rounded_coord(payload.get("lat")),
|
|
"lon": _rounded_coord(payload.get("lon")),
|
|
"parent_station": _clean_text(payload.get("parent_station")),
|
|
}
|
|
|
|
|
|
def _main_count(session: Session, model, dataset_id: int) -> int:
|
|
return int(session.scalar(select(func.count()).select_from(model).where(model.dataset_id == dataset_id)) or 0)
|
|
|
|
|
|
def _stage_count(connection: sqlite3.Connection, table_name: str) -> int:
|
|
return int(connection.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()[0] or 0)
|
|
|
|
|
|
def _diff_key(source_id: int, previous_dataset_id: int, new_dataset_id: int, summary: dict[str, Any]) -> str:
|
|
payload = {
|
|
"source_id": source_id,
|
|
"previous_dataset_id": previous_dataset_id,
|
|
"new_dataset_id": new_dataset_id,
|
|
"routes": summary.get("routes"),
|
|
"stops": summary.get("stops"),
|
|
}
|
|
digest = hashlib.sha256(_json_dumps(payload).encode("utf-8")).hexdigest()
|
|
return f"gtfs-diff-{digest[:32]}"
|
|
|
|
|
|
def _change_type_order():
|
|
from sqlalchemy import case
|
|
|
|
return case(
|
|
(GtfsFeedDiffItem.change_type == "changed", 0),
|
|
(GtfsFeedDiffItem.change_type == "removed", 1),
|
|
(GtfsFeedDiffItem.change_type == "added", 2),
|
|
else_=9,
|
|
)
|
|
|
|
|
|
def _change_type_rank(value: str) -> int:
|
|
return {"changed": 0, "removed": 1, "added": 2}.get(value, 9)
|
|
|
|
|
|
def _clean_text(value: object) -> str | None:
|
|
if value is None:
|
|
return None
|
|
text = " ".join(str(value).strip().split())
|
|
return text or None
|
|
|
|
|
|
def _rounded_coord(value: object) -> float | None:
|
|
try:
|
|
return round(float(value), 7)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _json_object(value: str | None) -> dict[str, Any]:
|
|
if not value:
|
|
return {}
|
|
try:
|
|
payload = json.loads(value)
|
|
except json.JSONDecodeError:
|
|
return {}
|
|
return payload if isinstance(payload, dict) else {}
|
|
|
|
|
|
def _json_dumps(value: dict[str, Any]) -> str:
|
|
return json.dumps(value, sort_keys=True, separators=(",", ":"), default=str)
|