add-ideas/meubility-workbench
1
0
Fork 0
Code Issues 122 Pull Requests Actions Packages Projects Releases Wiki Activity

Next alpha stage commit

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-06 20:54:46 +02:00
parent e23387738b
commit 1d7ec956d2
42 changed files with 11603 additions and 2837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

418
app/main.py
View File

@@ -1,5 +1,7 @@
from __future__ import annotations
import csv
import io
import json
from contextlib import asynccontextmanager
from datetime import datetime, timezone
@@ -24,8 +26,17 @@ from app.dataset_search import search_datasets
from app.db import engine, get_db, init_db
from app.db_lock import DatabaseWriteBusy, database_write_lock, database_write_status
from app.geofabrik import create_geofabrik_source, geofabrik_catalog
from app.gtfs_diff import diff_run_payload, gtfs_update_diff_detail, gtfs_update_diff_summary, latest_gtfs_update_diffs
from app.gtfs_storage import scheduled_stop_ids
from app.harmonization import GTFS_QA_NOTE_PREFIX, gtfs_harmonization_feed_detail, gtfs_harmonization_inventory
from app.harmonization import (
GTFS_QA_NOTE_PREFIX,
gtfs_harmonization_feed_detail,
gtfs_harmonization_inventory,
gtfs_harmonized_snapshot,
gtfs_harmonized_snapshot_diagnostics,
gtfs_qa_review_payload,
list_active_harmonized_snapshot_routes,
)
from app.itineraries import generate_itineraries, itinerary_payload, recent_itineraries, set_itinerary_saved, set_leg_locked
from app.journey import find_journeys, nearest_scheduled_stops, search_scheduled_stops
from app.journey_search import cancel_journey_search, journey_search_payload, start_journey_search
@@ -37,6 +48,8 @@ from app.jobs import (
cancel_job,
create_dataset_delete_job,
create_address_index_rebuild_job,
create_gtfs_harmonized_snapshot_job,
create_map_gtfs_review_job,
create_maintenance_job,
create_osm_relabel_job,
create_route_matching_job,
@@ -69,6 +82,7 @@ from app.models import (
ItineraryLeg,
Job,
MatchRule,
MapGtfsReviewItem,
OsmFeature,
PipelineRun,
RouteMatch,
@@ -106,7 +120,21 @@ from app.source_catalog import (
source_catalog_rows,
source_catalog_summary,
)
from app.worker_supervisor import queue_worker_status, start_queue_workers, stop_queue_workers
from app.worker_supervisor import (
queue_worker_status,
restart_queue_workers,
start_queue_workers,
stop_configured_queue_workers,
stop_queue_workers,
)
from app.workbench import (
generate_map_gtfs_review_items,
list_map_gtfs_review_items,
map_gtfs_review_summary,
record_route_match_decision,
review_item_payload,
update_map_gtfs_review_item,
)
@asynccontextmanager
async def lifespan(_app: FastAPI):
@@ -125,6 +153,7 @@ async def lifespan(_app: FastAPI):
app = FastAPI(title="Mobility Workbench", version="0.1.0", lifespan=lifespan)
app.mount("/static", StaticFiles(directory=Path(__file__).parent / "static"), name="static")
templates = Jinja2Templates(directory=str(Path(__file__).parent / "templates"))
STATIC_ASSET_VERSION = "20260706-bike-mode"
MAP_FEATURE_LIMIT = 5000
MAP_FEATURE_LIMIT_MAX = 20000
@@ -146,6 +175,12 @@ class GtfsFeedReviewUpdate(BaseModel):
license: Optional[str] = None
review_status: Optional[str] = None
review_note: Optional[str] = None
can_import: Optional[str] = None
can_derive: Optional[str] = None
can_redistribute: Optional[str] = None
requires_attribution: Optional[str] = None
commercial_restrictions: Optional[str] = None
authority_level: Optional[str] = None
enabled: Optional[bool] = None
@@ -221,6 +256,11 @@ class JobPriorityRequest(BaseModel):
priority: int
class WorkbenchReviewUpdate(BaseModel):
status: Optional[str] = None
note: Optional[str] = None
def write_endpoint(operation: str):
def decorator(fn):
@wraps(fn)
@@ -249,7 +289,7 @@ async def database_operational_error_handler(_request: Request, exc: Operational
@app.get("/", response_class=HTMLResponse)
def index(request: Request) -> HTMLResponse:
return templates.TemplateResponse(request=request, name="index.html")
return templates.TemplateResponse(request=request, name="index.html", context={"static_version": STATIC_ASSET_VERSION})
@app.get("/api/sources")
@@ -623,6 +663,21 @@ def queue_route_matching(priority: int = 0, db: Session = Depends(get_db)) -> di
return job_payload(job)
@app.post("/api/jobs/map-gtfs-review")
@write_endpoint("queue Map/GTFS review generation")
def queue_map_gtfs_review(
source_id: Optional[int] = None,
dataset_id: Optional[int] = None,
limit: int = 1000,
priority: int = 0,
db: Session = Depends(get_db),
) -> dict:
job = create_map_gtfs_review_job(db, source_id=source_id, dataset_id=dataset_id, limit=limit, priority=priority)
db.commit()
db.refresh(job)
return job_payload(job)
@app.post("/api/jobs/osm-relabel")
@write_endpoint("queue OSM relabeling")
def queue_osm_relabel(
@@ -658,6 +713,28 @@ def list_jobs(
}
@app.post("/api/workers/start")
def start_workers_endpoint() -> dict:
started = start_queue_workers(force=True)
return _worker_control_payload("start", started=started)
@app.post("/api/workers/stop")
def stop_workers_endpoint() -> dict:
stopped = stop_configured_queue_workers()
return _worker_control_payload("stop", stopped=stopped)
@app.post("/api/workers/restart")
def restart_workers_endpoint() -> dict:
result = restart_queue_workers()
return _worker_control_payload(
"restart",
stopped=result.get("stopped", []),
started=result.get("started", []),
)
@app.get("/api/jobs/revision")
def get_jobs_revision(
response: Response,
@@ -870,6 +947,152 @@ def gtfs_harmonization_inventory_endpoint(db: Session = Depends(get_db)) -> dict
return gtfs_harmonization_inventory(db)
@app.get("/api/harmonization/gtfs/snapshot")
def gtfs_harmonized_snapshot_endpoint(db: Session = Depends(get_db)) -> dict:
return gtfs_harmonized_snapshot(db)
@app.get("/api/harmonization/gtfs/snapshot/routes")
def gtfs_harmonized_snapshot_routes_endpoint(
role: Optional[str] = None,
dataset_id: Optional[int] = None,
source_id: Optional[int] = None,
shadowed_by_dataset_id: Optional[int] = None,
mode: Optional[str] = None,
q: Optional[str] = None,
limit: int = 200,
offset: int = 0,
db: Session = Depends(get_db),
) -> dict:
if role is not None and role not in {"included", "shadowed", "excluded"}:
raise HTTPException(status_code=400, detail="role must be included, shadowed, or excluded")
return list_active_harmonized_snapshot_routes(
db,
role=role,
dataset_id=dataset_id,
source_id=source_id,
shadowed_by_dataset_id=shadowed_by_dataset_id,
mode=mode,
q=q,
limit=limit,
offset=offset,
)
@app.get("/api/harmonization/gtfs/snapshot/routes.csv")
def gtfs_harmonized_snapshot_routes_csv_endpoint(
role: Optional[str] = None,
dataset_id: Optional[int] = None,
source_id: Optional[int] = None,
shadowed_by_dataset_id: Optional[int] = None,
mode: Optional[str] = None,
q: Optional[str] = None,
limit: int = 10000,
offset: int = 0,
db: Session = Depends(get_db),
) -> Response:
if role is not None and role not in {"included", "shadowed", "excluded"}:
raise HTTPException(status_code=400, detail="role must be included, shadowed, or excluded")
payload = list_active_harmonized_snapshot_routes(
db,
role=role,
dataset_id=dataset_id,
source_id=source_id,
shadowed_by_dataset_id=shadowed_by_dataset_id,
mode=mode,
q=q,
limit=limit,
offset=offset,
max_limit=100_000,
)
output = io.StringIO()
fieldnames = [
"snapshot_id",
"gtfs_route_id",
"dataset_id",
"source_id",
"source_name",
"route_id",
"route_ref",
"route_name",
"mode",
"operator",
"overlap_key",
"role",
"reason",
"shadowed_by_gtfs_route_id",
"shadowed_by_dataset_id",
"shadowed_by_source_id",
]
writer = csv.DictWriter(output, fieldnames=fieldnames, extrasaction="ignore")
writer.writeheader()
for route in payload.get("routes", []):
writer.writerow(route)
snapshot = payload.get("snapshot") or {}
filename = f"gtfs_harmonized_snapshot_routes_{snapshot.get('id') or 'none'}.csv"
return Response(
content=output.getvalue(),
media_type="text/csv; charset=utf-8",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@app.get("/api/harmonization/gtfs/diffs")
def list_gtfs_update_diffs(
source_id: Optional[int] = None,
limit: int = 20,
db: Session = Depends(get_db),
) -> dict:
return {
"summary": gtfs_update_diff_summary(db),
"diffs": [diff_run_payload(diff) for diff in latest_gtfs_update_diffs(db, source_id=source_id, limit=limit)],
}
@app.get("/api/harmonization/gtfs/diffs/{diff_id}")
def get_gtfs_update_diff(diff_id: int, item_limit: int = 200, db: Session = Depends(get_db)) -> dict:
detail = gtfs_update_diff_detail(db, diff_id, item_limit=item_limit)
if detail is None:
raise HTTPException(status_code=404, detail="GTFS update diff not found")
return detail
@app.get("/api/workbench/gtfs-workflow")
def gtfs_workflow_status_endpoint(db: Session = Depends(get_db)) -> dict:
inventory = gtfs_harmonization_inventory(db)
snapshot_diagnostics = inventory.get("snapshot_diagnostics") or gtfs_harmonized_snapshot_diagnostics(db)
snapshot = inventory["snapshot"]
snapshot_summary = snapshot.get("summary") or {}
snapshot_route_preview = (
list_active_harmonized_snapshot_routes(db, role="shadowed", limit=8)
if snapshot.get("persisted") and int(snapshot_summary.get("snapshot_route_rows") or 0) > 0
else {"snapshot": None, "summary": {}, "routes": []}
)
review = map_gtfs_review_summary(db)
diff_summary = gtfs_update_diff_summary(db)
latest_diffs = [diff_run_payload(diff) for diff in latest_gtfs_update_diffs(db, limit=5)]
jobs = latest_jobs(db, limit=8)
return {
"inventory_summary": inventory["summary"],
"snapshot": snapshot,
"snapshot_diagnostics": snapshot_diagnostics,
"snapshot_route_preview": snapshot_route_preview,
"review_summary": review,
"diff_summary": diff_summary,
"latest_diffs": latest_diffs,
"jobs": [job_payload(job) for job in jobs],
}
@app.post("/api/jobs/gtfs-harmonized-snapshot")
@write_endpoint("queue harmonized GTFS snapshot build")
def queue_gtfs_harmonized_snapshot(activate: bool = True, priority: int = 0, db: Session = Depends(get_db)) -> dict:
job = create_gtfs_harmonized_snapshot_job(db, activate=activate, priority=priority)
db.commit()
db.refresh(job)
return job_payload(job)
@app.get("/api/harmonization/gtfs/sources/{source_id}")
def gtfs_harmonization_feed_detail_endpoint(source_id: int, db: Session = Depends(get_db)) -> dict:
detail = gtfs_harmonization_feed_detail(db, source_id)
@@ -886,15 +1109,48 @@ def update_gtfs_harmonization_feed_review(source_id: int, payload: GtfsFeedRevie
raise HTTPException(status_code=404, detail="GTFS source not found")
if payload.review_status is not None and payload.review_status not in {"unreviewed", "approved", "needs_review", "blocked", "rejected"}:
raise HTTPException(status_code=400, detail="review_status must be unreviewed, approved, needs_review, blocked, or rejected")
license_flags = {
"can_import": payload.can_import,
"can_derive": payload.can_derive,
"can_redistribute": payload.can_redistribute,
"requires_attribution": payload.requires_attribution,
"commercial_restrictions": payload.commercial_restrictions,
}
for field, value in license_flags.items():
if value is not None and value not in {"unknown", "yes", "no"}:
raise HTTPException(status_code=400, detail=f"{field} must be unknown, yes, or no")
if payload.authority_level is not None and payload.authority_level not in {
"unknown",
"national_official",
"regional_authority",
"operator",
"aggregator",
"mirror",
"secondary_discovery",
}:
raise HTTPException(status_code=400, detail="authority_level is not valid")
if payload.license is not None:
source.license = _truncate(payload.license, 255)
if payload.enabled is not None:
source.enabled = bool(payload.enabled)
if payload.review_status is not None or payload.review_note is not None:
existing_review = gtfs_qa_review_payload(source.notes)
review_fields_present = (
payload.review_status is not None
or payload.review_note is not None
or any(value is not None for value in license_flags.values())
or payload.authority_level is not None
)
if review_fields_present:
source.notes = _upsert_gtfs_qa_note(
source.notes,
status=payload.review_status or "unreviewed",
note=payload.review_note or "",
status=payload.review_status or existing_review["status"],
note=payload.review_note if payload.review_note is not None else existing_review["note"],
can_import=payload.can_import or existing_review["can_import"],
can_derive=payload.can_derive or existing_review["can_derive"],
can_redistribute=payload.can_redistribute or existing_review["can_redistribute"],
requires_attribution=payload.requires_attribution or existing_review["requires_attribution"],
commercial_restrictions=payload.commercial_restrictions or existing_review["commercial_restrictions"],
authority_level=payload.authority_level or existing_review["authority_level"],
)
db.commit()
detail = gtfs_harmonization_feed_detail(db, source_id)
@@ -903,6 +1159,84 @@ def update_gtfs_harmonization_feed_review(source_id: int, payload: GtfsFeedRevie
return detail
@app.get("/api/workbench/map-gtfs/summary")
def map_gtfs_workbench_summary_endpoint(db: Session = Depends(get_db)) -> dict:
return map_gtfs_review_summary(db)
@app.get("/api/workbench/map-gtfs/review-items")
def list_map_gtfs_workbench_review_items(
queue: Optional[str] = None,
status: Optional[str] = "open",
severity: Optional[str] = None,
mode: Optional[str] = None,
q: Optional[str] = None,
source_id: Optional[int] = None,
dataset_id: Optional[int] = None,
limit: int = 100,
db: Session = Depends(get_db),
) -> dict:
items = list_map_gtfs_review_items(
db,
queue=queue,
status=status,
severity=severity,
mode=mode,
q=q,
source_id=source_id,
dataset_id=dataset_id,
limit=limit,
)
return {
"summary": map_gtfs_review_summary(db),
"filters": {
"queue": queue,
"status": status,
"severity": severity,
"mode": mode,
"q": q,
"source_id": source_id,
"dataset_id": dataset_id,
},
"items": [review_item_payload(item) for item in items],
}
@app.get("/api/workbench/map-gtfs/review-items/{item_id}")
def get_map_gtfs_workbench_review_item(item_id: int, db: Session = Depends(get_db)) -> dict:
item = db.get(MapGtfsReviewItem, item_id)
if item is None:
raise HTTPException(status_code=404, detail="review item not found")
return review_item_payload(item)
@app.post("/api/workbench/map-gtfs/review-items/generate")
@write_endpoint("generate Map/GTFS review queue")
def generate_map_gtfs_workbench_review_items(
source_id: Optional[int] = None,
dataset_id: Optional[int] = None,
limit: int = 1000,
db: Session = Depends(get_db),
) -> dict:
result = generate_map_gtfs_review_items(db, source_id=source_id, dataset_id=dataset_id, limit=limit)
db.commit()
return {"summary": map_gtfs_review_summary(db), "result": result}
@app.patch("/api/workbench/map-gtfs/review-items/{item_id}")
@write_endpoint("update Map/GTFS review item")
def update_map_gtfs_workbench_review_item(item_id: int, payload: WorkbenchReviewUpdate, db: Session = Depends(get_db)) -> dict:
try:
item = update_map_gtfs_review_item(db, item_id, status=payload.status, note=payload.note)
except ValueError as exc:
detail = str(exc)
status_code = 404 if detail == "review item not found" else 400
raise HTTPException(status_code=status_code, detail=detail) from exc
db.commit()
db.refresh(item)
return review_item_payload(item)
@app.get("/api/matches")
def list_matches(
status: Optional[str] = None,
@@ -933,6 +1267,7 @@ def accept_match(match_id: int, db: Session = Depends(get_db)) -> dict:
match.rule_source = "manual"
match.updated_at = datetime.now(timezone.utc)
_persist_match_rule(db, match, "accept_match")
record_route_match_decision(db, match, note="Accepted from route match review.")
db.commit()
return {"id": match.id, "status": match.status}
@@ -947,6 +1282,7 @@ def reject_match(match_id: int, db: Session = Depends(get_db)) -> dict:
match.rule_source = "manual"
match.updated_at = datetime.now(timezone.utc)
_persist_match_rule(db, match, "reject_match")
record_route_match_decision(db, match, note="Rejected from route match review.")
db.commit()
return {"id": match.id, "status": match.status}
@@ -1030,6 +1366,7 @@ def accept_match_candidate(match_id: int, osm_feature_id: str, db: Session = Dep
match.reasons_json = json.dumps(reasons, separators=(",", ":"))
match.updated_at = datetime.now(timezone.utc)
_persist_match_rule(db, match, "accept_match")
record_route_match_decision(db, match, note="Accepted from route candidate review.")
db.commit()
db.refresh(match)
return {"id": match.id, "status": match.status, "confidence": match.confidence, "match": match_row(match)}
@@ -1423,6 +1760,7 @@ def journey_nearest_location(
source_ids=_csv_ints(source_id, "source_id"),
limit=1,
radius_m=max(5, min(float(stop_radius_m), 120)),
grouped=False,
)
location = stops[0] if stops else None
if location is not None:
@@ -1509,7 +1847,8 @@ def journey_search(
@app.post("/api/journey/searches")
def start_progressive_journey_search(payload: JourneySearchRequest) -> dict:
mode = payload.mode if payload.mode in {"transit", "walk", "drive", "car"} else "transit"
requested_mode = str(payload.mode or "transit").lower()
mode = "bike" if requested_mode in {"bike", "bicycle", "cycle", "cycling"} else requested_mode if requested_mode in {"transit", "walk", "drive", "car"} else "transit"
ranking = payload.ranking if payload.ranking in {"recommended", "earliest_arrival", "duration", "fewest_transfers"} else "recommended"
try:
return start_journey_search(
@@ -1978,6 +2317,29 @@ def _json_object_value(value: object, key: str) -> object:
return value.get(key) if isinstance(value, dict) else None
def _worker_control_payload(action: str, **groups: list[object]) -> dict:
return {
"action": action,
"operations": {
name: [_worker_handle_payload(handle) for handle in handles]
for name, handles in groups.items()
},
"workers": queue_worker_status(),
}
def _worker_handle_payload(handle: object) -> dict:
return {
"index": getattr(handle, "index", None),
"worker_id": getattr(handle, "worker_id", None),
"pid": getattr(handle, "pid", None),
"status": getattr(handle, "status", None),
"pid_file": str(getattr(handle, "pid_file", "")),
"log_file": str(getattr(handle, "log_file", "")),
"started_by_server": bool(getattr(handle, "started_by_server", False)),
}
def _job_queue_revision_payload(
db: Session,
*,
@@ -2018,19 +2380,51 @@ def _set_etag(response: Response, revision: str) -> None:
response.headers["ETag"] = f'W/"{revision}"'
def _upsert_gtfs_qa_note(notes: str | None, *, status: str, note: str) -> str | None:
def _upsert_gtfs_qa_note(
notes: str | None,
*,
status: str,
note: str,
can_import: str | None = None,
can_derive: str | None = None,
can_redistribute: str | None = None,
requires_attribution: str | None = None,
commercial_restrictions: str | None = None,
authority_level: str | None = None,
) -> str | None:
status_text = (status or "unreviewed").strip() or "unreviewed"
note_text = " ".join((note or "").strip().split())
updated_at = datetime.now(timezone.utc).isoformat()
marker = f"{GTFS_QA_NOTE_PREFIX} status={status_text}; updated_at={updated_at}"
if note_text:
marker = f"{marker}; note={note_text}"
payload = {
"status": status_text,
"updated_at": updated_at,
"note": note_text,
"can_import": can_import or "unknown",
"can_derive": can_derive or "unknown",
"can_redistribute": can_redistribute or "unknown",
"requires_attribution": requires_attribution or "unknown",
"commercial_restrictions": commercial_restrictions or "unknown",
"authority_level": authority_level or "unknown",
}
marker = f"{GTFS_QA_NOTE_PREFIX} {json.dumps(payload, sort_keys=True, separators=(',', ':'))}"
preserved = [
line
for line in str(notes or "").splitlines()
if line.strip() and not line.startswith(GTFS_QA_NOTE_PREFIX)
]
if status_text != "unreviewed" or note_text:
has_decision = (
status_text != "unreviewed"
or note_text
or any(payload[key] != "unknown" for key in [
"can_import",
"can_derive",
"can_redistribute",
"requires_attribution",
"commercial_restrictions",
"authority_level",
])
)
if has_decision:
preserved.insert(0, marker)
return "\n".join(preserved) or None