Next alpha stage commit

This commit is contained in:
2026-07-06 20:54:46 +02:00
parent e23387738b
commit 1d7ec956d2
42 changed files with 11603 additions and 2837 deletions

View File

@@ -1,19 +1,24 @@
from __future__ import annotations
import json
from pathlib import Path
from fastapi.testclient import TestClient
from sqlalchemy import select
from sqlalchemy import delete, select
import app.jobs as jobs_module
import app.main as main_module
from app.config import settings
from app.db import init_db, session_scope
from app.db_lock import DatabaseWriteBusy, database_write_lock
from app.jobs import run_worker_once
from app.harmonization import active_harmonized_gtfs_dataset_ids, active_harmonized_gtfs_shadowed_route_ids
from app.jobs import add_job_event, run_worker_once
from app.main import app
from app.models import Dataset, GtfsRoute, Job, Source
from app.models import Dataset, GtfsHarmonizedSnapshotRoute, GtfsRoute, Job, RouteMatch, Source
from app.pipeline.matcher import run_route_matching
from app.pipeline.route_layer import rebuild_route_layer
from app.source_catalog import import_ingestable_sources
from app.worker_supervisor import WorkerHandle
def test_api_sample_and_geojson():
@@ -127,6 +132,28 @@ def test_route_matching_job_endpoint_completes():
assert event_types[-1] == "completed"
def test_job_events_endpoint_returns_latest_window_in_chronological_order():
client = TestClient(app)
assert client.post("/api/sample/reset").status_code == 200
with session_scope() as session:
job = Job(kind="maintenance", status="running", description="event window test")
session.add(job)
session.flush()
for index in range(5):
add_job_event(
session,
job,
event_type=f"event_{index + 1}",
message=f"message {index + 1}",
progress_current=index + 1,
progress_total=5,
)
job_id = job.id
events = client.get(f"/api/jobs/{job_id}/events?limit=3").json()["events"]
assert [event["event_type"] for event in events] == ["event_3", "event_4", "event_5"]
def test_qa_summary_endpoint_exposes_harmonization_sections():
client = TestClient(app)
assert client.post("/api/sample/reset").status_code == 200
@@ -168,14 +195,342 @@ def test_gtfs_harmonization_inventory_and_detail():
reviewed = client.patch(
f"/api/harmonization/gtfs/sources/{feed['source']['id']}/review",
json={"license": "CC-BY-4.0", "review_status": "approved", "review_note": "Operator publication allowed.", "enabled": True},
json={
"license": "CC-BY-4.0",
"review_status": "approved",
"review_note": "Operator publication allowed.",
"can_import": "yes",
"can_derive": "yes",
"can_redistribute": "yes",
"requires_attribution": "yes",
"commercial_restrictions": "no",
"authority_level": "regional_authority",
"enabled": True,
},
).json()
assert reviewed["source"]["license"] == "CC-BY-4.0"
assert reviewed["source"]["qa_review"]["status"] == "approved"
assert reviewed["source"]["qa_review"]["note"] == "Operator publication allowed."
assert reviewed["source"]["qa_review"]["can_redistribute"] == "yes"
assert reviewed["source"]["qa_review"]["authority_level"] == "regional_authority"
assert reviewed["license"]["redistribution_status"] == "allowed"
assert reviewed["source"]["enabled"] is True
def test_gtfs_harmonized_snapshot_shadows_lower_authority_duplicate_feed():
client = TestClient(app)
assert client.post("/api/sample/reset").status_code == 200
with session_scope() as session:
primary_source = session.scalar(select(Source).where(Source.kind == "gtfs"))
primary_dataset = next(dataset for dataset in primary_source.datasets if dataset.kind == "gtfs" and dataset.is_active)
primary_source.priority = "P0"
primary_source.notes = _gtfs_review_note("national_official")
duplicate_source = Source(
name="Sample Berlin regional duplicate",
kind="gtfs",
url="https://example.invalid/regional-duplicate.zip",
country="DE",
license="sample",
priority="P1",
notes=_gtfs_review_note("regional_authority"),
)
session.add(duplicate_source)
session.flush()
duplicate_dataset = Dataset(
source_id=duplicate_source.id,
kind="gtfs",
local_path="data/sample/regional-duplicate.gtfs.zip",
sha256="d" * 64,
is_active=True,
status="imported",
)
session.add(duplicate_dataset)
session.flush()
routes = session.scalars(select(GtfsRoute).where(GtfsRoute.dataset_id == primary_dataset.id)).all()
for route in routes:
session.add(
GtfsRoute(
dataset_id=duplicate_dataset.id,
route_id=f"dup_{route.route_id}",
agency_id=route.agency_id,
short_name=route.short_name,
long_name=route.long_name,
route_type=route.route_type,
mode=route.mode,
route_scope=route.route_scope,
operator_name=route.operator_name,
route_key=route.route_key,
operator_key=route.operator_key,
min_lon=route.min_lon,
min_lat=route.min_lat,
max_lon=route.max_lon,
max_lat=route.max_lat,
)
)
primary_dataset_id = primary_dataset.id
duplicate_dataset_id = duplicate_dataset.id
snapshot = client.get("/api/harmonization/gtfs/snapshot").json()
assert snapshot["summary"]["raw_active_datasets"] == 2
assert snapshot["summary"]["included_datasets"] == 1
assert snapshot["summary"]["shadowed_datasets"] == 1
assert snapshot["dataset_ids"] == [primary_dataset_id]
shadowed = next(item for item in snapshot["datasets"] if item["dataset_id"] == duplicate_dataset_id)
assert shadowed["role"] == "shadowed"
assert shadowed["shadowed_by_dataset_id"] == primary_dataset_id
assert shadowed["route_key_overlap_ratio"] >= 0.8
with session_scope() as session:
assert active_harmonized_gtfs_dataset_ids(session) == [primary_dataset_id]
queued = client.post("/api/jobs/gtfs-harmonized-snapshot?activate=true").json()
assert queued["kind"] == "gtfs_harmonized_snapshot"
assert run_worker_once(worker_id="test-worker")["processed"] == 1
job = client.get(f"/api/jobs/{queued['id']}").json()
assert job["status"] == "completed"
assert job["result"]["snapshot"]["persisted"] is True
assert job["result"]["snapshot"]["dataset_ids"] == [primary_dataset_id]
persisted_snapshot = client.get("/api/harmonization/gtfs/snapshot").json()
assert persisted_snapshot["persisted"] is True
assert persisted_snapshot["dataset_ids"] == [primary_dataset_id]
workflow = client.get("/api/workbench/gtfs-workflow").json()
assert workflow["snapshot"]["persisted"] is True
assert workflow["inventory_summary"]["snapshot_shadowed_datasets"] == 1
diffs = client.get("/api/harmonization/gtfs/diffs").json()
assert diffs["summary"]["runs"] == 0
dedup_job = client.post("/api/jobs/map-gtfs-review?limit=50").json()
assert run_worker_once(worker_id="test-worker")["processed"] == 1
dedup_result = client.get(f"/api/jobs/{dedup_job['id']}").json()["result"]["review_result"]
assert dedup_result["dedup_items"] > 0
dedup_items = client.get("/api/workbench/map-gtfs/review-items?queue=gtfs_deduplication&status=open&limit=20").json()["items"]
assert dedup_items
assert dedup_items[0]["metadata"]["data_differs"] is True
def test_gtfs_harmonized_snapshot_partially_shadows_aggregator_routes_by_bbox():
client = TestClient(app)
assert client.post("/api/sample/reset").status_code == 200
with session_scope() as session:
aggregator_source = session.scalar(select(Source).where(Source.kind == "gtfs"))
aggregator_source.notes = _gtfs_review_note("aggregator")
aggregator_dataset = next(dataset for dataset in aggregator_source.datasets if dataset.kind == "gtfs" and dataset.is_active)
local_route = session.scalar(select(GtfsRoute).where(GtfsRoute.dataset_id == aggregator_dataset.id).order_by(GtfsRoute.id))
assert local_route is not None
local_route.min_lon = 13.35
local_route.min_lat = 52.48
local_route.max_lon = 13.45
local_route.max_lat = 52.55
far_route = GtfsRoute(
dataset_id=aggregator_dataset.id,
route_id="far-same-key",
agency_id=local_route.agency_id,
short_name=local_route.short_name,
long_name=local_route.long_name,
route_type=local_route.route_type,
mode=local_route.mode,
route_scope=local_route.route_scope,
operator_name=local_route.operator_name,
route_key=local_route.route_key,
operator_key=local_route.operator_key,
min_lon=8.35,
min_lat=49.35,
max_lon=8.45,
max_lat=49.45,
)
session.add(far_route)
regional_source = Source(
name="Sample regional operator overlay",
kind="gtfs",
url="https://example.invalid/regional-overlay.zip",
country="DE",
license="sample",
priority="P1",
notes=_gtfs_review_note("operator"),
)
session.add(regional_source)
session.flush()
regional_dataset = Dataset(
source_id=regional_source.id,
kind="gtfs",
local_path="data/sample/regional-overlay.gtfs.zip",
sha256="e" * 64,
is_active=True,
status="imported",
)
session.add(regional_dataset)
session.flush()
session.add(
GtfsRoute(
dataset_id=regional_dataset.id,
route_id="regional-same-key",
agency_id=local_route.agency_id,
short_name=local_route.short_name,
long_name=local_route.long_name,
route_type=local_route.route_type,
mode=local_route.mode,
route_scope=local_route.route_scope,
operator_name="Regional Operator",
route_key=local_route.route_key,
operator_key="regional operator",
min_lon=13.36,
min_lat=52.49,
max_lon=13.44,
max_lat=52.54,
)
)
session.flush()
local_route_id = local_route.id
far_route_id = far_route.id
aggregator_dataset_id = aggregator_dataset.id
regional_dataset_id = regional_dataset.id
snapshot = client.get("/api/harmonization/gtfs/snapshot").json()
assert snapshot["summary"]["included_datasets"] == 2
assert snapshot["summary"]["shadowed_datasets"] == 0
assert snapshot["summary"]["route_shadowed_routes"] == 1
aggregator_item = next(item for item in snapshot["datasets"] if item["dataset_id"] == aggregator_dataset_id)
assert aggregator_item["role"] == "included"
assert aggregator_item["route_shadow_count"] == 1
assert aggregator_item["route_shadowed_by_dataset_ids"] == [regional_dataset_id]
workflow = client.get("/api/workbench/gtfs-workflow").json()
assert workflow["snapshot_diagnostics"]["needs_rebuild"] is True
assert workflow["snapshot_diagnostics"]["computed"]["route_shadowed_routes"] == 1
with session_scope() as session:
shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session)
assert local_route_id in shadowed_route_ids
assert far_route_id not in shadowed_route_ids
snapshot_job = client.post("/api/jobs/gtfs-harmonized-snapshot?activate=true").json()
assert run_worker_once(worker_id="test-worker")["processed"] == 1
snapshot_result = client.get(f"/api/jobs/{snapshot_job['id']}").json()["result"]["snapshot"]
assert snapshot_result["persisted"] is True
assert snapshot_result["summary"]["snapshot_route_rows"] >= 1
assert snapshot_result["summary"]["snapshot_route_included_rows"] >= 1
assert snapshot_result["summary"]["snapshot_route_shadowed_rows"] == 1
assert snapshot_result["summary"]["route_shadowed_routes"] == 1
workflow = client.get("/api/workbench/gtfs-workflow").json()
assert workflow["snapshot_diagnostics"]["needs_rebuild"] is False
assert workflow["snapshot_diagnostics"]["active"]["route_rows"] >= 1
assert workflow["snapshot_route_preview"]["summary"]["filtered_total"] == 1
assert workflow["snapshot_route_preview"]["routes"][0]["gtfs_route_id"] == local_route_id
with session_scope() as session:
session.execute(delete(GtfsHarmonizedSnapshotRoute).where(GtfsHarmonizedSnapshotRoute.snapshot_id == snapshot_result["id"]))
legacy_workflow = client.get("/api/workbench/gtfs-workflow").json()
assert legacy_workflow["snapshot_diagnostics"]["needs_rebuild"] is True
assert "route_ownership_not_materialized" in legacy_workflow["snapshot_diagnostics"]["reasons"]
assert legacy_workflow["snapshot_route_preview"]["routes"] == []
backfill_job = client.post("/api/jobs/gtfs-harmonized-snapshot?activate=true").json()
assert run_worker_once(worker_id="test-worker")["processed"] == 1
backfilled_snapshot = client.get(f"/api/jobs/{backfill_job['id']}").json()["result"]["snapshot"]
assert backfilled_snapshot["id"] == snapshot_result["id"]
assert backfilled_snapshot["summary"]["snapshot_route_rows"] >= 1
assert backfilled_snapshot["summary"]["route_shadowed_routes"] == 1
backfilled_workflow = client.get("/api/workbench/gtfs-workflow").json()
assert backfilled_workflow["snapshot_route_preview"]["summary"]["filtered_total"] == 1
assert backfilled_workflow["snapshot_route_preview"]["routes"][0]["gtfs_route_id"] == local_route_id
route_rows = client.get("/api/harmonization/gtfs/snapshot/routes?role=shadowed&limit=20").json()
assert route_rows["summary"]["filtered_total"] == 1
assert route_rows["routes"][0]["gtfs_route_id"] == local_route_id
assert route_rows["routes"][0]["shadowed_by_dataset_id"] == regional_dataset_id
filtered_route_rows = client.get(
"/api/harmonization/gtfs/snapshot/routes",
params={"role": "shadowed", "mode": route_rows["routes"][0]["mode"], "q": route_rows["routes"][0]["route_ref"], "limit": 20},
).json()
assert filtered_route_rows["summary"]["filtered_total"] == 1
assert filtered_route_rows["routes"][0]["gtfs_route_id"] == local_route_id
csv_response = client.get("/api/harmonization/gtfs/snapshot/routes.csv?role=shadowed&limit=20")
assert csv_response.status_code == 200
assert "text/csv" in csv_response.headers["content-type"]
assert "gtfs_route_id" in csv_response.text.splitlines()[0]
assert str(local_route_id) in csv_response.text
with session_scope() as session:
persisted_shadowed_route_ids = active_harmonized_gtfs_shadowed_route_ids(session)
assert local_route_id in persisted_shadowed_route_ids
assert far_route_id not in persisted_shadowed_route_ids
session.execute(delete(RouteMatch).where(RouteMatch.gtfs_route_id == local_route_id))
match_result = run_route_matching(session, batch_size=50)
assert match_result["shadowed_skipped"] == 1
assert session.scalar(select(RouteMatch).where(RouteMatch.gtfs_route_id == local_route_id)) is None
route_layer_result = rebuild_route_layer(session, commit_between_steps=False)
assert route_layer_result["shadowed_routes_skipped"] >= 1
dedup_job = client.post("/api/jobs/map-gtfs-review?limit=100").json()
assert run_worker_once(worker_id="test-worker")["processed"] == 1
dedup_result = client.get(f"/api/jobs/{dedup_job['id']}").json()["result"]["review_result"]
assert dedup_result["dedup_items"] > 0
dedup_items = client.get("/api/workbench/map-gtfs/review-items?queue=gtfs_deduplication&status=open&limit=100").json()["items"]
assert any(
item["gtfs_route"]["id"] == local_route_id
and item["metadata"]["snapshot"]["reason"] == "route_covered_by_higher_precedence_dataset"
for item in dedup_items
)
dedup_detail_id = next(item["id"] for item in dedup_items if item["gtfs_route"]["id"] == local_route_id)
dedup_detail = client.get(f"/api/workbench/map-gtfs/review-items/{dedup_detail_id}").json()
assert dedup_detail["metadata"]["shadowed_dataset"]["dataset_id"] == aggregator_dataset_id
assert dedup_detail["metadata"]["included_dataset"]["dataset_id"] == regional_dataset_id
filtered_dedup = client.get(
"/api/workbench/map-gtfs/review-items",
params={
"queue": "gtfs_deduplication",
"status": "open",
"severity": "warn",
"mode": route_rows["routes"][0]["mode"],
"q": route_rows["routes"][0]["route_ref"],
"limit": 20,
},
).json()
assert any(item["gtfs_route"]["id"] == local_route_id for item in filtered_dedup["items"])
def test_map_gtfs_workbench_review_queue_and_decision_replay_foundation():
client = TestClient(app)
assert client.post("/api/sample/reset").status_code == 200
queued = client.post("/api/jobs/map-gtfs-review?limit=20").json()
assert queued["kind"] == "map_gtfs_review"
assert run_worker_once(worker_id="test-worker")["processed"] == 1
job = client.get(f"/api/jobs/{queued['id']}").json()
assert job["status"] == "completed"
assert job["result"]["review_result"]["route_items"] > 0
listed = client.get("/api/workbench/map-gtfs/review-items?queue=route_matching&status=open&limit=20").json()
assert listed["items"]
route_item = next(item for item in listed["items"] if item["route_match_id"])
assert route_item["queue"] == "route_matching"
assert route_item["gtfs_route"]
accepted = client.post(f"/api/matches/{route_item['route_match_id']}/accept").json()
assert accepted["status"] == "accepted"
summary = client.get("/api/workbench/map-gtfs/summary").json()
assert summary["active_decisions"] >= 1
resolved = client.get("/api/workbench/map-gtfs/review-items?queue=route_matching&status=resolved&limit=20").json()
assert any(item["fingerprint"] == route_item["fingerprint"] and item["decision_id"] for item in resolved["items"])
def _gtfs_review_note(authority_level: str) -> str:
return "[GTFS QA] " + json.dumps(
{
"status": "approved",
"note": "",
"can_import": "yes",
"can_derive": "yes",
"can_redistribute": "yes",
"requires_attribution": "yes",
"commercial_restrictions": "no",
"authority_level": authority_level,
},
sort_keys=True,
separators=(",", ":"),
)
def test_terminal_jobs_can_be_dismissed_from_default_view():
client = TestClient(app)
assert client.post("/api/sample/reset").status_code == 200
@@ -276,6 +631,73 @@ def test_worker_once_returns_idle_when_claim_is_busy(monkeypatch):
assert jobs_module.run_worker_once(worker_id="test-worker") == {"worker_id": "test-worker", "processed": 0}
def test_worker_control_endpoints(monkeypatch):
calls = []
handle = WorkerHandle(
index=0,
worker_id="server-worker-1",
pid=1234,
status="started",
pid_file=Path("data/workers/server-worker-1.pid"),
log_file=Path("data/workers/server-worker-1.log"),
started_by_server=True,
)
stopped = WorkerHandle(
index=0,
worker_id="server-worker-1",
pid=None,
status="stopped",
pid_file=Path("data/workers/server-worker-1.pid"),
log_file=Path("data/workers/server-worker-1.log"),
)
def fake_start_queue_workers(*, force=False):
calls.append(("start", force))
return [handle]
def fake_stop_configured_queue_workers():
calls.append(("stop", None))
return [stopped]
def fake_restart_queue_workers():
calls.append(("restart", None))
return {"stopped": [stopped], "started": [handle]}
monkeypatch.setattr(main_module, "start_queue_workers", fake_start_queue_workers)
monkeypatch.setattr(main_module, "stop_configured_queue_workers", fake_stop_configured_queue_workers)
monkeypatch.setattr(main_module, "restart_queue_workers", fake_restart_queue_workers)
monkeypatch.setattr(
main_module,
"queue_worker_status",
lambda: [
{
"index": 0,
"worker_id": "server-worker-1",
"pid": 1234,
"running": True,
"stale": False,
"status": "running",
"pid_file": "data/workers/server-worker-1.pid",
"log_file": "data/workers/server-worker-1.log",
}
],
)
client = TestClient(app)
started = client.post("/api/workers/start").json()
assert ("start", True) in calls
assert started["operations"]["started"][0]["status"] == "started"
assert started["workers"][0]["running"] is True
stopped_response = client.post("/api/workers/stop").json()
assert ("stop", None) in calls
assert stopped_response["operations"]["stopped"][0]["status"] == "stopped"
restarted = client.post("/api/workers/restart").json()
assert ("restart", None) in calls
assert restarted["operations"]["started"][0]["worker_id"] == "server-worker-1"
def test_running_job_can_be_stopped_while_write_lock_is_held():
client = TestClient(app)
assert client.post("/api/sample/reset").status_code == 200