add-ideas/meubility-workbench
1
0
Fork 0
Code Issues 122 Pull Requests Actions Packages Projects Releases Wiki Activity

Next alpha stage commit

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-06 20:54:46 +02:00
parent e23387738b
commit 1d7ec956d2
42 changed files with 11603 additions and 2837 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
app/data_management.py
View File

@@ -13,6 +13,8 @@ from app.models import (
GtfsAgency,
GtfsCalendar,
GtfsCalendarDate,
GtfsHarmonizedSnapshotDataset,
GtfsHarmonizedSnapshotRoute,
GtfsRoute,
GtfsRoutePatternLink,
GtfsShape,
@@ -20,6 +22,8 @@ from app.models import (
GtfsStopTime,
GtfsTripRoutePatternLink,
GtfsTrip,
MapGtfsDecision,
MapGtfsReviewItem,
OsmDiffState,
OsmFeature,
RouteMatch,
@@ -136,6 +140,8 @@ def delete_source(session: Session, source_id: int) -> dict[str, object]:
_delete_dataset_files(dataset)
session.delete(dataset)
session.execute(delete(OsmDiffState).where(OsmDiffState.source_id == source.id))
session.execute(delete(MapGtfsReviewItem).where(MapGtfsReviewItem.source_id == source.id))
session.execute(delete(MapGtfsDecision).where(MapGtfsDecision.source_id == source.id))
session.delete(source)
session.flush()
return {"deleted": True, "source_id": source_id, "datasets": dataset_results}
@@ -217,14 +223,41 @@ def prune_inactive_datasets(session: Session, dry_run: bool = True) -> dict[str,
for dataset_id in dataset_ids:
_detach_update_checks_for_dataset(session, dataset_id)
for dataset in session.scalars(select(Dataset).where(Dataset.id.in_(dataset_ids))).all():
_delete_workbench_rows_for_dataset(session, dataset)
if match_filters:
session.execute(delete(RouteMatch).where(or_(*match_filters)))
if gtfs_ids:
route_ids = select(GtfsRoute.id).where(GtfsRoute.dataset_id.in_(gtfs_ids))
pattern_ids = select(RoutePattern.id).where(RoutePattern.gtfs_route_id.in_(route_ids))
session.execute(
delete(GtfsHarmonizedSnapshotRoute).where(
or_(
GtfsHarmonizedSnapshotRoute.dataset_id.in_(gtfs_ids),
GtfsHarmonizedSnapshotRoute.shadowed_by_dataset_id.in_(gtfs_ids),
GtfsHarmonizedSnapshotRoute.gtfs_route_id.in_(route_ids),
GtfsHarmonizedSnapshotRoute.shadowed_by_gtfs_route_id.in_(route_ids),
)
)
)
session.execute(delete(GtfsHarmonizedSnapshotDataset).where(GtfsHarmonizedSnapshotDataset.dataset_id.in_(gtfs_ids)))
session.execute(delete(RoutePatternStop).where(RoutePatternStop.route_pattern_id.in_(pattern_ids)))
session.execute(delete(GtfsTripRoutePatternLink).where(GtfsTripRoutePatternLink.dataset_id.in_(gtfs_ids)))
session.execute(delete(GtfsRoutePatternLink).where(GtfsRoutePatternLink.dataset_id.in_(gtfs_ids)))
session.execute(
delete(GtfsTripRoutePatternLink).where(
or_(
GtfsTripRoutePatternLink.dataset_id.in_(gtfs_ids),
GtfsTripRoutePatternLink.route_pattern_id.in_(pattern_ids),
)
)
)
session.execute(
delete(GtfsRoutePatternLink).where(
or_(
GtfsRoutePatternLink.dataset_id.in_(gtfs_ids),
GtfsRoutePatternLink.route_pattern_id.in_(pattern_ids),
)
)
)
session.execute(delete(RoutePattern).where(RoutePattern.gtfs_route_id.in_(route_ids)))
session.execute(delete(CanonicalStopLink).where(CanonicalStopLink.dataset_id.in_(gtfs_ids), CanonicalStopLink.object_type == "gtfs_stop"))
for model in [GtfsStopTime, GtfsShape, GtfsTrip, GtfsCalendarDate, GtfsCalendar, GtfsRoute, GtfsStop, GtfsAgency]:
@@ -246,13 +279,39 @@ def prune_inactive_datasets(session: Session, dry_run: bool = True) -> dict[str,
def _delete_dataset_rows(session: Session, dataset: Dataset) -> None:
_delete_workbench_rows_for_dataset(session, dataset)
if dataset.kind == "gtfs":
route_ids = select(GtfsRoute.id).where(GtfsRoute.dataset_id == dataset.id)
pattern_ids = select(RoutePattern.id).where(RoutePattern.gtfs_route_id.in_(route_ids))
session.execute(
delete(GtfsHarmonizedSnapshotRoute).where(
or_(
GtfsHarmonizedSnapshotRoute.dataset_id == dataset.id,
GtfsHarmonizedSnapshotRoute.shadowed_by_dataset_id == dataset.id,
GtfsHarmonizedSnapshotRoute.gtfs_route_id.in_(route_ids),
GtfsHarmonizedSnapshotRoute.shadowed_by_gtfs_route_id.in_(route_ids),
)
)
)
session.execute(delete(GtfsHarmonizedSnapshotDataset).where(GtfsHarmonizedSnapshotDataset.dataset_id == dataset.id))
session.execute(delete(RouteMatch).where(RouteMatch.gtfs_route_id.in_(route_ids)))
session.execute(delete(RoutePatternStop).where(RoutePatternStop.route_pattern_id.in_(pattern_ids)))
session.execute(delete(GtfsTripRoutePatternLink).where(GtfsTripRoutePatternLink.dataset_id == dataset.id))
session.execute(delete(GtfsRoutePatternLink).where(GtfsRoutePatternLink.dataset_id == dataset.id))
session.execute(
delete(GtfsTripRoutePatternLink).where(
or_(
GtfsTripRoutePatternLink.dataset_id == dataset.id,
GtfsTripRoutePatternLink.route_pattern_id.in_(pattern_ids),
)
)
)
session.execute(
delete(GtfsRoutePatternLink).where(
or_(
GtfsRoutePatternLink.dataset_id == dataset.id,
GtfsRoutePatternLink.route_pattern_id.in_(pattern_ids),
)
)
)
session.execute(delete(RoutePattern).where(RoutePattern.gtfs_route_id.in_(route_ids)))
session.execute(delete(CanonicalStopLink).where(CanonicalStopLink.dataset_id == dataset.id, CanonicalStopLink.object_type == "gtfs_stop"))
for model in [GtfsStopTime, GtfsShape, GtfsTrip, GtfsCalendarDate, GtfsCalendar, GtfsRoute, GtfsStop, GtfsAgency]:
@@ -269,6 +328,55 @@ def _delete_dataset_rows(session: Session, dataset: Dataset) -> None:
session.execute(delete(OsmFeature).where(OsmFeature.dataset_id == dataset.id))
def _delete_workbench_rows_for_dataset(session: Session, dataset: Dataset) -> None:
review_filters = [MapGtfsReviewItem.dataset_id == dataset.id]
decision_filters = [
MapGtfsDecision.gtfs_dataset_id == dataset.id,
MapGtfsDecision.osm_dataset_id == dataset.id,
]
if dataset.kind == "gtfs":
route_ids = select(GtfsRoute.id).where(GtfsRoute.dataset_id == dataset.id)
stop_ids = select(GtfsStop.id).where(GtfsStop.dataset_id == dataset.id)
match_ids = select(RouteMatch.id).where(RouteMatch.gtfs_route_id.in_(route_ids))
link_ids = select(CanonicalStopLink.id).where(CanonicalStopLink.dataset_id == dataset.id, CanonicalStopLink.object_type == "gtfs_stop")
review_filters.extend(
[
MapGtfsReviewItem.gtfs_route_id.in_(route_ids),
MapGtfsReviewItem.gtfs_stop_id.in_(stop_ids),
MapGtfsReviewItem.route_match_id.in_(match_ids),
MapGtfsReviewItem.canonical_stop_link_id.in_(link_ids),
]
)
decision_filters.extend(
[
MapGtfsDecision.gtfs_route_id.in_(route_ids),
MapGtfsDecision.gtfs_stop_id.in_(stop_ids),
MapGtfsDecision.route_match_id.in_(match_ids),
MapGtfsDecision.canonical_stop_link_id.in_(link_ids),
]
)
elif dataset.kind == "osm_geojson":
osm_feature_ids = select(OsmFeature.id).where(OsmFeature.dataset_id == dataset.id)
match_ids = select(RouteMatch.id).where(RouteMatch.osm_feature_id.in_(osm_feature_ids))
link_ids = select(CanonicalStopLink.id).where(CanonicalStopLink.dataset_id == dataset.id, CanonicalStopLink.object_type == "osm_feature")
review_filters.extend(
[
MapGtfsReviewItem.osm_feature_id.in_(osm_feature_ids),
MapGtfsReviewItem.route_match_id.in_(match_ids),
MapGtfsReviewItem.canonical_stop_link_id.in_(link_ids),
]
)
decision_filters.extend(
[
MapGtfsDecision.osm_feature_id.in_(osm_feature_ids),
MapGtfsDecision.route_match_id.in_(match_ids),
MapGtfsDecision.canonical_stop_link_id.in_(link_ids),
]
)
session.execute(delete(MapGtfsReviewItem).where(or_(*review_filters)))
session.execute(delete(MapGtfsDecision).where(or_(*decision_filters)))
def _delete_dataset_files(dataset: Dataset) -> None:
for path in dataset_sidecar_paths(dataset):
try: