1564 lines
58 KiB
Python
1564 lines
58 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import re
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from html import unescape
|
|
from html.parser import HTMLParser
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
from urllib.parse import parse_qs, urljoin, urlparse
|
|
|
|
import requests
|
|
|
|
|
|
MOBILITY_DATABASE_FEEDS_URL = "https://files.mobilitydatabase.org/feeds_v2.csv"
|
|
MOBILITY_DATABASE_ACCEPTANCE_TEST_URL = (
|
|
"https://raw.githubusercontent.com/MobilityData/gtfs-validator/master/"
|
|
"scripts/mobility-database-harvester/acceptance_test_feed_list.csv"
|
|
)
|
|
PTNA_GTFS_INDEX_URL = "https://ptna.openstreetmap.de/gtfs/index.html"
|
|
PTNA_COUNTRY_URL_TEMPLATE = "https://ptna.openstreetmap.de/gtfs/{country}/index.php"
|
|
OPENDATA_OEPNV_DATASETS_URL = "https://www.opendata-oepnv.de/ht/de/datensaetze"
|
|
|
|
DEFAULT_DISCOVERY_COUNTRIES = ["DE", "AT", "CH", "NL", "DK", "FR", "BE", "LU", "NO", "SE", "FI", "IE", "GB"]
|
|
CURATED_TEST_COUNTRIES = ["DE", "CH", "AT", "NL", "DK", "FI", "NO", "SE", "IE", "GB", "FR", "BE", "LU"]
|
|
DIRECT_INGEST_HEADERS = ["name", "kind", "url", "country", "license", "mode_scope", "source_basis", "priority", "notes"]
|
|
CANONICAL_HEADERS = [
|
|
"candidate_id",
|
|
"discovery_source",
|
|
"country",
|
|
"subdivision",
|
|
"provider",
|
|
"feed_name",
|
|
"authority_level",
|
|
"coverage_scope",
|
|
"overlap_role",
|
|
"import_policy",
|
|
"overlap_group",
|
|
"stable_id",
|
|
"ptna_feed_id",
|
|
"data_type",
|
|
"status",
|
|
"is_official",
|
|
"selected_url",
|
|
"direct_download_url",
|
|
"latest_url",
|
|
"original_release_url",
|
|
"license_url",
|
|
"license_text",
|
|
"osm_license_text",
|
|
"details_url",
|
|
"routes_url",
|
|
"valid_from",
|
|
"valid_to",
|
|
"release_date",
|
|
"feed_version",
|
|
"bbox",
|
|
"features",
|
|
"priority",
|
|
"availability_status",
|
|
"http_status",
|
|
"content_type",
|
|
"content_length",
|
|
"final_url",
|
|
"source_basis",
|
|
"notes",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class FeedCandidate:
|
|
discovery_source: str
|
|
country: str = ""
|
|
subdivision: str = ""
|
|
provider: str = ""
|
|
feed_name: str = ""
|
|
authority_level: str = ""
|
|
coverage_scope: str = ""
|
|
overlap_role: str = ""
|
|
import_policy: str = ""
|
|
overlap_group: str = ""
|
|
stable_id: str = ""
|
|
ptna_feed_id: str = ""
|
|
data_type: str = "gtfs"
|
|
status: str = ""
|
|
is_official: str = ""
|
|
selected_url: str = ""
|
|
direct_download_url: str = ""
|
|
latest_url: str = ""
|
|
original_release_url: str = ""
|
|
license_url: str = ""
|
|
license_text: str = ""
|
|
osm_license_text: str = ""
|
|
details_url: str = ""
|
|
routes_url: str = ""
|
|
valid_from: str = ""
|
|
valid_to: str = ""
|
|
release_date: str = ""
|
|
feed_version: str = ""
|
|
bbox: str = ""
|
|
features: str = ""
|
|
priority: str = ""
|
|
availability_status: str = "unchecked"
|
|
http_status: str = ""
|
|
content_type: str = ""
|
|
content_length: str = ""
|
|
final_url: str = ""
|
|
source_basis: str = ""
|
|
notes: str = ""
|
|
evidence_sources: list[str] = field(default_factory=list)
|
|
|
|
def key(self) -> str:
|
|
if self.stable_id:
|
|
return f"stable:{self.stable_id}"
|
|
if self.selected_url:
|
|
return f"url:{_normalize_url_key(self.selected_url)}"
|
|
if self.ptna_feed_id:
|
|
return f"ptna:{self.ptna_feed_id}"
|
|
return "hash:" + hashlib.sha256(json.dumps(self.row(), sort_keys=True).encode("utf-8")).hexdigest()
|
|
|
|
def candidate_id(self) -> str:
|
|
seed = "|".join(
|
|
[
|
|
self.discovery_source,
|
|
self.country,
|
|
self.stable_id,
|
|
self.ptna_feed_id,
|
|
self.selected_url,
|
|
self.provider,
|
|
self.feed_name,
|
|
]
|
|
)
|
|
return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:16]
|
|
|
|
def row(self) -> dict[str, str]:
|
|
payload = {header: _string(getattr(self, header, "")) for header in CANONICAL_HEADERS if header != "candidate_id"}
|
|
payload["candidate_id"] = self.candidate_id()
|
|
return payload
|
|
|
|
def ingestable_row(self) -> dict[str, str]:
|
|
name = _feed_source_name(self.country, self.provider or self.feed_name)
|
|
license_value = self.license_text or (f"see {self.license_url}" if self.license_url else "")
|
|
basis_parts = [self.source_basis or self.discovery_source]
|
|
if self.details_url:
|
|
basis_parts.append(f"details: {self.details_url}")
|
|
if self.original_release_url and self.original_release_url != self.selected_url:
|
|
basis_parts.append(f"release: {self.original_release_url}")
|
|
notes = self.notes or ""
|
|
if self.latest_url and self.latest_url != self.selected_url:
|
|
notes = _join_notes(notes, f"Mobility Database mirror: {self.latest_url}")
|
|
if self.osm_license_text:
|
|
notes = _join_notes(notes, f"OSM permission note: {_truncate(self.osm_license_text, 240)}")
|
|
if self.import_policy or self.overlap_role:
|
|
notes = _join_notes(
|
|
notes,
|
|
f"Import policy: {self.import_policy or 'review'}; overlap role: {self.overlap_role or 'review'}.",
|
|
)
|
|
return {
|
|
"name": _truncate(name, 240),
|
|
"kind": "gtfs",
|
|
"url": self.selected_url,
|
|
"country": self.country,
|
|
"license": _truncate(license_value, 240),
|
|
"mode_scope": _mode_scope_from_features(self.features),
|
|
"source_basis": _truncate("; ".join(part for part in basis_parts if part), 500),
|
|
"priority": self.priority or _candidate_priority(self),
|
|
"notes": _truncate(notes, 1200),
|
|
}
|
|
|
|
|
|
def default_generated_dir() -> Path:
|
|
return Path(__file__).resolve().parents[1] / "docs" / "generated"
|
|
|
|
|
|
def build_gtfs_discovery_manifests(
|
|
*,
|
|
output_dir: Path | str | None = None,
|
|
countries: Iterable[str] | None = None,
|
|
include_mobility_database: bool = True,
|
|
include_acceptance_test_list: bool = True,
|
|
include_ptna: bool = True,
|
|
include_opendata_oepnv: bool = True,
|
|
max_ptna_details: int = 80,
|
|
max_opendata_oepnv_details: int = 80,
|
|
test_limit: int = 24,
|
|
check_urls: bool = False,
|
|
timeout: float = 30.0,
|
|
) -> dict[str, object]:
|
|
selected_countries = _normalize_countries(countries)
|
|
out_dir = Path(output_dir) if output_dir is not None else default_generated_dir()
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
candidates: list[FeedCandidate] = []
|
|
candidates.extend(load_curated_ingestable_seed(countries=selected_countries))
|
|
if include_mobility_database:
|
|
candidates.extend(fetch_mobility_database_candidates(countries=selected_countries, timeout=timeout))
|
|
if include_acceptance_test_list:
|
|
candidates.extend(fetch_mobility_acceptance_candidates(countries=selected_countries, timeout=timeout))
|
|
if include_ptna:
|
|
candidates.extend(fetch_ptna_candidates(countries=selected_countries, max_details=max_ptna_details, timeout=timeout))
|
|
if include_opendata_oepnv:
|
|
candidates.extend(
|
|
fetch_opendata_oepnv_candidates(
|
|
countries=selected_countries,
|
|
max_details=max_opendata_oepnv_details,
|
|
timeout=timeout,
|
|
)
|
|
)
|
|
|
|
merged = merge_candidates(candidates)
|
|
for candidate in merged:
|
|
apply_source_hierarchy(candidate)
|
|
ingestable = [candidate for candidate in merged if candidate.selected_url and candidate.data_type == "gtfs"]
|
|
if check_urls:
|
|
for candidate in ingestable:
|
|
annotate_url_availability(candidate, timeout=min(timeout, 12.0))
|
|
test_run = select_test_run_candidates(ingestable, limit=test_limit)
|
|
|
|
candidates_path = out_dir / "gtfs_feed_candidates.csv"
|
|
ingestable_path = out_dir / "gtfs_ingestable_sources.csv"
|
|
test_path = out_dir / "gtfs_test_run_sources.csv"
|
|
report_path = out_dir / "gtfs_discovery_report.json"
|
|
|
|
_write_csv(candidates_path, CANONICAL_HEADERS, [candidate.row() for candidate in merged])
|
|
_write_csv(ingestable_path, DIRECT_INGEST_HEADERS, [candidate.ingestable_row() for candidate in ingestable])
|
|
_write_csv(test_path, DIRECT_INGEST_HEADERS, [candidate.ingestable_row() for candidate in test_run])
|
|
|
|
by_source = _count_by(merged, lambda item: item.discovery_source)
|
|
by_country = _count_by(ingestable, lambda item: item.country or "unknown")
|
|
report = {
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"countries": selected_countries or "all",
|
|
"sources": {
|
|
"mobility_database": MOBILITY_DATABASE_FEEDS_URL if include_mobility_database else None,
|
|
"mobility_acceptance_test_list": MOBILITY_DATABASE_ACCEPTANCE_TEST_URL if include_acceptance_test_list else None,
|
|
"ptna": PTNA_GTFS_INDEX_URL if include_ptna else None,
|
|
"opendata_oepnv": OPENDATA_OEPNV_DATASETS_URL if include_opendata_oepnv else None,
|
|
},
|
|
"counts": {
|
|
"candidates": len(merged),
|
|
"ingestable": len(ingestable),
|
|
"test_run": len(test_run),
|
|
"by_source": by_source,
|
|
"ingestable_by_country": by_country,
|
|
},
|
|
"files": {
|
|
"candidates": str(candidates_path),
|
|
"ingestable": str(ingestable_path),
|
|
"test_run": str(test_path),
|
|
},
|
|
}
|
|
report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
return report
|
|
|
|
|
|
def fetch_mobility_database_candidates(
|
|
*,
|
|
countries: list[str] | None = None,
|
|
timeout: float = 30.0,
|
|
url: str = MOBILITY_DATABASE_FEEDS_URL,
|
|
) -> list[FeedCandidate]:
|
|
text = _fetch_text(url, timeout=timeout)
|
|
rows = csv.DictReader(text.splitlines())
|
|
candidates: list[FeedCandidate] = []
|
|
for row in rows:
|
|
if _value(row, "data_type").lower() != "gtfs":
|
|
continue
|
|
country = _value(row, "location.country_code").upper()
|
|
if countries and country not in countries:
|
|
continue
|
|
direct_url = _normalize_feed_url(_value(row, "urls.direct_download"))
|
|
latest_url = _normalize_feed_url(_value(row, "urls.latest"))
|
|
selected_url = _choose_feed_url(direct_url, latest_url)
|
|
candidate = FeedCandidate(
|
|
discovery_source="mobility_database",
|
|
country=country,
|
|
subdivision=_value(row, "location.subdivision_name"),
|
|
provider=_value(row, "provider"),
|
|
feed_name=_value(row, "name"),
|
|
stable_id=_value(row, "id"),
|
|
data_type="gtfs",
|
|
status=_value(row, "status"),
|
|
is_official=_value(row, "is_official"),
|
|
selected_url=selected_url,
|
|
direct_download_url=direct_url,
|
|
latest_url=latest_url,
|
|
license_url=_value(row, "urls.license"),
|
|
bbox=_bbox_from_mobility_row(row),
|
|
features=_value(row, "features"),
|
|
source_basis="Mobility Database feed catalog",
|
|
notes=_value(row, "note"),
|
|
)
|
|
normalize_candidate_geography(candidate)
|
|
apply_known_download_overrides(candidate)
|
|
apply_source_hierarchy(candidate)
|
|
candidate.priority = _candidate_priority(candidate)
|
|
candidates.append(candidate)
|
|
return candidates
|
|
|
|
|
|
def fetch_mobility_acceptance_candidates(
|
|
*,
|
|
countries: list[str] | None = None,
|
|
timeout: float = 30.0,
|
|
url: str = MOBILITY_DATABASE_ACCEPTANCE_TEST_URL,
|
|
) -> list[FeedCandidate]:
|
|
text = _fetch_text(url, timeout=timeout)
|
|
rows = csv.DictReader(text.splitlines())
|
|
candidates: list[FeedCandidate] = []
|
|
for row in rows:
|
|
country = _value(row, "country_code").upper()
|
|
if countries and country not in countries:
|
|
continue
|
|
latest_url = _normalize_feed_url(_value(row, "urls.latest"))
|
|
if not latest_url:
|
|
continue
|
|
candidate = FeedCandidate(
|
|
discovery_source="mobility_validator_acceptance",
|
|
country=country,
|
|
subdivision=_value(row, "subdivision_name"),
|
|
provider=_value(row, "provider"),
|
|
feed_name=_value(row, "provider"),
|
|
stable_id=_value(row, "stable_id"),
|
|
status="acceptance_test",
|
|
selected_url=latest_url,
|
|
latest_url=latest_url,
|
|
source_basis="MobilityData validator acceptance-test feed list",
|
|
notes="Useful smoke-test feed list; prefer Mobility Database feeds_v2 metadata for production source review.",
|
|
priority="P3",
|
|
)
|
|
normalize_candidate_geography(candidate)
|
|
apply_known_download_overrides(candidate)
|
|
apply_source_hierarchy(candidate)
|
|
candidates.append(candidate)
|
|
return candidates
|
|
|
|
|
|
def fetch_ptna_candidates(
|
|
*,
|
|
countries: list[str] | None = None,
|
|
max_details: int = 80,
|
|
timeout: float = 30.0,
|
|
) -> list[FeedCandidate]:
|
|
country_codes = countries or DEFAULT_DISCOVERY_COUNTRIES
|
|
if not country_codes:
|
|
country_codes = discover_ptna_country_codes(timeout=timeout)
|
|
candidates: list[FeedCandidate] = []
|
|
detail_fetches = 0
|
|
for country in country_codes:
|
|
country_url = PTNA_COUNTRY_URL_TEMPLATE.format(country=country)
|
|
try:
|
|
html = _fetch_text(country_url, timeout=timeout)
|
|
except requests.RequestException:
|
|
continue
|
|
for candidate in parse_ptna_country_page(html, country=country, page_url=country_url):
|
|
if candidate.details_url and detail_fetches < max_details:
|
|
try:
|
|
detail_html = _fetch_text(candidate.details_url, timeout=timeout)
|
|
enrich_ptna_candidate_from_details(candidate, detail_html, candidate.details_url)
|
|
detail_fetches += 1
|
|
except requests.RequestException:
|
|
candidate.notes = _join_notes(candidate.notes, "PTNA detail page could not be fetched during discovery.")
|
|
apply_source_hierarchy(candidate)
|
|
candidate.priority = _candidate_priority(candidate)
|
|
candidates.append(candidate)
|
|
return candidates
|
|
|
|
|
|
def discover_ptna_country_codes(*, timeout: float = 30.0) -> list[str]:
|
|
html = _fetch_text(PTNA_GTFS_INDEX_URL, timeout=timeout)
|
|
links = _all_links(html, PTNA_GTFS_INDEX_URL)
|
|
codes: list[str] = []
|
|
for link in links:
|
|
match = re.search(r"/gtfs/([A-Z]{2})/index\.php$", urlparse(link).path)
|
|
if match and match.group(1) not in codes:
|
|
codes.append(match.group(1))
|
|
return codes
|
|
|
|
|
|
def parse_ptna_country_page(html: str, *, country: str, page_url: str) -> list[FeedCandidate]:
|
|
rows = _parse_table_rows(html, page_url)
|
|
candidates: list[FeedCandidate] = []
|
|
for row in rows:
|
|
links = [link for cell in row.cells for link in cell.links]
|
|
routes_url = _first_link_matching(links, "routes.php?feed=")
|
|
details_url = _first_link_matching(links, "gtfs-details.php?feed=")
|
|
if not routes_url and not details_url:
|
|
continue
|
|
feed_id = _feed_id_from_url(routes_url or details_url)
|
|
if not feed_id:
|
|
continue
|
|
texts = [cell.text for cell in row.cells]
|
|
release_link = _normalize_feed_url(row.cells[6].first_external_link if len(row.cells) > 6 else "")
|
|
direct_url = release_link if _looks_like_download_url(release_link) else ""
|
|
candidate = FeedCandidate(
|
|
discovery_source="ptna",
|
|
country=country,
|
|
provider=texts[2] if len(texts) > 2 else "",
|
|
feed_name=texts[1] if len(texts) > 1 else feed_id,
|
|
ptna_feed_id=feed_id,
|
|
selected_url=direct_url,
|
|
direct_download_url=direct_url,
|
|
original_release_url=release_link,
|
|
details_url=details_url,
|
|
routes_url=routes_url,
|
|
valid_from=texts[3] if len(texts) > 3 else "",
|
|
valid_to=texts[4] if len(texts) > 4 else "",
|
|
feed_version=texts[5] if len(texts) > 5 else "",
|
|
release_date=texts[6] if len(texts) > 6 else "",
|
|
source_basis="PTNA GTFS analysis",
|
|
notes="PTNA candidate; use original publisher URL where available.",
|
|
)
|
|
normalize_candidate_geography(candidate)
|
|
apply_known_download_overrides(candidate)
|
|
apply_source_hierarchy(candidate)
|
|
candidates.append(candidate)
|
|
return candidates
|
|
|
|
|
|
def enrich_ptna_candidate_from_details(candidate: FeedCandidate, html: str, page_url: str) -> None:
|
|
fields = parse_ptna_detail_fields(html, page_url)
|
|
candidate.original_release_url = _normalize_feed_url(fields.get("release url href") or fields.get("release url") or candidate.original_release_url)
|
|
candidate.license_url = fields.get("publisher's license href") or candidate.license_url
|
|
candidate.license_text = fields.get("publisher's license") or candidate.license_text
|
|
candidate.osm_license_text = fields.get("license given for use in osm") or candidate.osm_license_text
|
|
candidate.valid_from = fields.get("feed start date") or candidate.valid_from
|
|
candidate.valid_to = fields.get("feed end date") or candidate.valid_to
|
|
candidate.feed_version = fields.get("feed version") or candidate.feed_version
|
|
candidate.release_date = fields.get("release date") or candidate.release_date
|
|
network_guid = fields.get('"network:guid"')
|
|
if network_guid:
|
|
candidate.notes = _join_notes(candidate.notes, f"PTNA network:guid={network_guid}")
|
|
if not candidate.selected_url and _looks_like_download_url(candidate.original_release_url):
|
|
candidate.selected_url = _normalize_feed_url(candidate.original_release_url)
|
|
candidate.direct_download_url = candidate.selected_url
|
|
normalize_candidate_geography(candidate)
|
|
|
|
|
|
def parse_ptna_detail_fields(html: str, page_url: str) -> dict[str, str]:
|
|
parsed: dict[str, str] = {}
|
|
for row in _parse_table_rows(html, page_url):
|
|
if len(row.cells) < 2:
|
|
continue
|
|
label = _clean_text(row.cells[0].text).lower()
|
|
if not label:
|
|
continue
|
|
detail = _clean_text(row.cells[1].text)
|
|
parsed[label] = detail
|
|
if row.cells[1].first_external_link:
|
|
parsed[f"{label} href"] = row.cells[1].first_external_link
|
|
return parsed
|
|
|
|
|
|
@dataclass
|
|
class OpenDataOepnvDatasetLink:
|
|
slug: str
|
|
url: str
|
|
title: str = ""
|
|
|
|
|
|
def fetch_opendata_oepnv_candidates(
|
|
*,
|
|
countries: list[str] | None = None,
|
|
max_details: int = 80,
|
|
timeout: float = 30.0,
|
|
url: str = OPENDATA_OEPNV_DATASETS_URL,
|
|
) -> list[FeedCandidate]:
|
|
if countries and "DE" not in countries:
|
|
return []
|
|
try:
|
|
html = _fetch_text(url, timeout=timeout)
|
|
except requests.RequestException:
|
|
return []
|
|
dataset_links = parse_opendata_oepnv_dataset_links(html, url)
|
|
candidates: list[FeedCandidate] = []
|
|
for dataset_link in dataset_links[:max_details]:
|
|
try:
|
|
detail_html = _fetch_text(dataset_link.url, timeout=timeout)
|
|
except requests.RequestException:
|
|
candidate = FeedCandidate(
|
|
discovery_source="opendata_oepnv",
|
|
country="DE",
|
|
provider=_opendata_oepnv_provider(dataset_link.url, dataset_link.slug),
|
|
feed_name=dataset_link.title or _title_from_slug(dataset_link.slug),
|
|
stable_id=f"opendata-oepnv:{dataset_link.slug}",
|
|
status="detail_fetch_failed",
|
|
details_url=dataset_link.url,
|
|
source_basis="OpenData ÖPNV dataset catalog",
|
|
notes="OpenData ÖPNV detail page could not be fetched during discovery.",
|
|
priority="P4",
|
|
)
|
|
apply_source_hierarchy(candidate)
|
|
candidates.append(candidate)
|
|
continue
|
|
candidate = parse_opendata_oepnv_detail_page(
|
|
detail_html,
|
|
detail_url=dataset_link.url,
|
|
slug=dataset_link.slug,
|
|
title_hint=dataset_link.title,
|
|
)
|
|
normalize_candidate_geography(candidate)
|
|
apply_known_download_overrides(candidate)
|
|
apply_source_hierarchy(candidate)
|
|
candidate.priority = candidate.priority or _candidate_priority(candidate)
|
|
candidates.append(candidate)
|
|
return candidates
|
|
|
|
|
|
def parse_opendata_oepnv_dataset_links(html: str, page_url: str = OPENDATA_OEPNV_DATASETS_URL) -> list[OpenDataOepnvDatasetLink]:
|
|
titles_by_slug: dict[str, str] = {}
|
|
title_pattern = re.compile(
|
|
r"<a\s+[^>]*href=[\"'](?P<href>[^\"']*(?:dataset_name%5D|dataset_name\])=[^\"']+)[\"'][^>]*>\s*"
|
|
r"<span[^>]*itemprop=[\"']headline[\"'][^>]*>(?P<title>.*?)</span>",
|
|
re.IGNORECASE | re.DOTALL,
|
|
)
|
|
for match in title_pattern.finditer(html):
|
|
href = urljoin(page_url, unescape(match.group("href")))
|
|
slug = _opendata_oepnv_dataset_slug_from_url(href)
|
|
if slug:
|
|
titles_by_slug[slug] = _html_text(match.group("title"))
|
|
|
|
by_slug: dict[str, OpenDataOepnvDatasetLink] = {}
|
|
for link in _all_links(html, page_url):
|
|
slug = _opendata_oepnv_dataset_slug_from_url(link)
|
|
if not slug:
|
|
continue
|
|
existing = by_slug.get(slug)
|
|
candidate = OpenDataOepnvDatasetLink(slug=slug, url=link, title=titles_by_slug.get(slug, ""))
|
|
if existing is None or _opendata_oepnv_link_score(candidate.url) > _opendata_oepnv_link_score(existing.url):
|
|
by_slug[slug] = candidate
|
|
return sorted(by_slug.values(), key=lambda item: (item.title.lower(), item.slug))
|
|
|
|
|
|
def parse_opendata_oepnv_detail_page(
|
|
html: str,
|
|
*,
|
|
detail_url: str,
|
|
slug: str = "",
|
|
title_hint: str = "",
|
|
) -> FeedCandidate:
|
|
text = _html_text(html)
|
|
title = _opendata_oepnv_title(html) or title_hint or _title_from_slug(slug)
|
|
provider = _opendata_oepnv_provider(detail_url, slug)
|
|
data_type = _opendata_oepnv_data_type(title, text)
|
|
download_urls = _opendata_oepnv_download_urls(html, detail_url)
|
|
selected_url = _choose_opendata_oepnv_download_url(download_urls, data_type)
|
|
login_required = _opendata_oepnv_requires_login(text)
|
|
license_url, license_text = _opendata_oepnv_license(html, detail_url)
|
|
status = "active" if selected_url else "review_required"
|
|
if login_required:
|
|
status = "registration_required"
|
|
notes = "Discovered from OpenData ÖPNV; review per-dataset license and authority before publication."
|
|
if selected_url:
|
|
notes = _join_notes(notes, "Selected the first current GTFS download URL exposed on the detail page.")
|
|
if login_required:
|
|
notes = _join_notes(notes, "Download requires an OpenData ÖPNV account; do not auto-import without credentials.")
|
|
candidate = FeedCandidate(
|
|
discovery_source="opendata_oepnv",
|
|
country="DE",
|
|
subdivision=_opendata_oepnv_subdivision(detail_url, slug),
|
|
provider=provider,
|
|
feed_name=title,
|
|
stable_id=f"opendata-oepnv:{slug}" if slug else "",
|
|
data_type=data_type,
|
|
status=status,
|
|
is_official="",
|
|
selected_url=selected_url,
|
|
direct_download_url=selected_url,
|
|
original_release_url=detail_url,
|
|
license_url=license_url,
|
|
license_text=license_text,
|
|
details_url=detail_url,
|
|
features=_opendata_oepnv_features(data_type, title, text),
|
|
priority=_opendata_oepnv_priority(provider, data_type, selected_url),
|
|
source_basis="OpenData ÖPNV dataset catalog",
|
|
notes=notes,
|
|
)
|
|
apply_source_hierarchy(candidate)
|
|
return candidate
|
|
|
|
|
|
def load_curated_ingestable_seed(
|
|
*,
|
|
countries: list[str] | None = None,
|
|
path: Path | str | None = None,
|
|
) -> list[FeedCandidate]:
|
|
seed_path = Path(path) if path is not None else Path(__file__).resolve().parents[1] / "docs" / "ingestable_sources_seed.csv"
|
|
if not seed_path.exists():
|
|
return []
|
|
candidates: list[FeedCandidate] = []
|
|
with seed_path.open("r", encoding="utf-8-sig", newline="") as handle:
|
|
for row in csv.DictReader(handle):
|
|
if _value(row, "kind").lower() != "gtfs":
|
|
continue
|
|
country = _value(row, "country").upper()
|
|
if countries and country not in countries and country != "EU":
|
|
continue
|
|
candidate = FeedCandidate(
|
|
discovery_source="curated_seed",
|
|
country=country,
|
|
provider=_value(row, "name").removesuffix(" GTFS"),
|
|
feed_name=_value(row, "name"),
|
|
selected_url=_normalize_feed_url(_value(row, "url")),
|
|
direct_download_url=_normalize_feed_url(_value(row, "url")),
|
|
license_text=_value(row, "license"),
|
|
features=_value(row, "mode_scope"),
|
|
priority=_value(row, "priority"),
|
|
source_basis=_value(row, "source_basis") or "curated seed",
|
|
notes=_value(row, "notes"),
|
|
)
|
|
normalize_candidate_geography(candidate)
|
|
apply_known_download_overrides(candidate)
|
|
apply_source_hierarchy(candidate)
|
|
candidates.append(candidate)
|
|
return candidates
|
|
|
|
|
|
def merge_candidates(candidates: Iterable[FeedCandidate]) -> list[FeedCandidate]:
|
|
by_key: dict[str, FeedCandidate] = {}
|
|
alias_to_key: dict[str, str] = {}
|
|
for candidate in candidates:
|
|
keys = _candidate_alias_keys(candidate)
|
|
primary_key = keys[0]
|
|
existing_key = next((alias_to_key[key] for key in keys if key in alias_to_key), None)
|
|
existing = by_key.get(existing_key) if existing_key is not None else None
|
|
if existing is None:
|
|
by_key[primary_key] = candidate
|
|
for key in keys:
|
|
alias_to_key[key] = primary_key
|
|
continue
|
|
_merge_candidate(existing, candidate)
|
|
for key in keys:
|
|
alias_to_key[key] = existing_key or primary_key
|
|
return sorted(by_key.values(), key=lambda item: (_priority_sort_key(item.priority), item.country, item.provider.lower(), item.feed_name.lower()))
|
|
|
|
|
|
def select_test_run_candidates(candidates: Iterable[FeedCandidate], *, limit: int = 24) -> list[FeedCandidate]:
|
|
sorted_candidates = sorted(
|
|
[
|
|
candidate
|
|
for candidate in candidates
|
|
if candidate.discovery_source != "mobility_validator_acceptance" and _test_candidate_eligible(candidate)
|
|
],
|
|
key=_test_candidate_sort_key,
|
|
)
|
|
selected: list[FeedCandidate] = []
|
|
seen_urls: set[str] = set()
|
|
per_country: dict[str, int] = {}
|
|
|
|
def add(candidate: FeedCandidate, *, force: bool = False) -> None:
|
|
if len(selected) >= limit:
|
|
return
|
|
url_key = _normalize_url_key(candidate.selected_url)
|
|
if not candidate.selected_url or url_key in seen_urls:
|
|
return
|
|
country = candidate.country or "unknown"
|
|
country_limit = 7 if force and country == "DE" else 3
|
|
if per_country.get(country, 0) >= country_limit:
|
|
return
|
|
selected.append(candidate)
|
|
seen_urls.add(url_key)
|
|
per_country[country] = per_country.get(country, 0) + 1
|
|
|
|
preferred_tokens = [
|
|
"opendata-oepnv.de",
|
|
"download.gtfs.de/germany/",
|
|
"vbb.de/vbbgtfs",
|
|
"rnv-online.de",
|
|
"vrn.de",
|
|
"gtfs.geops.ch",
|
|
"wienerlinien.at",
|
|
"gtfs.openov.nl",
|
|
"gtfs.ovapi.nl",
|
|
"rejseplanen.info",
|
|
"dev.hsl.fi/gtfs",
|
|
"hsldev.com/gtfs",
|
|
"rb_norway-aggregated-gtfs",
|
|
"data.bus-data.dft.gov.uk",
|
|
"transportforireland",
|
|
"gtfs.irail.be/de-lijn",
|
|
]
|
|
for candidate in sorted_candidates:
|
|
text = " ".join([candidate.provider, candidate.feed_name, candidate.source_basis, candidate.selected_url]).lower()
|
|
if any(token in text for token in preferred_tokens):
|
|
add(candidate, force=True)
|
|
for country in CURATED_TEST_COUNTRIES:
|
|
for candidate in sorted_candidates:
|
|
if candidate.country == country:
|
|
add(candidate)
|
|
if len(selected) >= limit:
|
|
break
|
|
if len(selected) >= limit:
|
|
break
|
|
for candidate in sorted_candidates:
|
|
add(candidate)
|
|
if len(selected) >= limit:
|
|
break
|
|
return selected
|
|
|
|
|
|
def _test_candidate_eligible(candidate: FeedCandidate) -> bool:
|
|
if not candidate.selected_url:
|
|
return False
|
|
if _priority_sort_key(candidate.priority) > 2:
|
|
return False
|
|
text = " ".join([candidate.status, candidate.selected_url, candidate.provider, candidate.feed_name, candidate.notes]).lower()
|
|
if "deprecated" in text or "inactive" in text or "{apikey}" in text:
|
|
return False
|
|
if "registration required" in text or "authentication" in text:
|
|
return False
|
|
return True
|
|
|
|
|
|
def annotate_url_availability(candidate: FeedCandidate, *, timeout: float = 10.0) -> FeedCandidate:
|
|
if not candidate.selected_url:
|
|
candidate.availability_status = "missing_url"
|
|
return candidate
|
|
headers = {"User-Agent": "meubility-workbench-feed-discovery/0.1"}
|
|
try:
|
|
response = requests.head(candidate.selected_url, allow_redirects=True, timeout=timeout, headers=headers)
|
|
if response.status_code in {405, 403} or response.status_code >= 500:
|
|
response = requests.get(
|
|
candidate.selected_url,
|
|
allow_redirects=True,
|
|
timeout=timeout,
|
|
headers={**headers, "Range": "bytes=0-0"},
|
|
stream=True,
|
|
)
|
|
candidate.http_status = str(response.status_code)
|
|
candidate.content_type = response.headers.get("content-type", "")
|
|
candidate.content_length = response.headers.get("content-length", "")
|
|
candidate.final_url = response.url
|
|
candidate.availability_status = "ok" if response.status_code < 400 else "error"
|
|
response.close()
|
|
except requests.RequestException as exc:
|
|
candidate.availability_status = "error"
|
|
candidate.notes = _join_notes(candidate.notes, f"Availability check failed: {exc}")
|
|
return candidate
|
|
|
|
|
|
def normalize_candidate_geography(candidate: FeedCandidate) -> None:
|
|
text = " ".join(
|
|
[
|
|
candidate.selected_url,
|
|
candidate.direct_download_url,
|
|
candidate.latest_url,
|
|
candidate.original_release_url,
|
|
candidate.provider,
|
|
candidate.feed_name,
|
|
candidate.source_basis,
|
|
]
|
|
).lower()
|
|
if "download.gtfs.de/germany/" in text or "gtfs for germany" in text:
|
|
candidate.country = "DE"
|
|
elif "storage.googleapis.com/marduk-production/outbound/gtfs/rb_norway" in text:
|
|
candidate.country = "NO"
|
|
elif "gtfs.ovapi.nl" in text or "openov.nl" in text:
|
|
candidate.country = "NL"
|
|
elif "www.nvbw.de/fileadmin/user_upload/service/open_data/" in text:
|
|
candidate.country = "DE"
|
|
|
|
|
|
def apply_known_download_overrides(candidate: FeedCandidate) -> None:
|
|
stale_direct_ids = {"mdb-684", "mdb-777"}
|
|
if candidate.stable_id in stale_direct_ids and candidate.latest_url:
|
|
candidate.selected_url = candidate.latest_url
|
|
candidate.notes = _join_notes(
|
|
candidate.notes,
|
|
"Selected Mobility Database latest.zip mirror because the catalog direct URL is known to be stale.",
|
|
)
|
|
if _opendata_oepnv_download_is_registration_gated(candidate):
|
|
gated_url = candidate.selected_url or candidate.direct_download_url
|
|
candidate.original_release_url = candidate.original_release_url or gated_url
|
|
candidate.selected_url = ""
|
|
candidate.direct_download_url = ""
|
|
candidate.status = "registration_required"
|
|
candidate.notes = _join_notes(
|
|
candidate.notes,
|
|
"OpenData ÖPNV marks this download as registration-required; kept as a review candidate instead of direct ingest.",
|
|
)
|
|
|
|
|
|
def apply_source_hierarchy(candidate: FeedCandidate) -> None:
|
|
text = _hierarchy_text(candidate)
|
|
authority_level = _infer_authority_level(candidate, text)
|
|
coverage_scope = _infer_coverage_scope(candidate, text)
|
|
overlap_role = _infer_overlap_role(authority_level, coverage_scope, candidate, text)
|
|
import_policy = _infer_import_policy(authority_level, overlap_role, candidate)
|
|
overlap_group = _infer_overlap_group(candidate, authority_level, coverage_scope, text)
|
|
|
|
candidate.authority_level = authority_level
|
|
candidate.coverage_scope = coverage_scope
|
|
candidate.overlap_role = overlap_role
|
|
candidate.import_policy = import_policy
|
|
candidate.overlap_group = overlap_group
|
|
|
|
|
|
def _hierarchy_text(candidate: FeedCandidate) -> str:
|
|
return " ".join(
|
|
[
|
|
candidate.discovery_source,
|
|
candidate.country,
|
|
candidate.subdivision,
|
|
candidate.provider,
|
|
candidate.feed_name,
|
|
candidate.stable_id,
|
|
candidate.ptna_feed_id,
|
|
candidate.status,
|
|
_url_hierarchy_text(candidate.selected_url),
|
|
_url_hierarchy_text(candidate.direct_download_url),
|
|
_url_hierarchy_text(candidate.latest_url),
|
|
_url_hierarchy_text(candidate.original_release_url),
|
|
_url_hierarchy_text(candidate.details_url),
|
|
candidate.source_basis,
|
|
candidate.notes,
|
|
]
|
|
).lower()
|
|
|
|
|
|
def _infer_authority_level(candidate: FeedCandidate, text: str) -> str:
|
|
if _is_mirror_candidate(candidate, text):
|
|
return "mirror"
|
|
discovery_sources = {part.strip() for part in candidate.discovery_source.split(";") if part.strip()}
|
|
if discovery_sources and discovery_sources <= {"ptna", "mobility_validator_acceptance"}:
|
|
return "secondary_discovery"
|
|
if _contains_any(
|
|
text,
|
|
[
|
|
"delfi",
|
|
"gtfs.de",
|
|
"mobilithek",
|
|
"germany-wide",
|
|
"deutschlandweite",
|
|
"entur",
|
|
"bod national",
|
|
"national gtfs",
|
|
"national feed",
|
|
"national timetable",
|
|
"opentransportdata.swiss",
|
|
"trafiklab",
|
|
"samtrafiken",
|
|
"transport for ireland",
|
|
"transportforireland",
|
|
"rejseplanen",
|
|
"gtfs.ovapi.nl",
|
|
],
|
|
):
|
|
return "national_official"
|
|
if _contains_any(text, ["operator", "verkehrsunternehmen", "rnv", "flixbus", "flixtrain", "sncf", "tfl"]):
|
|
return "operator_feed"
|
|
if _contains_any(
|
|
text,
|
|
[
|
|
"verkehrsverbund",
|
|
"transport association",
|
|
"regional authority",
|
|
"authority feed",
|
|
"vbb",
|
|
"vrr",
|
|
"vvs",
|
|
"vrs",
|
|
"mvv",
|
|
"hvv",
|
|
"rmv",
|
|
"nvv",
|
|
"nwl",
|
|
"avv",
|
|
"nah.sh",
|
|
"nah-sh",
|
|
"bayern",
|
|
"baden-württemberg",
|
|
"baden-wuerttemberg",
|
|
"nordrhein-westfalen",
|
|
"nrw",
|
|
"thüringen",
|
|
"thueringen",
|
|
],
|
|
):
|
|
return "regional_authority"
|
|
if "mobility_database" in candidate.discovery_source:
|
|
return "aggregator_catalog"
|
|
return "unknown"
|
|
|
|
|
|
def _infer_coverage_scope(candidate: FeedCandidate, text: str) -> str:
|
|
if _contains_any(
|
|
text,
|
|
[
|
|
"germany-wide",
|
|
"deutschlandweite",
|
|
"national gtfs",
|
|
"national feed",
|
|
"national timetable",
|
|
"delfi",
|
|
"gtfs.de",
|
|
"entur",
|
|
"transport for ireland",
|
|
"transportforireland",
|
|
"trafiklab",
|
|
"samtrafiken",
|
|
"opentransportdata.swiss",
|
|
"gtfs.ovapi.nl",
|
|
"rejseplanen",
|
|
],
|
|
):
|
|
return "national"
|
|
if _contains_any(text, ["europe", "eu/eea", "pan-europe", "flixbus", "flixtrain"]):
|
|
return "multi_country"
|
|
if _contains_any(
|
|
text,
|
|
[
|
|
"verkehrsverbund",
|
|
"regional",
|
|
"bundesland",
|
|
"vbb",
|
|
"vrr",
|
|
"vvs",
|
|
"vrs",
|
|
"mvv",
|
|
"hvv",
|
|
"rmv",
|
|
"nvv",
|
|
"nwl",
|
|
"avv",
|
|
"nah.sh",
|
|
"nah-sh",
|
|
],
|
|
):
|
|
return "regional"
|
|
if _contains_any(text, ["operator", "verkehrsunternehmen", "rnv"]):
|
|
return "operator"
|
|
if candidate.country:
|
|
return "unknown_country"
|
|
return "unknown"
|
|
|
|
|
|
def _infer_overlap_role(authority_level: str, coverage_scope: str, candidate: FeedCandidate, text: str) -> str:
|
|
if authority_level == "mirror":
|
|
return "bootstrap_mirror"
|
|
if authority_level == "secondary_discovery":
|
|
return "discovery_evidence"
|
|
if candidate.status == "registration_required":
|
|
return "review_baseline" if coverage_scope == "national" else "review_candidate"
|
|
if authority_level == "national_official" and coverage_scope == "national":
|
|
return "canonical_baseline"
|
|
if authority_level == "regional_authority":
|
|
return "regional_override_candidate"
|
|
if authority_level == "operator_feed":
|
|
return "operator_detail_candidate"
|
|
if authority_level == "aggregator_catalog":
|
|
return "catalog_metadata"
|
|
if _contains_any(text, ["active", "gtfs"]):
|
|
return "gap_fill_candidate"
|
|
return "review_candidate"
|
|
|
|
|
|
def _infer_import_policy(authority_level: str, overlap_role: str, candidate: FeedCandidate) -> str:
|
|
if candidate.status == "registration_required" or not candidate.selected_url:
|
|
return "review_before_import"
|
|
if overlap_role == "canonical_baseline":
|
|
return "canonical_candidate"
|
|
if overlap_role in {"regional_override_candidate", "operator_detail_candidate"}:
|
|
return "qa_or_override_not_additive"
|
|
if overlap_role == "bootstrap_mirror":
|
|
return "bootstrap_only"
|
|
if authority_level in {"secondary_discovery", "aggregator_catalog"}:
|
|
return "metadata_only"
|
|
return "review_before_import"
|
|
|
|
|
|
def _infer_overlap_group(candidate: FeedCandidate, authority_level: str, coverage_scope: str, text: str) -> str:
|
|
country = candidate.country or "unknown"
|
|
if coverage_scope == "national":
|
|
return f"{country}:national"
|
|
identity_text = _hierarchy_identity_text(candidate)
|
|
for token in [
|
|
"vbb",
|
|
"vrr",
|
|
"vvs",
|
|
"vrs",
|
|
"mvv",
|
|
"hvv",
|
|
"rmv",
|
|
"nvv",
|
|
"nwl",
|
|
"avv",
|
|
"nah-sh",
|
|
"nah.sh",
|
|
"rnv",
|
|
]:
|
|
if token in identity_text:
|
|
normalized = token.replace(".", "-")
|
|
return f"{country}:{coverage_scope}:{normalized}"
|
|
key = _source_key(candidate.provider or candidate.feed_name)
|
|
if key:
|
|
return f"{country}:{coverage_scope}:{key}"
|
|
return f"{country}:{coverage_scope}:{authority_level}"
|
|
|
|
|
|
def _is_mirror_candidate(candidate: FeedCandidate, text: str) -> bool:
|
|
if any(_url_is_mirror_url(url) for url in [candidate.selected_url, candidate.direct_download_url, candidate.latest_url if not candidate.selected_url else ""]):
|
|
return True
|
|
mirror_notes = " ".join([candidate.source_basis, candidate.notes]).lower()
|
|
return _contains_any(mirror_notes, ["european transport feeds mirror", "bootstrap mirror"])
|
|
|
|
|
|
def _source_key(value: str) -> str:
|
|
cleaned = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
|
|
return cleaned[:80]
|
|
|
|
|
|
def _contains_any(text: str, tokens: Iterable[str]) -> bool:
|
|
return any(token in text for token in tokens)
|
|
|
|
|
|
def _hierarchy_identity_text(candidate: FeedCandidate) -> str:
|
|
return " ".join(
|
|
[
|
|
candidate.provider,
|
|
candidate.feed_name,
|
|
candidate.stable_id,
|
|
candidate.ptna_feed_id,
|
|
_url_hierarchy_text(candidate.selected_url),
|
|
_url_hierarchy_text(candidate.direct_download_url),
|
|
_url_hierarchy_text(candidate.original_release_url),
|
|
_url_hierarchy_text(candidate.details_url),
|
|
]
|
|
).lower()
|
|
|
|
|
|
def _url_hierarchy_text(url: str) -> str:
|
|
parsed = urlparse(url or "")
|
|
return " ".join(part for part in [parsed.netloc, parsed.path] if part)
|
|
|
|
|
|
def _url_is_mirror_url(url: str) -> bool:
|
|
parsed = urlparse(url or "")
|
|
host_path = f"{parsed.netloc.lower()}{parsed.path.lower()}"
|
|
return _contains_any(host_path, ["files.mobilitydatabase.org", "openmobilitydata-data.s3", "scraped.data.public-transport.earth"])
|
|
|
|
|
|
@dataclass
|
|
class _HtmlCell:
|
|
text: str = ""
|
|
links: list[str] = field(default_factory=list)
|
|
|
|
@property
|
|
def first_external_link(self) -> str:
|
|
for link in self.links:
|
|
parsed = urlparse(link)
|
|
if parsed.scheme in {"http", "https"} and "ptna.openstreetmap.de" not in parsed.netloc:
|
|
return link
|
|
return ""
|
|
|
|
|
|
@dataclass
|
|
class _HtmlRow:
|
|
cells: list[_HtmlCell] = field(default_factory=list)
|
|
|
|
|
|
class _TableParser(HTMLParser):
|
|
def __init__(self, base_url: str):
|
|
super().__init__(convert_charrefs=True)
|
|
self.base_url = base_url
|
|
self.rows: list[_HtmlRow] = []
|
|
self._row: _HtmlRow | None = None
|
|
self._cell: _HtmlCell | None = None
|
|
self._active_link: str = ""
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
attrs_dict = {key: value or "" for key, value in attrs}
|
|
if tag == "tr":
|
|
self._row = _HtmlRow()
|
|
elif tag in {"td", "th"} and self._row is not None:
|
|
self._cell = _HtmlCell()
|
|
elif tag == "a" and self._cell is not None:
|
|
href = attrs_dict.get("href", "")
|
|
if href:
|
|
self._active_link = urljoin(self.base_url, href)
|
|
self._cell.links.append(self._active_link)
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if tag in {"td", "th"} and self._row is not None and self._cell is not None:
|
|
self._cell.text = _clean_text(self._cell.text)
|
|
self._row.cells.append(self._cell)
|
|
self._cell = None
|
|
self._active_link = ""
|
|
elif tag == "a":
|
|
self._active_link = ""
|
|
elif tag == "tr":
|
|
if self._row is not None and self._row.cells:
|
|
self.rows.append(self._row)
|
|
self._row = None
|
|
self._cell = None
|
|
self._active_link = ""
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if self._cell is not None:
|
|
self._cell.text += data
|
|
|
|
|
|
class _LinkParser(HTMLParser):
|
|
def __init__(self, base_url: str):
|
|
super().__init__(convert_charrefs=True)
|
|
self.base_url = base_url
|
|
self.links: list[str] = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
if tag != "a":
|
|
return
|
|
for key, value in attrs:
|
|
if key == "href" and value:
|
|
self.links.append(urljoin(self.base_url, value))
|
|
|
|
|
|
def _parse_table_rows(html: str, base_url: str) -> list[_HtmlRow]:
|
|
parser = _TableParser(base_url)
|
|
parser.feed(html)
|
|
return parser.rows
|
|
|
|
|
|
def _all_links(html: str, base_url: str) -> list[str]:
|
|
parser = _LinkParser(base_url)
|
|
parser.feed(html)
|
|
return parser.links
|
|
|
|
|
|
def _fetch_text(url: str, *, timeout: float) -> str:
|
|
response = requests.get(url, timeout=timeout, headers={"User-Agent": "meubility-workbench-feed-discovery/0.1"})
|
|
response.raise_for_status()
|
|
return response.text
|
|
|
|
|
|
def _first_link_matching(links: Iterable[str], needle: str) -> str:
|
|
for link in links:
|
|
if needle in link:
|
|
return link
|
|
return ""
|
|
|
|
|
|
def _feed_id_from_url(url: str) -> str:
|
|
query = parse_qs(urlparse(url).query)
|
|
return (query.get("feed") or [""])[0]
|
|
|
|
|
|
def _looks_like_download_url(url: str) -> bool:
|
|
if not url:
|
|
return False
|
|
parsed = urlparse(url)
|
|
lower_path = parsed.path.lower()
|
|
lower_url = url.lower()
|
|
if lower_path.endswith(".zip"):
|
|
return True
|
|
if "exportformat=gtfs" in lower_url or "google_transit" in lower_url:
|
|
return True
|
|
if lower_path.rstrip("/").endswith(("current_gtfs", "gtfs")):
|
|
return True
|
|
if "gtfs.ovapi.nl" in parsed.netloc.lower() and "gtfs" in lower_path:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _opendata_oepnv_dataset_slug_from_url(url: str) -> str:
|
|
query = parse_qs(urlparse(unescape(url)).query)
|
|
return _clean_text((query.get("tx_vrrkit_view[dataset_name]") or [""])[0])
|
|
|
|
|
|
def _opendata_oepnv_link_score(url: str) -> int:
|
|
parsed = urlparse(url)
|
|
score = 0
|
|
if "/organisation/" in parsed.path:
|
|
score += 2
|
|
if parsed.path.rstrip("/").endswith("startseite"):
|
|
score += 1
|
|
return score
|
|
|
|
|
|
def _opendata_oepnv_title(html: str) -> str:
|
|
matches = re.findall(r"<div[^>]*class=[\"'][^\"']*element-header[^\"']*[\"'][^>]*>(.*?)</div>", html, re.IGNORECASE | re.DOTALL)
|
|
for raw in reversed(matches):
|
|
title = _html_text(raw)
|
|
if title and title.lower() not in {"hinweise", "spielregeln", "lizenzen"}:
|
|
return title
|
|
match = re.search(r"<title>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
|
|
return _html_text(match.group(1)) if match else ""
|
|
|
|
|
|
def _opendata_oepnv_download_urls(html: str, detail_url: str) -> list[str]:
|
|
urls: list[str] = []
|
|
for link in _all_links(html, detail_url):
|
|
normalized = _normalize_feed_url(link)
|
|
if normalized and _looks_like_download_url(normalized) and normalized not in urls:
|
|
urls.append(normalized)
|
|
return urls
|
|
|
|
|
|
def _choose_opendata_oepnv_download_url(urls: list[str], data_type: str) -> str:
|
|
if data_type == "gtfs_rt":
|
|
for url in urls:
|
|
lower = url.lower()
|
|
if "realtime" in lower or "gtfs-rt" in lower or "gtfs_rt" in lower:
|
|
return url
|
|
return ""
|
|
if data_type != "gtfs":
|
|
return ""
|
|
if not urls:
|
|
return ""
|
|
static_urls = [url for url in urls if "realtime" not in url.lower() and "gtfs-rt" not in url.lower() and "gtfs_rt" not in url.lower()]
|
|
if not static_urls:
|
|
return ""
|
|
for url in static_urls:
|
|
lower = url.lower()
|
|
if "latest" in lower or "aktueller" in lower or lower.rstrip("/").endswith("/gtfs.zip"):
|
|
return url
|
|
return static_urls[0]
|
|
|
|
|
|
def _opendata_oepnv_license(html: str, detail_url: str) -> tuple[str, str]:
|
|
match = re.search(
|
|
r"id=[\"']dodp_license[\"'][^>]*>.*?<a\s+[^>]*href=[\"'](?P<href>[^\"']+)[\"'][^>]*>(?P<text>.*?)</a>",
|
|
html,
|
|
re.IGNORECASE | re.DOTALL,
|
|
)
|
|
if not match:
|
|
return "", ""
|
|
return urljoin(detail_url, unescape(match.group("href"))), _html_text(match.group("text"))
|
|
|
|
|
|
def _opendata_oepnv_requires_login(text: str) -> bool:
|
|
lower = text.lower()
|
|
return (
|
|
"available for registered users" in lower
|
|
or "download is only available for registered users" in lower
|
|
or "this download is only available for registered users" in lower
|
|
or "anmeldung erforderlich" in lower
|
|
or "nur für registrierte nutzer" in lower
|
|
or "nur fuer registrierte nutzer" in lower
|
|
)
|
|
|
|
|
|
def _opendata_oepnv_data_type(title: str, text: str) -> str:
|
|
title_lower = title.lower()
|
|
lower = f"{title} {text}".lower()
|
|
if "gtfs-rt" in lower or "gtfs realtime" in lower or "gtfs-realtime" in lower:
|
|
return "gtfs_rt"
|
|
if "gtfs" in title_lower:
|
|
return "gtfs"
|
|
if "haltestell" in title_lower or "zhv" in title_lower:
|
|
return "stops"
|
|
if "netex" in title_lower:
|
|
return "netex"
|
|
if "trias" in lower:
|
|
return "api"
|
|
if "netex" in lower:
|
|
return "netex"
|
|
if "gtfs" in lower:
|
|
return "gtfs"
|
|
if "haltestell" in lower or "zhv" in lower:
|
|
return "stops"
|
|
if "linien" in lower:
|
|
return "line_metadata"
|
|
return "dataset"
|
|
|
|
|
|
def _opendata_oepnv_features(data_type: str, title: str, text: str) -> str:
|
|
features = ["OpenData ÖPNV"]
|
|
if data_type == "gtfs":
|
|
features.extend(["GTFS", "Schedules"])
|
|
elif data_type == "netex":
|
|
features.extend(["NeTEx", "Schedules"])
|
|
elif data_type == "gtfs_rt":
|
|
features.extend(["GTFS-Realtime"])
|
|
elif data_type == "stops":
|
|
features.append("Stops")
|
|
lower = f"{title} {text}".lower()
|
|
if "regionalbahn" in lower or "s-bahn" in lower or "bahn" in lower:
|
|
features.append("Rail")
|
|
if "u-bahn" in lower or "metro" in lower:
|
|
features.append("Metro")
|
|
if "tram" in lower or "straßenbahn" in lower:
|
|
features.append("Tram")
|
|
if "bus" in lower:
|
|
features.append("Bus")
|
|
return "|".join(dict.fromkeys(features))
|
|
|
|
|
|
def _opendata_oepnv_priority(provider: str, data_type: str, selected_url: str) -> str:
|
|
if provider == "DELFI e.V." and data_type in {"gtfs", "netex", "stops"}:
|
|
return "P0"
|
|
if data_type == "gtfs" and selected_url:
|
|
return "P1"
|
|
if data_type in {"netex", "gtfs_rt", "stops"}:
|
|
return "P2"
|
|
return "P3"
|
|
|
|
|
|
def _opendata_oepnv_provider(detail_url: str, slug: str) -> str:
|
|
path = urlparse(detail_url).path.lower()
|
|
haystack = f"{path}/{slug.lower()}/"
|
|
providers = {
|
|
"delfi": "DELFI e.V.",
|
|
"nah-sh": "NAH.SH",
|
|
"avv": "Aachener Verkehrsverbund",
|
|
"baden-wuerttemberg": "Baden-Württemberg",
|
|
"bawue": "Baden-Württemberg",
|
|
"hvv": "Hamburger Verkehrsverbund",
|
|
"mvv": "Münchner Verkehrs- und Tarifverbund",
|
|
"nvv": "Nordhessischer VerkehrsVerbund",
|
|
"nwl": "Nahverkehr Westfalen-Lippe",
|
|
"nordrhein-westfalen": "Nordrhein-Westfalen",
|
|
"nrw": "Nordrhein-Westfalen",
|
|
"rmv": "Rhein-Main-Verkehrsverbund",
|
|
"rnv": "Rhein-Neckar-Verkehr",
|
|
"thueringen": "Thüringen",
|
|
"vbb": "Verkehrsverbund Berlin-Brandenburg",
|
|
"vrr": "Verkehrsverbund Rhein-Ruhr",
|
|
"vrs": "Verkehrsverbund Rhein-Sieg",
|
|
"vvs": "Verkehrs- und Tarifverbund Stuttgart",
|
|
}
|
|
for token, provider in providers.items():
|
|
if f"/{token}/" in haystack or f"-{token}" in haystack or f"_{token}" in haystack:
|
|
return provider
|
|
return "OpenData ÖPNV"
|
|
|
|
|
|
def _opendata_oepnv_subdivision(detail_url: str, slug: str) -> str:
|
|
text = f"{urlparse(detail_url).path} {slug}".lower()
|
|
subdivisions = {
|
|
"baden-wuerttemberg": "Baden-Württemberg",
|
|
"bawue": "Baden-Württemberg",
|
|
"thueringen": "Thüringen",
|
|
"nrw": "Nordrhein-Westfalen",
|
|
"nordrhein-westfalen": "Nordrhein-Westfalen",
|
|
"vbb": "Berlin-Brandenburg",
|
|
"hvv": "Hamburg",
|
|
"mvv": "Bayern",
|
|
"vrr": "Nordrhein-Westfalen",
|
|
"vrs": "Nordrhein-Westfalen",
|
|
"nwl": "Nordrhein-Westfalen",
|
|
"avv": "Nordrhein-Westfalen",
|
|
"rmv": "Hessen",
|
|
"nvv": "Hessen",
|
|
"rnv": "Baden-Württemberg/Rheinland-Pfalz/Hessen",
|
|
"vvs": "Baden-Württemberg",
|
|
}
|
|
for token, subdivision in subdivisions.items():
|
|
if token in text:
|
|
return subdivision
|
|
return ""
|
|
|
|
|
|
def _opendata_oepnv_download_is_registration_gated(candidate: FeedCandidate) -> bool:
|
|
url_text = " ".join([candidate.selected_url, candidate.direct_download_url, candidate.original_release_url]).lower()
|
|
if "opendata-oepnv.de" not in url_text:
|
|
return False
|
|
descriptive_text = " ".join([candidate.provider, candidate.feed_name, candidate.status, candidate.notes]).lower()
|
|
return "registration required" in descriptive_text or "user registration required" in descriptive_text
|
|
|
|
|
|
def _html_text(value: str) -> str:
|
|
return _clean_text(re.sub(r"<[^>]+>", " ", value or ""))
|
|
|
|
|
|
def _title_from_slug(slug: str) -> str:
|
|
return _clean_text(slug.replace("_", " ").replace("-", " ").title())
|
|
|
|
|
|
def _normalize_feed_url(url: str) -> str:
|
|
cleaned = _clean_text(url)
|
|
if not cleaned:
|
|
return ""
|
|
parsed = urlparse(cleaned)
|
|
if parsed.scheme:
|
|
return cleaned
|
|
first = cleaned.split("/", 1)[0]
|
|
if "." in first:
|
|
return f"https://{cleaned}"
|
|
return cleaned
|
|
|
|
|
|
def _choose_feed_url(direct_url: str, latest_url: str) -> str:
|
|
if direct_url:
|
|
return direct_url
|
|
return latest_url
|
|
|
|
|
|
def _candidate_priority(candidate: FeedCandidate) -> str:
|
|
status = candidate.status.lower()
|
|
official = candidate.is_official.lower() == "true"
|
|
if candidate.discovery_source == "curated_seed":
|
|
return candidate.priority or "P1"
|
|
if status == "active" and official and candidate.direct_download_url:
|
|
return "P0"
|
|
if status == "active" and candidate.direct_download_url:
|
|
return "P1"
|
|
if status == "active" and candidate.latest_url:
|
|
return "P2"
|
|
if candidate.discovery_source == "ptna":
|
|
return "P2" if candidate.selected_url else "P4"
|
|
return "P3"
|
|
|
|
|
|
def _test_candidate_sort_key(candidate: FeedCandidate) -> tuple[int, int, str, str]:
|
|
source_bonus = 0 if candidate.discovery_source == "curated_seed" else 1
|
|
country_bonus = CURATED_TEST_COUNTRIES.index(candidate.country) if candidate.country in CURATED_TEST_COUNTRIES else 99
|
|
return (_priority_sort_key(candidate.priority), source_bonus + country_bonus, candidate.country, candidate.provider.lower())
|
|
|
|
|
|
def _priority_sort_key(priority: str) -> int:
|
|
match = re.match(r"P(\d+)", priority or "")
|
|
return int(match.group(1)) if match else 9
|
|
|
|
|
|
def _candidate_alias_keys(candidate: FeedCandidate) -> list[str]:
|
|
keys = [candidate.key()]
|
|
if candidate.stable_id:
|
|
keys.append(f"stable:{candidate.stable_id}")
|
|
for url in [candidate.selected_url, candidate.direct_download_url, candidate.latest_url]:
|
|
if url:
|
|
keys.append(f"url:{_normalize_url_key(url)}")
|
|
if candidate.ptna_feed_id:
|
|
keys.append(f"ptna:{candidate.ptna_feed_id}")
|
|
deduped: list[str] = []
|
|
for key in keys:
|
|
if key not in deduped:
|
|
deduped.append(key)
|
|
return deduped
|
|
|
|
|
|
def _merge_candidate(existing: FeedCandidate, incoming: FeedCandidate) -> None:
|
|
if incoming.discovery_source == "curated_seed":
|
|
for field_name in ["country", "provider", "feed_name", "license_text", "features", "source_basis", "notes"]:
|
|
new_value = getattr(incoming, field_name, "")
|
|
if new_value:
|
|
setattr(existing, field_name, new_value)
|
|
if existing.selected_url and incoming.selected_url and _url_is_mirror_url(existing.selected_url) and not _url_is_mirror_url(incoming.selected_url):
|
|
existing.selected_url = incoming.selected_url
|
|
existing.direct_download_url = incoming.direct_download_url or incoming.selected_url
|
|
existing.notes = _join_notes(existing.notes, "Selected non-mirror publisher URL from merged source evidence.")
|
|
existing.discovery_source = _join_unique(existing.discovery_source, incoming.discovery_source)
|
|
for field_name in CANONICAL_HEADERS:
|
|
if field_name == "candidate_id":
|
|
continue
|
|
current = getattr(existing, field_name, "")
|
|
new_value = getattr(incoming, field_name, "")
|
|
if not current and new_value:
|
|
setattr(existing, field_name, new_value)
|
|
existing.priority = _better_priority(existing.priority, incoming.priority)
|
|
existing.source_basis = _join_unique(existing.source_basis, incoming.source_basis)
|
|
existing.notes = _join_notes(existing.notes, incoming.notes)
|
|
|
|
|
|
def _better_priority(left: str, right: str) -> str:
|
|
return left if _priority_sort_key(left) <= _priority_sort_key(right) else right
|
|
|
|
|
|
def _join_unique(left: str, right: str) -> str:
|
|
parts: list[str] = []
|
|
for value in [left, right]:
|
|
for part in value.split(";"):
|
|
cleaned = part.strip()
|
|
if cleaned and cleaned not in parts:
|
|
parts.append(cleaned)
|
|
return "; ".join(parts)
|
|
|
|
|
|
def _join_notes(left: str, right: str) -> str:
|
|
return _join_unique(left, right)
|
|
|
|
|
|
def _compact_name(value: str) -> str:
|
|
return re.sub(r"\s+", " ", _clean_text(value)).strip()
|
|
|
|
|
|
def _feed_source_name(country: str, value: str) -> str:
|
|
base = _compact_name(value) or "GTFS feed"
|
|
prefix = country.upper()
|
|
display = base
|
|
if prefix and not base.upper().startswith(f"{prefix} "):
|
|
display = f"{prefix} {base}"
|
|
if "gtfs" not in display.lower():
|
|
display = f"{display} GTFS"
|
|
return display
|
|
|
|
|
|
def _clean_text(value: str) -> str:
|
|
cleaned = unescape(value or "").replace("\xa0", " ")
|
|
cleaned = re.sub(r"\s+", " ", cleaned)
|
|
return cleaned.strip()
|
|
|
|
|
|
def _mode_scope_from_features(features: str) -> str:
|
|
lower = features.lower()
|
|
modes = []
|
|
if "rail" in lower or "train" in lower:
|
|
modes.append("rail")
|
|
if "tram" in lower or "light_rail" in lower:
|
|
modes.append("tram")
|
|
if "subway" in lower or "metro" in lower:
|
|
modes.append("metro")
|
|
if "bus" in lower or not modes:
|
|
modes.append("bus")
|
|
if "ferry" in lower:
|
|
modes.append("ferry")
|
|
return ",".join(dict.fromkeys(modes))
|
|
|
|
|
|
def _bbox_from_mobility_row(row: dict[str, str]) -> str:
|
|
min_lat = _value(row, "location.bounding_box.minimum_latitude")
|
|
max_lat = _value(row, "location.bounding_box.maximum_latitude")
|
|
min_lon = _value(row, "location.bounding_box.minimum_longitude")
|
|
max_lon = _value(row, "location.bounding_box.maximum_longitude")
|
|
if not all([min_lat, max_lat, min_lon, max_lon]):
|
|
return ""
|
|
return f"{min_lon},{min_lat},{max_lon},{max_lat}"
|
|
|
|
|
|
def _normalize_countries(countries: Iterable[str] | None) -> list[str] | None:
|
|
if countries is None:
|
|
return DEFAULT_DISCOVERY_COUNTRIES
|
|
normalized = [country.strip().upper() for country in countries if country and country.strip()]
|
|
if any(country == "ALL" for country in normalized):
|
|
return None
|
|
return normalized
|
|
|
|
|
|
def _normalize_url_key(url: str) -> str:
|
|
parsed = urlparse(url.strip())
|
|
scheme = parsed.scheme.lower()
|
|
netloc = parsed.netloc.lower()
|
|
path = parsed.path.rstrip("/")
|
|
query = parsed.query
|
|
return f"{scheme}://{netloc}{path}" + (f"?{query}" if query else "")
|
|
|
|
|
|
def _write_csv(path: Path, headers: list[str], rows: list[dict[str, str]]) -> None:
|
|
with path.open("w", encoding="utf-8", newline="") as handle:
|
|
writer = csv.DictWriter(handle, fieldnames=headers, extrasaction="ignore")
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
|
|
def _count_by(items: Iterable[FeedCandidate], key_fn) -> dict[str, int]:
|
|
counts: dict[str, int] = {}
|
|
for item in items:
|
|
key = key_fn(item)
|
|
counts[key] = counts.get(key, 0) + 1
|
|
return dict(sorted(counts.items()))
|
|
|
|
|
|
def _value(row: dict[str, str], key: str) -> str:
|
|
return _clean_text(row.get(key, ""))
|
|
|
|
|
|
def _string(value: object) -> str:
|
|
return "" if value is None else str(value)
|
|
|
|
|
|
def _truncate(value: str, length: int) -> str:
|
|
return value[:length] if value else ""
|