from __future__ import annotations import csv import hashlib import json import re from dataclasses import dataclass, field from datetime import datetime, timezone from html import unescape from html.parser import HTMLParser from pathlib import Path from typing import Iterable from urllib.parse import parse_qs, urljoin, urlparse import requests MOBILITY_DATABASE_FEEDS_URL = "https://files.mobilitydatabase.org/feeds_v2.csv" MOBILITY_DATABASE_ACCEPTANCE_TEST_URL = ( "https://raw.githubusercontent.com/MobilityData/gtfs-validator/master/" "scripts/mobility-database-harvester/acceptance_test_feed_list.csv" ) PTNA_GTFS_INDEX_URL = "https://ptna.openstreetmap.de/gtfs/index.html" PTNA_COUNTRY_URL_TEMPLATE = "https://ptna.openstreetmap.de/gtfs/{country}/index.php" OPENDATA_OEPNV_DATASETS_URL = "https://www.opendata-oepnv.de/ht/de/datensaetze" DEFAULT_DISCOVERY_COUNTRIES = ["DE", "AT", "CH", "NL", "DK", "FR", "BE", "LU", "NO", "SE", "FI", "IE", "GB"] CURATED_TEST_COUNTRIES = ["DE", "CH", "AT", "NL", "DK", "FI", "NO", "SE", "IE", "GB", "FR", "BE", "LU"] DIRECT_INGEST_HEADERS = ["name", "kind", "url", "country", "license", "mode_scope", "source_basis", "priority", "notes"] CANONICAL_HEADERS = [ "candidate_id", "discovery_source", "country", "subdivision", "provider", "feed_name", "authority_level", "coverage_scope", "overlap_role", "import_policy", "overlap_group", "stable_id", "ptna_feed_id", "data_type", "status", "is_official", "selected_url", "direct_download_url", "latest_url", "original_release_url", "license_url", "license_text", "osm_license_text", "details_url", "routes_url", "valid_from", "valid_to", "release_date", "feed_version", "bbox", "features", "priority", "availability_status", "http_status", "content_type", "content_length", "final_url", "source_basis", "notes", ] @dataclass class FeedCandidate: discovery_source: str country: str = "" subdivision: str = "" provider: str = "" feed_name: str = "" authority_level: str = "" coverage_scope: str = "" overlap_role: str = "" import_policy: str = "" overlap_group: str = "" stable_id: str = "" ptna_feed_id: str = "" data_type: str = "gtfs" status: str = "" is_official: str = "" selected_url: str = "" direct_download_url: str = "" latest_url: str = "" original_release_url: str = "" license_url: str = "" license_text: str = "" osm_license_text: str = "" details_url: str = "" routes_url: str = "" valid_from: str = "" valid_to: str = "" release_date: str = "" feed_version: str = "" bbox: str = "" features: str = "" priority: str = "" availability_status: str = "unchecked" http_status: str = "" content_type: str = "" content_length: str = "" final_url: str = "" source_basis: str = "" notes: str = "" evidence_sources: list[str] = field(default_factory=list) def key(self) -> str: if self.stable_id: return f"stable:{self.stable_id}" if self.selected_url: return f"url:{_normalize_url_key(self.selected_url)}" if self.ptna_feed_id: return f"ptna:{self.ptna_feed_id}" return "hash:" + hashlib.sha256(json.dumps(self.row(), sort_keys=True).encode("utf-8")).hexdigest() def candidate_id(self) -> str: seed = "|".join( [ self.discovery_source, self.country, self.stable_id, self.ptna_feed_id, self.selected_url, self.provider, self.feed_name, ] ) return hashlib.sha256(seed.encode("utf-8")).hexdigest()[:16] def row(self) -> dict[str, str]: payload = {header: _string(getattr(self, header, "")) for header in CANONICAL_HEADERS if header != "candidate_id"} payload["candidate_id"] = self.candidate_id() return payload def ingestable_row(self) -> dict[str, str]: name = _feed_source_name(self.country, self.provider or self.feed_name) license_value = self.license_text or (f"see {self.license_url}" if self.license_url else "") basis_parts = [self.source_basis or self.discovery_source] if self.details_url: basis_parts.append(f"details: {self.details_url}") if self.original_release_url and self.original_release_url != self.selected_url: basis_parts.append(f"release: {self.original_release_url}") notes = self.notes or "" if self.latest_url and self.latest_url != self.selected_url: notes = _join_notes(notes, f"Mobility Database mirror: {self.latest_url}") if self.osm_license_text: notes = _join_notes(notes, f"OSM permission note: {_truncate(self.osm_license_text, 240)}") if self.import_policy or self.overlap_role: notes = _join_notes( notes, f"Import policy: {self.import_policy or 'review'}; overlap role: {self.overlap_role or 'review'}.", ) return { "name": _truncate(name, 240), "kind": "gtfs", "url": self.selected_url, "country": self.country, "license": _truncate(license_value, 240), "mode_scope": _mode_scope_from_features(self.features), "source_basis": _truncate("; ".join(part for part in basis_parts if part), 500), "priority": self.priority or _candidate_priority(self), "notes": _truncate(notes, 1200), } def default_generated_dir() -> Path: return Path(__file__).resolve().parents[1] / "docs" / "generated" def build_gtfs_discovery_manifests( *, output_dir: Path | str | None = None, countries: Iterable[str] | None = None, include_mobility_database: bool = True, include_acceptance_test_list: bool = True, include_ptna: bool = True, include_opendata_oepnv: bool = True, max_ptna_details: int = 80, max_opendata_oepnv_details: int = 80, test_limit: int = 24, check_urls: bool = False, timeout: float = 30.0, ) -> dict[str, object]: selected_countries = _normalize_countries(countries) out_dir = Path(output_dir) if output_dir is not None else default_generated_dir() out_dir.mkdir(parents=True, exist_ok=True) candidates: list[FeedCandidate] = [] candidates.extend(load_curated_ingestable_seed(countries=selected_countries)) if include_mobility_database: candidates.extend(fetch_mobility_database_candidates(countries=selected_countries, timeout=timeout)) if include_acceptance_test_list: candidates.extend(fetch_mobility_acceptance_candidates(countries=selected_countries, timeout=timeout)) if include_ptna: candidates.extend(fetch_ptna_candidates(countries=selected_countries, max_details=max_ptna_details, timeout=timeout)) if include_opendata_oepnv: candidates.extend( fetch_opendata_oepnv_candidates( countries=selected_countries, max_details=max_opendata_oepnv_details, timeout=timeout, ) ) merged = merge_candidates(candidates) for candidate in merged: apply_source_hierarchy(candidate) ingestable = [candidate for candidate in merged if candidate.selected_url and candidate.data_type == "gtfs"] if check_urls: for candidate in ingestable: annotate_url_availability(candidate, timeout=min(timeout, 12.0)) test_run = select_test_run_candidates(ingestable, limit=test_limit) candidates_path = out_dir / "gtfs_feed_candidates.csv" ingestable_path = out_dir / "gtfs_ingestable_sources.csv" test_path = out_dir / "gtfs_test_run_sources.csv" report_path = out_dir / "gtfs_discovery_report.json" _write_csv(candidates_path, CANONICAL_HEADERS, [candidate.row() for candidate in merged]) _write_csv(ingestable_path, DIRECT_INGEST_HEADERS, [candidate.ingestable_row() for candidate in ingestable]) _write_csv(test_path, DIRECT_INGEST_HEADERS, [candidate.ingestable_row() for candidate in test_run]) by_source = _count_by(merged, lambda item: item.discovery_source) by_country = _count_by(ingestable, lambda item: item.country or "unknown") report = { "generated_at": datetime.now(timezone.utc).isoformat(), "countries": selected_countries or "all", "sources": { "mobility_database": MOBILITY_DATABASE_FEEDS_URL if include_mobility_database else None, "mobility_acceptance_test_list": MOBILITY_DATABASE_ACCEPTANCE_TEST_URL if include_acceptance_test_list else None, "ptna": PTNA_GTFS_INDEX_URL if include_ptna else None, "opendata_oepnv": OPENDATA_OEPNV_DATASETS_URL if include_opendata_oepnv else None, }, "counts": { "candidates": len(merged), "ingestable": len(ingestable), "test_run": len(test_run), "by_source": by_source, "ingestable_by_country": by_country, }, "files": { "candidates": str(candidates_path), "ingestable": str(ingestable_path), "test_run": str(test_path), }, } report_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") return report def fetch_mobility_database_candidates( *, countries: list[str] | None = None, timeout: float = 30.0, url: str = MOBILITY_DATABASE_FEEDS_URL, ) -> list[FeedCandidate]: text = _fetch_text(url, timeout=timeout) rows = csv.DictReader(text.splitlines()) candidates: list[FeedCandidate] = [] for row in rows: if _value(row, "data_type").lower() != "gtfs": continue country = _value(row, "location.country_code").upper() if countries and country not in countries: continue direct_url = _normalize_feed_url(_value(row, "urls.direct_download")) latest_url = _normalize_feed_url(_value(row, "urls.latest")) selected_url = _choose_feed_url(direct_url, latest_url) candidate = FeedCandidate( discovery_source="mobility_database", country=country, subdivision=_value(row, "location.subdivision_name"), provider=_value(row, "provider"), feed_name=_value(row, "name"), stable_id=_value(row, "id"), data_type="gtfs", status=_value(row, "status"), is_official=_value(row, "is_official"), selected_url=selected_url, direct_download_url=direct_url, latest_url=latest_url, license_url=_value(row, "urls.license"), bbox=_bbox_from_mobility_row(row), features=_value(row, "features"), source_basis="Mobility Database feed catalog", notes=_value(row, "note"), ) normalize_candidate_geography(candidate) apply_known_download_overrides(candidate) apply_source_hierarchy(candidate) candidate.priority = _candidate_priority(candidate) candidates.append(candidate) return candidates def fetch_mobility_acceptance_candidates( *, countries: list[str] | None = None, timeout: float = 30.0, url: str = MOBILITY_DATABASE_ACCEPTANCE_TEST_URL, ) -> list[FeedCandidate]: text = _fetch_text(url, timeout=timeout) rows = csv.DictReader(text.splitlines()) candidates: list[FeedCandidate] = [] for row in rows: country = _value(row, "country_code").upper() if countries and country not in countries: continue latest_url = _normalize_feed_url(_value(row, "urls.latest")) if not latest_url: continue candidate = FeedCandidate( discovery_source="mobility_validator_acceptance", country=country, subdivision=_value(row, "subdivision_name"), provider=_value(row, "provider"), feed_name=_value(row, "provider"), stable_id=_value(row, "stable_id"), status="acceptance_test", selected_url=latest_url, latest_url=latest_url, source_basis="MobilityData validator acceptance-test feed list", notes="Useful smoke-test feed list; prefer Mobility Database feeds_v2 metadata for production source review.", priority="P3", ) normalize_candidate_geography(candidate) apply_known_download_overrides(candidate) apply_source_hierarchy(candidate) candidates.append(candidate) return candidates def fetch_ptna_candidates( *, countries: list[str] | None = None, max_details: int = 80, timeout: float = 30.0, ) -> list[FeedCandidate]: country_codes = countries or DEFAULT_DISCOVERY_COUNTRIES if not country_codes: country_codes = discover_ptna_country_codes(timeout=timeout) candidates: list[FeedCandidate] = [] detail_fetches = 0 for country in country_codes: country_url = PTNA_COUNTRY_URL_TEMPLATE.format(country=country) try: html = _fetch_text(country_url, timeout=timeout) except requests.RequestException: continue for candidate in parse_ptna_country_page(html, country=country, page_url=country_url): if candidate.details_url and detail_fetches < max_details: try: detail_html = _fetch_text(candidate.details_url, timeout=timeout) enrich_ptna_candidate_from_details(candidate, detail_html, candidate.details_url) detail_fetches += 1 except requests.RequestException: candidate.notes = _join_notes(candidate.notes, "PTNA detail page could not be fetched during discovery.") apply_source_hierarchy(candidate) candidate.priority = _candidate_priority(candidate) candidates.append(candidate) return candidates def discover_ptna_country_codes(*, timeout: float = 30.0) -> list[str]: html = _fetch_text(PTNA_GTFS_INDEX_URL, timeout=timeout) links = _all_links(html, PTNA_GTFS_INDEX_URL) codes: list[str] = [] for link in links: match = re.search(r"/gtfs/([A-Z]{2})/index\.php$", urlparse(link).path) if match and match.group(1) not in codes: codes.append(match.group(1)) return codes def parse_ptna_country_page(html: str, *, country: str, page_url: str) -> list[FeedCandidate]: rows = _parse_table_rows(html, page_url) candidates: list[FeedCandidate] = [] for row in rows: links = [link for cell in row.cells for link in cell.links] routes_url = _first_link_matching(links, "routes.php?feed=") details_url = _first_link_matching(links, "gtfs-details.php?feed=") if not routes_url and not details_url: continue feed_id = _feed_id_from_url(routes_url or details_url) if not feed_id: continue texts = [cell.text for cell in row.cells] release_link = _normalize_feed_url(row.cells[6].first_external_link if len(row.cells) > 6 else "") direct_url = release_link if _looks_like_download_url(release_link) else "" candidate = FeedCandidate( discovery_source="ptna", country=country, provider=texts[2] if len(texts) > 2 else "", feed_name=texts[1] if len(texts) > 1 else feed_id, ptna_feed_id=feed_id, selected_url=direct_url, direct_download_url=direct_url, original_release_url=release_link, details_url=details_url, routes_url=routes_url, valid_from=texts[3] if len(texts) > 3 else "", valid_to=texts[4] if len(texts) > 4 else "", feed_version=texts[5] if len(texts) > 5 else "", release_date=texts[6] if len(texts) > 6 else "", source_basis="PTNA GTFS analysis", notes="PTNA candidate; use original publisher URL where available.", ) normalize_candidate_geography(candidate) apply_known_download_overrides(candidate) apply_source_hierarchy(candidate) candidates.append(candidate) return candidates def enrich_ptna_candidate_from_details(candidate: FeedCandidate, html: str, page_url: str) -> None: fields = parse_ptna_detail_fields(html, page_url) candidate.original_release_url = _normalize_feed_url(fields.get("release url href") or fields.get("release url") or candidate.original_release_url) candidate.license_url = fields.get("publisher's license href") or candidate.license_url candidate.license_text = fields.get("publisher's license") or candidate.license_text candidate.osm_license_text = fields.get("license given for use in osm") or candidate.osm_license_text candidate.valid_from = fields.get("feed start date") or candidate.valid_from candidate.valid_to = fields.get("feed end date") or candidate.valid_to candidate.feed_version = fields.get("feed version") or candidate.feed_version candidate.release_date = fields.get("release date") or candidate.release_date network_guid = fields.get('"network:guid"') if network_guid: candidate.notes = _join_notes(candidate.notes, f"PTNA network:guid={network_guid}") if not candidate.selected_url and _looks_like_download_url(candidate.original_release_url): candidate.selected_url = _normalize_feed_url(candidate.original_release_url) candidate.direct_download_url = candidate.selected_url normalize_candidate_geography(candidate) def parse_ptna_detail_fields(html: str, page_url: str) -> dict[str, str]: parsed: dict[str, str] = {} for row in _parse_table_rows(html, page_url): if len(row.cells) < 2: continue label = _clean_text(row.cells[0].text).lower() if not label: continue detail = _clean_text(row.cells[1].text) parsed[label] = detail if row.cells[1].first_external_link: parsed[f"{label} href"] = row.cells[1].first_external_link return parsed @dataclass class OpenDataOepnvDatasetLink: slug: str url: str title: str = "" def fetch_opendata_oepnv_candidates( *, countries: list[str] | None = None, max_details: int = 80, timeout: float = 30.0, url: str = OPENDATA_OEPNV_DATASETS_URL, ) -> list[FeedCandidate]: if countries and "DE" not in countries: return [] try: html = _fetch_text(url, timeout=timeout) except requests.RequestException: return [] dataset_links = parse_opendata_oepnv_dataset_links(html, url) candidates: list[FeedCandidate] = [] for dataset_link in dataset_links[:max_details]: try: detail_html = _fetch_text(dataset_link.url, timeout=timeout) except requests.RequestException: candidate = FeedCandidate( discovery_source="opendata_oepnv", country="DE", provider=_opendata_oepnv_provider(dataset_link.url, dataset_link.slug), feed_name=dataset_link.title or _title_from_slug(dataset_link.slug), stable_id=f"opendata-oepnv:{dataset_link.slug}", status="detail_fetch_failed", details_url=dataset_link.url, source_basis="OpenData ÖPNV dataset catalog", notes="OpenData ÖPNV detail page could not be fetched during discovery.", priority="P4", ) apply_source_hierarchy(candidate) candidates.append(candidate) continue candidate = parse_opendata_oepnv_detail_page( detail_html, detail_url=dataset_link.url, slug=dataset_link.slug, title_hint=dataset_link.title, ) normalize_candidate_geography(candidate) apply_known_download_overrides(candidate) apply_source_hierarchy(candidate) candidate.priority = candidate.priority or _candidate_priority(candidate) candidates.append(candidate) return candidates def parse_opendata_oepnv_dataset_links(html: str, page_url: str = OPENDATA_OEPNV_DATASETS_URL) -> list[OpenDataOepnvDatasetLink]: titles_by_slug: dict[str, str] = {} title_pattern = re.compile( r"]*href=[\"'](?P[^\"']*(?:dataset_name%5D|dataset_name\])=[^\"']+)[\"'][^>]*>\s*" r"]*itemprop=[\"']headline[\"'][^>]*>(?P.*?)</span>", re.IGNORECASE | re.DOTALL, ) for match in title_pattern.finditer(html): href = urljoin(page_url, unescape(match.group("href"))) slug = _opendata_oepnv_dataset_slug_from_url(href) if slug: titles_by_slug[slug] = _html_text(match.group("title")) by_slug: dict[str, OpenDataOepnvDatasetLink] = {} for link in _all_links(html, page_url): slug = _opendata_oepnv_dataset_slug_from_url(link) if not slug: continue existing = by_slug.get(slug) candidate = OpenDataOepnvDatasetLink(slug=slug, url=link, title=titles_by_slug.get(slug, "")) if existing is None or _opendata_oepnv_link_score(candidate.url) > _opendata_oepnv_link_score(existing.url): by_slug[slug] = candidate return sorted(by_slug.values(), key=lambda item: (item.title.lower(), item.slug)) def parse_opendata_oepnv_detail_page( html: str, *, detail_url: str, slug: str = "", title_hint: str = "", ) -> FeedCandidate: text = _html_text(html) title = _opendata_oepnv_title(html) or title_hint or _title_from_slug(slug) provider = _opendata_oepnv_provider(detail_url, slug) data_type = _opendata_oepnv_data_type(title, text) download_urls = _opendata_oepnv_download_urls(html, detail_url) selected_url = _choose_opendata_oepnv_download_url(download_urls, data_type) login_required = _opendata_oepnv_requires_login(text) license_url, license_text = _opendata_oepnv_license(html, detail_url) status = "active" if selected_url else "review_required" if login_required: status = "registration_required" notes = "Discovered from OpenData ÖPNV; review per-dataset license and authority before publication." if selected_url: notes = _join_notes(notes, "Selected the first current GTFS download URL exposed on the detail page.") if login_required: notes = _join_notes(notes, "Download requires an OpenData ÖPNV account; do not auto-import without credentials.") candidate = FeedCandidate( discovery_source="opendata_oepnv", country="DE", subdivision=_opendata_oepnv_subdivision(detail_url, slug), provider=provider, feed_name=title, stable_id=f"opendata-oepnv:{slug}" if slug else "", data_type=data_type, status=status, is_official="", selected_url=selected_url, direct_download_url=selected_url, original_release_url=detail_url, license_url=license_url, license_text=license_text, details_url=detail_url, features=_opendata_oepnv_features(data_type, title, text), priority=_opendata_oepnv_priority(provider, data_type, selected_url), source_basis="OpenData ÖPNV dataset catalog", notes=notes, ) apply_source_hierarchy(candidate) return candidate def load_curated_ingestable_seed( *, countries: list[str] | None = None, path: Path | str | None = None, ) -> list[FeedCandidate]: seed_path = Path(path) if path is not None else Path(__file__).resolve().parents[1] / "docs" / "ingestable_sources_seed.csv" if not seed_path.exists(): return [] candidates: list[FeedCandidate] = [] with seed_path.open("r", encoding="utf-8-sig", newline="") as handle: for row in csv.DictReader(handle): if _value(row, "kind").lower() != "gtfs": continue country = _value(row, "country").upper() if countries and country not in countries and country != "EU": continue candidate = FeedCandidate( discovery_source="curated_seed", country=country, provider=_value(row, "name").removesuffix(" GTFS"), feed_name=_value(row, "name"), selected_url=_normalize_feed_url(_value(row, "url")), direct_download_url=_normalize_feed_url(_value(row, "url")), license_text=_value(row, "license"), features=_value(row, "mode_scope"), priority=_value(row, "priority"), source_basis=_value(row, "source_basis") or "curated seed", notes=_value(row, "notes"), ) normalize_candidate_geography(candidate) apply_known_download_overrides(candidate) apply_source_hierarchy(candidate) candidates.append(candidate) return candidates def merge_candidates(candidates: Iterable[FeedCandidate]) -> list[FeedCandidate]: by_key: dict[str, FeedCandidate] = {} alias_to_key: dict[str, str] = {} for candidate in candidates: keys = _candidate_alias_keys(candidate) primary_key = keys[0] existing_key = next((alias_to_key[key] for key in keys if key in alias_to_key), None) existing = by_key.get(existing_key) if existing_key is not None else None if existing is None: by_key[primary_key] = candidate for key in keys: alias_to_key[key] = primary_key continue _merge_candidate(existing, candidate) for key in keys: alias_to_key[key] = existing_key or primary_key return sorted(by_key.values(), key=lambda item: (_priority_sort_key(item.priority), item.country, item.provider.lower(), item.feed_name.lower())) def select_test_run_candidates(candidates: Iterable[FeedCandidate], *, limit: int = 24) -> list[FeedCandidate]: sorted_candidates = sorted( [ candidate for candidate in candidates if candidate.discovery_source != "mobility_validator_acceptance" and _test_candidate_eligible(candidate) ], key=_test_candidate_sort_key, ) selected: list[FeedCandidate] = [] seen_urls: set[str] = set() per_country: dict[str, int] = {} def add(candidate: FeedCandidate, *, force: bool = False) -> None: if len(selected) >= limit: return url_key = _normalize_url_key(candidate.selected_url) if not candidate.selected_url or url_key in seen_urls: return country = candidate.country or "unknown" country_limit = 7 if force and country == "DE" else 3 if per_country.get(country, 0) >= country_limit: return selected.append(candidate) seen_urls.add(url_key) per_country[country] = per_country.get(country, 0) + 1 preferred_tokens = [ "opendata-oepnv.de", "download.gtfs.de/germany/", "vbb.de/vbbgtfs", "rnv-online.de", "vrn.de", "gtfs.geops.ch", "wienerlinien.at", "gtfs.openov.nl", "gtfs.ovapi.nl", "rejseplanen.info", "dev.hsl.fi/gtfs", "hsldev.com/gtfs", "rb_norway-aggregated-gtfs", "data.bus-data.dft.gov.uk", "transportforireland", "gtfs.irail.be/de-lijn", ] for candidate in sorted_candidates: text = " ".join([candidate.provider, candidate.feed_name, candidate.source_basis, candidate.selected_url]).lower() if any(token in text for token in preferred_tokens): add(candidate, force=True) for country in CURATED_TEST_COUNTRIES: for candidate in sorted_candidates: if candidate.country == country: add(candidate) if len(selected) >= limit: break if len(selected) >= limit: break for candidate in sorted_candidates: add(candidate) if len(selected) >= limit: break return selected def _test_candidate_eligible(candidate: FeedCandidate) -> bool: if not candidate.selected_url: return False if _priority_sort_key(candidate.priority) > 2: return False text = " ".join([candidate.status, candidate.selected_url, candidate.provider, candidate.feed_name, candidate.notes]).lower() if "deprecated" in text or "inactive" in text or "{apikey}" in text: return False if "registration required" in text or "authentication" in text: return False return True def annotate_url_availability(candidate: FeedCandidate, *, timeout: float = 10.0) -> FeedCandidate: if not candidate.selected_url: candidate.availability_status = "missing_url" return candidate headers = {"User-Agent": "meubility-workbench-feed-discovery/0.1"} try: response = requests.head(candidate.selected_url, allow_redirects=True, timeout=timeout, headers=headers) if response.status_code in {405, 403} or response.status_code >= 500: response = requests.get( candidate.selected_url, allow_redirects=True, timeout=timeout, headers={**headers, "Range": "bytes=0-0"}, stream=True, ) candidate.http_status = str(response.status_code) candidate.content_type = response.headers.get("content-type", "") candidate.content_length = response.headers.get("content-length", "") candidate.final_url = response.url candidate.availability_status = "ok" if response.status_code < 400 else "error" response.close() except requests.RequestException as exc: candidate.availability_status = "error" candidate.notes = _join_notes(candidate.notes, f"Availability check failed: {exc}") return candidate def normalize_candidate_geography(candidate: FeedCandidate) -> None: text = " ".join( [ candidate.selected_url, candidate.direct_download_url, candidate.latest_url, candidate.original_release_url, candidate.provider, candidate.feed_name, candidate.source_basis, ] ).lower() if "download.gtfs.de/germany/" in text or "gtfs for germany" in text: candidate.country = "DE" elif "storage.googleapis.com/marduk-production/outbound/gtfs/rb_norway" in text: candidate.country = "NO" elif "gtfs.ovapi.nl" in text or "openov.nl" in text: candidate.country = "NL" elif "www.nvbw.de/fileadmin/user_upload/service/open_data/" in text: candidate.country = "DE" def apply_known_download_overrides(candidate: FeedCandidate) -> None: stale_direct_ids = {"mdb-684", "mdb-777"} if candidate.stable_id in stale_direct_ids and candidate.latest_url: candidate.selected_url = candidate.latest_url candidate.notes = _join_notes( candidate.notes, "Selected Mobility Database latest.zip mirror because the catalog direct URL is known to be stale.", ) if _opendata_oepnv_download_is_registration_gated(candidate): gated_url = candidate.selected_url or candidate.direct_download_url candidate.original_release_url = candidate.original_release_url or gated_url candidate.selected_url = "" candidate.direct_download_url = "" candidate.status = "registration_required" candidate.notes = _join_notes( candidate.notes, "OpenData ÖPNV marks this download as registration-required; kept as a review candidate instead of direct ingest.", ) def apply_source_hierarchy(candidate: FeedCandidate) -> None: text = _hierarchy_text(candidate) authority_level = _infer_authority_level(candidate, text) coverage_scope = _infer_coverage_scope(candidate, text) overlap_role = _infer_overlap_role(authority_level, coverage_scope, candidate, text) import_policy = _infer_import_policy(authority_level, overlap_role, candidate) overlap_group = _infer_overlap_group(candidate, authority_level, coverage_scope, text) candidate.authority_level = authority_level candidate.coverage_scope = coverage_scope candidate.overlap_role = overlap_role candidate.import_policy = import_policy candidate.overlap_group = overlap_group def _hierarchy_text(candidate: FeedCandidate) -> str: return " ".join( [ candidate.discovery_source, candidate.country, candidate.subdivision, candidate.provider, candidate.feed_name, candidate.stable_id, candidate.ptna_feed_id, candidate.status, _url_hierarchy_text(candidate.selected_url), _url_hierarchy_text(candidate.direct_download_url), _url_hierarchy_text(candidate.latest_url), _url_hierarchy_text(candidate.original_release_url), _url_hierarchy_text(candidate.details_url), candidate.source_basis, candidate.notes, ] ).lower() def _infer_authority_level(candidate: FeedCandidate, text: str) -> str: if _is_mirror_candidate(candidate, text): return "mirror" discovery_sources = {part.strip() for part in candidate.discovery_source.split(";") if part.strip()} if discovery_sources and discovery_sources <= {"ptna", "mobility_validator_acceptance"}: return "secondary_discovery" if _contains_any( text, [ "delfi", "gtfs.de", "mobilithek", "germany-wide", "deutschlandweite", "entur", "bod national", "national gtfs", "national feed", "national timetable", "opentransportdata.swiss", "trafiklab", "samtrafiken", "transport for ireland", "transportforireland", "rejseplanen", "gtfs.ovapi.nl", ], ): return "national_official" if _contains_any(text, ["operator", "verkehrsunternehmen", "rnv", "flixbus", "flixtrain", "sncf", "tfl"]): return "operator_feed" if _contains_any( text, [ "verkehrsverbund", "transport association", "regional authority", "authority feed", "vbb", "vrr", "vvs", "vrs", "mvv", "hvv", "rmv", "nvv", "nwl", "avv", "nah.sh", "nah-sh", "bayern", "baden-württemberg", "baden-wuerttemberg", "nordrhein-westfalen", "nrw", "thüringen", "thueringen", ], ): return "regional_authority" if "mobility_database" in candidate.discovery_source: return "aggregator_catalog" return "unknown" def _infer_coverage_scope(candidate: FeedCandidate, text: str) -> str: if _contains_any( text, [ "germany-wide", "deutschlandweite", "national gtfs", "national feed", "national timetable", "delfi", "gtfs.de", "entur", "transport for ireland", "transportforireland", "trafiklab", "samtrafiken", "opentransportdata.swiss", "gtfs.ovapi.nl", "rejseplanen", ], ): return "national" if _contains_any(text, ["europe", "eu/eea", "pan-europe", "flixbus", "flixtrain"]): return "multi_country" if _contains_any( text, [ "verkehrsverbund", "regional", "bundesland", "vbb", "vrr", "vvs", "vrs", "mvv", "hvv", "rmv", "nvv", "nwl", "avv", "nah.sh", "nah-sh", ], ): return "regional" if _contains_any(text, ["operator", "verkehrsunternehmen", "rnv"]): return "operator" if candidate.country: return "unknown_country" return "unknown" def _infer_overlap_role(authority_level: str, coverage_scope: str, candidate: FeedCandidate, text: str) -> str: if authority_level == "mirror": return "bootstrap_mirror" if authority_level == "secondary_discovery": return "discovery_evidence" if candidate.status == "registration_required": return "review_baseline" if coverage_scope == "national" else "review_candidate" if authority_level == "national_official" and coverage_scope == "national": return "canonical_baseline" if authority_level == "regional_authority": return "regional_override_candidate" if authority_level == "operator_feed": return "operator_detail_candidate" if authority_level == "aggregator_catalog": return "catalog_metadata" if _contains_any(text, ["active", "gtfs"]): return "gap_fill_candidate" return "review_candidate" def _infer_import_policy(authority_level: str, overlap_role: str, candidate: FeedCandidate) -> str: if candidate.status == "registration_required" or not candidate.selected_url: return "review_before_import" if overlap_role == "canonical_baseline": return "canonical_candidate" if overlap_role in {"regional_override_candidate", "operator_detail_candidate"}: return "qa_or_override_not_additive" if overlap_role == "bootstrap_mirror": return "bootstrap_only" if authority_level in {"secondary_discovery", "aggregator_catalog"}: return "metadata_only" return "review_before_import" def _infer_overlap_group(candidate: FeedCandidate, authority_level: str, coverage_scope: str, text: str) -> str: country = candidate.country or "unknown" if coverage_scope == "national": return f"{country}:national" identity_text = _hierarchy_identity_text(candidate) for token in [ "vbb", "vrr", "vvs", "vrs", "mvv", "hvv", "rmv", "nvv", "nwl", "avv", "nah-sh", "nah.sh", "rnv", ]: if token in identity_text: normalized = token.replace(".", "-") return f"{country}:{coverage_scope}:{normalized}" key = _source_key(candidate.provider or candidate.feed_name) if key: return f"{country}:{coverage_scope}:{key}" return f"{country}:{coverage_scope}:{authority_level}" def _is_mirror_candidate(candidate: FeedCandidate, text: str) -> bool: if any(_url_is_mirror_url(url) for url in [candidate.selected_url, candidate.direct_download_url, candidate.latest_url if not candidate.selected_url else ""]): return True mirror_notes = " ".join([candidate.source_basis, candidate.notes]).lower() return _contains_any(mirror_notes, ["european transport feeds mirror", "bootstrap mirror"]) def _source_key(value: str) -> str: cleaned = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") return cleaned[:80] def _contains_any(text: str, tokens: Iterable[str]) -> bool: return any(token in text for token in tokens) def _hierarchy_identity_text(candidate: FeedCandidate) -> str: return " ".join( [ candidate.provider, candidate.feed_name, candidate.stable_id, candidate.ptna_feed_id, _url_hierarchy_text(candidate.selected_url), _url_hierarchy_text(candidate.direct_download_url), _url_hierarchy_text(candidate.original_release_url), _url_hierarchy_text(candidate.details_url), ] ).lower() def _url_hierarchy_text(url: str) -> str: parsed = urlparse(url or "") return " ".join(part for part in [parsed.netloc, parsed.path] if part) def _url_is_mirror_url(url: str) -> bool: parsed = urlparse(url or "") host_path = f"{parsed.netloc.lower()}{parsed.path.lower()}" return _contains_any(host_path, ["files.mobilitydatabase.org", "openmobilitydata-data.s3", "scraped.data.public-transport.earth"]) @dataclass class _HtmlCell: text: str = "" links: list[str] = field(default_factory=list) @property def first_external_link(self) -> str: for link in self.links: parsed = urlparse(link) if parsed.scheme in {"http", "https"} and "ptna.openstreetmap.de" not in parsed.netloc: return link return "" @dataclass class _HtmlRow: cells: list[_HtmlCell] = field(default_factory=list) class _TableParser(HTMLParser): def __init__(self, base_url: str): super().__init__(convert_charrefs=True) self.base_url = base_url self.rows: list[_HtmlRow] = [] self._row: _HtmlRow | None = None self._cell: _HtmlCell | None = None self._active_link: str = "" def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: attrs_dict = {key: value or "" for key, value in attrs} if tag == "tr": self._row = _HtmlRow() elif tag in {"td", "th"} and self._row is not None: self._cell = _HtmlCell() elif tag == "a" and self._cell is not None: href = attrs_dict.get("href", "") if href: self._active_link = urljoin(self.base_url, href) self._cell.links.append(self._active_link) def handle_endtag(self, tag: str) -> None: if tag in {"td", "th"} and self._row is not None and self._cell is not None: self._cell.text = _clean_text(self._cell.text) self._row.cells.append(self._cell) self._cell = None self._active_link = "" elif tag == "a": self._active_link = "" elif tag == "tr": if self._row is not None and self._row.cells: self.rows.append(self._row) self._row = None self._cell = None self._active_link = "" def handle_data(self, data: str) -> None: if self._cell is not None: self._cell.text += data class _LinkParser(HTMLParser): def __init__(self, base_url: str): super().__init__(convert_charrefs=True) self.base_url = base_url self.links: list[str] = [] def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag != "a": return for key, value in attrs: if key == "href" and value: self.links.append(urljoin(self.base_url, value)) def _parse_table_rows(html: str, base_url: str) -> list[_HtmlRow]: parser = _TableParser(base_url) parser.feed(html) return parser.rows def _all_links(html: str, base_url: str) -> list[str]: parser = _LinkParser(base_url) parser.feed(html) return parser.links def _fetch_text(url: str, *, timeout: float) -> str: response = requests.get(url, timeout=timeout, headers={"User-Agent": "meubility-workbench-feed-discovery/0.1"}) response.raise_for_status() return response.text def _first_link_matching(links: Iterable[str], needle: str) -> str: for link in links: if needle in link: return link return "" def _feed_id_from_url(url: str) -> str: query = parse_qs(urlparse(url).query) return (query.get("feed") or [""])[0] def _looks_like_download_url(url: str) -> bool: if not url: return False parsed = urlparse(url) lower_path = parsed.path.lower() lower_url = url.lower() if lower_path.endswith(".zip"): return True if "exportformat=gtfs" in lower_url or "google_transit" in lower_url: return True if lower_path.rstrip("/").endswith(("current_gtfs", "gtfs")): return True if "gtfs.ovapi.nl" in parsed.netloc.lower() and "gtfs" in lower_path: return True return False def _opendata_oepnv_dataset_slug_from_url(url: str) -> str: query = parse_qs(urlparse(unescape(url)).query) return _clean_text((query.get("tx_vrrkit_view[dataset_name]") or [""])[0]) def _opendata_oepnv_link_score(url: str) -> int: parsed = urlparse(url) score = 0 if "/organisation/" in parsed.path: score += 2 if parsed.path.rstrip("/").endswith("startseite"): score += 1 return score def _opendata_oepnv_title(html: str) -> str: matches = re.findall(r"<div[^>]*class=[\"'][^\"']*element-header[^\"']*[\"'][^>]*>(.*?)</div>", html, re.IGNORECASE | re.DOTALL) for raw in reversed(matches): title = _html_text(raw) if title and title.lower() not in {"hinweise", "spielregeln", "lizenzen"}: return title match = re.search(r"<title>(.*?)", html, re.IGNORECASE | re.DOTALL) return _html_text(match.group(1)) if match else "" def _opendata_oepnv_download_urls(html: str, detail_url: str) -> list[str]: urls: list[str] = [] for link in _all_links(html, detail_url): normalized = _normalize_feed_url(link) if normalized and _looks_like_download_url(normalized) and normalized not in urls: urls.append(normalized) return urls def _choose_opendata_oepnv_download_url(urls: list[str], data_type: str) -> str: if data_type == "gtfs_rt": for url in urls: lower = url.lower() if "realtime" in lower or "gtfs-rt" in lower or "gtfs_rt" in lower: return url return "" if data_type != "gtfs": return "" if not urls: return "" static_urls = [url for url in urls if "realtime" not in url.lower() and "gtfs-rt" not in url.lower() and "gtfs_rt" not in url.lower()] if not static_urls: return "" for url in static_urls: lower = url.lower() if "latest" in lower or "aktueller" in lower or lower.rstrip("/").endswith("/gtfs.zip"): return url return static_urls[0] def _opendata_oepnv_license(html: str, detail_url: str) -> tuple[str, str]: match = re.search( r"id=[\"']dodp_license[\"'][^>]*>.*?]*href=[\"'](?P[^\"']+)[\"'][^>]*>(?P.*?)", html, re.IGNORECASE | re.DOTALL, ) if not match: return "", "" return urljoin(detail_url, unescape(match.group("href"))), _html_text(match.group("text")) def _opendata_oepnv_requires_login(text: str) -> bool: lower = text.lower() return ( "available for registered users" in lower or "download is only available for registered users" in lower or "this download is only available for registered users" in lower or "anmeldung erforderlich" in lower or "nur für registrierte nutzer" in lower or "nur fuer registrierte nutzer" in lower ) def _opendata_oepnv_data_type(title: str, text: str) -> str: title_lower = title.lower() lower = f"{title} {text}".lower() if "gtfs-rt" in lower or "gtfs realtime" in lower or "gtfs-realtime" in lower: return "gtfs_rt" if "gtfs" in title_lower: return "gtfs" if "haltestell" in title_lower or "zhv" in title_lower: return "stops" if "netex" in title_lower: return "netex" if "trias" in lower: return "api" if "netex" in lower: return "netex" if "gtfs" in lower: return "gtfs" if "haltestell" in lower or "zhv" in lower: return "stops" if "linien" in lower: return "line_metadata" return "dataset" def _opendata_oepnv_features(data_type: str, title: str, text: str) -> str: features = ["OpenData ÖPNV"] if data_type == "gtfs": features.extend(["GTFS", "Schedules"]) elif data_type == "netex": features.extend(["NeTEx", "Schedules"]) elif data_type == "gtfs_rt": features.extend(["GTFS-Realtime"]) elif data_type == "stops": features.append("Stops") lower = f"{title} {text}".lower() if "regionalbahn" in lower or "s-bahn" in lower or "bahn" in lower: features.append("Rail") if "u-bahn" in lower or "metro" in lower: features.append("Metro") if "tram" in lower or "straßenbahn" in lower: features.append("Tram") if "bus" in lower: features.append("Bus") return "|".join(dict.fromkeys(features)) def _opendata_oepnv_priority(provider: str, data_type: str, selected_url: str) -> str: if provider == "DELFI e.V." and data_type in {"gtfs", "netex", "stops"}: return "P0" if data_type == "gtfs" and selected_url: return "P1" if data_type in {"netex", "gtfs_rt", "stops"}: return "P2" return "P3" def _opendata_oepnv_provider(detail_url: str, slug: str) -> str: path = urlparse(detail_url).path.lower() haystack = f"{path}/{slug.lower()}/" providers = { "delfi": "DELFI e.V.", "nah-sh": "NAH.SH", "avv": "Aachener Verkehrsverbund", "baden-wuerttemberg": "Baden-Württemberg", "bawue": "Baden-Württemberg", "hvv": "Hamburger Verkehrsverbund", "mvv": "Münchner Verkehrs- und Tarifverbund", "nvv": "Nordhessischer VerkehrsVerbund", "nwl": "Nahverkehr Westfalen-Lippe", "nordrhein-westfalen": "Nordrhein-Westfalen", "nrw": "Nordrhein-Westfalen", "rmv": "Rhein-Main-Verkehrsverbund", "rnv": "Rhein-Neckar-Verkehr", "thueringen": "Thüringen", "vbb": "Verkehrsverbund Berlin-Brandenburg", "vrr": "Verkehrsverbund Rhein-Ruhr", "vrs": "Verkehrsverbund Rhein-Sieg", "vvs": "Verkehrs- und Tarifverbund Stuttgart", } for token, provider in providers.items(): if f"/{token}/" in haystack or f"-{token}" in haystack or f"_{token}" in haystack: return provider return "OpenData ÖPNV" def _opendata_oepnv_subdivision(detail_url: str, slug: str) -> str: text = f"{urlparse(detail_url).path} {slug}".lower() subdivisions = { "baden-wuerttemberg": "Baden-Württemberg", "bawue": "Baden-Württemberg", "thueringen": "Thüringen", "nrw": "Nordrhein-Westfalen", "nordrhein-westfalen": "Nordrhein-Westfalen", "vbb": "Berlin-Brandenburg", "hvv": "Hamburg", "mvv": "Bayern", "vrr": "Nordrhein-Westfalen", "vrs": "Nordrhein-Westfalen", "nwl": "Nordrhein-Westfalen", "avv": "Nordrhein-Westfalen", "rmv": "Hessen", "nvv": "Hessen", "rnv": "Baden-Württemberg/Rheinland-Pfalz/Hessen", "vvs": "Baden-Württemberg", } for token, subdivision in subdivisions.items(): if token in text: return subdivision return "" def _opendata_oepnv_download_is_registration_gated(candidate: FeedCandidate) -> bool: url_text = " ".join([candidate.selected_url, candidate.direct_download_url, candidate.original_release_url]).lower() if "opendata-oepnv.de" not in url_text: return False descriptive_text = " ".join([candidate.provider, candidate.feed_name, candidate.status, candidate.notes]).lower() return "registration required" in descriptive_text or "user registration required" in descriptive_text def _html_text(value: str) -> str: return _clean_text(re.sub(r"<[^>]+>", " ", value or "")) def _title_from_slug(slug: str) -> str: return _clean_text(slug.replace("_", " ").replace("-", " ").title()) def _normalize_feed_url(url: str) -> str: cleaned = _clean_text(url) if not cleaned: return "" parsed = urlparse(cleaned) if parsed.scheme: return cleaned first = cleaned.split("/", 1)[0] if "." in first: return f"https://{cleaned}" return cleaned def _choose_feed_url(direct_url: str, latest_url: str) -> str: if direct_url: return direct_url return latest_url def _candidate_priority(candidate: FeedCandidate) -> str: status = candidate.status.lower() official = candidate.is_official.lower() == "true" if candidate.discovery_source == "curated_seed": return candidate.priority or "P1" if status == "active" and official and candidate.direct_download_url: return "P0" if status == "active" and candidate.direct_download_url: return "P1" if status == "active" and candidate.latest_url: return "P2" if candidate.discovery_source == "ptna": return "P2" if candidate.selected_url else "P4" return "P3" def _test_candidate_sort_key(candidate: FeedCandidate) -> tuple[int, int, str, str]: source_bonus = 0 if candidate.discovery_source == "curated_seed" else 1 country_bonus = CURATED_TEST_COUNTRIES.index(candidate.country) if candidate.country in CURATED_TEST_COUNTRIES else 99 return (_priority_sort_key(candidate.priority), source_bonus + country_bonus, candidate.country, candidate.provider.lower()) def _priority_sort_key(priority: str) -> int: match = re.match(r"P(\d+)", priority or "") return int(match.group(1)) if match else 9 def _candidate_alias_keys(candidate: FeedCandidate) -> list[str]: keys = [candidate.key()] if candidate.stable_id: keys.append(f"stable:{candidate.stable_id}") for url in [candidate.selected_url, candidate.direct_download_url, candidate.latest_url]: if url: keys.append(f"url:{_normalize_url_key(url)}") if candidate.ptna_feed_id: keys.append(f"ptna:{candidate.ptna_feed_id}") deduped: list[str] = [] for key in keys: if key not in deduped: deduped.append(key) return deduped def _merge_candidate(existing: FeedCandidate, incoming: FeedCandidate) -> None: if incoming.discovery_source == "curated_seed": for field_name in ["country", "provider", "feed_name", "license_text", "features", "source_basis", "notes"]: new_value = getattr(incoming, field_name, "") if new_value: setattr(existing, field_name, new_value) if existing.selected_url and incoming.selected_url and _url_is_mirror_url(existing.selected_url) and not _url_is_mirror_url(incoming.selected_url): existing.selected_url = incoming.selected_url existing.direct_download_url = incoming.direct_download_url or incoming.selected_url existing.notes = _join_notes(existing.notes, "Selected non-mirror publisher URL from merged source evidence.") existing.discovery_source = _join_unique(existing.discovery_source, incoming.discovery_source) for field_name in CANONICAL_HEADERS: if field_name == "candidate_id": continue current = getattr(existing, field_name, "") new_value = getattr(incoming, field_name, "") if not current and new_value: setattr(existing, field_name, new_value) existing.priority = _better_priority(existing.priority, incoming.priority) existing.source_basis = _join_unique(existing.source_basis, incoming.source_basis) existing.notes = _join_notes(existing.notes, incoming.notes) def _better_priority(left: str, right: str) -> str: return left if _priority_sort_key(left) <= _priority_sort_key(right) else right def _join_unique(left: str, right: str) -> str: parts: list[str] = [] for value in [left, right]: for part in value.split(";"): cleaned = part.strip() if cleaned and cleaned not in parts: parts.append(cleaned) return "; ".join(parts) def _join_notes(left: str, right: str) -> str: return _join_unique(left, right) def _compact_name(value: str) -> str: return re.sub(r"\s+", " ", _clean_text(value)).strip() def _feed_source_name(country: str, value: str) -> str: base = _compact_name(value) or "GTFS feed" prefix = country.upper() display = base if prefix and not base.upper().startswith(f"{prefix} "): display = f"{prefix} {base}" if "gtfs" not in display.lower(): display = f"{display} GTFS" return display def _clean_text(value: str) -> str: cleaned = unescape(value or "").replace("\xa0", " ") cleaned = re.sub(r"\s+", " ", cleaned) return cleaned.strip() def _mode_scope_from_features(features: str) -> str: lower = features.lower() modes = [] if "rail" in lower or "train" in lower: modes.append("rail") if "tram" in lower or "light_rail" in lower: modes.append("tram") if "subway" in lower or "metro" in lower: modes.append("metro") if "bus" in lower or not modes: modes.append("bus") if "ferry" in lower: modes.append("ferry") return ",".join(dict.fromkeys(modes)) def _bbox_from_mobility_row(row: dict[str, str]) -> str: min_lat = _value(row, "location.bounding_box.minimum_latitude") max_lat = _value(row, "location.bounding_box.maximum_latitude") min_lon = _value(row, "location.bounding_box.minimum_longitude") max_lon = _value(row, "location.bounding_box.maximum_longitude") if not all([min_lat, max_lat, min_lon, max_lon]): return "" return f"{min_lon},{min_lat},{max_lon},{max_lat}" def _normalize_countries(countries: Iterable[str] | None) -> list[str] | None: if countries is None: return DEFAULT_DISCOVERY_COUNTRIES normalized = [country.strip().upper() for country in countries if country and country.strip()] if any(country == "ALL" for country in normalized): return None return normalized def _normalize_url_key(url: str) -> str: parsed = urlparse(url.strip()) scheme = parsed.scheme.lower() netloc = parsed.netloc.lower() path = parsed.path.rstrip("/") query = parsed.query return f"{scheme}://{netloc}{path}" + (f"?{query}" if query else "") def _write_csv(path: Path, headers: list[str], rows: list[dict[str, str]]) -> None: with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=headers, extrasaction="ignore") writer.writeheader() writer.writerows(rows) def _count_by(items: Iterable[FeedCandidate], key_fn) -> dict[str, int]: counts: dict[str, int] = {} for item in items: key = key_fn(item) counts[key] = counts.get(key, 0) + 1 return dict(sorted(counts.items())) def _value(row: dict[str, str], key: str) -> str: return _clean_text(row.get(key, "")) def _string(value: object) -> str: return "" if value is None else str(value) def _truncate(value: str, length: int) -> str: return value[:length] if value else ""