Next alpha stage commit

2026-07-06 20:54:46 +02:00
parent e23387738b
commit 1d7ec956d2
42 changed files with 11603 additions and 2837 deletions
--- a/app/feed_discovery.py
+++ b/app/feed_discovery.py
@@ -22,6 +22,7 @@ MOBILITY_DATABASE_ACCEPTANCE_TEST_URL = (
 )
 PTNA_GTFS_INDEX_URL = "https://ptna.openstreetmap.de/gtfs/index.html"
 PTNA_COUNTRY_URL_TEMPLATE = "https://ptna.openstreetmap.de/gtfs/{country}/index.php"
+OPENDATA_OEPNV_DATASETS_URL = "https://www.opendata-oepnv.de/ht/de/datensaetze"

 DEFAULT_DISCOVERY_COUNTRIES = ["DE", "AT", "CH", "NL", "DK", "FR", "BE", "LU", "NO", "SE", "FI", "IE", "GB"]
 CURATED_TEST_COUNTRIES = ["DE", "CH", "AT", "NL", "DK", "FI", "NO", "SE", "IE", "GB", "FR", "BE", "LU"]
@@ -33,6 +34,11 @@ CANONICAL_HEADERS = [
    "subdivision",
    "provider",
    "feed_name",
+    "authority_level",
+    "coverage_scope",
+    "overlap_role",
+    "import_policy",
+    "overlap_group",
    "stable_id",
    "ptna_feed_id",
    "data_type",
@@ -71,6 +77,11 @@ class FeedCandidate:
    subdivision: str = ""
    provider: str = ""
    feed_name: str = ""
+    authority_level: str = ""
+    coverage_scope: str = ""
+    overlap_role: str = ""
+    import_policy: str = ""
+    overlap_group: str = ""
    stable_id: str = ""
    ptna_feed_id: str = ""
    data_type: str = "gtfs"
@@ -142,6 +153,11 @@ class FeedCandidate:
            notes = _join_notes(notes, f"Mobility Database mirror: {self.latest_url}")
        if self.osm_license_text:
            notes = _join_notes(notes, f"OSM permission note: {_truncate(self.osm_license_text, 240)}")
+        if self.import_policy or self.overlap_role:
+            notes = _join_notes(
+                notes,
+                f"Import policy: {self.import_policy or 'review'}; overlap role: {self.overlap_role or 'review'}.",
+            )
        return {
            "name": _truncate(name, 240),
            "kind": "gtfs",
@@ -166,7 +182,9 @@ def build_gtfs_discovery_manifests(
    include_mobility_database: bool = True,
    include_acceptance_test_list: bool = True,
    include_ptna: bool = True,
+    include_opendata_oepnv: bool = True,
    max_ptna_details: int = 80,
+    max_opendata_oepnv_details: int = 80,
    test_limit: int = 24,
    check_urls: bool = False,
    timeout: float = 30.0,
@@ -183,8 +201,18 @@ def build_gtfs_discovery_manifests(
        candidates.extend(fetch_mobility_acceptance_candidates(countries=selected_countries, timeout=timeout))
    if include_ptna:
        candidates.extend(fetch_ptna_candidates(countries=selected_countries, max_details=max_ptna_details, timeout=timeout))
+    if include_opendata_oepnv:
+        candidates.extend(
+            fetch_opendata_oepnv_candidates(
+                countries=selected_countries,
+                max_details=max_opendata_oepnv_details,
+                timeout=timeout,
+            )
+        )

    merged = merge_candidates(candidates)
+    for candidate in merged:
+        apply_source_hierarchy(candidate)
    ingestable = [candidate for candidate in merged if candidate.selected_url and candidate.data_type == "gtfs"]
    if check_urls:
        for candidate in ingestable:
@@ -209,6 +237,7 @@ def build_gtfs_discovery_manifests(
            "mobility_database": MOBILITY_DATABASE_FEEDS_URL if include_mobility_database else None,
            "mobility_acceptance_test_list": MOBILITY_DATABASE_ACCEPTANCE_TEST_URL if include_acceptance_test_list else None,
            "ptna": PTNA_GTFS_INDEX_URL if include_ptna else None,
+            "opendata_oepnv": OPENDATA_OEPNV_DATASETS_URL if include_opendata_oepnv else None,
        },
        "counts": {
            "candidates": len(merged),
@@ -266,6 +295,7 @@ def fetch_mobility_database_candidates(
        )
        normalize_candidate_geography(candidate)
        apply_known_download_overrides(candidate)
+        apply_source_hierarchy(candidate)
        candidate.priority = _candidate_priority(candidate)
        candidates.append(candidate)
    return candidates
@@ -303,6 +333,7 @@ def fetch_mobility_acceptance_candidates(
        )
        normalize_candidate_geography(candidate)
        apply_known_download_overrides(candidate)
+        apply_source_hierarchy(candidate)
        candidates.append(candidate)
    return candidates

@@ -332,6 +363,7 @@ def fetch_ptna_candidates(
                    detail_fetches += 1
                except requests.RequestException:
                    candidate.notes = _join_notes(candidate.notes, "PTNA detail page could not be fetched during discovery.")
+            apply_source_hierarchy(candidate)
            candidate.priority = _candidate_priority(candidate)
            candidates.append(candidate)
    return candidates
@@ -383,6 +415,7 @@ def parse_ptna_country_page(html: str, *, country: str, page_url: str) -> list[F
        )
        normalize_candidate_geography(candidate)
        apply_known_download_overrides(candidate)
+        apply_source_hierarchy(candidate)
        candidates.append(candidate)
    return candidates

@@ -421,6 +454,134 @@ def parse_ptna_detail_fields(html: str, page_url: str) -> dict[str, str]:
    return parsed


+@dataclass
+class OpenDataOepnvDatasetLink:
+    slug: str
+    url: str
+    title: str = ""
+
+
+def fetch_opendata_oepnv_candidates(
+    *,
+    countries: list[str] | None = None,
+    max_details: int = 80,
+    timeout: float = 30.0,
+    url: str = OPENDATA_OEPNV_DATASETS_URL,
+) -> list[FeedCandidate]:
+    if countries and "DE" not in countries:
+        return []
+    try:
+        html = _fetch_text(url, timeout=timeout)
+    except requests.RequestException:
+        return []
+    dataset_links = parse_opendata_oepnv_dataset_links(html, url)
+    candidates: list[FeedCandidate] = []
+    for dataset_link in dataset_links[:max_details]:
+        try:
+            detail_html = _fetch_text(dataset_link.url, timeout=timeout)
+        except requests.RequestException:
+            candidate = FeedCandidate(
+                discovery_source="opendata_oepnv",
+                country="DE",
+                provider=_opendata_oepnv_provider(dataset_link.url, dataset_link.slug),
+                feed_name=dataset_link.title or _title_from_slug(dataset_link.slug),
+                stable_id=f"opendata-oepnv:{dataset_link.slug}",
+                status="detail_fetch_failed",
+                details_url=dataset_link.url,
+                source_basis="OpenData ÖPNV dataset catalog",
+                notes="OpenData ÖPNV detail page could not be fetched during discovery.",
+                priority="P4",
+            )
+            apply_source_hierarchy(candidate)
+            candidates.append(candidate)
+            continue
+        candidate = parse_opendata_oepnv_detail_page(
+            detail_html,
+            detail_url=dataset_link.url,
+            slug=dataset_link.slug,
+            title_hint=dataset_link.title,
+        )
+        normalize_candidate_geography(candidate)
+        apply_known_download_overrides(candidate)
+        apply_source_hierarchy(candidate)
+        candidate.priority = candidate.priority or _candidate_priority(candidate)
+        candidates.append(candidate)
+    return candidates
+
+
+def parse_opendata_oepnv_dataset_links(html: str, page_url: str = OPENDATA_OEPNV_DATASETS_URL) -> list[OpenDataOepnvDatasetLink]:
+    titles_by_slug: dict[str, str] = {}
+    title_pattern = re.compile(
+        r"<a\s+[^>]*href=[\"'](?P<href>[^\"']*(?:dataset_name%5D|dataset_name\])=[^\"']+)[\"'][^>]*>\s*"
+        r"<span[^>]*itemprop=[\"']headline[\"'][^>]*>(?P<title>.*?)</span>",
+        re.IGNORECASE | re.DOTALL,
+    )
+    for match in title_pattern.finditer(html):
+        href = urljoin(page_url, unescape(match.group("href")))
+        slug = _opendata_oepnv_dataset_slug_from_url(href)
+        if slug:
+            titles_by_slug[slug] = _html_text(match.group("title"))
+
+    by_slug: dict[str, OpenDataOepnvDatasetLink] = {}
+    for link in _all_links(html, page_url):
+        slug = _opendata_oepnv_dataset_slug_from_url(link)
+        if not slug:
+            continue
+        existing = by_slug.get(slug)
+        candidate = OpenDataOepnvDatasetLink(slug=slug, url=link, title=titles_by_slug.get(slug, ""))
+        if existing is None or _opendata_oepnv_link_score(candidate.url) > _opendata_oepnv_link_score(existing.url):
+            by_slug[slug] = candidate
+    return sorted(by_slug.values(), key=lambda item: (item.title.lower(), item.slug))
+
+
+def parse_opendata_oepnv_detail_page(
+    html: str,
+    *,
+    detail_url: str,
+    slug: str = "",
+    title_hint: str = "",
+) -> FeedCandidate:
+    text = _html_text(html)
+    title = _opendata_oepnv_title(html) or title_hint or _title_from_slug(slug)
+    provider = _opendata_oepnv_provider(detail_url, slug)
+    data_type = _opendata_oepnv_data_type(title, text)
+    download_urls = _opendata_oepnv_download_urls(html, detail_url)
+    selected_url = _choose_opendata_oepnv_download_url(download_urls, data_type)
+    login_required = _opendata_oepnv_requires_login(text)
+    license_url, license_text = _opendata_oepnv_license(html, detail_url)
+    status = "active" if selected_url else "review_required"
+    if login_required:
+        status = "registration_required"
+    notes = "Discovered from OpenData ÖPNV; review per-dataset license and authority before publication."
+    if selected_url:
+        notes = _join_notes(notes, "Selected the first current GTFS download URL exposed on the detail page.")
+    if login_required:
+        notes = _join_notes(notes, "Download requires an OpenData ÖPNV account; do not auto-import without credentials.")
+    candidate = FeedCandidate(
+        discovery_source="opendata_oepnv",
+        country="DE",
+        subdivision=_opendata_oepnv_subdivision(detail_url, slug),
+        provider=provider,
+        feed_name=title,
+        stable_id=f"opendata-oepnv:{slug}" if slug else "",
+        data_type=data_type,
+        status=status,
+        is_official="",
+        selected_url=selected_url,
+        direct_download_url=selected_url,
+        original_release_url=detail_url,
+        license_url=license_url,
+        license_text=license_text,
+        details_url=detail_url,
+        features=_opendata_oepnv_features(data_type, title, text),
+        priority=_opendata_oepnv_priority(provider, data_type, selected_url),
+        source_basis="OpenData ÖPNV dataset catalog",
+        notes=notes,
+    )
+    apply_source_hierarchy(candidate)
+    return candidate
+
+
 def load_curated_ingestable_seed(
    *,
    countries: list[str] | None = None,
@@ -452,6 +613,7 @@ def load_curated_ingestable_seed(
            )
            normalize_candidate_geography(candidate)
            apply_known_download_overrides(candidate)
+            apply_source_hierarchy(candidate)
            candidates.append(candidate)
    return candidates

@@ -609,6 +771,275 @@ def apply_known_download_overrides(candidate: FeedCandidate) -> None:
            candidate.notes,
            "Selected Mobility Database latest.zip mirror because the catalog direct URL is known to be stale.",
        )
+    if _opendata_oepnv_download_is_registration_gated(candidate):
+        gated_url = candidate.selected_url or candidate.direct_download_url
+        candidate.original_release_url = candidate.original_release_url or gated_url
+        candidate.selected_url = ""
+        candidate.direct_download_url = ""
+        candidate.status = "registration_required"
+        candidate.notes = _join_notes(
+            candidate.notes,
+            "OpenData ÖPNV marks this download as registration-required; kept as a review candidate instead of direct ingest.",
+        )
+
+
+def apply_source_hierarchy(candidate: FeedCandidate) -> None:
+    text = _hierarchy_text(candidate)
+    authority_level = _infer_authority_level(candidate, text)
+    coverage_scope = _infer_coverage_scope(candidate, text)
+    overlap_role = _infer_overlap_role(authority_level, coverage_scope, candidate, text)
+    import_policy = _infer_import_policy(authority_level, overlap_role, candidate)
+    overlap_group = _infer_overlap_group(candidate, authority_level, coverage_scope, text)
+
+    candidate.authority_level = authority_level
+    candidate.coverage_scope = coverage_scope
+    candidate.overlap_role = overlap_role
+    candidate.import_policy = import_policy
+    candidate.overlap_group = overlap_group
+
+
+def _hierarchy_text(candidate: FeedCandidate) -> str:
+    return " ".join(
+        [
+            candidate.discovery_source,
+            candidate.country,
+            candidate.subdivision,
+            candidate.provider,
+            candidate.feed_name,
+            candidate.stable_id,
+            candidate.ptna_feed_id,
+            candidate.status,
+            _url_hierarchy_text(candidate.selected_url),
+            _url_hierarchy_text(candidate.direct_download_url),
+            _url_hierarchy_text(candidate.latest_url),
+            _url_hierarchy_text(candidate.original_release_url),
+            _url_hierarchy_text(candidate.details_url),
+            candidate.source_basis,
+            candidate.notes,
+        ]
+    ).lower()
+
+
+def _infer_authority_level(candidate: FeedCandidate, text: str) -> str:
+    if _is_mirror_candidate(candidate, text):
+        return "mirror"
+    discovery_sources = {part.strip() for part in candidate.discovery_source.split(";") if part.strip()}
+    if discovery_sources and discovery_sources <= {"ptna", "mobility_validator_acceptance"}:
+        return "secondary_discovery"
+    if _contains_any(
+        text,
+        [
+            "delfi",
+            "gtfs.de",
+            "mobilithek",
+            "germany-wide",
+            "deutschlandweite",
+            "entur",
+            "bod national",
+            "national gtfs",
+            "national feed",
+            "national timetable",
+            "opentransportdata.swiss",
+            "trafiklab",
+            "samtrafiken",
+            "transport for ireland",
+            "transportforireland",
+            "rejseplanen",
+            "gtfs.ovapi.nl",
+        ],
+    ):
+        return "national_official"
+    if _contains_any(text, ["operator", "verkehrsunternehmen", "rnv", "flixbus", "flixtrain", "sncf", "tfl"]):
+        return "operator_feed"
+    if _contains_any(
+        text,
+        [
+            "verkehrsverbund",
+            "transport association",
+            "regional authority",
+            "authority feed",
+            "vbb",
+            "vrr",
+            "vvs",
+            "vrs",
+            "mvv",
+            "hvv",
+            "rmv",
+            "nvv",
+            "nwl",
+            "avv",
+            "nah.sh",
+            "nah-sh",
+            "bayern",
+            "baden-württemberg",
+            "baden-wuerttemberg",
+            "nordrhein-westfalen",
+            "nrw",
+            "thüringen",
+            "thueringen",
+        ],
+    ):
+        return "regional_authority"
+    if "mobility_database" in candidate.discovery_source:
+        return "aggregator_catalog"
+    return "unknown"
+
+
+def _infer_coverage_scope(candidate: FeedCandidate, text: str) -> str:
+    if _contains_any(
+        text,
+        [
+            "germany-wide",
+            "deutschlandweite",
+            "national gtfs",
+            "national feed",
+            "national timetable",
+            "delfi",
+            "gtfs.de",
+            "entur",
+            "transport for ireland",
+            "transportforireland",
+            "trafiklab",
+            "samtrafiken",
+            "opentransportdata.swiss",
+            "gtfs.ovapi.nl",
+            "rejseplanen",
+        ],
+    ):
+        return "national"
+    if _contains_any(text, ["europe", "eu/eea", "pan-europe", "flixbus", "flixtrain"]):
+        return "multi_country"
+    if _contains_any(
+        text,
+        [
+            "verkehrsverbund",
+            "regional",
+            "bundesland",
+            "vbb",
+            "vrr",
+            "vvs",
+            "vrs",
+            "mvv",
+            "hvv",
+            "rmv",
+            "nvv",
+            "nwl",
+            "avv",
+            "nah.sh",
+            "nah-sh",
+        ],
+    ):
+        return "regional"
+    if _contains_any(text, ["operator", "verkehrsunternehmen", "rnv"]):
+        return "operator"
+    if candidate.country:
+        return "unknown_country"
+    return "unknown"
+
+
+def _infer_overlap_role(authority_level: str, coverage_scope: str, candidate: FeedCandidate, text: str) -> str:
+    if authority_level == "mirror":
+        return "bootstrap_mirror"
+    if authority_level == "secondary_discovery":
+        return "discovery_evidence"
+    if candidate.status == "registration_required":
+        return "review_baseline" if coverage_scope == "national" else "review_candidate"
+    if authority_level == "national_official" and coverage_scope == "national":
+        return "canonical_baseline"
+    if authority_level == "regional_authority":
+        return "regional_override_candidate"
+    if authority_level == "operator_feed":
+        return "operator_detail_candidate"
+    if authority_level == "aggregator_catalog":
+        return "catalog_metadata"
+    if _contains_any(text, ["active", "gtfs"]):
+        return "gap_fill_candidate"
+    return "review_candidate"
+
+
+def _infer_import_policy(authority_level: str, overlap_role: str, candidate: FeedCandidate) -> str:
+    if candidate.status == "registration_required" or not candidate.selected_url:
+        return "review_before_import"
+    if overlap_role == "canonical_baseline":
+        return "canonical_candidate"
+    if overlap_role in {"regional_override_candidate", "operator_detail_candidate"}:
+        return "qa_or_override_not_additive"
+    if overlap_role == "bootstrap_mirror":
+        return "bootstrap_only"
+    if authority_level in {"secondary_discovery", "aggregator_catalog"}:
+        return "metadata_only"
+    return "review_before_import"
+
+
+def _infer_overlap_group(candidate: FeedCandidate, authority_level: str, coverage_scope: str, text: str) -> str:
+    country = candidate.country or "unknown"
+    if coverage_scope == "national":
+        return f"{country}:national"
+    identity_text = _hierarchy_identity_text(candidate)
+    for token in [
+        "vbb",
+        "vrr",
+        "vvs",
+        "vrs",
+        "mvv",
+        "hvv",
+        "rmv",
+        "nvv",
+        "nwl",
+        "avv",
+        "nah-sh",
+        "nah.sh",
+        "rnv",
+    ]:
+        if token in identity_text:
+            normalized = token.replace(".", "-")
+            return f"{country}:{coverage_scope}:{normalized}"
+    key = _source_key(candidate.provider or candidate.feed_name)
+    if key:
+        return f"{country}:{coverage_scope}:{key}"
+    return f"{country}:{coverage_scope}:{authority_level}"
+
+
+def _is_mirror_candidate(candidate: FeedCandidate, text: str) -> bool:
+    if any(_url_is_mirror_url(url) for url in [candidate.selected_url, candidate.direct_download_url, candidate.latest_url if not candidate.selected_url else ""]):
+        return True
+    mirror_notes = " ".join([candidate.source_basis, candidate.notes]).lower()
+    return _contains_any(mirror_notes, ["european transport feeds mirror", "bootstrap mirror"])
+
+
+def _source_key(value: str) -> str:
+    cleaned = re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-")
+    return cleaned[:80]
+
+
+def _contains_any(text: str, tokens: Iterable[str]) -> bool:
+    return any(token in text for token in tokens)
+
+
+def _hierarchy_identity_text(candidate: FeedCandidate) -> str:
+    return " ".join(
+        [
+            candidate.provider,
+            candidate.feed_name,
+            candidate.stable_id,
+            candidate.ptna_feed_id,
+            _url_hierarchy_text(candidate.selected_url),
+            _url_hierarchy_text(candidate.direct_download_url),
+            _url_hierarchy_text(candidate.original_release_url),
+            _url_hierarchy_text(candidate.details_url),
+        ]
+    ).lower()
+
+
+def _url_hierarchy_text(url: str) -> str:
+    parsed = urlparse(url or "")
+    return " ".join(part for part in [parsed.netloc, parsed.path] if part)
+
+
+def _url_is_mirror_url(url: str) -> bool:
+    parsed = urlparse(url or "")
+    host_path = f"{parsed.netloc.lower()}{parsed.path.lower()}"
+    return _contains_any(host_path, ["files.mobilitydatabase.org", "openmobilitydata-data.s3", "scraped.data.public-transport.earth"])


@dataclass
@@ -732,6 +1163,211 @@ def _looks_like_download_url(url: str) -> bool:
    return False


+def _opendata_oepnv_dataset_slug_from_url(url: str) -> str:
+    query = parse_qs(urlparse(unescape(url)).query)
+    return _clean_text((query.get("tx_vrrkit_view[dataset_name]") or [""])[0])
+
+
+def _opendata_oepnv_link_score(url: str) -> int:
+    parsed = urlparse(url)
+    score = 0
+    if "/organisation/" in parsed.path:
+        score += 2
+    if parsed.path.rstrip("/").endswith("startseite"):
+        score += 1
+    return score
+
+
+def _opendata_oepnv_title(html: str) -> str:
+    matches = re.findall(r"<div[^>]*class=[\"'][^\"']*element-header[^\"']*[\"'][^>]*>(.*?)</div>", html, re.IGNORECASE | re.DOTALL)
+    for raw in reversed(matches):
+        title = _html_text(raw)
+        if title and title.lower() not in {"hinweise", "spielregeln", "lizenzen"}:
+            return title
+    match = re.search(r"<title>(.*?)</title>", html, re.IGNORECASE | re.DOTALL)
+    return _html_text(match.group(1)) if match else ""
+
+
+def _opendata_oepnv_download_urls(html: str, detail_url: str) -> list[str]:
+    urls: list[str] = []
+    for link in _all_links(html, detail_url):
+        normalized = _normalize_feed_url(link)
+        if normalized and _looks_like_download_url(normalized) and normalized not in urls:
+            urls.append(normalized)
+    return urls
+
+
+def _choose_opendata_oepnv_download_url(urls: list[str], data_type: str) -> str:
+    if data_type == "gtfs_rt":
+        for url in urls:
+            lower = url.lower()
+            if "realtime" in lower or "gtfs-rt" in lower or "gtfs_rt" in lower:
+                return url
+        return ""
+    if data_type != "gtfs":
+        return ""
+    if not urls:
+        return ""
+    static_urls = [url for url in urls if "realtime" not in url.lower() and "gtfs-rt" not in url.lower() and "gtfs_rt" not in url.lower()]
+    if not static_urls:
+        return ""
+    for url in static_urls:
+        lower = url.lower()
+        if "latest" in lower or "aktueller" in lower or lower.rstrip("/").endswith("/gtfs.zip"):
+            return url
+    return static_urls[0]
+
+
+def _opendata_oepnv_license(html: str, detail_url: str) -> tuple[str, str]:
+    match = re.search(
+        r"id=[\"']dodp_license[\"'][^>]*>.*?<a\s+[^>]*href=[\"'](?P<href>[^\"']+)[\"'][^>]*>(?P<text>.*?)</a>",
+        html,
+        re.IGNORECASE | re.DOTALL,
+    )
+    if not match:
+        return "", ""
+    return urljoin(detail_url, unescape(match.group("href"))), _html_text(match.group("text"))
+
+
+def _opendata_oepnv_requires_login(text: str) -> bool:
+    lower = text.lower()
+    return (
+        "available for registered users" in lower
+        or "download is only available for registered users" in lower
+        or "this download is only available for registered users" in lower
+        or "anmeldung erforderlich" in lower
+        or "nur für registrierte nutzer" in lower
+        or "nur fuer registrierte nutzer" in lower
+    )
+
+
+def _opendata_oepnv_data_type(title: str, text: str) -> str:
+    title_lower = title.lower()
+    lower = f"{title} {text}".lower()
+    if "gtfs-rt" in lower or "gtfs realtime" in lower or "gtfs-realtime" in lower:
+        return "gtfs_rt"
+    if "gtfs" in title_lower:
+        return "gtfs"
+    if "haltestell" in title_lower or "zhv" in title_lower:
+        return "stops"
+    if "netex" in title_lower:
+        return "netex"
+    if "trias" in lower:
+        return "api"
+    if "netex" in lower:
+        return "netex"
+    if "gtfs" in lower:
+        return "gtfs"
+    if "haltestell" in lower or "zhv" in lower:
+        return "stops"
+    if "linien" in lower:
+        return "line_metadata"
+    return "dataset"
+
+
+def _opendata_oepnv_features(data_type: str, title: str, text: str) -> str:
+    features = ["OpenData ÖPNV"]
+    if data_type == "gtfs":
+        features.extend(["GTFS", "Schedules"])
+    elif data_type == "netex":
+        features.extend(["NeTEx", "Schedules"])
+    elif data_type == "gtfs_rt":
+        features.extend(["GTFS-Realtime"])
+    elif data_type == "stops":
+        features.append("Stops")
+    lower = f"{title} {text}".lower()
+    if "regionalbahn" in lower or "s-bahn" in lower or "bahn" in lower:
+        features.append("Rail")
+    if "u-bahn" in lower or "metro" in lower:
+        features.append("Metro")
+    if "tram" in lower or "straßenbahn" in lower:
+        features.append("Tram")
+    if "bus" in lower:
+        features.append("Bus")
+    return "|".join(dict.fromkeys(features))
+
+
+def _opendata_oepnv_priority(provider: str, data_type: str, selected_url: str) -> str:
+    if provider == "DELFI e.V." and data_type in {"gtfs", "netex", "stops"}:
+        return "P0"
+    if data_type == "gtfs" and selected_url:
+        return "P1"
+    if data_type in {"netex", "gtfs_rt", "stops"}:
+        return "P2"
+    return "P3"
+
+
+def _opendata_oepnv_provider(detail_url: str, slug: str) -> str:
+    path = urlparse(detail_url).path.lower()
+    haystack = f"{path}/{slug.lower()}/"
+    providers = {
+        "delfi": "DELFI e.V.",
+        "nah-sh": "NAH.SH",
+        "avv": "Aachener Verkehrsverbund",
+        "baden-wuerttemberg": "Baden-Württemberg",
+        "bawue": "Baden-Württemberg",
+        "hvv": "Hamburger Verkehrsverbund",
+        "mvv": "Münchner Verkehrs- und Tarifverbund",
+        "nvv": "Nordhessischer VerkehrsVerbund",
+        "nwl": "Nahverkehr Westfalen-Lippe",
+        "nordrhein-westfalen": "Nordrhein-Westfalen",
+        "nrw": "Nordrhein-Westfalen",
+        "rmv": "Rhein-Main-Verkehrsverbund",
+        "rnv": "Rhein-Neckar-Verkehr",
+        "thueringen": "Thüringen",
+        "vbb": "Verkehrsverbund Berlin-Brandenburg",
+        "vrr": "Verkehrsverbund Rhein-Ruhr",
+        "vrs": "Verkehrsverbund Rhein-Sieg",
+        "vvs": "Verkehrs- und Tarifverbund Stuttgart",
+    }
+    for token, provider in providers.items():
+        if f"/{token}/" in haystack or f"-{token}" in haystack or f"_{token}" in haystack:
+            return provider
+    return "OpenData ÖPNV"
+
+
+def _opendata_oepnv_subdivision(detail_url: str, slug: str) -> str:
+    text = f"{urlparse(detail_url).path} {slug}".lower()
+    subdivisions = {
+        "baden-wuerttemberg": "Baden-Württemberg",
+        "bawue": "Baden-Württemberg",
+        "thueringen": "Thüringen",
+        "nrw": "Nordrhein-Westfalen",
+        "nordrhein-westfalen": "Nordrhein-Westfalen",
+        "vbb": "Berlin-Brandenburg",
+        "hvv": "Hamburg",
+        "mvv": "Bayern",
+        "vrr": "Nordrhein-Westfalen",
+        "vrs": "Nordrhein-Westfalen",
+        "nwl": "Nordrhein-Westfalen",
+        "avv": "Nordrhein-Westfalen",
+        "rmv": "Hessen",
+        "nvv": "Hessen",
+        "rnv": "Baden-Württemberg/Rheinland-Pfalz/Hessen",
+        "vvs": "Baden-Württemberg",
+    }
+    for token, subdivision in subdivisions.items():
+        if token in text:
+            return subdivision
+    return ""
+
+
+def _opendata_oepnv_download_is_registration_gated(candidate: FeedCandidate) -> bool:
+    url_text = " ".join([candidate.selected_url, candidate.direct_download_url, candidate.original_release_url]).lower()
+    if "opendata-oepnv.de" not in url_text:
+        return False
+    descriptive_text = " ".join([candidate.provider, candidate.feed_name, candidate.status, candidate.notes]).lower()
+    return "registration required" in descriptive_text or "user registration required" in descriptive_text
+
+
+def _html_text(value: str) -> str:
+    return _clean_text(re.sub(r"<[^>]+>", " ", value or ""))
+
+
+def _title_from_slug(slug: str) -> str:
+    return _clean_text(slug.replace("_", " ").replace("-", " ").title())
+
+
 def _normalize_feed_url(url: str) -> str:
    cleaned = _clean_text(url)
    if not cleaned:
@@ -800,6 +1436,10 @@ def _merge_candidate(existing: FeedCandidate, incoming: FeedCandidate) -> None:
            new_value = getattr(incoming, field_name, "")
            if new_value:
                setattr(existing, field_name, new_value)
+    if existing.selected_url and incoming.selected_url and _url_is_mirror_url(existing.selected_url) and not _url_is_mirror_url(incoming.selected_url):
+        existing.selected_url = incoming.selected_url
+        existing.direct_download_url = incoming.direct_download_url or incoming.selected_url
+        existing.notes = _join_notes(existing.notes, "Selected non-mirror publisher URL from merged source evidence.")
    existing.discovery_source = _join_unique(existing.discovery_source, incoming.discovery_source)
    for field_name in CANONICAL_HEADERS:
        if field_name == "candidate_id":