chore: sync GovOPlaN module split state

This commit is contained in:
2026-07-10 12:51:21 +02:00
parent 18e6c3eb9b
commit 13fd7fc3bd
50 changed files with 12678 additions and 1109 deletions

View File

@@ -0,0 +1,600 @@
from __future__ import annotations
import os
from collections.abc import Mapping
from dataclasses import dataclass, field
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
from importlib import import_module
import mimetypes
from typing import Any
from urllib.parse import quote, unquote, urljoin, urlsplit
import xml.etree.ElementTree as ET
import httpx
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
class ConnectorBrowseError(RuntimeError):
pass
class ConnectorBrowseUnsupported(ConnectorBrowseError):
pass
@dataclass(frozen=True, slots=True)
class ConnectorBrowseItem:
kind: str
name: str
path: str
external_id: str | None = None
external_url: str | None = None
size_bytes: int | None = None
content_type: str | None = None
modified_at: str | None = None
etag: str | None = None
metadata: Mapping[str, Any] = field(default_factory=dict)
def to_response(self) -> dict[str, Any]:
return {
"kind": self.kind,
"name": self.name,
"path": self.path,
"external_id": self.external_id,
"external_url": self.external_url,
"size_bytes": self.size_bytes,
"content_type": self.content_type,
"modified_at": self.modified_at,
"etag": self.etag,
"metadata": dict(self.metadata),
}
def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = None, library_id: str | None = None) -> list[ConnectorBrowseItem]:
browse_path = normalize_connector_browse_path(path)
static_items = _static_listing(profile, path=browse_path, library_id=library_id)
if static_items is not None:
return static_items
if profile.provider == "seafile":
if _metadata_string(profile, "browse_protocol") == "webdav" or _metadata_string(profile, "webdav_endpoint_url"):
return _browse_webdav(profile, path=browse_path)
return _browse_seafile(profile, path=browse_path, library_id=library_id)
if profile.provider in {"webdav", "nextcloud"} or _metadata_string(profile, "browse_protocol") == "webdav":
return _browse_webdav(profile, path=browse_path)
if profile.provider == "smb":
return _browse_smb(profile, path=browse_path)
raise ConnectorBrowseUnsupported(f"Read-only browsing is not implemented for {profile.provider} connector profiles yet")
def parse_webdav_multistatus(*, root_url: str, current_path: str, payload: str | bytes) -> list[ConnectorBrowseItem]:
try:
root = ET.fromstring(payload)
except ET.ParseError as exc:
raise ConnectorBrowseError("Connector returned invalid WebDAV XML") from exc
root_path = _url_path_root(root_url)
current = normalize_connector_browse_path(current_path)
items: list[ConnectorBrowseItem] = []
for response in root.findall("{DAV:}response"):
href = response.findtext("{DAV:}href")
if not href:
continue
relative_path = _relative_href_path(root_path, href)
if relative_path == current:
continue
if current and not relative_path.startswith(current.rstrip("/") + "/"):
continue
parent = relative_path.rsplit("/", 1)[0] if "/" in relative_path else ""
if parent != current:
continue
prop = _webdav_prop(response)
is_collection = prop.find("{DAV:}resourcetype/{DAV:}collection") is not None if prop is not None else False
name = _webdav_display_name(prop) or _path_name(relative_path)
if not name:
continue
items.append(
ConnectorBrowseItem(
kind="folder" if is_collection else "file",
name=name,
path=relative_path,
external_id=_text(prop, "{DAV:}getetag") or relative_path,
size_bytes=None if is_collection else _int(_text(prop, "{DAV:}getcontentlength")),
content_type=None if is_collection else _text(prop, "{DAV:}getcontenttype"),
modified_at=_http_date(_text(prop, "{DAV:}getlastmodified")),
etag=_text(prop, "{DAV:}getetag"),
)
)
return sorted(items, key=lambda item: (item.kind != "library", item.kind != "folder", item.name.casefold(), item.path.casefold()))
def seafile_libraries_from_payload(payload: object) -> list[ConnectorBrowseItem]:
if not isinstance(payload, list):
raise ConnectorBrowseError("Seafile library response must be a list")
libraries = [_seafile_library_item(item) for item in payload if isinstance(item, Mapping)]
return sorted(libraries, key=lambda item: item.name.casefold())
def seafile_directory_items_from_payload(payload: object, *, path: str | None = None) -> list[ConnectorBrowseItem]:
if payload == "uptodate":
return []
if not isinstance(payload, list):
raise ConnectorBrowseError("Seafile directory response must be a list")
browse_path = normalize_connector_browse_path(path)
items = [_seafile_directory_item(item, parent_path=browse_path) for item in payload if isinstance(item, Mapping)]
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
def _browse_seafile(profile: ConnectorProfile, *, path: str, library_id: str | None) -> list[ConnectorBrowseItem]:
repo_id, dir_path = _seafile_repo_and_path(path=path, library_id=library_id)
headers = _seafile_headers(profile)
if not repo_id:
payload = _request_json("GET", _seafile_url(profile, "api2/repos/"), headers=headers)
return seafile_libraries_from_payload(payload)
payload = _request_json(
"GET",
_seafile_url(profile, f"api2/repos/{quote(repo_id, safe='')}/dir/"),
headers=headers,
params={"p": "/" + dir_path if dir_path else "/"},
)
return seafile_directory_items_from_payload(payload, path=dir_path)
def _browse_webdav(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowseItem]:
root_url = _metadata_string(profile, "webdav_endpoint_url") or profile.endpoint_url
if not root_url:
raise ConnectorBrowseError("Connector profile does not define an endpoint URL")
url = _webdav_url(root_url, path)
headers = {"Depth": "1", "Content-Type": "application/xml; charset=utf-8"}
auth: tuple[str, str] | None = None
password = _profile_password(profile)
token = _profile_token(profile)
if profile.username and password:
auth = (profile.username, password)
elif token:
headers["Authorization"] = f"Bearer {token}"
elif profile.credential_mode.casefold() not in {"", "none", "anonymous"} and profile.secret_ref:
raise ConnectorBrowseError("Secret-ref connector credentials need a runtime secret resolver before live browsing")
body = """<?xml version="1.0" encoding="utf-8"?>
<propfind xmlns="DAV:">
<prop>
<displayname />
<resourcetype />
<getcontentlength />
<getcontenttype />
<getlastmodified />
<getetag />
</prop>
</propfind>"""
try:
response = httpx.request("PROPFIND", url, headers=headers, content=body, auth=auth, timeout=15.0)
except httpx.HTTPError as exc:
raise ConnectorBrowseError(f"Connector browse failed: {exc}") from exc
if response.status_code in {401, 403}:
raise ConnectorBrowseError("Connector credentials were rejected")
if response.status_code not in {200, 207}:
raise ConnectorBrowseError(f"Connector browse failed with HTTP {response.status_code}")
return parse_webdav_multistatus(root_url=root_url, current_path=path, payload=response.text)
def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowseItem]:
location = _smb_location(profile)
unc_path = _smb_unc_path(location, path)
smbclient = _smbclient_module()
try:
entries = smbclient.scandir(unc_path, **_smb_client_kwargs(profile, location))
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"SMB connector browse failed: {exc}") from exc
items: list[ConnectorBrowseItem] = []
try:
with entries as iterator:
for entry in iterator:
name = _clean(getattr(entry, "name", None))
if not name or name in {".", ".."}:
continue
try:
is_dir = bool(entry.is_dir())
except Exception:
is_dir = False
stat_result = _smb_entry_stat(entry)
item_path = _join_browse_path(path, name)
items.append(
ConnectorBrowseItem(
kind="folder" if is_dir else "file",
name=name,
path=item_path,
external_id=f"{location.share}:{item_path}",
size_bytes=None if is_dir else _smb_stat_size(stat_result),
content_type=None if is_dir else mimetypes.guess_type(name)[0],
modified_at=_smb_stat_modified_at(stat_result),
etag=_smb_stat_revision(stat_result),
metadata={
"share": location.share,
**({"server": location.server} if _metadata_bool(profile, "expose_server_metadata", default=False) else {}),
},
)
)
except ConnectorBrowseError:
raise
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
raise ConnectorBrowseError(f"SMB connector browse failed: {exc}") from exc
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
def _seafile_headers(profile: ConnectorProfile) -> dict[str, str]:
token = _seafile_token(profile)
return {"Authorization": f"Token {token}", "Accept": "application/json"}
def _seafile_token(profile: ConnectorProfile) -> str:
token = _profile_token(profile)
if token:
return token
password = _profile_password(profile)
if profile.username and password:
payload = _request_json(
"POST",
_seafile_url(profile, "api2/auth-token/"),
data={"username": profile.username, "password": password},
)
if not isinstance(payload, Mapping) or not _clean(payload.get("token")):
raise ConnectorBrowseError("Seafile did not return an account token")
return _clean(payload.get("token")) or ""
if profile.secret_ref:
raise ConnectorBrowseError("Secret-ref Seafile credentials need a runtime secret resolver before live browsing")
raise ConnectorBrowseError("Seafile connector profiles require token credentials or username plus password credentials")
def _seafile_url(profile: ConnectorProfile, suffix: str) -> str:
if not profile.endpoint_url:
raise ConnectorBrowseError("Seafile connector profile does not define endpoint_url")
return urljoin(profile.endpoint_url.rstrip("/") + "/", suffix.lstrip("/"))
def _request_json(
method: str,
url: str,
*,
headers: Mapping[str, str] | None = None,
params: Mapping[str, str] | None = None,
data: Mapping[str, str] | None = None,
) -> object:
try:
response = httpx.request(method, url, headers=dict(headers or {}), params=params, data=data, timeout=15.0)
except httpx.HTTPError as exc:
raise ConnectorBrowseError(f"Connector browse failed: {exc}") from exc
if response.status_code in {401, 403}:
raise ConnectorBrowseError("Connector credentials were rejected")
if response.status_code not in {200, 201}:
raise ConnectorBrowseError(f"Connector browse failed with HTTP {response.status_code}")
try:
return response.json()
except ValueError as exc:
raise ConnectorBrowseError("Connector returned invalid JSON") from exc
def _seafile_repo_and_path(*, path: str, library_id: str | None) -> tuple[str | None, str]:
repo_id = _clean(library_id)
if repo_id:
return repo_id, path
if not path:
return None, ""
first, _, rest = path.partition("/")
return first, rest
def _seafile_library_item(value: Mapping[str, Any]) -> ConnectorBrowseItem:
repo_id = _clean(value.get("id") or value.get("repo_id"))
name = _clean(value.get("name") or value.get("repo_name") or repo_id)
if not repo_id or not name:
raise ConnectorBrowseError("Seafile library entries require id and name")
return ConnectorBrowseItem(
kind="library",
name=name,
path=repo_id,
external_id=repo_id,
size_bytes=_int(value.get("size") or value.get("repo_size")),
modified_at=_timestamp(value.get("mtime")),
metadata={
key: value[key]
for key in ("type", "permission", "encrypted", "owner", "file_count")
if key in value
},
)
def _seafile_directory_item(value: Mapping[str, Any], *, parent_path: str) -> ConnectorBrowseItem:
name = _clean(value.get("name") or value.get("obj_name"))
if not name:
raise ConnectorBrowseError("Seafile directory entries require name")
item_type = str(value.get("type") or ("dir" if value.get("is_dir") else "file")).casefold()
kind = "folder" if item_type in {"dir", "folder"} else "file"
path = _join_browse_path(parent_path, name)
content_type = None if kind == "folder" else mimetypes.guess_type(name)[0]
return ConnectorBrowseItem(
kind=kind,
name=name,
path=path,
external_id=_clean(value.get("id") or value.get("obj_id")),
size_bytes=None if kind == "folder" else _int(value.get("size")),
content_type=content_type,
modified_at=_timestamp(value.get("mtime") or value.get("modified")),
etag=_clean(value.get("id") or value.get("obj_id")),
metadata={
key: value[key]
for key in ("permission", "modifier_email", "modifier_name")
if key in value
},
)
def _static_listing(profile: ConnectorProfile, *, path: str, library_id: str | None) -> list[ConnectorBrowseItem] | None:
listing = profile.metadata.get("static_listing")
if listing is None:
return None
if isinstance(listing, list):
raw_items = listing if path == "" else []
elif isinstance(listing, Mapping):
key = library_id or path
raw_items = listing.get(key)
if raw_items is None and key == "":
raw_items = listing.get("/")
else:
raise ConnectorBrowseError("Connector static_listing metadata must be a list or object")
if raw_items is None:
return []
if not isinstance(raw_items, list):
raise ConnectorBrowseError("Connector static_listing entries must be lists")
return sorted(
[_static_item(item, parent_path=path) for item in raw_items if isinstance(item, Mapping)],
key=lambda item: (item.kind != "library", item.kind != "folder", item.name.casefold(), item.path.casefold()),
)
def _static_item(value: Mapping[str, Any], *, parent_path: str) -> ConnectorBrowseItem:
kind = str(value.get("kind") or "file").strip().casefold()
if kind not in {"library", "folder", "file"}:
raise ConnectorBrowseError("Connector browse items must be library, folder or file")
name = str(value.get("name") or value.get("path") or "").strip()
if not name:
raise ConnectorBrowseError("Connector browse items require name")
path = normalize_connector_browse_path(value.get("path"))
if not path:
path = _join_browse_path(parent_path, name)
return ConnectorBrowseItem(
kind=kind,
name=name,
path=path,
external_id=_clean(value.get("external_id")),
external_url=_clean(value.get("external_url")),
size_bytes=_int(value.get("size_bytes")),
content_type=_clean(value.get("content_type")),
modified_at=_clean(value.get("modified_at")),
etag=_clean(value.get("etag")),
metadata=value.get("metadata") if isinstance(value.get("metadata"), Mapping) else {},
)
def _webdav_url(root_url: str, path: str) -> str:
base = root_url if root_url.endswith("/") else root_url + "/"
if not path:
return base
quoted = "/".join(quote(part, safe="") for part in path.split("/") if part)
return urljoin(base, quoted + "/")
def _url_path_root(root_url: str) -> str:
path = unquote(urlsplit(root_url).path or "/")
return path if path.endswith("/") else path + "/"
def _relative_href_path(root_path: str, href: str) -> str:
href_path = unquote(urlsplit(href).path or href).strip()
if href_path.startswith(root_path):
return normalize_connector_browse_path(href_path[len(root_path):])
return normalize_connector_browse_path(href_path.rsplit("/", 1)[-1])
def _webdav_prop(response: ET.Element) -> ET.Element:
prop = response.find("{DAV:}propstat/{DAV:}prop")
if prop is not None:
return prop
prop = response.find(".//{DAV:}prop")
return prop if prop is not None else ET.Element("prop")
def _webdav_display_name(prop: ET.Element | None) -> str | None:
return _clean(_text(prop, "{DAV:}displayname"))
def _text(prop: ET.Element | None, tag: str) -> str | None:
if prop is None:
return None
child = prop.find(tag)
return child.text.strip() if child is not None and child.text else None
def _http_date(value: str | None) -> str | None:
if not value:
return None
try:
return parsedate_to_datetime(value).isoformat()
except (TypeError, ValueError):
return value
def _timestamp(value: object) -> str | None:
number = _int(value)
if number is None:
return _clean(value)
return datetime.fromtimestamp(number, tz=timezone.utc).isoformat()
def _metadata_string(profile: ConnectorProfile, key: str) -> str | None:
return _clean(profile.metadata.get(key))
def _metadata_bool(profile: ConnectorProfile, key: str, *, default: bool = False) -> bool:
value = profile.metadata.get(key)
if value is None:
return default
if isinstance(value, bool):
return value
return str(value).strip().casefold() in {"1", "true", "yes", "on"}
def _env_required(name: str, profile_id: str) -> str:
value = _clean(os.environ.get(name))
if not value:
raise ConnectorBrowseError(f"Connector profile {profile_id} is missing required credential environment")
return value
def normalize_connector_browse_path(value: object) -> str:
if value is None:
return ""
path = str(value).replace("\\", "/").strip().strip("/")
parts = [part for part in path.split("/") if part and part not in {"."}]
if any(part == ".." for part in parts):
raise ConnectorBrowseError("Connector browse paths cannot contain parent directory segments")
return "/".join(parts)
def _join_browse_path(parent: str, name: str) -> str:
child = normalize_connector_browse_path(name)
if not parent:
return child
return f"{parent.rstrip('/')}/{child}"
def _path_name(path: str) -> str:
return path.rstrip("/").rsplit("/", 1)[-1]
def _int(value: object) -> int | None:
if value is None or value == "":
return None
try:
return int(value)
except (TypeError, ValueError):
return None
def _clean(value: object) -> str | None:
if value is None:
return None
text = str(value).strip()
return text or None
@dataclass(frozen=True, slots=True)
class _SmbLocation:
server: str
share: str
port: int
root_path: str = ""
def _smb_location(profile: ConnectorProfile) -> _SmbLocation:
if not profile.endpoint_url:
raise ConnectorBrowseError("SMB connector profile does not define endpoint_url")
parsed = urlsplit(profile.endpoint_url)
if parsed.scheme.casefold() != "smb":
raise ConnectorBrowseError("SMB connector endpoint_url must use smb://")
server = _clean(parsed.hostname)
if not server:
raise ConnectorBrowseError("SMB connector endpoint_url must include a server")
path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part]
if not path_parts:
raise ConnectorBrowseError("SMB connector endpoint_url must include a share name")
root_parts = path_parts[1:]
if profile.base_path:
root_parts.extend(normalize_connector_browse_path(profile.base_path).split("/"))
return _SmbLocation(
server=server,
share=path_parts[0],
port=parsed.port or _int(profile.metadata.get("port")) or 445,
root_path=normalize_connector_browse_path("/".join(root_parts)),
)
def _smb_unc_path(location: _SmbLocation, path: str) -> str:
parts = [part for part in (location.root_path, normalize_connector_browse_path(path)) if part]
suffix = "\\".join(part.replace("/", "\\") for part in parts)
base = f"\\\\{location.server}\\{location.share}"
return f"{base}\\{suffix}" if suffix else base
def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]:
kwargs: dict[str, object] = {
"port": location.port,
"require_signing": _metadata_bool(profile, "require_signing", default=True),
"auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm",
}
if "encrypt" in profile.metadata:
kwargs["encrypt"] = _metadata_bool(profile, "encrypt", default=False)
password = _profile_password(profile)
token = _profile_token(profile)
if profile.username and password:
kwargs["username"] = profile.username
kwargs["password"] = password
elif token:
raise ConnectorBrowseError("SMB connector profiles do not support bearer-token credentials")
elif profile.credential_mode.casefold() not in {"", "none", "anonymous"} and profile.secret_ref:
raise ConnectorBrowseError("Secret-ref SMB credentials need a runtime secret resolver before live browsing")
return kwargs
def _profile_password(profile: ConnectorProfile) -> str | None:
if profile.password_value:
return profile.password_value
if profile.password_env:
return _env_required(profile.password_env, profile.id)
return None
def _profile_token(profile: ConnectorProfile) -> str | None:
if profile.token_value:
return profile.token_value
if profile.token_env:
return _env_required(profile.token_env, profile.id)
return None
def _smbclient_module() -> Any:
try:
return import_module("smbclient")
except ImportError as exc:
raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc
def _smb_entry_stat(entry: object) -> object | None:
try:
return entry.stat() # type: ignore[attr-defined]
except Exception:
return None
def _smb_stat_size(stat_result: object | None) -> int | None:
return _int(getattr(stat_result, "st_size", None))
def _smb_stat_modified_at(stat_result: object | None) -> str | None:
value = getattr(stat_result, "st_mtime", None)
if value is None:
return None
try:
return datetime.fromtimestamp(float(value), tz=timezone.utc).isoformat()
except (TypeError, ValueError, OSError, OverflowError):
return None
def _smb_stat_revision(stat_result: object | None) -> str | None:
mtime_ns = getattr(stat_result, "st_mtime_ns", None)
size = getattr(stat_result, "st_size", None)
if mtime_ns is not None:
return f"{mtime_ns}:{size or 0}"
modified = getattr(stat_result, "st_mtime", None)
if modified is not None:
return f"{modified}:{size or 0}"
return None