883 lines
34 KiB
Python
883 lines
34 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from collections.abc import Mapping
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from email.utils import parsedate_to_datetime
|
|
from importlib import import_module
|
|
import mimetypes
|
|
from typing import Any
|
|
from urllib.parse import quote, unquote, urljoin, urlsplit
|
|
import xml.etree.ElementTree as ET # nosec B405 - typing/element creation only; parsing uses defusedxml below.
|
|
|
|
from defusedxml import ElementTree as SafeElementTree
|
|
|
|
from govoplan_core.security.outbound_http import (
|
|
OutboundHttpError,
|
|
validate_unpinned_sdk_host,
|
|
validate_unpinned_sdk_http_url,
|
|
)
|
|
|
|
from govoplan_files.backend.storage.http_client import ConnectorHttpError, request_connector_bytes
|
|
from govoplan_files.backend.storage.connector_deployment import (
|
|
ConnectorDeploymentConfigurationError,
|
|
connector_ca_bundle_path,
|
|
connector_secret_env_value,
|
|
validate_connector_tls_metadata,
|
|
)
|
|
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
|
|
|
|
|
class ConnectorBrowseError(RuntimeError):
|
|
pass
|
|
|
|
|
|
class ConnectorBrowseUnsupported(ConnectorBrowseError):
|
|
pass
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class ConnectorBrowseItem:
|
|
kind: str
|
|
name: str
|
|
path: str
|
|
external_id: str | None = None
|
|
external_url: str | None = None
|
|
size_bytes: int | None = None
|
|
content_type: str | None = None
|
|
modified_at: str | None = None
|
|
etag: str | None = None
|
|
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
|
|
def to_response(self) -> dict[str, Any]:
|
|
return {
|
|
"kind": self.kind,
|
|
"name": self.name,
|
|
"path": self.path,
|
|
"external_id": self.external_id,
|
|
"external_url": self.external_url,
|
|
"size_bytes": self.size_bytes,
|
|
"content_type": self.content_type,
|
|
"modified_at": self.modified_at,
|
|
"etag": self.etag,
|
|
"metadata": dict(self.metadata),
|
|
}
|
|
|
|
|
|
def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = None, library_id: str | None = None, continuation_token: str | None = None) -> list[ConnectorBrowseItem]:
|
|
browse_path = normalize_connector_browse_path(path)
|
|
static_items = _static_listing(profile, path=browse_path, library_id=library_id)
|
|
if static_items is not None:
|
|
return static_items
|
|
if profile.provider == "seafile":
|
|
if _metadata_string(profile, "browse_protocol") == "webdav" or _metadata_string(profile, "webdav_endpoint_url"):
|
|
return _browse_webdav(profile, path=browse_path)
|
|
return _browse_seafile(profile, path=browse_path, library_id=library_id)
|
|
if profile.provider in {"webdav", "nextcloud"} or _metadata_string(profile, "browse_protocol") == "webdav":
|
|
return _browse_webdav(profile, path=browse_path)
|
|
if profile.provider == "smb":
|
|
return _browse_smb(profile, path=browse_path)
|
|
if profile.provider == "s3":
|
|
return _browse_s3(profile, path=browse_path, library_id=library_id, continuation_token=continuation_token)
|
|
raise ConnectorBrowseUnsupported(f"Read-only browsing is not implemented for {profile.provider} connector profiles yet")
|
|
|
|
|
|
def parse_webdav_multistatus(*, root_url: str, current_path: str, payload: str | bytes) -> list[ConnectorBrowseItem]:
|
|
try:
|
|
root = SafeElementTree.fromstring(payload)
|
|
except SafeElementTree.ParseError as exc:
|
|
raise ConnectorBrowseError("Connector returned invalid WebDAV XML") from exc
|
|
root_path = _url_path_root(root_url)
|
|
current = normalize_connector_browse_path(current_path)
|
|
items: list[ConnectorBrowseItem] = []
|
|
for response in root.findall("{DAV:}response"):
|
|
href = response.findtext("{DAV:}href")
|
|
if not href:
|
|
continue
|
|
relative_path = _relative_href_path(root_path, href)
|
|
if relative_path == current:
|
|
continue
|
|
if current and not relative_path.startswith(current.rstrip("/") + "/"):
|
|
continue
|
|
parent = relative_path.rsplit("/", 1)[0] if "/" in relative_path else ""
|
|
if parent != current:
|
|
continue
|
|
prop = _webdav_prop(response)
|
|
is_collection = prop.find("{DAV:}resourcetype/{DAV:}collection") is not None if prop is not None else False
|
|
name = _webdav_display_name(prop) or _path_name(relative_path)
|
|
if not name:
|
|
continue
|
|
items.append(
|
|
ConnectorBrowseItem(
|
|
kind="folder" if is_collection else "file",
|
|
name=name,
|
|
path=relative_path,
|
|
external_id=_text(prop, "{DAV:}getetag") or relative_path,
|
|
size_bytes=None if is_collection else _int(_text(prop, "{DAV:}getcontentlength")),
|
|
content_type=None if is_collection else _text(prop, "{DAV:}getcontenttype"),
|
|
modified_at=_http_date(_text(prop, "{DAV:}getlastmodified")),
|
|
etag=_text(prop, "{DAV:}getetag"),
|
|
)
|
|
)
|
|
return sorted(items, key=lambda item: (item.kind != "library", item.kind != "folder", item.name.casefold(), item.path.casefold()))
|
|
|
|
|
|
def seafile_libraries_from_payload(payload: object) -> list[ConnectorBrowseItem]:
|
|
if not isinstance(payload, list):
|
|
raise ConnectorBrowseError("Seafile library response must be a list")
|
|
libraries = [_seafile_library_item(item) for item in payload if isinstance(item, Mapping)]
|
|
return sorted(libraries, key=lambda item: item.name.casefold())
|
|
|
|
|
|
def seafile_directory_items_from_payload(payload: object, *, path: str | None = None) -> list[ConnectorBrowseItem]:
|
|
if payload == "uptodate":
|
|
return []
|
|
if not isinstance(payload, list):
|
|
raise ConnectorBrowseError("Seafile directory response must be a list")
|
|
browse_path = normalize_connector_browse_path(path)
|
|
items = [_seafile_directory_item(item, parent_path=browse_path) for item in payload if isinstance(item, Mapping)]
|
|
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
|
|
|
|
|
|
def _browse_seafile(profile: ConnectorProfile, *, path: str, library_id: str | None) -> list[ConnectorBrowseItem]:
|
|
repo_id, dir_path = _seafile_repo_and_path(path=path, library_id=library_id)
|
|
headers = _seafile_headers(profile)
|
|
if not repo_id:
|
|
payload = _request_json("GET", _seafile_url(profile, "api2/repos/"), headers=headers)
|
|
return seafile_libraries_from_payload(payload)
|
|
payload = _request_json(
|
|
"GET",
|
|
_seafile_url(profile, f"api2/repos/{quote(repo_id, safe='')}/dir/"),
|
|
headers=headers,
|
|
params={"p": "/" + dir_path if dir_path else "/"},
|
|
)
|
|
return seafile_directory_items_from_payload(payload, path=dir_path)
|
|
|
|
|
|
def _browse_webdav(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowseItem]:
|
|
root_url = _metadata_string(profile, "webdav_endpoint_url") or profile.endpoint_url
|
|
if not root_url:
|
|
raise ConnectorBrowseError("Connector profile does not define an endpoint URL")
|
|
url = _webdav_url(root_url, path)
|
|
headers = {"Depth": "1", "Content-Type": "application/xml; charset=utf-8"}
|
|
auth: tuple[str, str] | None = None
|
|
password = _profile_password(profile)
|
|
token = _profile_token(profile)
|
|
if profile.username and password:
|
|
auth = (profile.username, password)
|
|
elif token:
|
|
headers["Authorization"] = f"Bearer {token}"
|
|
elif profile.credential_mode.casefold() not in {"", "none", "anonymous"} and profile.secret_ref:
|
|
raise ConnectorBrowseError("Secret-ref connector credentials need a runtime secret resolver before live browsing")
|
|
body = """<?xml version="1.0" encoding="utf-8"?>
|
|
<propfind xmlns="DAV:">
|
|
<prop>
|
|
<displayname />
|
|
<resourcetype />
|
|
<getcontentlength />
|
|
<getcontenttype />
|
|
<getlastmodified />
|
|
<getetag />
|
|
</prop>
|
|
</propfind>"""
|
|
try:
|
|
response = request_connector_bytes(
|
|
"PROPFIND",
|
|
url,
|
|
headers=headers,
|
|
content=body,
|
|
auth=auth,
|
|
timeout=15.0,
|
|
label="WebDAV browse",
|
|
)
|
|
except ConnectorHttpError as exc:
|
|
raise ConnectorBrowseError(f"Connector browse failed: {exc}") from exc
|
|
if response.status_code in {401, 403}:
|
|
raise ConnectorBrowseError("Connector credentials were rejected")
|
|
if response.status_code not in {200, 207}:
|
|
raise ConnectorBrowseError(f"Connector browse failed with HTTP {response.status_code}")
|
|
return parse_webdav_multistatus(root_url=root_url, current_path=path, payload=response.content)
|
|
|
|
|
|
def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowseItem]:
|
|
location = _smb_location(profile)
|
|
unc_path = _smb_unc_path(location, path)
|
|
smbclient = _smbclient_module()
|
|
try:
|
|
entries = smbclient.scandir(unc_path, **_smb_client_kwargs(profile, location))
|
|
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
|
|
raise ConnectorBrowseError(f"SMB connector browse failed: {exc}") from exc
|
|
items: list[ConnectorBrowseItem] = []
|
|
try:
|
|
with entries as iterator:
|
|
for entry in iterator:
|
|
name = _clean(getattr(entry, "name", None))
|
|
if not name or name in {".", ".."}:
|
|
continue
|
|
try:
|
|
is_dir = bool(entry.is_dir())
|
|
except Exception:
|
|
is_dir = False
|
|
stat_result = _smb_entry_stat(entry)
|
|
item_path = _join_browse_path(path, name)
|
|
items.append(
|
|
ConnectorBrowseItem(
|
|
kind="folder" if is_dir else "file",
|
|
name=name,
|
|
path=item_path,
|
|
external_id=f"{location.share}:{item_path}",
|
|
size_bytes=None if is_dir else _smb_stat_size(stat_result),
|
|
content_type=None if is_dir else mimetypes.guess_type(name)[0],
|
|
modified_at=_smb_stat_modified_at(stat_result),
|
|
etag=_smb_stat_revision(stat_result),
|
|
metadata={
|
|
"share": location.share,
|
|
**({"server": location.server} if _metadata_bool(profile, "expose_server_metadata", default=False) else {}),
|
|
},
|
|
)
|
|
)
|
|
except ConnectorBrowseError:
|
|
raise
|
|
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
|
|
raise ConnectorBrowseError(f"SMB connector browse failed: {exc}") from exc
|
|
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
|
|
|
|
|
|
def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]:
|
|
client = _s3_client(profile)
|
|
bucket = _s3_bucket(profile, library_id)
|
|
if not bucket:
|
|
try:
|
|
payload = client.list_buckets()
|
|
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
|
|
raise ConnectorBrowseError(f"S3 connector browse failed: {exc}") from exc
|
|
buckets = payload.get("Buckets") if isinstance(payload, Mapping) else None
|
|
if not isinstance(buckets, list):
|
|
raise ConnectorBrowseError("S3 connector returned invalid bucket list")
|
|
return sorted(
|
|
(
|
|
ConnectorBrowseItem(
|
|
kind="library",
|
|
name=_clean(item.get("Name")) or "",
|
|
path=_clean(item.get("Name")) or "",
|
|
external_id=_clean(item.get("Name")),
|
|
modified_at=_timestamp(item.get("CreationDate")),
|
|
metadata={"bucket": _clean(item.get("Name"))},
|
|
)
|
|
for item in buckets
|
|
if isinstance(item, Mapping) and _clean(item.get("Name"))
|
|
),
|
|
key=lambda item: item.name.casefold(),
|
|
)
|
|
prefix = _s3_object_key(profile, path, directory=True)
|
|
params: dict[str, object] = {
|
|
"Bucket": bucket,
|
|
"Prefix": prefix,
|
|
"Delimiter": "/",
|
|
"MaxKeys": _s3_max_keys(profile),
|
|
}
|
|
next_page_request = _clean(continuation_token) or _metadata_string(profile, "continuation_token")
|
|
if next_page_request:
|
|
params["ContinuationToken"] = next_page_request
|
|
try:
|
|
payload = client.list_objects_v2(**params)
|
|
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
|
|
raise ConnectorBrowseError(f"S3 connector browse failed: {exc}") from exc
|
|
if not isinstance(payload, Mapping):
|
|
raise ConnectorBrowseError("S3 connector returned invalid object listing")
|
|
items = [
|
|
*_s3_prefix_items(bucket=bucket, browse_path=path, base_prefix=prefix, prefixes=payload.get("CommonPrefixes")),
|
|
*_s3_object_items(bucket=bucket, browse_path=path, base_prefix=prefix, objects=payload.get("Contents")),
|
|
]
|
|
next_token = _clean(payload.get("NextContinuationToken"))
|
|
if next_token and items:
|
|
last = items[-1]
|
|
items[-1] = ConnectorBrowseItem(
|
|
kind=last.kind,
|
|
name=last.name,
|
|
path=last.path,
|
|
external_id=last.external_id,
|
|
external_url=last.external_url,
|
|
size_bytes=last.size_bytes,
|
|
content_type=last.content_type,
|
|
modified_at=last.modified_at,
|
|
etag=last.etag,
|
|
metadata={**dict(last.metadata), "next_continuation_token": next_token, "listing_truncated": True},
|
|
)
|
|
return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold()))
|
|
|
|
|
|
def _s3_client(profile: ConnectorProfile) -> Any:
|
|
if profile.secret_ref:
|
|
raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing")
|
|
if profile.endpoint_url:
|
|
try:
|
|
endpoint_url = validate_unpinned_sdk_http_url(
|
|
profile.endpoint_url,
|
|
label="S3 connector endpoint",
|
|
)
|
|
except OutboundHttpError as exc:
|
|
raise ConnectorBrowseError(str(exc)) from exc
|
|
else:
|
|
raise ConnectorBrowseError(
|
|
"S3 connector endpoint discovery uses an SDK transport that cannot guarantee connection-time DNS/IP "
|
|
"pinning; live S3 access is disabled until that transport supports pinning"
|
|
)
|
|
try:
|
|
boto3 = import_module("boto3")
|
|
config_module = import_module("botocore.config")
|
|
except ImportError as exc:
|
|
raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc
|
|
kwargs: dict[str, object] = {"endpoint_url": endpoint_url}
|
|
region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region")
|
|
if region:
|
|
kwargs["region_name"] = region
|
|
access_key = profile.username or _metadata_env(profile, "access_key_id_env") or _metadata_string(profile, "access_key_id")
|
|
secret_key = _profile_password(profile) or _metadata_env(profile, "secret_access_key_env")
|
|
session_token = _profile_token(profile) or _metadata_env(profile, "session_token_env")
|
|
if access_key:
|
|
kwargs["aws_access_key_id"] = access_key
|
|
if secret_key:
|
|
kwargs["aws_secret_access_key"] = secret_key
|
|
if session_token:
|
|
kwargs["aws_session_token"] = session_token
|
|
verify = _s3_verify(profile)
|
|
if verify is not None:
|
|
kwargs["verify"] = verify
|
|
addressing_style = _s3_addressing_style(profile)
|
|
if addressing_style:
|
|
kwargs["config"] = config_module.Config(s3={"addressing_style": addressing_style})
|
|
try:
|
|
return boto3.client("s3", **kwargs)
|
|
except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific
|
|
raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc
|
|
|
|
|
|
def _s3_bucket(profile: ConnectorProfile, library_id: str | None = None) -> str | None:
|
|
return _metadata_string(profile, "bucket") or _metadata_string(profile, "bucket_name") or _clean(library_id)
|
|
|
|
|
|
def _s3_object_key(profile: ConnectorProfile, path: str, *, directory: bool = False) -> str:
|
|
base_prefix = normalize_connector_browse_path(profile.base_path or _metadata_string(profile, "base_prefix"))
|
|
browse_path = normalize_connector_browse_path(path)
|
|
key = "/".join(part for part in (base_prefix, browse_path) if part)
|
|
if directory and key:
|
|
return key.rstrip("/") + "/"
|
|
return key
|
|
|
|
|
|
def _s3_prefix_items(
|
|
*,
|
|
bucket: str,
|
|
browse_path: str,
|
|
base_prefix: str,
|
|
prefixes: object,
|
|
) -> list[ConnectorBrowseItem]:
|
|
if not isinstance(prefixes, list):
|
|
return []
|
|
items: list[ConnectorBrowseItem] = []
|
|
for item in prefixes:
|
|
if not isinstance(item, Mapping):
|
|
continue
|
|
key = _clean(item.get("Prefix"))
|
|
if not key:
|
|
continue
|
|
relative = _s3_relative_key(key, base_prefix=base_prefix)
|
|
name = _path_name(relative)
|
|
if not name:
|
|
continue
|
|
path = _join_browse_path(browse_path, name)
|
|
items.append(
|
|
ConnectorBrowseItem(
|
|
kind="folder",
|
|
name=name,
|
|
path=path,
|
|
external_id=f"{bucket}:{key}",
|
|
metadata={"bucket": bucket, "key": key},
|
|
)
|
|
)
|
|
return items
|
|
|
|
|
|
def _s3_object_items(
|
|
*,
|
|
bucket: str,
|
|
browse_path: str,
|
|
base_prefix: str,
|
|
objects: object,
|
|
) -> list[ConnectorBrowseItem]:
|
|
if not isinstance(objects, list):
|
|
return []
|
|
items: list[ConnectorBrowseItem] = []
|
|
for item in objects:
|
|
if not isinstance(item, Mapping):
|
|
continue
|
|
key = _clean(item.get("Key"))
|
|
if not key or key == base_prefix:
|
|
continue
|
|
relative = _s3_relative_key(key, base_prefix=base_prefix)
|
|
name = _path_name(relative)
|
|
if not name:
|
|
continue
|
|
path = _join_browse_path(browse_path, name)
|
|
items.append(
|
|
ConnectorBrowseItem(
|
|
kind="file",
|
|
name=name,
|
|
path=path,
|
|
external_id=f"{bucket}:{key}",
|
|
size_bytes=_int(item.get("Size")),
|
|
content_type=mimetypes.guess_type(name)[0],
|
|
modified_at=_timestamp(item.get("LastModified")),
|
|
etag=_clean(item.get("ETag")),
|
|
metadata={
|
|
"bucket": bucket,
|
|
"key": key,
|
|
**({"storage_class": item["StorageClass"]} if "StorageClass" in item else {}),
|
|
},
|
|
)
|
|
)
|
|
return items
|
|
|
|
|
|
def _s3_relative_key(key: str, *, base_prefix: str) -> str:
|
|
clean_key = key.rstrip("/")
|
|
clean_base = base_prefix.rstrip("/")
|
|
if clean_base and clean_key.startswith(f"{clean_base}/"):
|
|
return clean_key[len(clean_base) + 1 :]
|
|
return clean_key
|
|
|
|
|
|
def _s3_max_keys(profile: ConnectorProfile) -> int:
|
|
configured = _int(profile.metadata.get("max_keys"))
|
|
if configured is None:
|
|
return 1000
|
|
return max(1, min(configured, 1000))
|
|
|
|
|
|
def _s3_verify(profile: ConnectorProfile) -> bool | str | None:
|
|
try:
|
|
validate_connector_tls_metadata(profile.metadata)
|
|
except ConnectorDeploymentConfigurationError as exc:
|
|
raise ConnectorBrowseError(str(exc)) from exc
|
|
ca_bundle = _metadata_string(profile, "ca_bundle")
|
|
if ca_bundle:
|
|
try:
|
|
return connector_ca_bundle_path(ca_bundle)
|
|
except ConnectorDeploymentConfigurationError as exc:
|
|
raise ConnectorBrowseError(str(exc)) from exc
|
|
if "verify_tls" in profile.metadata:
|
|
return _metadata_bool(profile, "verify_tls", default=True)
|
|
if "tls_verify" in profile.metadata:
|
|
return _metadata_bool(profile, "tls_verify", default=True)
|
|
return None
|
|
|
|
|
|
def _s3_addressing_style(profile: ConnectorProfile) -> str | None:
|
|
style = _metadata_string(profile, "addressing_style")
|
|
if style in {"path", "virtual", "auto"}:
|
|
return style
|
|
if _metadata_bool(profile, "path_style", default=False):
|
|
return "path"
|
|
return None
|
|
|
|
|
|
def _seafile_headers(profile: ConnectorProfile) -> dict[str, str]:
|
|
token = _seafile_token(profile)
|
|
return {"Authorization": f"Token {token}", "Accept": "application/json"}
|
|
|
|
|
|
def _seafile_token(profile: ConnectorProfile) -> str:
|
|
token = _profile_token(profile)
|
|
if token:
|
|
return token
|
|
password = _profile_password(profile)
|
|
if profile.username and password:
|
|
payload = _request_json(
|
|
"POST",
|
|
_seafile_url(profile, "api2/auth-token/"),
|
|
data={"username": profile.username, "password": password},
|
|
)
|
|
if not isinstance(payload, Mapping) or not _clean(payload.get("token")):
|
|
raise ConnectorBrowseError("Seafile did not return an account token")
|
|
return _clean(payload.get("token")) or ""
|
|
if profile.secret_ref:
|
|
raise ConnectorBrowseError("Secret-ref Seafile credentials need a runtime secret resolver before live browsing")
|
|
raise ConnectorBrowseError("Seafile connector profiles require token credentials or username plus password credentials")
|
|
|
|
|
|
def _seafile_url(profile: ConnectorProfile, suffix: str) -> str:
|
|
if not profile.endpoint_url:
|
|
raise ConnectorBrowseError("Seafile connector profile does not define endpoint_url")
|
|
return urljoin(profile.endpoint_url.rstrip("/") + "/", suffix.lstrip("/"))
|
|
|
|
|
|
def _request_json(
|
|
method: str,
|
|
url: str,
|
|
*,
|
|
headers: Mapping[str, str] | None = None,
|
|
params: Mapping[str, str] | None = None,
|
|
data: Mapping[str, str] | None = None,
|
|
) -> object:
|
|
try:
|
|
response = request_connector_bytes(
|
|
method,
|
|
url,
|
|
headers=dict(headers or {}),
|
|
params=params,
|
|
data=data,
|
|
timeout=15.0,
|
|
label="Seafile API",
|
|
)
|
|
except ConnectorHttpError as exc:
|
|
raise ConnectorBrowseError(f"Connector browse failed: {exc}") from exc
|
|
if response.status_code in {401, 403}:
|
|
raise ConnectorBrowseError("Connector credentials were rejected")
|
|
if response.status_code not in {200, 201}:
|
|
raise ConnectorBrowseError(f"Connector browse failed with HTTP {response.status_code}")
|
|
try:
|
|
return json.loads(response.content)
|
|
except (UnicodeDecodeError, ValueError) as exc:
|
|
raise ConnectorBrowseError("Connector returned invalid JSON") from exc
|
|
|
|
|
|
def _seafile_repo_and_path(*, path: str, library_id: str | None) -> tuple[str | None, str]:
|
|
repo_id = _clean(library_id)
|
|
if repo_id:
|
|
return repo_id, path
|
|
if not path:
|
|
return None, ""
|
|
first, _, rest = path.partition("/")
|
|
return first, rest
|
|
|
|
|
|
def _seafile_library_item(value: Mapping[str, Any]) -> ConnectorBrowseItem:
|
|
repo_id = _clean(value.get("id") or value.get("repo_id"))
|
|
name = _clean(value.get("name") or value.get("repo_name") or repo_id)
|
|
if not repo_id or not name:
|
|
raise ConnectorBrowseError("Seafile library entries require id and name")
|
|
return ConnectorBrowseItem(
|
|
kind="library",
|
|
name=name,
|
|
path=repo_id,
|
|
external_id=repo_id,
|
|
size_bytes=_int(value.get("size") or value.get("repo_size")),
|
|
modified_at=_timestamp(value.get("mtime")),
|
|
metadata={
|
|
key: value[key]
|
|
for key in ("type", "permission", "encrypted", "owner", "file_count")
|
|
if key in value
|
|
},
|
|
)
|
|
|
|
|
|
def _seafile_directory_item(value: Mapping[str, Any], *, parent_path: str) -> ConnectorBrowseItem:
|
|
name = _clean(value.get("name") or value.get("obj_name"))
|
|
if not name:
|
|
raise ConnectorBrowseError("Seafile directory entries require name")
|
|
item_type = str(value.get("type") or ("dir" if value.get("is_dir") else "file")).casefold()
|
|
kind = "folder" if item_type in {"dir", "folder"} else "file"
|
|
path = _join_browse_path(parent_path, name)
|
|
content_type = None if kind == "folder" else mimetypes.guess_type(name)[0]
|
|
return ConnectorBrowseItem(
|
|
kind=kind,
|
|
name=name,
|
|
path=path,
|
|
external_id=_clean(value.get("id") or value.get("obj_id")),
|
|
size_bytes=None if kind == "folder" else _int(value.get("size")),
|
|
content_type=content_type,
|
|
modified_at=_timestamp(value.get("mtime") or value.get("modified")),
|
|
etag=_clean(value.get("id") or value.get("obj_id")),
|
|
metadata={
|
|
key: value[key]
|
|
for key in ("permission", "modifier_email", "modifier_name")
|
|
if key in value
|
|
},
|
|
)
|
|
|
|
|
|
def _static_listing(profile: ConnectorProfile, *, path: str, library_id: str | None) -> list[ConnectorBrowseItem] | None:
|
|
listing = profile.metadata.get("static_listing")
|
|
if listing is None:
|
|
return None
|
|
if isinstance(listing, list):
|
|
raw_items = listing if path == "" else []
|
|
elif isinstance(listing, Mapping):
|
|
key = library_id or path
|
|
raw_items = listing.get(key)
|
|
if raw_items is None and key == "":
|
|
raw_items = listing.get("/")
|
|
else:
|
|
raise ConnectorBrowseError("Connector static_listing metadata must be a list or object")
|
|
if raw_items is None:
|
|
return []
|
|
if not isinstance(raw_items, list):
|
|
raise ConnectorBrowseError("Connector static_listing entries must be lists")
|
|
return sorted(
|
|
[_static_item(item, parent_path=path) for item in raw_items if isinstance(item, Mapping)],
|
|
key=lambda item: (item.kind != "library", item.kind != "folder", item.name.casefold(), item.path.casefold()),
|
|
)
|
|
|
|
|
|
def _static_item(value: Mapping[str, Any], *, parent_path: str) -> ConnectorBrowseItem:
|
|
kind = str(value.get("kind") or "file").strip().casefold()
|
|
if kind not in {"library", "folder", "file"}:
|
|
raise ConnectorBrowseError("Connector browse items must be library, folder or file")
|
|
name = str(value.get("name") or value.get("path") or "").strip()
|
|
if not name:
|
|
raise ConnectorBrowseError("Connector browse items require name")
|
|
path = normalize_connector_browse_path(value.get("path"))
|
|
if not path:
|
|
path = _join_browse_path(parent_path, name)
|
|
return ConnectorBrowseItem(
|
|
kind=kind,
|
|
name=name,
|
|
path=path,
|
|
external_id=_clean(value.get("external_id")),
|
|
external_url=_clean(value.get("external_url")),
|
|
size_bytes=_int(value.get("size_bytes")),
|
|
content_type=_clean(value.get("content_type")),
|
|
modified_at=_clean(value.get("modified_at")),
|
|
etag=_clean(value.get("etag")),
|
|
metadata=value.get("metadata") if isinstance(value.get("metadata"), Mapping) else {},
|
|
)
|
|
|
|
|
|
def _webdav_url(root_url: str, path: str) -> str:
|
|
base = root_url if root_url.endswith("/") else root_url + "/"
|
|
if not path:
|
|
return base
|
|
quoted = "/".join(quote(part, safe="") for part in path.split("/") if part)
|
|
return urljoin(base, quoted + "/")
|
|
|
|
|
|
def _url_path_root(root_url: str) -> str:
|
|
path = unquote(urlsplit(root_url).path or "/")
|
|
return path if path.endswith("/") else path + "/"
|
|
|
|
|
|
def _relative_href_path(root_path: str, href: str) -> str:
|
|
href_path = unquote(urlsplit(href).path or href).strip()
|
|
if href_path.startswith(root_path):
|
|
return normalize_connector_browse_path(href_path[len(root_path):])
|
|
return normalize_connector_browse_path(href_path.rsplit("/", 1)[-1])
|
|
|
|
|
|
def _webdav_prop(response: ET.Element) -> ET.Element:
|
|
prop = response.find("{DAV:}propstat/{DAV:}prop")
|
|
if prop is not None:
|
|
return prop
|
|
prop = response.find(".//{DAV:}prop")
|
|
return prop if prop is not None else ET.Element("prop")
|
|
|
|
|
|
def _webdav_display_name(prop: ET.Element | None) -> str | None:
|
|
return _clean(_text(prop, "{DAV:}displayname"))
|
|
|
|
|
|
def _text(prop: ET.Element | None, tag: str) -> str | None:
|
|
if prop is None:
|
|
return None
|
|
child = prop.find(tag)
|
|
return child.text.strip() if child is not None and child.text else None
|
|
|
|
|
|
def _http_date(value: str | None) -> str | None:
|
|
if not value:
|
|
return None
|
|
try:
|
|
return parsedate_to_datetime(value).isoformat()
|
|
except (TypeError, ValueError):
|
|
return value
|
|
|
|
|
|
def _timestamp(value: object) -> str | None:
|
|
number = _int(value)
|
|
if number is None:
|
|
return _clean(value)
|
|
return datetime.fromtimestamp(number, tz=timezone.utc).isoformat()
|
|
|
|
|
|
def _metadata_string(profile: ConnectorProfile, key: str) -> str | None:
|
|
return _clean(profile.metadata.get(key))
|
|
|
|
|
|
def _metadata_bool(profile: ConnectorProfile, key: str, *, default: bool = False) -> bool:
|
|
value = profile.metadata.get(key)
|
|
if value is None:
|
|
return default
|
|
if isinstance(value, bool):
|
|
return value
|
|
return str(value).strip().casefold() in {"1", "true", "yes", "on"}
|
|
|
|
|
|
def _metadata_env(profile: ConnectorProfile, key: str) -> str | None:
|
|
env_name = _metadata_string(profile, key)
|
|
if not env_name:
|
|
return None
|
|
return _env_required(env_name, profile)
|
|
|
|
|
|
def _env_required(name: str, profile: ConnectorProfile) -> str:
|
|
try:
|
|
return connector_secret_env_value(name, source_kind=profile.source_kind)
|
|
except ConnectorDeploymentConfigurationError as exc:
|
|
raise ConnectorBrowseError(f"Connector profile {profile.id}: {exc}") from exc
|
|
|
|
|
|
def normalize_connector_browse_path(value: object) -> str:
|
|
if value is None:
|
|
return ""
|
|
path = str(value).replace("\\", "/").strip().strip("/")
|
|
parts = [part for part in path.split("/") if part and part not in {"."}]
|
|
if any(part == ".." for part in parts):
|
|
raise ConnectorBrowseError("Connector browse paths cannot contain parent directory segments")
|
|
return "/".join(parts)
|
|
|
|
|
|
def _join_browse_path(parent: str, name: str) -> str:
|
|
child = normalize_connector_browse_path(name)
|
|
if not parent:
|
|
return child
|
|
return f"{parent.rstrip('/')}/{child}"
|
|
|
|
|
|
def _path_name(path: str) -> str:
|
|
return path.rstrip("/").rsplit("/", 1)[-1]
|
|
|
|
|
|
def _int(value: object) -> int | None:
|
|
if value is None or value == "":
|
|
return None
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _clean(value: object) -> str | None:
|
|
if value is None:
|
|
return None
|
|
text = str(value).strip()
|
|
return text or None
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _SmbLocation:
|
|
server: str
|
|
share: str
|
|
port: int
|
|
root_path: str = ""
|
|
|
|
|
|
def _smb_location(profile: ConnectorProfile) -> _SmbLocation:
|
|
if not profile.endpoint_url:
|
|
raise ConnectorBrowseError("SMB connector profile does not define endpoint_url")
|
|
parsed = urlsplit(profile.endpoint_url)
|
|
if parsed.scheme.casefold() != "smb":
|
|
raise ConnectorBrowseError("SMB connector endpoint_url must use smb://")
|
|
server = _clean(parsed.hostname)
|
|
if not server:
|
|
raise ConnectorBrowseError("SMB connector endpoint_url must include a server")
|
|
port = parsed.port or _int(profile.metadata.get("port")) or 445
|
|
try:
|
|
validate_unpinned_sdk_host(server, port=port, label="SMB connector endpoint")
|
|
except OutboundHttpError as exc:
|
|
raise ConnectorBrowseError(str(exc)) from exc
|
|
path_parts = [part for part in unquote(parsed.path or "").strip("/").split("/") if part]
|
|
if not path_parts:
|
|
raise ConnectorBrowseError("SMB connector endpoint_url must include a share name")
|
|
root_parts = path_parts[1:]
|
|
if profile.base_path:
|
|
root_parts.extend(normalize_connector_browse_path(profile.base_path).split("/"))
|
|
return _SmbLocation(
|
|
server=server,
|
|
share=path_parts[0],
|
|
port=port,
|
|
root_path=normalize_connector_browse_path("/".join(root_parts)),
|
|
)
|
|
|
|
|
|
def _smb_unc_path(location: _SmbLocation, path: str) -> str:
|
|
parts = [part for part in (location.root_path, normalize_connector_browse_path(path)) if part]
|
|
suffix = "\\".join(part.replace("/", "\\") for part in parts)
|
|
base = f"\\\\{location.server}\\{location.share}"
|
|
return f"{base}\\{suffix}" if suffix else base
|
|
|
|
|
|
def _smb_client_kwargs(profile: ConnectorProfile, location: _SmbLocation) -> dict[str, object]:
|
|
kwargs: dict[str, object] = {
|
|
"port": location.port,
|
|
"require_signing": _metadata_bool(profile, "require_signing", default=True),
|
|
"auth_protocol": _metadata_string(profile, "auth_protocol") or "ntlm",
|
|
}
|
|
if "encrypt" in profile.metadata:
|
|
kwargs["encrypt"] = _metadata_bool(profile, "encrypt", default=False)
|
|
password = _profile_password(profile)
|
|
token = _profile_token(profile)
|
|
if profile.username and password:
|
|
kwargs["username"] = profile.username
|
|
kwargs["password"] = password
|
|
elif token:
|
|
raise ConnectorBrowseError("SMB connector profiles do not support bearer-token credentials")
|
|
elif profile.credential_mode.casefold() not in {"", "none", "anonymous"} and profile.secret_ref:
|
|
raise ConnectorBrowseError("Secret-ref SMB credentials need a runtime secret resolver before live browsing")
|
|
return kwargs
|
|
|
|
|
|
def _profile_password(profile: ConnectorProfile) -> str | None:
|
|
if profile.password_value:
|
|
return profile.password_value
|
|
if profile.password_env:
|
|
return _env_required(profile.password_env, profile)
|
|
return None
|
|
|
|
|
|
def _profile_token(profile: ConnectorProfile) -> str | None:
|
|
if profile.token_value:
|
|
return profile.token_value
|
|
if profile.token_env:
|
|
return _env_required(profile.token_env, profile)
|
|
return None
|
|
|
|
|
|
def _smbclient_module() -> Any:
|
|
try:
|
|
return import_module("smbclient")
|
|
except ImportError as exc:
|
|
raise ConnectorBrowseUnsupported("SMB connector browsing requires the optional smbprotocol dependency") from exc
|
|
|
|
|
|
def _smb_entry_stat(entry: object) -> object | None:
|
|
try:
|
|
return entry.stat() # type: ignore[attr-defined]
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _smb_stat_size(stat_result: object | None) -> int | None:
|
|
return _int(getattr(stat_result, "st_size", None))
|
|
|
|
|
|
def _smb_stat_modified_at(stat_result: object | None) -> str | None:
|
|
value = getattr(stat_result, "st_mtime", None)
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return datetime.fromtimestamp(float(value), tz=timezone.utc).isoformat()
|
|
except (TypeError, ValueError, OSError, OverflowError):
|
|
return None
|
|
|
|
|
|
def _smb_stat_revision(stat_result: object | None) -> str | None:
|
|
mtime_ns = getattr(stat_result, "st_mtime_ns", None)
|
|
size = getattr(stat_result, "st_size", None)
|
|
if mtime_ns is not None:
|
|
return f"{mtime_ns}:{size or 0}"
|
|
modified = getattr(stat_result, "st_mtime", None)
|
|
if modified is not None:
|
|
return f"{modified}:{size or 0}"
|
|
return None
|