govoplan-files/src/govoplan_files/backend/route_support.py

from __future__ import annotations

import json
import os
import tempfile
from typing import Any, Literal
from urllib.parse import quote, urljoin, urlsplit, urlunsplit
from fastapi import HTTPException, UploadFile, status
from sqlalchemy import func, or_
from sqlalchemy.orm import Session

from govoplan_core.auth import ApiPrincipal, has_scope
from govoplan_core.api.v1.schemas import DeltaDeletedItem
from govoplan_core.audit.logging import audit_from_principal
from govoplan_core.core.campaigns import (
    CAPABILITY_CAMPAIGNS_ACCESS,
    CampaignAccessProvider,
)
from govoplan_core.core.change_sequence import (
    ChangeSequenceEntry,
    decode_sequence_watermark,
    encode_sequence_watermark,
    record_change,
    sequence_watermark_is_expired,
)
from govoplan_files.backend.change_tracking import (
    FILES_CONNECTOR_CREDENTIALS_COLLECTION,
    FILES_CONNECTOR_POLICIES_COLLECTION,
    FILES_CONNECTOR_PROFILES_COLLECTION,
    FILES_CONNECTOR_SPACES_COLLECTION,
    FILES_MODULE_ID,
)
from govoplan_files.backend.schemas import (
    FileAssetResponse,
    FileConnectorCredentialResponse,
    FileConnectorDiscoveryRequest,
    FileConnectorSpaceResponse,
    FileConnectorImportRequest,
    FileFolderResponse,
    FileShareResponse,
    FileSpaceResponse,
)
from govoplan_files.backend.db.models import (
    CampaignAttachmentUse,
    FileAsset,
    FileBlob,
    FileConnectorSpace,
    FileFolder,
    FileShare,
    FileVersion,
)
from govoplan_files.backend.runtime import get_registry, settings
from govoplan_files.backend.storage.access import ensure_group_access, user_group_ids
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_credential_store import (
    connector_credential_from_row,
    get_connector_credential_row,
    list_database_connector_credentials,
    resolve_reusable_connector_credential,
    reusable_credential_id,
)
from govoplan_files.backend.storage.connector_browse import (
    normalize_connector_browse_path,
)
from govoplan_files.backend.storage.connector_imports import read_connector_file
from govoplan_files.backend.storage.connector_deployment import (
    connector_effective_endpoint_url,
)
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
from govoplan_files.backend.storage.connector_visibility import (
    visible_connector_profiles_for_actor,
)
from govoplan_files.backend.storage.connector_spaces import (
    connector_space_owner_id,
    list_connector_spaces_for_user,
)
from govoplan_files.backend.storage.connector_policy import (
    ConnectorAccessRequest,
    ConnectorPolicyDenied,
    connector_policy_decision,
    connector_policy_sources_for_fields,
    connector_policy_sources_from_payload,
    ensure_connector_policy_allows,
)
from govoplan_files.backend.storage.connector_policy_store import (
    effective_connector_policy_sources,
    parent_connector_policy,
    validate_connector_policy_allowed_by_parent,
)
from govoplan_files.backend.storage.files import (
    asset_is_audit_relevant,
    current_version_and_blob,
)
from govoplan_files.backend.storage.provenance import (
    source_metadata,
    source_provenance_from_metadata,
    source_revision_from_metadata,
)
from govoplan_files.backend.storage.share_state import file_share_is_active


FILES_CONNECTOR_SETTINGS_COLLECTIONS = (
    FILES_CONNECTOR_PROFILES_COLLECTION,
    FILES_CONNECTOR_CREDENTIALS_COLLECTION,
    FILES_CONNECTOR_POLICIES_COLLECTION,
    FILES_CONNECTOR_SPACES_COLLECTION,
)
FILES_CONNECTOR_PROFILE_RESOURCE = "file_connector_profile"
FILES_CONNECTOR_CREDENTIAL_RESOURCE = "file_connector_credential"
FILES_CONNECTOR_POLICY_RESOURCE = "file_connector_policy"
FILES_CONNECTOR_SPACE_RESOURCE = "file_connector_space"


def _campaign_access_provider() -> CampaignAccessProvider:
    registry = get_registry()
    if (
        registry is None
        or not hasattr(registry, "has_capability")
        or not registry.has_capability(CAPABILITY_CAMPAIGNS_ACCESS)
    ):
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST,
            detail="Campaign module is not installed",
        )
    capability = registry.require_capability(CAPABILITY_CAMPAIGNS_ACCESS)
    if not isinstance(capability, CampaignAccessProvider):
        raise HTTPException(
            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
            detail="Campaign access capability is invalid",
        )
    return capability


def _is_admin(principal: ApiPrincipal) -> bool:
    return has_scope(principal, "files:file:admin")


def _can_read_disabled_connector_profiles(principal: ApiPrincipal) -> bool:
    return any(
        has_scope(principal, scope)
        for scope in (
            "files:file:admin",
            "system:settings:read",
            "system:settings:write",
            "admin:settings:read",
            "admin:settings:write",
        )
    )


def _record_connector_settings_change(
    session: Session,
    *,
    collection: str,
    resource_type: str,
    resource_id: str | None,
    operation: str,
    principal: ApiPrincipal,
    tenant_id: str | None,
    payload: dict[str, object] | None = None,
) -> None:
    if not resource_id:
        return
    record_change(
        session,
        module_id=FILES_MODULE_ID,
        collection=collection,
        resource_type=resource_type,
        resource_id=resource_id,
        operation=operation,
        tenant_id=tenant_id,
        actor_type="user",
        actor_id=principal.user.id,
        payload=payload or {},
    )


def _file_connector_settings_query(
    session: Session, *, tenant_id: str, since_sequence: int
):
    return session.query(ChangeSequenceEntry).filter(
        ChangeSequenceEntry.id > since_sequence,
        ChangeSequenceEntry.module_id == FILES_MODULE_ID,
        ChangeSequenceEntry.collection.in_(FILES_CONNECTOR_SETTINGS_COLLECTIONS),
        or_(
            ChangeSequenceEntry.tenant_id == tenant_id,
            ChangeSequenceEntry.tenant_id.is_(None),
        ),
    )


def _file_connector_settings_watermark(session: Session, *, tenant_id: str) -> str:
    sequence_id = (
        _file_connector_settings_query(session, tenant_id=tenant_id, since_sequence=0)
        .with_entities(func.max(ChangeSequenceEntry.id))
        .scalar()
    )
    return encode_sequence_watermark(int(sequence_id or 0))


def _file_connector_settings_entries(
    session: Session, *, tenant_id: str, since: str, limit: int
):
    try:
        since_sequence = decode_sequence_watermark(since)
    except ValueError as exc:
        raise HTTPException(
            status_code=status.HTTP_400_BAD_REQUEST, detail=str(exc)
        ) from exc
    expired_for_tenant = sequence_watermark_is_expired(
        session,
        since=since_sequence,
        tenant_id=tenant_id,
        module_id=FILES_MODULE_ID,
        collections=FILES_CONNECTOR_SETTINGS_COLLECTIONS,
    )
    expired_for_system = sequence_watermark_is_expired(
        session,
        since=since_sequence,
        tenant_id=None,
        module_id=FILES_MODULE_ID,
        collections=FILES_CONNECTOR_SETTINGS_COLLECTIONS,
    )
    if expired_for_tenant or expired_for_system:
        return None, False
    entries_plus_one = (
        _file_connector_settings_query(
            session,
            tenant_id=tenant_id,
            since_sequence=since_sequence,
        )
        .order_by(ChangeSequenceEntry.id.asc())
        .limit(limit + 1)
        .all()
    )
    has_more = len(entries_plus_one) > limit
    return entries_plus_one[:limit], has_more


def _file_connector_settings_response_watermark(
    session: Session, *, tenant_id: str, entries, has_more: bool
) -> str:
    return (
        encode_sequence_watermark(entries[-1].id)
        if has_more and entries
        else _file_connector_settings_watermark(session, tenant_id=tenant_id)
    )


def _connector_deleted_item(entry) -> DeltaDeletedItem:
    return DeltaDeletedItem(
        id=entry.resource_id,
        resource_type=entry.resource_type,
        revision=encode_sequence_watermark(entry.id),
        deleted_at=entry.created_at if entry.operation == "deleted" else None,
    )


def _visible_connector_credentials(
    session: Session,
    principal: ApiPrincipal,
    *,
    provider: str | None = None,
    include_disabled: bool = False,
):
    provider_norm = provider.strip().casefold() if provider else None
    credentials = list_database_connector_credentials(
        session,
        tenant_id=principal.tenant_id,
        include_disabled=include_disabled
        and _can_read_disabled_connector_profiles(principal),
    )
    return [
        credential
        for credential in credentials
        if provider_norm is None or credential.provider in {None, provider_norm}
    ]


def _visible_connector_spaces(
    session: Session,
    principal: ApiPrincipal,
    *,
    owner_type: Literal["user", "group"] | None = None,
    owner_id: str | None = None,
    include_inactive: bool = False,
) -> list[FileConnectorSpace]:
    if not (
        has_scope(principal, "files:file:read")
        or has_scope(principal, "files:file:admin")
    ):
        return []
    return list_connector_spaces_for_user(
        session,
        tenant_id=principal.tenant_id,
        user_id=principal.user.id,
        owner_type=owner_type,
        owner_id=owner_id,
        include_inactive=include_inactive and _is_admin(principal),
        is_admin=_is_admin(principal),
    )


def _file_connector_policy_resource_id(scope_type: str, scope_id: str | None) -> str:
    return f"{scope_type.strip().casefold()}:{scope_id or ''}"


def _read_limited_upload(upload: UploadFile, *, max_bytes: int) -> bytes:
    data = upload.file.read(max_bytes + 1)
    if len(data) > max_bytes:
        raise HTTPException(
            status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
            detail=f"Upload exceeds limit of {max_bytes} bytes",
        )
    return data


def _spool_limited_upload_to_temp(
    upload: UploadFile, *, max_bytes: int, suffix: str = ".upload"
) -> str:
    tmp = tempfile.NamedTemporaryFile(
        prefix="govoplan-upload-", suffix=suffix, delete=False
    )
    total = 0
    try:
        while True:
            chunk = upload.file.read(1024 * 1024)
            if not chunk:
                break
            total += len(chunk)
            if total > max_bytes:
                raise HTTPException(
                    status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
                    detail=f"Upload exceeds limit of {max_bytes} bytes",
                )
            tmp.write(chunk)
    except Exception:
        tmp.close()
        _cleanup_temp_file(tmp.name)
        raise
    tmp.close()
    return tmp.name


def _cleanup_temp_file(path: str) -> None:
    try:
        os.unlink(path)
    except FileNotFoundError:
        pass


def _attachment_disposition(filename: str) -> str:
    safe = (
        filename.replace("\\", "_")
        .replace("/", "_")
        .replace("\r", "_")
        .replace("\n", "_")
        .strip()
        or "download"
    )
    ascii_name = (
        "".join(
            char if 32 <= ord(char) < 127 and char not in {'"', "\\", ";"} else "_"
            for char in safe
        ).strip()
        or "download"
    )
    encoded = quote(safe, safe="")
    return f'attachment; filename="{ascii_name}"; filename*=UTF-8' + "''" + encoded


def _ensure_campaign_file_access(
    session: Session, principal: ApiPrincipal, campaign_id: str | None
) -> None:
    if not campaign_id:
        return
    if not has_scope(principal, "campaigns:campaign:read"):
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: campaign:read"
        )
    provider = _campaign_access_provider()
    if not provider.campaign_exists(
        session, tenant_id=principal.tenant_id, campaign_id=campaign_id
    ):
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND, detail="Campaign not found"
        )
    group_ids = set(
        user_group_ids(
            session, tenant_id=principal.tenant_id, user_id=principal.user.id
        )
    )
    if provider.can_read_campaign(
        session,
        tenant_id=principal.tenant_id,
        campaign_id=campaign_id,
        user_id=principal.user.id,
        group_ids=group_ids,
        tenant_admin=has_scope(principal, "tenant:*"),
    ):
        return
    raise HTTPException(
        status_code=status.HTTP_403_FORBIDDEN, detail="No access to this campaign"
    )


def _http_error(exc: Exception, *, not_found: bool = False) -> HTTPException:
    code = status.HTTP_404_NOT_FOUND if not_found else status.HTTP_400_BAD_REQUEST
    return HTTPException(status_code=code, detail=str(exc))


def _connector_policy_error(exc: ConnectorPolicyDenied) -> HTTPException:
    return HTTPException(
        status_code=status.HTTP_403_FORBIDDEN, detail=exc.decision.to_dict()
    )


def _owner_id(asset: FileAsset) -> str:
    return asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id  # type: ignore[return-value]


def _source_metadata_from_form(
    source_provenance_json: str | None, source_revision: str | None
) -> dict[str, object] | None:
    source_provenance = (
        json.loads(source_provenance_json) if source_provenance_json else None
    )
    return source_metadata(
        source_provenance=source_provenance, source_revision=source_revision
    )


def _connector_policy_sources_from_form(connector_policy_json: str | None):
    if not connector_policy_json:
        return []
    return connector_policy_sources_from_payload(json.loads(connector_policy_json))


def _enforce_connector_policy(
    source_provenance_json: str | None,
    connector_policy_json: str | None,
    *,
    operation: str,
) -> None:
    if not source_provenance_json:
        return
    sources = _connector_policy_sources_from_form(connector_policy_json)
    if not sources:
        return
    provenance = json.loads(source_provenance_json)
    request = ConnectorAccessRequest.from_provenance(
        provenance if isinstance(provenance, dict) else {}, operation=operation
    )
    ensure_connector_policy_allows(request, sources)


def _visible_connector_profiles(
    session: Session,
    principal: ApiPrincipal,
    *,
    provider: str | None = None,
    campaign_id: str | None = None,
    include_disabled: bool = False,
    include_admin_scopes: bool = False,
    include_effective_policy: bool = True,
) -> list[ConnectorProfile]:
    group_ids = user_group_ids(
        session,
        tenant_id=principal.tenant_id,
        user_id=principal.user.id,
        include_admin_groups=_is_admin(principal),
    )

    def campaign_visible(profile_campaign_id: str) -> bool:
        try:
            _ensure_campaign_file_access(session, principal, profile_campaign_id)
        except HTTPException:
            return False
        return True

    return visible_connector_profiles_for_actor(
        session,
        tenant_id=principal.tenant_id,
        user_id=principal.user.id,
        group_ids=group_ids,
        settings=settings,
        provider=provider,
        campaign_id=campaign_id,
        campaign_visible=campaign_visible,
        include_disabled=include_disabled,
        include_admin_scopes=include_admin_scopes,
        include_effective_policy=include_effective_policy,
    )


def _webdav_discovery_candidates(payload: FileConnectorDiscoveryRequest) -> list[str]:
    endpoint_url = str(payload.endpoint_url or "").strip()
    if not endpoint_url:
        return []
    direct_url = endpoint_url if endpoint_url.endswith("/") else endpoint_url + "/"
    candidates = [direct_url]
    if payload.provider == "nextcloud":
        root_url = _nextcloud_root_url(direct_url)
        username = (payload.credentials.username or "").strip()
        if username:
            candidates.append(
                urljoin(root_url, f"remote.php/dav/files/{quote(username, safe='')}/")
            )
        candidates.append(urljoin(root_url, "remote.php/webdav/"))
    seen: set[str] = set()
    unique: list[str] = []
    for candidate in candidates:
        normalized = candidate.rstrip("/")
        if not normalized or normalized in seen:
            continue
        seen.add(normalized)
        unique.append(candidate)
    return unique


def _nextcloud_root_url(endpoint_url: str) -> str:
    marker = "/remote.php/"
    if marker in endpoint_url:
        return endpoint_url.split(marker, 1)[0].rstrip("/") + "/"
    return endpoint_url.rstrip("/") + "/"


def _same_endpoint(left: str, right: str) -> bool:
    return left.strip().rstrip("/") == right.strip().rstrip("/")


def _discovery_profile_from_payload(
    payload: FileConnectorDiscoveryRequest,
    endpoint_url: str,
    *,
    principal: ApiPrincipal,
) -> ConnectorProfile:
    credentials = payload.credentials
    return ConnectorProfile(
        id=f"discovery-{principal.tenant_id}",
        label="Connector discovery",
        provider=payload.provider,
        endpoint_url=endpoint_url,
        base_path=payload.base_path,
        scope_type="tenant",
        scope_id=principal.tenant_id,
        credential_mode=payload.credential_mode,
        username=credentials.username,
        password_env=credentials.password_env,
        token_env=credentials.token_env,
        secret_ref=credentials.secret_ref,
        password_value=credentials.password,
        token_value=credentials.token,
        capabilities=("browse",),
        # Discovery metadata is untrusted API input and must not be able to
        # replace the candidate endpoint or inject a static/fake listing.
        metadata={},
        source_kind="discovery",
    )


def _audit_connector_discovery_attempt(
    session: Session,
    principal: ApiPrincipal,
    *,
    provider: str,
    endpoint_url: str,
    base_path: str | None,
) -> None:
    audit_from_principal(
        session,
        principal,
        action="files.connector.discovery_attempted",
        object_type="connector_endpoint",
        object_id=provider,
        details={
            "provider": provider,
            "endpoint_origin": _redacted_connector_url(endpoint_url),
            "base_path": base_path,
        },
        commit=True,
    )


def _redacted_connector_url(value: str) -> str:
    try:
        parsed = urlsplit(value)
        hostname = parsed.hostname or ""
        if ":" in hostname:
            hostname = f"[{hostname}]"
        netloc = f"{hostname}:{parsed.port}" if parsed.port is not None else hostname
        return urlunsplit((parsed.scheme, netloc, "", "", ""))
    except ValueError:
        return "<invalid connector URL>"


def _visible_connector_profile(
    session: Session,
    principal: ApiPrincipal,
    profile_id: str,
    *,
    campaign_id: str | None = None,
    include_disabled: bool = False,
    include_effective_policy: bool = True,
) -> ConnectorProfile:
    for profile in _visible_connector_profiles(
        session,
        principal,
        campaign_id=campaign_id,
        include_disabled=include_disabled,
        include_admin_scopes=False,
        include_effective_policy=include_effective_policy,
    ):
        if profile.id == profile_id:
            return profile
    raise HTTPException(
        status_code=status.HTTP_404_NOT_FOUND, detail="Connector profile not found"
    )


def _require_connector_profile_write(principal: ApiPrincipal, scope_type: str) -> None:
    clean_scope = scope_type.strip().casefold()
    if clean_scope == "system":
        if has_scope(principal, "system:settings:write") or has_scope(
            principal, "files:file:admin"
        ):
            return
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Missing scope: system:settings:write",
        )
    if (
        has_scope(principal, "files:file:admin")
        or has_scope(principal, "admin:settings:write")
        or has_scope(principal, "system:settings:write")
    ):
        return
    raise HTTPException(
        status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: files:file:admin"
    )


def _require_connector_credential_write(
    principal: ApiPrincipal, scope_type: str
) -> None:
    _require_connector_profile_write(principal, scope_type)


def _require_connector_policy_read(principal: ApiPrincipal, scope_type: str) -> None:
    clean_scope = scope_type.strip().casefold()
    if clean_scope == "system":
        if any(
            has_scope(principal, scope)
            for scope in (
                "files:file:admin",
                "system:settings:read",
                "system:settings:write",
            )
        ):
            return
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="Missing scope: system:settings:read",
        )
    if any(
        has_scope(principal, scope)
        for scope in (
            "files:file:admin",
            "admin:settings:read",
            "admin:settings:write",
            "system:settings:read",
            "system:settings:write",
        )
    ):
        return
    raise HTTPException(
        status_code=status.HTTP_403_FORBIDDEN, detail="Missing scope: files:file:admin"
    )


def _connector_policy_configure_sources(
    session: Session,
    principal: ApiPrincipal,
    *,
    scope_type: str,
    scope_id: str | None,
    fields: tuple[str, ...],
):
    sources = effective_connector_policy_sources(
        session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id
    )
    return connector_policy_sources_for_fields(sources, fields)


def _ensure_connector_configuration_allowed(
    session: Session,
    principal: ApiPrincipal,
    *,
    connector_id: str | None,
    credential_id: str | None,
    provider: str | None,
    endpoint_url: str | None,
    base_path: str | None,
    scope_type: str,
    scope_id: str | None,
    operation: str,
) -> None:
    sources = _connector_policy_configure_sources(
        session,
        principal,
        scope_type=scope_type,
        scope_id=scope_id,
        fields=(
            "connectors",
            "credentials",
            "providers",
            "external_paths",
            "external_urls",
        ),
    )
    if not sources:
        return
    ensure_connector_policy_allows(
        ConnectorAccessRequest(
            connector_id=connector_id,
            credential_id=credential_id,
            provider=provider,
            external_path=base_path,
            external_url=endpoint_url,
            operation=operation,
        ),
        sources,
    )


def _ensure_connector_credential_configuration_allowed(
    session: Session,
    principal: ApiPrincipal,
    *,
    credential_id: str,
    provider: str | None,
    scope_type: str,
    scope_id: str | None,
    operation: str,
) -> None:
    sources = _connector_policy_configure_sources(
        session,
        principal,
        scope_type=scope_type,
        scope_id=scope_id,
        fields=("credentials", "providers"),
    )
    if not sources:
        return
    ensure_connector_policy_allows(
        ConnectorAccessRequest(
            credential_id=credential_id,
            provider=provider,
            operation=operation,
        ),
        sources,
    )


def _ensure_connector_local_policy_allowed(
    session: Session,
    principal: ApiPrincipal,
    *,
    scope_type: str,
    scope_id: str | None,
    policy: dict[str, Any] | None,
) -> None:
    if policy is None:
        return
    validate_connector_policy_allowed_by_parent(
        parent_connector_policy(
            session,
            tenant_id=principal.tenant_id,
            scope_type=scope_type,
            scope_id=scope_id,
        ),
        policy,
    )


def _connector_credential_response(row) -> FileConnectorCredentialResponse:
    return FileConnectorCredentialResponse(
        **connector_credential_from_row(row).to_response()
    )


def _credential_row_for_profile(
    session: Session,
    principal: ApiPrincipal,
    *,
    credential_profile_id: str | None,
    provider: str,
    profile_id: str | None = None,
    scope_type: str = "tenant",
    scope_id: str | None = None,
    include_disabled: bool = False,
):
    credential_id = (credential_profile_id or "").strip()
    if not credential_id:
        return None
    if reusable_credential_id(credential_id):
        if not profile_id:
            raise FileStorageError(
                "File connection id is required for reusable credentials"
            )
        return resolve_reusable_connector_credential(
            session,
            tenant_id=principal.tenant_id,
            credential_ref=credential_id,
            profile_id=profile_id,
            scope_type=scope_type,
            scope_id=scope_id,
        )
    row = get_connector_credential_row(
        session,
        tenant_id=principal.tenant_id,
        credential_id=credential_id,
        include_disabled=include_disabled,
    )
    if row.provider and row.provider != provider:
        raise FileStorageError(
            f"Connector credential {credential_id} is limited to {row.provider} connections"
        )
    return row


def _download_connector_payload(
    profile: ConnectorProfile,
    payload: FileConnectorImportRequest,
    *,
    operation: str,
) -> tuple[str, Any, dict[str, Any]]:
    source_path = normalize_connector_browse_path(payload.path)
    decision = connector_policy_decision(
        ConnectorAccessRequest(
            connector_id=profile.id,
            credential_id=profile.credential_profile_id,
            provider=profile.provider,
            external_id=f"{payload.library_id}:{source_path}",
            external_path=source_path,
            external_url=connector_effective_endpoint_url(
                provider=profile.provider,
                endpoint_url=profile.endpoint_url,
                metadata=profile.metadata,
            ),
            operation=operation,
        ),
        profile.policy_sources,
    )
    if not decision.allowed:
        raise ConnectorPolicyDenied(decision)
    downloaded = read_connector_file(
        profile,
        library_id=payload.library_id,
        path=source_path,
        max_bytes=settings.file_upload_max_bytes,
    )
    provenance_metadata = {
        "profile_id": profile.id,
        "library_id": payload.library_id,
        "library_path": source_path,
        **downloaded.metadata,
        **payload.metadata,
    }
    metadata = source_metadata(
        source_provenance={
            "source_type": "connector",
            "connector_id": profile.id,
            "provider": profile.provider,
            "external_id": downloaded.external_id
            or f"{payload.library_id}:{source_path}",
            "external_path": source_path,
            "external_url": downloaded.external_url,
            "metadata": provenance_metadata,
        },
        source_revision=payload.source_revision or downloaded.revision,
    )
    return source_path, downloaded, metadata or {}


def _connector_browse_next_token(items: list[Any]) -> str | None:
    for item in items:
        token = (
            item.metadata.get("next_continuation_token")
            if hasattr(item, "metadata")
            else None
        )
        if token:
            return str(token)
    return None


def _connector_audit_details(
    asset: FileAsset, version: FileVersion, blob: FileBlob, *, operation: str
) -> dict[str, object] | None:
    metadata = asset.metadata_ or {}
    provenance = source_provenance_from_metadata(metadata)
    if not provenance:
        return None
    return {
        "operation": operation,
        "file_id": asset.id,
        "display_path": asset.display_path,
        "version_id": version.id,
        "blob_id": blob.id,
        "checksum_sha256": blob.checksum_sha256,
        "size_bytes": blob.size_bytes,
        "source_revision": source_revision_from_metadata(metadata),
        "source_provenance": provenance,
    }


def _audit_connector_event(
    session: Session,
    principal: ApiPrincipal,
    *,
    action: str,
    asset: FileAsset,
    version: FileVersion,
    blob: FileBlob,
    operation: str,
    extra_details: dict[str, object] | None = None,
    commit: bool = False,
) -> bool:
    details = _connector_audit_details(asset, version, blob, operation=operation)
    if details is None:
        return False
    if extra_details:
        details.update(extra_details)
    audit_from_principal(
        session,
        principal,
        action=action,
        object_type="file",
        object_id=asset.id,
        details=details,
        commit=commit,
    )
    return True


def _audit_connector_imports(
    session: Session, principal: ApiPrincipal, assets: list[FileAsset]
) -> None:
    for asset in assets:
        version, blob = current_version_and_blob(session, asset)
        _audit_connector_event(
            session,
            principal,
            action="files.connector.imported",
            asset=asset,
            version=version,
            blob=blob,
            operation="import",
        )


def _audit_connector_sync(
    session: Session,
    principal: ApiPrincipal,
    asset: FileAsset,
    *,
    sync_action: str,
    previous_version_id: str | None,
) -> None:
    version, blob = current_version_and_blob(session, asset)
    _audit_connector_event(
        session,
        principal,
        action="files.connector.synced",
        asset=asset,
        version=version,
        blob=blob,
        operation="sync",
        extra_details={
            "sync_action": sync_action,
            "previous_version_id": previous_version_id,
        },
    )


def _audit_connector_access(
    session: Session,
    principal: ApiPrincipal,
    assets: list[FileAsset],
    *,
    operation: str,
) -> None:
    recorded = False
    for asset in assets:
        version, blob = current_version_and_blob(session, asset)
        recorded = (
            _audit_connector_event(
                session,
                principal,
                action="files.connector.accessed",
                asset=asset,
                version=version,
                blob=blob,
                operation=operation,
            )
            or recorded
        )
    if recorded:
        session.commit()


def _asset_response(
    session: Session, asset: FileAsset, *, include_shares: bool = False
) -> FileAssetResponse:
    version, blob = current_version_and_blob(session, asset)
    metadata = asset.metadata_ or {}
    shares: list[FileShareResponse] = []
    if include_shares:
        rows = (
            session.query(FileShare)
            .filter(FileShare.file_asset_id == asset.id)
            .order_by(FileShare.created_at.desc())
            .all()
        )
        shares = [_file_share_response(row) for row in rows]
    return FileAssetResponse(
        id=asset.id,
        tenant_id=asset.tenant_id,
        owner_type=asset.owner_type,
        owner_id=_owner_id(asset),
        display_path=asset.display_path,
        filename=asset.filename,
        description=asset.description,
        size_bytes=blob.size_bytes,
        content_type=blob.content_type,
        checksum_sha256=blob.checksum_sha256,
        version_id=version.id,
        created_at=asset.created_at.isoformat(),
        updated_at=asset.updated_at.isoformat(),
        deleted_at=asset.deleted_at.isoformat() if asset.deleted_at else None,
        audit_relevant=asset_is_audit_relevant(session, asset),
        metadata=metadata,
        source_provenance=source_provenance_from_metadata(metadata),
        source_revision=source_revision_from_metadata(metadata),
        shares=shares,
    )


def _file_share_response(row: FileShare) -> FileShareResponse:
    return FileShareResponse(
        id=row.id,
        file_asset_id=row.file_asset_id,
        target_type=row.target_type,
        target_id=row.target_id,
        permission=row.permission,
        created_by_user_id=row.created_by_user_id,
        created_at=row.created_at.isoformat(),
        expires_at=row.expires_at.isoformat() if row.expires_at else None,
        revoked_at=row.revoked_at.isoformat() if row.revoked_at else None,
        revoked_by_user_id=row.revoked_by_user_id,
        active=file_share_is_active(row),
    )


def _asset_versions_and_blobs(
    session: Session, assets: list[FileAsset]
) -> dict[str, tuple[FileVersion, FileBlob]]:
    version_ids = [
        asset.current_version_id for asset in assets if asset.current_version_id
    ]
    if len(version_ids) != len(assets):
        raise FileStorageError("File has no current version")
    rows: list[tuple[FileVersion, FileBlob]] = []
    for chunk in _chunks(version_ids):
        rows.extend(
            session.query(FileVersion, FileBlob)
            .join(FileBlob, FileBlob.id == FileVersion.blob_id)
            .filter(FileVersion.id.in_(chunk))
            .all()
        )
    return {version.id: (version, blob) for version, blob in rows}


def _asset_shares_by_id(
    session: Session, asset_ids: list[str], *, include_shares: bool
) -> dict[str, list[FileShareResponse]]:
    shares_by_asset_id: dict[str, list[FileShareResponse]] = {
        asset_id: [] for asset_id in asset_ids
    }
    if not include_shares:
        return shares_by_asset_id
    for chunk in _chunks(asset_ids):
        rows = (
            session.query(FileShare)
            .filter(FileShare.file_asset_id.in_(chunk))
            .order_by(FileShare.created_at.desc())
            .all()
        )
        for row in rows:
            shares_by_asset_id.setdefault(row.file_asset_id, []).append(
                _file_share_response(row)
            )
    return shares_by_asset_id


def _sent_campaign_asset_ids(session: Session, asset_ids: list[str]) -> set[str]:
    sent_asset_ids: set[str] = set()
    for chunk in _chunks(asset_ids):
        rows = (
            session.query(CampaignAttachmentUse.file_asset_id)
            .filter(
                CampaignAttachmentUse.file_asset_id.in_(chunk),
                CampaignAttachmentUse.use_stage == "sent",
            )
            .distinct()
            .all()
        )
        sent_asset_ids.update(row[0] for row in rows)
    return sent_asset_ids


def _loaded_asset_response(
    asset: FileAsset,
    *,
    version: FileVersion,
    blob: FileBlob,
    shares: list[FileShareResponse],
    sent_asset_ids: set[str],
) -> FileAssetResponse:
    metadata = asset.metadata_ or {}
    return FileAssetResponse(
        id=asset.id,
        tenant_id=asset.tenant_id,
        owner_type=asset.owner_type,
        owner_id=_owner_id(asset),
        display_path=asset.display_path,
        filename=asset.filename,
        description=asset.description,
        size_bytes=blob.size_bytes,
        content_type=blob.content_type,
        checksum_sha256=blob.checksum_sha256,
        version_id=version.id,
        created_at=asset.created_at.isoformat(),
        updated_at=asset.updated_at.isoformat(),
        deleted_at=asset.deleted_at.isoformat() if asset.deleted_at else None,
        audit_relevant=asset.id in sent_asset_ids,
        metadata=metadata,
        source_provenance=source_provenance_from_metadata(metadata),
        source_revision=source_revision_from_metadata(metadata),
        shares=shares,
    )


def _asset_list_response(
    session: Session, assets: list[FileAsset], *, include_shares: bool = False
) -> list[FileAssetResponse]:
    if not assets:
        return []

    asset_ids = [asset.id for asset in assets]
    versions_and_blobs = _asset_versions_and_blobs(session, assets)
    shares_by_asset_id = _asset_shares_by_id(
        session, asset_ids, include_shares=include_shares
    )
    sent_asset_ids = _sent_campaign_asset_ids(session, asset_ids)

    responses: list[FileAssetResponse] = []
    for asset in assets:
        version_and_blob = versions_and_blobs.get(asset.current_version_id or "")
        if version_and_blob is None:
            raise FileStorageError("File version not found")
        version, blob = version_and_blob
        responses.append(
            _loaded_asset_response(
                asset,
                version=version,
                blob=blob,
                shares=shares_by_asset_id.get(asset.id, []),
                sent_asset_ids=sent_asset_ids,
            )
        )
    return responses


def _chunks(values: list[str], size: int = 900):
    for index in range(0, len(values), size):
        yield values[index : index + size]


def _folder_owner_id(folder: FileFolder) -> str:
    return (
        folder.owner_user_id if folder.owner_type == "user" else folder.owner_group_id
    )  # type: ignore[return-value]


def _folder_response(folder: FileFolder) -> FileFolderResponse:
    return FileFolderResponse(
        id=folder.id,
        tenant_id=folder.tenant_id,
        owner_type=folder.owner_type,
        owner_id=_folder_owner_id(folder),
        path=folder.path,
        created_at=folder.created_at.isoformat(),
        updated_at=folder.updated_at.isoformat(),
        deleted_at=folder.deleted_at.isoformat() if folder.deleted_at else None,
    )


def _connector_space_response(space: FileConnectorSpace) -> FileConnectorSpaceResponse:
    return FileConnectorSpaceResponse(
        id=space.id,
        tenant_id=space.tenant_id,
        owner_type=space.owner_type,  # type: ignore[arg-type]
        owner_id=connector_space_owner_id(space),
        label=space.label,
        connector_profile_id=space.connector_profile_id,
        provider=space.provider,
        library_id=space.library_id,
        remote_path=space.remote_path,
        sync_mode=space.sync_mode,
        read_only=space.read_only,
        is_active=space.is_active,
        created_at=space.created_at.isoformat(),
        updated_at=space.updated_at.isoformat(),
        deleted_at=space.deleted_at.isoformat() if space.deleted_at else None,
        metadata=space.metadata_ or {},
    )


def _connector_space_file_space_response(
    space: FileConnectorSpace,
) -> FileSpaceResponse:
    owner_id = connector_space_owner_id(space)
    return FileSpaceResponse(
        id=f"connector:{space.id}",
        label=space.label,
        owner_type=space.owner_type,  # type: ignore[arg-type]
        owner_id=owner_id,
        description=f"Linked {space.provider} folder.",
        space_type="connector",
        connector_space_id=space.id,
        connector_profile_id=space.connector_profile_id,
        provider=space.provider,
        library_id=space.library_id,
        remote_path=space.remote_path,
        sync_mode=space.sync_mode,
        read_only=space.read_only,
    )


def _connector_space_policy_decision(
    profile: ConnectorProfile,
    *,
    library_id: str | None,
    remote_path: str | None,
    operation: str,
):
    browse_path = normalize_connector_browse_path(remote_path)
    external_id = (
        f"{library_id}:{browse_path}" if library_id else browse_path or profile.id
    )
    return connector_policy_decision(
        ConnectorAccessRequest(
            connector_id=profile.id,
            credential_id=profile.credential_profile_id,
            provider=profile.provider,
            external_id=external_id,
            external_path=browse_path,
            external_url=connector_effective_endpoint_url(
                provider=profile.provider,
                endpoint_url=profile.endpoint_url,
                metadata=profile.metadata,
            ),
            operation=operation,
        ),
        profile.policy_sources,
    )


def _ensure_list_owner_access(
    session: Session,
    principal: ApiPrincipal,
    owner_type: str | None,
    owner_id: str | None,
) -> None:
    if not owner_type:
        return
    if (
        owner_type == "user"
        and owner_id
        and owner_id != principal.user.id
        and not _is_admin(principal)
    ):
        raise HTTPException(
            status_code=status.HTTP_403_FORBIDDEN,
            detail="No access to this user file space",
        )
    if owner_type == "group" and owner_id:
        try:
            ensure_group_access(
                session,
                tenant_id=principal.tenant_id,
                group_id=owner_id,
                user_id=principal.user.id,
                is_admin=_is_admin(principal),
            )
        except FileStorageError as exc:
            raise HTTPException(
                status_code=status.HTTP_403_FORBIDDEN, detail=str(exc)
            ) from exc