diff --git a/dev/connectors/.env.example b/dev/connectors/.env.example index 15ed158..c8812a4 100644 --- a/dev/connectors/.env.example +++ b/dev/connectors/.env.example @@ -17,3 +17,9 @@ SMB_HOST_PORT=1445 SMB_SHARE_NAME=files SMB_USER=govoplan SMB_PASSWORD=govoplan-smb + +MINIO_API_HOST_PORT=9000 +MINIO_CONSOLE_HOST_PORT=9001 +MINIO_ROOT_USER=govoplan +MINIO_ROOT_PASSWORD=govoplan-minio +MINIO_BUCKET=govoplan diff --git a/dev/connectors/README.md b/dev/connectors/README.md index 1087b37..6accbd9 100644 --- a/dev/connectors/README.md +++ b/dev/connectors/README.md @@ -10,7 +10,7 @@ binds services to localhost high ports. cd /mnt/DATA/git/govoplan-files/dev/connectors cp .env.example .env mkdir -p data/smb data/webdav -docker compose up -d nextcloud nextcloud-db webdav smb +docker compose up -d nextcloud nextcloud-db webdav smb minio docker compose up -d seafile-db seafile-memcached seafile ``` @@ -30,6 +30,8 @@ Endpoints: `http://127.0.0.1:9082/seafdav/` - WebDAV: `http://127.0.0.1:9083` - SMB: `smb://127.0.0.1:1445/files` +- MinIO/S3 API: `http://127.0.0.1:9000`, console + `http://127.0.0.1:9001` The local fixture data under `data/` is ignored by git. @@ -93,6 +95,25 @@ Example local profile file: "password_env": "SMB_PASSWORD", "capabilities": ["browse", "import"], "policy": { "allow": { "providers": ["smb"] } } + }, + { + "id": "dev-s3", + "label": "Dev MinIO", + "provider": "s3", + "endpoint_url": "http://127.0.0.1:9000", + "base_path": "", + "scope_type": "system", + "credential_mode": "basic", + "username": "govoplan", + "password_env": "MINIO_ROOT_PASSWORD", + "capabilities": ["browse", "import"], + "metadata": { + "bucket": "govoplan", + "region": "us-east-1", + "path_style": true, + "verify_tls": false + }, + "policy": { "allow": { "providers": ["s3"] } } } ] } @@ -115,9 +136,11 @@ cd /mnt/DATA/git/govoplan-files/dev/connectors ``` The script reads `.env` from this directory when present, seeds tiny WebDAV, -Nextcloud, and SMB fixtures, then browses and imports them through the connector -helper layer. Pass `--require-smb` after recreating the SMB container to fail on -SMB access errors instead of reporting them as an optional skip. +Nextcloud, SMB, and MinIO fixtures, then browses and imports them through the +connector helper layer. Pass `--require-smb` after recreating the SMB container +to fail on SMB access errors instead of reporting them as an optional skip. Pass +`--require-s3` after starting MinIO and installing `govoplan-files[s3]` to fail +on S3 access errors instead of reporting them as an optional skip. SMB smoke checks need the optional Python dependency in the environment running the script: @@ -126,6 +149,13 @@ the script: /mnt/DATA/git/govoplan-core/.venv/bin/python -m pip install -e /mnt/DATA/git/govoplan-files[smb] ``` +S3 smoke checks need the optional boto3 dependency in the environment running +the script: + +```bash +/mnt/DATA/git/govoplan-core/.venv/bin/python -m pip install -e /mnt/DATA/git/govoplan-files[s3] +``` + The SMB service is built from `dev/connectors/smb/` so the development share is deterministic: one `files` share backed by `data/smb`, with the credentials from `.env`. diff --git a/dev/connectors/docker-compose.yml b/dev/connectors/docker-compose.yml index 2921ae3..32661fa 100644 --- a/dev/connectors/docker-compose.yml +++ b/dev/connectors/docker-compose.yml @@ -99,8 +99,22 @@ services: volumes: - ./data/smb:/storage + minio: + image: minio/minio:latest + restart: unless-stopped + command: server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-govoplan} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-govoplan-minio} + ports: + - "127.0.0.1:${MINIO_API_HOST_PORT:-9000}:9000" + - "127.0.0.1:${MINIO_CONSOLE_HOST_PORT:-9001}:9001" + volumes: + - minio:/data + volumes: nextcloud-db: nextcloud: seafile-db: seafile-data: + minio: diff --git a/dev/connectors/smoke.py b/dev/connectors/smoke.py index ec8938e..bcdb3fd 100644 --- a/dev/connectors/smoke.py +++ b/dev/connectors/smoke.py @@ -18,16 +18,17 @@ ROOT = Path(__file__).resolve().parent def main() -> int: parser = argparse.ArgumentParser(description="Smoke-test GovOPlaN connector helpers against the local dev compose stack.") parser.add_argument("--require-smb", action="store_true", help="Fail if the SMB connector cannot browse/import.") + parser.add_argument("--require-s3", action="store_true", help="Fail if the S3 connector cannot browse/import.") parser.add_argument("--debug-smb", action="store_true", help="Print direct smbclient probes before the connector smoke check.") args = parser.parse_args() _load_dotenv() _default_env() - preflight_failures = _preflight_services(require_smb=args.require_smb) + preflight_failures = _preflight_services(require_smb=args.require_smb, require_s3=args.require_s3) if preflight_failures: for failure in preflight_failures: print(f"FAIL {failure}") - print("Start or recreate the connector stack from this directory with: docker compose up -d nextcloud nextcloud-db webdav smb") + print("Start or recreate the connector stack from this directory with: docker compose up -d nextcloud nextcloud-db webdav smb minio") return 1 _seed_webdav_fixture() _seed_smb_fixture() @@ -36,6 +37,13 @@ def main() -> int: except httpx.HTTPError as exc: print(f"FAIL dev-nextcloud seed failed: {exc}") return 1 + try: + _seed_s3_fixture() + except Exception as exc: + if args.require_s3: + print(f"FAIL dev-s3 seed failed: {exc}") + return 1 + print(f"SKIP dev-s3 seed failed: {exc}") profiles = {profile.id: profile for profile in connector_profiles_from_payload({"profiles": _profile_payloads()})} if args.debug_smb: @@ -59,6 +67,15 @@ def main() -> int: else: print(f"SKIP {message}") + try: + _exercise_profile(profiles["dev-s3"], folder="GovOPlaN", file_path="GovOPlaN/s3-live.txt", expected="s3 live fixture") + except Exception as exc: + message = f"dev-s3: {exc}" + if args.require_s3: + failures.append(message) + else: + print(f"SKIP {message}") + if failures: for failure in failures: print(f"FAIL {failure}") @@ -76,6 +93,9 @@ def _default_env() -> None: "SMB_SHARE_NAME": "files", "SMB_USER": "govoplan", "SMB_PASSWORD": "govoplan-smb", + "MINIO_ROOT_USER": "govoplan", + "MINIO_ROOT_PASSWORD": "govoplan-minio", + "MINIO_BUCKET": "govoplan", } for key, value in defaults.items(): os.environ.setdefault(key, value) @@ -96,7 +116,7 @@ def _load_dotenv() -> None: os.environ[key] = value.strip().strip("\"'") -def _preflight_services(*, require_smb: bool) -> list[str]: +def _preflight_services(*, require_smb: bool, require_s3: bool) -> list[str]: failures: list[str] = [] nextcloud_url = f"http://127.0.0.1:{os.getenv('NEXTCLOUD_HOST_PORT', '9081')}/status.php" try: @@ -121,6 +141,14 @@ def _preflight_services(*, require_smb: bool) -> list[str]: pass except OSError as exc: failures.append(f"dev-smb is not reachable at 127.0.0.1:{smb_port}: {exc}") + if require_s3: + minio_url = f"http://127.0.0.1:{os.getenv('MINIO_API_HOST_PORT', '9000')}/minio/health/live" + try: + response = httpx.get(minio_url, timeout=3.0) + if response.status_code != 200: + failures.append(f"dev-s3 expected HTTP 200 at {minio_url}, got HTTP {response.status_code}") + except httpx.HTTPError as exc: + failures.append(f"dev-s3 is not reachable at {minio_url}: {exc}") return failures @@ -150,6 +178,20 @@ def _profile_payloads() -> list[dict[str, object]]: "username": os.getenv("SMB_USER", "govoplan"), "password_env": "SMB_PASSWORD", }, + { + "id": "dev-s3", + "provider": "s3", + "endpoint_url": f"http://127.0.0.1:{os.getenv('MINIO_API_HOST_PORT', '9000')}", + "credential_mode": "basic", + "username": os.getenv("MINIO_ROOT_USER", "govoplan"), + "password_env": "MINIO_ROOT_PASSWORD", + "metadata": { + "bucket": os.getenv("MINIO_BUCKET", "govoplan"), + "region": "us-east-1", + "path_style": True, + "verify_tls": False, + }, + }, ] @@ -211,5 +253,30 @@ def _seed_nextcloud_fixture() -> None: raise RuntimeError(f"Nextcloud PUT failed with HTTP {response.status_code}: {response.text[:200]}") +def _seed_s3_fixture() -> None: + try: + import boto3 + from botocore.config import Config + from botocore.exceptions import ClientError + except ImportError as exc: + raise RuntimeError("boto3 is not installed; install govoplan-files[s3]") from exc + bucket = os.getenv("MINIO_BUCKET", "govoplan") + client = boto3.client( + "s3", + endpoint_url=f"http://127.0.0.1:{os.getenv('MINIO_API_HOST_PORT', '9000')}", + aws_access_key_id=os.getenv("MINIO_ROOT_USER", "govoplan"), + aws_secret_access_key=os.getenv("MINIO_ROOT_PASSWORD", "govoplan-minio"), + region_name="us-east-1", + config=Config(s3={"addressing_style": "path"}), + ) + try: + client.create_bucket(Bucket=bucket) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}: + raise + client.put_object(Bucket=bucket, Key="GovOPlaN/s3-live.txt", Body=b"s3 live fixture\n", ContentType="text/plain") + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 008d97f..4129291 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,9 @@ dependencies = [ ] [project.optional-dependencies] +s3 = [ + "boto3>=1.34,<2", +] smb = [ "smbprotocol>=1.13", ] diff --git a/src/govoplan_files/backend/change_tracking.py b/src/govoplan_files/backend/change_tracking.py index a0adcd0..8d5b8c3 100644 --- a/src/govoplan_files/backend/change_tracking.py +++ b/src/govoplan_files/backend/change_tracking.py @@ -1,12 +1,18 @@ from __future__ import annotations from datetime import datetime -from typing import Any -from sqlalchemy import event, inspect +from sqlalchemy import event from sqlalchemy.orm import Session as OrmSession from govoplan_core.core.change_sequence import record_change +from govoplan_core.core.sqlalchemy_change_tracking import ( + ensure_object_id, + has_attr_changes, + object_state, + operation_for_soft_deletable, + previous_value, +) from govoplan_files.backend.db.models import FileAsset, FileFolder, FileShare, new_uuid FILES_MODULE_ID = "files" @@ -39,7 +45,7 @@ def _record_files_changes(session: OrmSession, _flush_context: object, _instance def _record_asset_change(session: OrmSession, asset: FileAsset) -> None: - operation = _operation_for_soft_deletable( + operation = operation_for_soft_deletable( asset, changed_attrs=( "owner_type", @@ -70,7 +76,7 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None: "owner_type": asset.owner_type, "owner_id": _owner_id(asset.owner_type, asset.owner_user_id, asset.owner_group_id), "path": asset.display_path, - "previous_path": _previous_value(asset, "display_path"), + "previous_path": previous_value(asset, "display_path"), "filename": asset.filename, "deleted_at": _isoformat(asset.deleted_at), }, @@ -78,7 +84,7 @@ def _record_asset_change(session: OrmSession, asset: FileAsset) -> None: def _record_folder_change(session: OrmSession, folder: FileFolder) -> None: - operation = _operation_for_soft_deletable( + operation = operation_for_soft_deletable( folder, changed_attrs=("owner_type", "owner_user_id", "owner_group_id", "path", "deleted_at", "metadata_"), ) @@ -99,17 +105,17 @@ def _record_folder_change(session: OrmSession, folder: FileFolder) -> None: "owner_type": folder.owner_type, "owner_id": _owner_id(folder.owner_type, folder.owner_user_id, folder.owner_group_id), "path": folder.path, - "previous_path": _previous_value(folder, "path"), + "previous_path": previous_value(folder, "path"), "deleted_at": _isoformat(folder.deleted_at), }, ) def _record_share_visibility_change(session: OrmSession, share: FileShare) -> None: - state = inspect(share) + state = object_state(share) if not share.file_asset_id: return - if not state.pending and not _has_attr_changes( + if not state.pending and not has_attr_changes( state, ("file_asset_id", "target_type", "target_id", "permission", "revoked_at"), ): @@ -135,44 +141,8 @@ def _record_share_visibility_change(session: OrmSession, share: FileShare) -> No ) -def _operation_for_soft_deletable(obj: object, *, changed_attrs: tuple[str, ...]) -> str | None: - state = inspect(obj) - if state.pending: - return "created" - if not _has_attr_changes(state, changed_attrs): - return None - if "deleted_at" in state.attrs: - history = state.attrs.deleted_at.history - if history.has_changes(): - if any(value is not None for value in history.added): - return "deleted" - if any(value is not None for value in history.deleted): - return "created" - return "updated" - - -def _has_attr_changes(state: Any, attrs: tuple[str, ...]) -> bool: - return any(name in state.attrs and state.attrs[name].history.has_changes() for name in attrs) - - -def _previous_value(obj: object, attr_name: str) -> str | None: - state = inspect(obj) - if attr_name not in state.attrs: - return None - history = state.attrs[attr_name].history - if not history.has_changes() or not history.deleted: - return None - value = history.deleted[0] - return str(value) if value is not None else None - - def _ensure_id(obj: object) -> str: - resource_id = getattr(obj, "id", None) - if resource_id: - return str(resource_id) - resource_id = new_uuid() - setattr(obj, "id", resource_id) - return resource_id + return ensure_object_id(obj, new_uuid) def _owner_id(owner_type: str, owner_user_id: str | None, owner_group_id: str | None) -> str | None: diff --git a/src/govoplan_files/backend/router.py b/src/govoplan_files/backend/router.py index ae4754c..9fc3cc1 100644 --- a/src/govoplan_files/backend/router.py +++ b/src/govoplan_files/backend/router.py @@ -105,6 +105,7 @@ from govoplan_files.backend.storage.access import ensure_group_access, group_ref from govoplan_files.backend.storage.archives import create_zip_file, extract_zip_upload from govoplan_files.backend.storage.common import FileStorageError from govoplan_files.backend.storage.connector_credential_store import ( + ConnectorCredential, connector_credential_from_row, create_connector_credential_row, deactivate_connector_credential_row, @@ -802,6 +803,14 @@ def _download_connector_payload( return source_path, downloaded, metadata or {} +def _connector_browse_next_token(items: list[Any]) -> str | None: + for item in items: + token = item.metadata.get("next_continuation_token") if hasattr(item, "metadata") else None + if token: + return str(token) + return None + + def _connector_audit_details(asset: FileAsset, version: FileVersion, blob: FileBlob, *, operation: str) -> dict[str, object] | None: metadata = asset.metadata_ or {} provenance = source_provenance_from_metadata(metadata) @@ -2148,6 +2157,148 @@ def _full_file_connector_settings_delta_response( ) +def _changed_file_connector_setting_ids(entries: list[ChangeSequenceEntry]) -> tuple[set[str], set[str], set[str], bool]: + changed_profile_ids = { + entry.resource_id + for entry in entries + if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE + } + changed_credential_ids = { + entry.resource_id + for entry in entries + if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE + } + changed_space_ids = { + entry.resource_id + for entry in entries + if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE + } + policy_changed = any(entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries) + return changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed + + +def _file_connector_settings_changed_sections( + *, + changed_profile_ids: set[str], + changed_credential_ids: set[str], + changed_space_ids: set[str], + policy_changed: bool, + profiles: list[ConnectorProfile], +) -> list[str]: + changed_sections = [] + if changed_profile_ids or any(profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids for profile in profiles): + changed_sections.append("profiles") + if changed_credential_ids: + changed_sections.append("credentials") + if changed_space_ids: + changed_sections.append("spaces") + if policy_changed: + changed_sections.append("policy") + return changed_sections + + +def _file_connector_settings_deleted_items( + entries: list[ChangeSequenceEntry], + *, + visible_profiles: dict[str, ConnectorProfile], + visible_credentials: dict[str, ConnectorCredential], + visible_spaces: dict[str, FileConnectorSpace], +) -> list[DeltaDeletedItem]: + return [ + _connector_deleted_item(entry) + for entry in entries + if ( + entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE + and entry.resource_id not in visible_profiles + ) + or ( + entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE + and entry.resource_id not in visible_credentials + ) + or ( + entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE + and entry.resource_id not in visible_spaces + ) + ] + + +def _incremental_file_connector_settings_delta_response( + session: Session, + principal: ApiPrincipal, + *, + entries: list[ChangeSequenceEntry], + has_more: bool, + scope_type: str, + scope_id: str | None, + provider: str | None, + campaign_id: str | None, + include_disabled: bool, + include_inactive: bool, + owner_type: Literal["user", "group"] | None, + owner_id: str | None, +) -> FileConnectorSettingsDeltaResponse: + changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed = _changed_file_connector_setting_ids(entries) + profiles = _visible_connector_profiles( + session, + principal, + provider=provider, + campaign_id=campaign_id, + include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal), + include_admin_scopes=_can_read_disabled_connector_profiles(principal), + include_effective_policy=False, + ) + visible_profiles = { + profile.id: profile + for profile in profiles + if profile.id in changed_profile_ids or (profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids) + } + credentials = _visible_connector_credentials( + session, + principal, + provider=provider, + include_disabled=include_disabled, + ) + visible_credentials = { + credential.id: credential + for credential in credentials + if credential.id in changed_credential_ids + } + spaces = _visible_connector_spaces( + session, + principal, + owner_type=owner_type, + owner_id=owner_id, + include_inactive=include_inactive, + ) + visible_spaces = { + space.id: space + for space in spaces + if space.id in changed_space_ids + } + return FileConnectorSettingsDeltaResponse( + profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in visible_profiles.values()], + credentials=[FileConnectorCredentialResponse(**credential.to_response()) for credential in visible_credentials.values()], + spaces=[_connector_space_response(space) for space in visible_spaces.values()], + policy=FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) if policy_changed else None, + changed_sections=_file_connector_settings_changed_sections( + changed_profile_ids=changed_profile_ids, + changed_credential_ids=changed_credential_ids, + changed_space_ids=changed_space_ids, + policy_changed=policy_changed, + profiles=profiles, + ), + deleted=_file_connector_settings_deleted_items( + entries, + visible_profiles=visible_profiles, + visible_credentials=visible_credentials, + visible_spaces=visible_spaces, + ), + watermark=_file_connector_settings_response_watermark(session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more), + has_more=has_more, + full=False, + ) + + @router.get("/connectors/settings/delta", response_model=FileConnectorSettingsDeltaResponse) def connector_settings_delta( scope_type: str = Query(default="tenant"), @@ -2193,94 +2344,19 @@ def connector_settings_delta( owner_type=owner_type, owner_id=owner_id, ) - changed_profile_ids = { - entry.resource_id - for entry in entries - if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE - } - changed_credential_ids = { - entry.resource_id - for entry in entries - if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE - } - changed_space_ids = { - entry.resource_id - for entry in entries - if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE - } - policy_changed = any(entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries) - profiles = _visible_connector_profiles( + return _incremental_file_connector_settings_delta_response( session, principal, + entries=entries, + has_more=has_more, + scope_type=scope_type, + scope_id=scope_id, provider=provider, campaign_id=campaign_id, - include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal), - include_admin_scopes=_can_read_disabled_connector_profiles(principal), - include_effective_policy=False, - ) - visible_profiles = { - profile.id: profile - for profile in profiles - if profile.id in changed_profile_ids or (profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids) - } - credentials = _visible_connector_credentials( - session, - principal, - provider=provider, include_disabled=include_disabled, - ) - visible_credentials = { - credential.id: credential - for credential in credentials - if credential.id in changed_credential_ids - } - spaces = _visible_connector_spaces( - session, - principal, + include_inactive=include_inactive, owner_type=owner_type, owner_id=owner_id, - include_inactive=include_inactive, - ) - visible_spaces = { - space.id: space - for space in spaces - if space.id in changed_space_ids - } - changed_sections = [] - if changed_profile_ids or any(profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids for profile in profiles): - changed_sections.append("profiles") - if changed_credential_ids: - changed_sections.append("credentials") - if changed_space_ids: - changed_sections.append("spaces") - if policy_changed: - changed_sections.append("policy") - deleted = [ - _connector_deleted_item(entry) - for entry in entries - if ( - entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE - and entry.resource_id not in visible_profiles - ) - or ( - entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE - and entry.resource_id not in visible_credentials - ) - or ( - entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE - and entry.resource_id not in visible_spaces - ) - ] - return FileConnectorSettingsDeltaResponse( - profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in visible_profiles.values()], - credentials=[FileConnectorCredentialResponse(**credential.to_response()) for credential in visible_credentials.values()], - spaces=[_connector_space_response(space) for space in visible_spaces.values()], - policy=FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) if policy_changed else None, - changed_sections=changed_sections, - deleted=deleted, - watermark=_file_connector_settings_response_watermark(session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more), - has_more=has_more, - full=False, ) except (FileStorageError, ValueError, json.JSONDecodeError) as exc: raise _http_error(exc) from exc @@ -2819,6 +2895,7 @@ def browse_connector_profile_items( profile_id: str, path: str | None = None, library_id: str | None = None, + continuation_token: str | None = None, campaign_id: str | None = None, session: Session = Depends(get_session), principal: ApiPrincipal = Depends(require_any_scope("files:file:read", "files:file:upload", "files:file:download", "files:file:admin", "system:settings:read", "admin:settings:read")), @@ -2839,7 +2916,7 @@ def browse_connector_profile_items( ) if not decision.allowed: raise ConnectorPolicyDenied(decision) - items = browse_connector_profile(profile, path=browse_path, library_id=library_id) + items = browse_connector_profile(profile, path=browse_path, library_id=library_id, continuation_token=continuation_token) except ConnectorPolicyDenied as exc: raise _connector_policy_error(exc) from exc except ConnectorBrowseUnsupported as exc: @@ -2851,6 +2928,8 @@ def browse_connector_profile_items( provider=profile.provider, path=browse_path, library_id=library_id, + next_continuation_token=_connector_browse_next_token(items), + has_more=any(bool(item.metadata.get("listing_truncated")) for item in items), decision=decision.to_dict(), items=[FileConnectorBrowseItem(**item.to_response()) for item in items], ) @@ -3230,7 +3309,7 @@ def download_archive( get_asset_for_user(session, tenant_id=principal.tenant_id, user_id=principal.user.id, asset_id=file_id, is_admin=_is_admin(principal)) for file_id in payload.file_ids ] - tmp = tempfile.NamedTemporaryFile(prefix="multimailer-files-", suffix=".zip", delete=False) + tmp = tempfile.NamedTemporaryFile(prefix="govoplan-files-", suffix=".zip", delete=False) tmp_path = tmp.name tmp.close() try: diff --git a/src/govoplan_files/backend/runtime.py b/src/govoplan_files/backend/runtime.py index 6678ce8..8a4f525 100644 --- a/src/govoplan_files/backend/runtime.py +++ b/src/govoplan_files/backend/runtime.py @@ -1,36 +1,10 @@ from __future__ import annotations -from typing import Any +from govoplan_core.core.runtime import ModuleRuntimeState -_runtime_registry: object | None = None -_runtime_settings: object | None = None +_runtime = ModuleRuntimeState("Files") - -def configure_runtime(*, registry: object | None = None, settings: object | None = None) -> None: - global _runtime_registry, _runtime_settings - if registry is not None: - _runtime_registry = registry - if settings is not None: - _runtime_settings = settings - - -def get_registry() -> object | None: - return _runtime_registry - - -def get_settings() -> object: - if _runtime_settings is not None: - return _runtime_settings - try: - from govoplan_core.settings import settings as legacy_settings - except ModuleNotFoundError as exc: - raise RuntimeError("GovOPlaN Files runtime settings are not configured") from exc - return legacy_settings - - -class SettingsProxy: - def __getattr__(self, name: str) -> Any: - return getattr(get_settings(), name) - - -settings = SettingsProxy() +configure_runtime = _runtime.configure_runtime +get_registry = _runtime.get_registry +get_settings = _runtime.get_settings +settings = _runtime.settings diff --git a/src/govoplan_files/backend/schemas.py b/src/govoplan_files/backend/schemas.py index bd4496e..8ecb8c3 100644 --- a/src/govoplan_files/backend/schemas.py +++ b/src/govoplan_files/backend/schemas.py @@ -287,7 +287,7 @@ class FileConnectorDiscoveryCandidate(BaseModel): class FileConnectorDiscoveryRequest(BaseModel): - provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] + provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] endpoint_url: str base_path: str | None = None credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] = "none" @@ -309,7 +309,7 @@ class FileConnectorDiscoveryResponse(BaseModel): class FileConnectorProfileCreateRequest(BaseModel): id: str label: str - provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] + provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] endpoint_url: str | None = None base_path: str | None = None enabled: bool = True @@ -325,7 +325,7 @@ class FileConnectorProfileCreateRequest(BaseModel): class FileConnectorProfileUpdateRequest(BaseModel): label: str | None = None - provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None + provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] | None = None endpoint_url: str | None = None base_path: str | None = None enabled: bool | None = None @@ -342,7 +342,7 @@ class FileConnectorProfileUpdateRequest(BaseModel): class FileConnectorCredentialCreateRequest(BaseModel): id: str label: str - provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None + provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] | None = None enabled: bool = True scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "tenant" scope_id: str | None = None @@ -354,7 +354,7 @@ class FileConnectorCredentialCreateRequest(BaseModel): class FileConnectorCredentialUpdateRequest(BaseModel): label: str | None = None - provider: Literal["seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"] | None = None + provider: Literal["seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"] | None = None enabled: bool | None = None credential_mode: Literal["none", "anonymous", "basic", "token", "secret_ref"] | None = None credentials: FileConnectorProfileCredentialsRequest | None = None @@ -404,6 +404,8 @@ class FileConnectorBrowseResponse(BaseModel): path: str = "" library_id: str | None = None read_only: bool = True + next_continuation_token: str | None = None + has_more: bool = False decision: dict[str, Any] items: list[FileConnectorBrowseItem] diff --git a/src/govoplan_files/backend/storage/campaign_attachments.py b/src/govoplan_files/backend/storage/campaign_attachments.py index 09ed152..4e08b92 100644 --- a/src/govoplan_files/backend/storage/campaign_attachments.py +++ b/src/govoplan_files/backend/storage/campaign_attachments.py @@ -273,7 +273,7 @@ def prepared_campaign_snapshot( campaign_id: str, raw_json: dict[str, Any], include_bytes: bool, - prefix: str = "multimailer-managed-campaign-", + prefix: str = "govoplan-managed-campaign-", ) -> Iterator[PreparedCampaignSnapshot]: temp_dir = Path(tempfile.mkdtemp(prefix=prefix)) try: diff --git a/src/govoplan_files/backend/storage/campaign_usage.py b/src/govoplan_files/backend/storage/campaign_usage.py index c06aa1a..16cffa1 100644 --- a/src/govoplan_files/backend/storage/campaign_usage.py +++ b/src/govoplan_files/backend/storage/campaign_usage.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections import defaultdict +from dataclasses import dataclass, field from pathlib import PurePosixPath from typing import Any, Iterable @@ -24,6 +25,15 @@ def _candidate_match_keys(raw_match: str) -> set[str]: AttachmentUseKey = tuple[str, str, str, str] +@dataclass(slots=True) +class _AttachmentBatchRefs: + managed_by_job: dict[str, list[dict[str, object]]] = field(default_factory=lambda: defaultdict(list)) + fallback_attachments_by_job: dict[str, list[dict[str, object]]] = field(default_factory=lambda: defaultdict(list)) + asset_ids: set[str] = field(default_factory=set) + version_ids: set[str] = field(default_factory=set) + blob_ids: set[str] = field(default_factory=set) + + def _attachment_use_key(*, job_id: str, file_version_id: str, filename_used: str, stage: str) -> AttachmentUseKey: return job_id, file_version_id, filename_used, stage @@ -134,62 +144,91 @@ def record_campaign_attachment_uses_for_jobs( if not job_list: return - managed_by_job: dict[str, list[dict[str, object]]] = defaultdict(list) - fallback_attachments_by_job: dict[str, list[dict[str, object]]] = defaultdict(list) + refs = _collect_attachment_batch_refs(job_list) job_by_id = {job.id: job for job in job_list} - asset_ids: set[str] = set() - version_ids: set[str] = set() - blob_ids: set[str] = set() + known_keys = _known_use_keys_for_jobs(session, job_list, stage=stage) + assets_by_id, versions_by_id, blobs_by_id = _load_managed_attachment_entities(session, refs) - for job in job_list: + _record_managed_attachment_uses( + session, + job_by_id=job_by_id, + managed_by_job=refs.managed_by_job, + assets_by_id=assets_by_id, + versions_by_id=versions_by_id, + blobs_by_id=blobs_by_id, + stage=stage, + known_keys=known_keys, + ) + _record_fallback_attachment_uses( + session, + job_by_id=job_by_id, + fallback_attachments_by_job=refs.fallback_attachments_by_job, + stage=stage, + known_keys=known_keys, + ) + + +def _collect_attachment_batch_refs(jobs: list[CampaignJobLike]) -> _AttachmentBatchRefs: + refs = _AttachmentBatchRefs() + for job in jobs: attachments = job.resolved_attachments or [] if not isinstance(attachments, list): continue for attachment in attachments: if not isinstance(attachment, dict): continue - managed_matches = attachment.get("managed_matches") - if isinstance(managed_matches, list) and managed_matches: - for item in managed_matches: - if not isinstance(item, dict): - continue - asset_id = str(item.get("asset_id") or "") - version_id = str(item.get("version_id") or "") - blob_id = str(item.get("blob_id") or "") - if not asset_id or not version_id or not blob_id: - continue - managed_by_job[job.id].append(item) - asset_ids.add(asset_id) - version_ids.add(version_id) - blob_ids.add(blob_id) - else: - fallback_attachments_by_job[job.id].append(attachment) + if not _collect_managed_attachment_refs(refs, job.id, attachment): + refs.fallback_attachments_by_job[job.id].append(attachment) + return refs - assets_by_id = { - item.id: item - for item in session.query(FileAsset).filter(FileAsset.id.in_(asset_ids)).all() - } if asset_ids else {} - versions_by_id = { - item.id: item - for item in session.query(FileVersion).filter(FileVersion.id.in_(version_ids)).all() - } if version_ids else {} - blobs_by_id = { - item.id: item - for item in session.query(FileBlob).filter(FileBlob.id.in_(blob_ids)).all() - } if blob_ids else {} - known_keys = _known_use_keys_for_jobs(session, job_list, stage=stage) +def _collect_managed_attachment_refs(refs: _AttachmentBatchRefs, job_id: str, attachment: dict[str, object]) -> bool: + managed_matches = attachment.get("managed_matches") + if not isinstance(managed_matches, list) or not managed_matches: + return False + for item in managed_matches: + if not isinstance(item, dict): + continue + asset_id = str(item.get("asset_id") or "") + version_id = str(item.get("version_id") or "") + blob_id = str(item.get("blob_id") or "") + if not asset_id or not version_id or not blob_id: + continue + refs.managed_by_job[job_id].append(item) + refs.asset_ids.add(asset_id) + refs.version_ids.add(version_id) + refs.blob_ids.add(blob_id) + return True + + +def _load_managed_attachment_entities( + session: Session, + refs: _AttachmentBatchRefs, +) -> tuple[dict[str, FileAsset], dict[str, FileVersion], dict[str, FileBlob]]: + assets_by_id = {item.id: item for item in session.query(FileAsset).filter(FileAsset.id.in_(refs.asset_ids)).all()} if refs.asset_ids else {} + versions_by_id = {item.id: item for item in session.query(FileVersion).filter(FileVersion.id.in_(refs.version_ids)).all()} if refs.version_ids else {} + blobs_by_id = {item.id: item for item in session.query(FileBlob).filter(FileBlob.id.in_(refs.blob_ids)).all()} if refs.blob_ids else {} + return assets_by_id, versions_by_id, blobs_by_id + + +def _record_managed_attachment_uses( + session: Session, + *, + job_by_id: dict[str, CampaignJobLike], + managed_by_job: dict[str, list[dict[str, object]]], + assets_by_id: dict[str, FileAsset], + versions_by_id: dict[str, FileVersion], + blobs_by_id: dict[str, FileBlob], + stage: str, + known_keys: set[AttachmentUseKey], +) -> None: for job_id, items in managed_by_job.items(): job = job_by_id[job_id] for item in items: asset = assets_by_id.get(str(item.get("asset_id") or "")) version = versions_by_id.get(str(item.get("version_id") or "")) blob = blobs_by_id.get(str(item.get("blob_id") or "")) - if not asset or not version or not blob: - continue - if asset.tenant_id != job.tenant_id or version.tenant_id != job.tenant_id or blob.tenant_id != job.tenant_id: - continue - if version.file_asset_id != asset.id or version.blob_id != blob.id: + if not _managed_attachment_entities_match_job(job, asset, version, blob): continue _add_use( session, @@ -202,50 +241,100 @@ def record_campaign_attachment_uses_for_jobs( known_keys=known_keys, ) + +def _managed_attachment_entities_match_job( + job: CampaignJobLike, + asset: FileAsset | None, + version: FileVersion | None, + blob: FileBlob | None, +) -> bool: + if not asset or not version or not blob: + return False + if asset.tenant_id != job.tenant_id or version.tenant_id != job.tenant_id or blob.tenant_id != job.tenant_id: + return False + return version.file_asset_id == asset.id and version.blob_id == blob.id + + +def _record_fallback_attachment_uses( + session: Session, + *, + job_by_id: dict[str, CampaignJobLike], + fallback_attachments_by_job: dict[str, list[dict[str, object]]], + stage: str, + known_keys: set[AttachmentUseKey], +) -> None: assets_by_campaign: dict[tuple[str, str], dict[str, FileAsset]] = {} version_blobs_by_campaign: dict[tuple[str, str], dict[str, tuple[FileVersion, FileBlob]]] = {} for job_id, attachments in fallback_attachments_by_job.items(): job = job_by_id[job_id] campaign_key = (job.tenant_id, job.campaign_id) - by_key = assets_by_campaign.get(campaign_key) - if by_key is None: - assets = list_assets_for_user( - session, - tenant_id=job.tenant_id, - user_id="", - campaign_id=job.campaign_id, - is_admin=True, - ) - by_key = {} - for asset in assets: - by_key[asset.display_path.strip("/")] = asset - by_key.setdefault(asset.filename, asset) - assets_by_campaign[campaign_key] = by_key - version_blobs_by_campaign[campaign_key] = current_versions_and_blobs(session, assets) - version_blobs = version_blobs_by_campaign[campaign_key] - + by_key, version_blobs = _fallback_campaign_assets( + session, + job, + campaign_key=campaign_key, + assets_by_campaign=assets_by_campaign, + version_blobs_by_campaign=version_blobs_by_campaign, + ) for attachment in attachments: - matches = attachment.get("matches") if isinstance(attachment.get("matches"), list) else [] - for raw in matches: - if not isinstance(raw, str): - continue - asset = next((by_key[key] for key in _candidate_match_keys(raw) if key in by_key), None) - if not asset: - continue - version_blob = version_blobs.get(asset.id) - if not version_blob: - continue - version, blob = version_blob - _add_use( - session, - job, - asset=asset, - version=version, - blob=blob, - filename_used=asset.filename, - stage=stage, - known_keys=known_keys, - ) + _record_fallback_attachment(session, job, attachment, by_key=by_key, version_blobs=version_blobs, stage=stage, known_keys=known_keys) + + +def _fallback_campaign_assets( + session: Session, + job: CampaignJobLike, + *, + campaign_key: tuple[str, str], + assets_by_campaign: dict[tuple[str, str], dict[str, FileAsset]], + version_blobs_by_campaign: dict[tuple[str, str], dict[str, tuple[FileVersion, FileBlob]]], +) -> tuple[dict[str, FileAsset], dict[str, tuple[FileVersion, FileBlob]]]: + by_key = assets_by_campaign.get(campaign_key) + if by_key is None: + assets = list_assets_for_user(session, tenant_id=job.tenant_id, user_id="", campaign_id=job.campaign_id, is_admin=True) + by_key = _asset_lookup_by_path_and_filename(assets) + assets_by_campaign[campaign_key] = by_key + version_blobs_by_campaign[campaign_key] = current_versions_and_blobs(session, assets) + return by_key, version_blobs_by_campaign[campaign_key] + + +def _asset_lookup_by_path_and_filename(assets: list[FileAsset]) -> dict[str, FileAsset]: + by_key: dict[str, FileAsset] = {} + for asset in assets: + by_key[asset.display_path.strip("/")] = asset + by_key.setdefault(asset.filename, asset) + return by_key + + +def _record_fallback_attachment( + session: Session, + job: CampaignJobLike, + attachment: dict[str, object], + *, + by_key: dict[str, FileAsset], + version_blobs: dict[str, tuple[FileVersion, FileBlob]], + stage: str, + known_keys: set[AttachmentUseKey], +) -> None: + matches = attachment.get("matches") if isinstance(attachment.get("matches"), list) else [] + for raw in matches: + if not isinstance(raw, str): + continue + asset = next((by_key[key] for key in _candidate_match_keys(raw) if key in by_key), None) + if not asset: + continue + version_blob = version_blobs.get(asset.id) + if not version_blob: + continue + version, blob = version_blob + _add_use( + session, + job, + asset=asset, + version=version, + blob=blob, + filename_used=asset.filename, + stage=stage, + known_keys=known_keys, + ) def record_campaign_attachment_uses_for_job(session: Session, job: CampaignJobLike, *, stage: str = "built") -> None: diff --git a/src/govoplan_files/backend/storage/connector_browse.py b/src/govoplan_files/backend/storage/connector_browse.py index 55fee42..0dc85b1 100644 --- a/src/govoplan_files/backend/storage/connector_browse.py +++ b/src/govoplan_files/backend/storage/connector_browse.py @@ -53,7 +53,7 @@ class ConnectorBrowseItem: } -def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = None, library_id: str | None = None) -> list[ConnectorBrowseItem]: +def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = None, library_id: str | None = None, continuation_token: str | None = None) -> list[ConnectorBrowseItem]: browse_path = normalize_connector_browse_path(path) static_items = _static_listing(profile, path=browse_path, library_id=library_id) if static_items is not None: @@ -66,6 +66,8 @@ def browse_connector_profile(profile: ConnectorProfile, *, path: str | None = No return _browse_webdav(profile, path=browse_path) if profile.provider == "smb": return _browse_smb(profile, path=browse_path) + if profile.provider == "s3": + return _browse_s3(profile, path=browse_path, library_id=library_id, continuation_token=continuation_token) raise ConnectorBrowseUnsupported(f"Read-only browsing is not implemented for {profile.provider} connector profiles yet") @@ -222,6 +224,227 @@ def _browse_smb(profile: ConnectorProfile, *, path: str) -> list[ConnectorBrowse return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold())) +def _browse_s3(profile: ConnectorProfile, *, path: str, library_id: str | None, continuation_token: str | None) -> list[ConnectorBrowseItem]: + client = _s3_client(profile) + bucket = _s3_bucket(profile, library_id) + if not bucket: + try: + payload = client.list_buckets() + except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific + raise ConnectorBrowseError(f"S3 connector browse failed: {exc}") from exc + buckets = payload.get("Buckets") if isinstance(payload, Mapping) else None + if not isinstance(buckets, list): + raise ConnectorBrowseError("S3 connector returned invalid bucket list") + return sorted( + ( + ConnectorBrowseItem( + kind="library", + name=_clean(item.get("Name")) or "", + path=_clean(item.get("Name")) or "", + external_id=_clean(item.get("Name")), + modified_at=_timestamp(item.get("CreationDate")), + metadata={"bucket": _clean(item.get("Name"))}, + ) + for item in buckets + if isinstance(item, Mapping) and _clean(item.get("Name")) + ), + key=lambda item: item.name.casefold(), + ) + prefix = _s3_object_key(profile, path, directory=True) + params: dict[str, object] = { + "Bucket": bucket, + "Prefix": prefix, + "Delimiter": "/", + "MaxKeys": _s3_max_keys(profile), + } + next_page_request = _clean(continuation_token) or _metadata_string(profile, "continuation_token") + if next_page_request: + params["ContinuationToken"] = next_page_request + try: + payload = client.list_objects_v2(**params) + except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific + raise ConnectorBrowseError(f"S3 connector browse failed: {exc}") from exc + if not isinstance(payload, Mapping): + raise ConnectorBrowseError("S3 connector returned invalid object listing") + items = [ + *_s3_prefix_items(bucket=bucket, browse_path=path, base_prefix=prefix, prefixes=payload.get("CommonPrefixes")), + *_s3_object_items(bucket=bucket, browse_path=path, base_prefix=prefix, objects=payload.get("Contents")), + ] + next_token = _clean(payload.get("NextContinuationToken")) + if next_token and items: + last = items[-1] + items[-1] = ConnectorBrowseItem( + kind=last.kind, + name=last.name, + path=last.path, + external_id=last.external_id, + external_url=last.external_url, + size_bytes=last.size_bytes, + content_type=last.content_type, + modified_at=last.modified_at, + etag=last.etag, + metadata={**dict(last.metadata), "next_continuation_token": next_token, "listing_truncated": True}, + ) + return sorted(items, key=lambda item: (item.kind != "folder", item.name.casefold(), item.path.casefold())) + + +def _s3_client(profile: ConnectorProfile) -> Any: + if profile.secret_ref: + raise ConnectorBrowseError("Secret-ref S3 credentials need a runtime secret resolver before live browsing") + try: + boto3 = import_module("boto3") + config_module = import_module("botocore.config") + except ImportError as exc: + raise ConnectorBrowseUnsupported("S3 connector browsing requires the optional boto3 dependency") from exc + kwargs: dict[str, object] = {} + if profile.endpoint_url: + kwargs["endpoint_url"] = profile.endpoint_url + region = _metadata_string(profile, "region") or _metadata_string(profile, "aws_region") + if region: + kwargs["region_name"] = region + access_key = profile.username or _metadata_env(profile, "access_key_id_env") or _metadata_string(profile, "access_key_id") + secret_key = _profile_password(profile) or _metadata_env(profile, "secret_access_key_env") + session_token = _profile_token(profile) or _metadata_env(profile, "session_token_env") + if access_key: + kwargs["aws_access_key_id"] = access_key + if secret_key: + kwargs["aws_secret_access_key"] = secret_key + if session_token: + kwargs["aws_session_token"] = session_token + verify = _s3_verify(profile) + if verify is not None: + kwargs["verify"] = verify + addressing_style = _s3_addressing_style(profile) + if addressing_style: + kwargs["config"] = config_module.Config(s3={"addressing_style": addressing_style}) + try: + return boto3.client("s3", **kwargs) + except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific + raise ConnectorBrowseError(f"S3 connector could not be initialized: {exc}") from exc + + +def _s3_bucket(profile: ConnectorProfile, library_id: str | None = None) -> str | None: + return _metadata_string(profile, "bucket") or _metadata_string(profile, "bucket_name") or _clean(library_id) + + +def _s3_object_key(profile: ConnectorProfile, path: str, *, directory: bool = False) -> str: + base_prefix = normalize_connector_browse_path(profile.base_path or _metadata_string(profile, "base_prefix")) + browse_path = normalize_connector_browse_path(path) + key = "/".join(part for part in (base_prefix, browse_path) if part) + if directory and key: + return key.rstrip("/") + "/" + return key + + +def _s3_prefix_items( + *, + bucket: str, + browse_path: str, + base_prefix: str, + prefixes: object, +) -> list[ConnectorBrowseItem]: + if not isinstance(prefixes, list): + return [] + items: list[ConnectorBrowseItem] = [] + for item in prefixes: + if not isinstance(item, Mapping): + continue + key = _clean(item.get("Prefix")) + if not key: + continue + relative = _s3_relative_key(key, base_prefix=base_prefix) + name = _path_name(relative) + if not name: + continue + path = _join_browse_path(browse_path, name) + items.append( + ConnectorBrowseItem( + kind="folder", + name=name, + path=path, + external_id=f"{bucket}:{key}", + metadata={"bucket": bucket, "key": key}, + ) + ) + return items + + +def _s3_object_items( + *, + bucket: str, + browse_path: str, + base_prefix: str, + objects: object, +) -> list[ConnectorBrowseItem]: + if not isinstance(objects, list): + return [] + items: list[ConnectorBrowseItem] = [] + for item in objects: + if not isinstance(item, Mapping): + continue + key = _clean(item.get("Key")) + if not key or key == base_prefix: + continue + relative = _s3_relative_key(key, base_prefix=base_prefix) + name = _path_name(relative) + if not name: + continue + path = _join_browse_path(browse_path, name) + items.append( + ConnectorBrowseItem( + kind="file", + name=name, + path=path, + external_id=f"{bucket}:{key}", + size_bytes=_int(item.get("Size")), + content_type=mimetypes.guess_type(name)[0], + modified_at=_timestamp(item.get("LastModified")), + etag=_clean(item.get("ETag")), + metadata={ + "bucket": bucket, + "key": key, + **({"storage_class": item["StorageClass"]} if "StorageClass" in item else {}), + }, + ) + ) + return items + + +def _s3_relative_key(key: str, *, base_prefix: str) -> str: + clean_key = key.rstrip("/") + clean_base = base_prefix.rstrip("/") + if clean_base and clean_key.startswith(f"{clean_base}/"): + return clean_key[len(clean_base) + 1 :] + return clean_key + + +def _s3_max_keys(profile: ConnectorProfile) -> int: + configured = _int(profile.metadata.get("max_keys")) + if configured is None: + return 1000 + return max(1, min(configured, 1000)) + + +def _s3_verify(profile: ConnectorProfile) -> bool | str | None: + ca_bundle = _metadata_string(profile, "ca_bundle") + if ca_bundle: + return ca_bundle + if "verify_tls" in profile.metadata: + return _metadata_bool(profile, "verify_tls", default=True) + if "tls_verify" in profile.metadata: + return _metadata_bool(profile, "tls_verify", default=True) + return None + + +def _s3_addressing_style(profile: ConnectorProfile) -> str | None: + style = _metadata_string(profile, "addressing_style") + if style in {"path", "virtual", "auto"}: + return style + if _metadata_bool(profile, "path_style", default=False): + return "path" + return None + + def _seafile_headers(profile: ConnectorProfile) -> dict[str, str]: token = _seafile_token(profile) return {"Authorization": f"Token {token}", "Accept": "application/json"} @@ -444,6 +667,13 @@ def _metadata_bool(profile: ConnectorProfile, key: str, *, default: bool = False return str(value).strip().casefold() in {"1", "true", "yes", "on"} +def _metadata_env(profile: ConnectorProfile, key: str) -> str | None: + env_name = _metadata_string(profile, key) + if not env_name: + return None + return _env_required(env_name, profile.id) + + def _env_required(name: str, profile_id: str) -> str: value = _clean(os.environ.get(name)) if not value: diff --git a/src/govoplan_files/backend/storage/connector_imports.py b/src/govoplan_files/backend/storage/connector_imports.py index 31a8833..d4a5157 100644 --- a/src/govoplan_files/backend/storage/connector_imports.py +++ b/src/govoplan_files/backend/storage/connector_imports.py @@ -21,6 +21,9 @@ from govoplan_files.backend.storage.connector_browse import ( _smb_stat_size, _smb_unc_path, _smbclient_module, + _s3_bucket, + _s3_client, + _s3_object_key, _seafile_headers, _seafile_url, _webdav_url, @@ -64,6 +67,8 @@ def read_connector_file( return _read_webdav_file(profile, path=path, max_bytes=max_bytes) if profile.provider == "smb": return _read_smb_file(profile, path=path, max_bytes=max_bytes) + if profile.provider == "s3": + return _read_s3_file(profile, library_id=library_id, path=path, max_bytes=max_bytes) raise ConnectorImportUnsupported(f"Connector file import is not implemented for {profile.provider} profiles yet") @@ -213,6 +218,63 @@ def _read_smb_file(profile: ConnectorProfile, *, path: str, max_bytes: int) -> C ) +def _read_s3_file(profile: ConnectorProfile, *, library_id: str, path: str, max_bytes: int) -> ConnectorDownloadedFile: + bucket = _s3_bucket(profile, library_id) + if not bucket: + raise ConnectorImportError("S3 import requires a bucket in library_id or profile metadata") + key = _s3_object_key(profile, path) + if not key: + raise ConnectorImportError("S3 import requires an object key") + try: + client = _s3_client(profile) + except ConnectorBrowseUnsupported as exc: + raise ConnectorImportUnsupported(str(exc)) from exc + except ConnectorBrowseError as exc: + raise ConnectorImportError(str(exc)) from exc + try: + detail = client.head_object(Bucket=bucket, Key=key) + except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific + raise ConnectorImportError(f"S3 object metadata lookup failed: {exc}") from exc + if not isinstance(detail, dict): + raise ConnectorImportError("S3 connector returned invalid object metadata") + size = _int(detail.get("ContentLength")) + if size is not None and size > max_bytes: + raise ConnectorImportError(f"S3 object exceeds limit of {max_bytes} bytes") + version_id = _clean(detail.get("VersionId")) + request: dict[str, object] = {"Bucket": bucket, "Key": key} + if version_id: + request["VersionId"] = version_id + try: + response = client.get_object(**request) + body = response.get("Body") + data = body.read(max_bytes + 1) if hasattr(body, "read") else bytes(response.get("Body") or b"") + except Exception as exc: # pragma: no cover - concrete exception types are dependency-version specific + raise ConnectorImportError(f"S3 object download failed: {exc}") from exc + if len(data) > max_bytes: + raise ConnectorImportError(f"S3 object exceeds limit of {max_bytes} bytes") + content_type = _clean(response.get("ContentType") if isinstance(response, dict) else None) or _clean(detail.get("ContentType")) or mimetypes.guess_type(key)[0] + etag = _clean(response.get("ETag") if isinstance(response, dict) else None) or _clean(detail.get("ETag")) + filename = filename_from_path(key) + return ConnectorDownloadedFile( + filename=filename, + data=data, + content_type=content_type, + revision=version_id or etag or _clean(detail.get("LastModified")), + external_id=f"{bucket}:{key}", + external_url=f"s3://{bucket}/{key}", + metadata={ + "bucket": bucket, + "key": key, + "version_id": version_id, + "etag": etag, + "size": len(data), + **({"checksum_sha256": detail["ChecksumSHA256"]} if "ChecksumSHA256" in detail else {}), + **({"checksum_crc32": detail["ChecksumCRC32"]} if "ChecksumCRC32" in detail else {}), + **({"storage_class": detail["StorageClass"]} if "StorageClass" in detail else {}), + }, + ) + + def _int(value: object) -> int | None: if value is None or value == "": return None diff --git a/src/govoplan_files/backend/storage/connector_profiles.py b/src/govoplan_files/backend/storage/connector_profiles.py index 77d3a85..ee26565 100644 --- a/src/govoplan_files/backend/storage/connector_profiles.py +++ b/src/govoplan_files/backend/storage/connector_profiles.py @@ -12,8 +12,17 @@ from govoplan_files.backend.storage.connector_policy import ConnectorPolicySourc _ENV_JSON_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_JSON", "FILES_CONNECTOR_PROFILES_JSON") _ENV_FILE_KEYS = ("GOVOPLAN_FILES_CONNECTOR_PROFILES_FILE", "FILES_CONNECTOR_PROFILES_FILE") -_SUPPORTED_PROVIDERS = {"seafile", "nextcloud", "webdav", "smb", "nfs", "dms", "generic"} -_INLINE_SECRET_FIELDS = ("password", "token", "api_key", "access_key", "secret_key") +_SUPPORTED_PROVIDERS = {"seafile", "nextcloud", "webdav", "smb", "s3", "sharepoint", "onedrive", "nfs", "dms", "generic"} +_INLINE_SECRET_FIELDS = ( + "password", + "token", + "api_key", + "access_key", + "access_key_id", + "secret_key", + "secret_access_key", + "session_token", +) def supported_connector_providers() -> set[str]: @@ -123,43 +132,19 @@ def connector_profiles_from_payload(value: object) -> list[ConnectorProfile]: def _profile_from_mapping(value: Mapping[str, Any]) -> ConnectorProfile: - profile_id = _clean(value.get("id") or value.get("connector_id") or value.get("name")) - if not profile_id: - raise ValueError("Connector profiles require id") - provider = (_clean(value.get("provider") or value.get("type")) or "generic").casefold() - if provider not in _SUPPORTED_PROVIDERS: - raise ValueError(f"Unsupported connector provider: {provider}") - scope_type = normalize_policy_scope_type(str(value.get("scope_type") or "system")) - scope_id = _clean(value.get("scope_id")) - if scope_type == "system": - scope_id = None - elif not scope_id: - raise ValueError(f"{scope_type} connector profiles require scope_id") - - credentials = value.get("credentials") - credentials = credentials if isinstance(credentials, Mapping) else {} + profile_id = _profile_id_from_mapping(value) + provider = _provider_from_mapping(value) + scope_type, scope_id = _scope_from_mapping(value) + credentials = _credentials_from_mapping(value) mode = _clean(value.get("credential_mode") or value.get("auth_type") or credentials.get("mode") or credentials.get("type")) or "none" - profile = ConnectorProfile( - id=profile_id, - label=_clean(value.get("label") or value.get("name")) or profile_id, + profile = _profile_from_normalized_mapping( + value, + profile_id=profile_id, provider=provider, - endpoint_url=_clean(value.get("endpoint_url") or value.get("base_url") or value.get("url")), - base_path=_clean(value.get("base_path") or value.get("path") or value.get("root")), - enabled=_bool(value.get("enabled"), default=True), scope_type=scope_type, scope_id=scope_id, - credential_profile_id=_clean(value.get("credential_profile_id") or value.get("credential_id")), - credential_profile_label=_clean(value.get("credential_profile_label") or value.get("credential_label")), credential_mode=mode, - username=_clean(value.get("username") or credentials.get("username")), - password_env=_clean(value.get("password_env") or credentials.get("password_env")), - token_env=_clean(value.get("token_env") or credentials.get("token_env")), - secret_ref=_clean(value.get("secret_ref") or credentials.get("secret_ref")), - password_value=_clean(value.get("password") or credentials.get("password")), - token_value=_clean(value.get("token") or credentials.get("token")), - has_inline_secret=any(_clean(value.get(field) or credentials.get(field)) for field in _INLINE_SECRET_FIELDS), - capabilities=tuple(_string_list(value.get("capabilities"))), - metadata=_public_metadata(value.get("metadata")), + credentials=credentials, ) return ConnectorProfile( id=profile.id, @@ -187,6 +172,69 @@ def _profile_from_mapping(value: Mapping[str, Any]) -> ConnectorProfile: ) +def _profile_id_from_mapping(value: Mapping[str, Any]) -> str: + profile_id = _clean(value.get("id") or value.get("connector_id") or value.get("name")) + if not profile_id: + raise ValueError("Connector profiles require id") + return profile_id + + +def _provider_from_mapping(value: Mapping[str, Any]) -> str: + provider = (_clean(value.get("provider") or value.get("type")) or "generic").casefold() + if provider not in _SUPPORTED_PROVIDERS: + raise ValueError(f"Unsupported connector provider: {provider}") + return provider + + +def _scope_from_mapping(value: Mapping[str, Any]) -> tuple[str, str | None]: + scope_type = normalize_policy_scope_type(str(value.get("scope_type") or "system")) + scope_id = _clean(value.get("scope_id")) + if scope_type == "system": + return scope_type, None + if not scope_id: + raise ValueError(f"{scope_type} connector profiles require scope_id") + return scope_type, scope_id + + +def _credentials_from_mapping(value: Mapping[str, Any]) -> Mapping[str, Any]: + credentials = value.get("credentials") + return credentials if isinstance(credentials, Mapping) else {} + + +def _profile_from_normalized_mapping( + value: Mapping[str, Any], + *, + profile_id: str, + provider: str, + scope_type: str, + scope_id: str | None, + credential_mode: str, + credentials: Mapping[str, Any], +) -> ConnectorProfile: + return ConnectorProfile( + id=profile_id, + label=_clean(value.get("label") or value.get("name")) or profile_id, + provider=provider, + endpoint_url=_clean(value.get("endpoint_url") or value.get("base_url") or value.get("url")), + base_path=_clean(value.get("base_path") or value.get("path") or value.get("root")), + enabled=_bool(value.get("enabled"), default=True), + scope_type=scope_type, + scope_id=scope_id, + credential_profile_id=_clean(value.get("credential_profile_id") or value.get("credential_id")), + credential_profile_label=_clean(value.get("credential_profile_label") or value.get("credential_label")), + credential_mode=credential_mode, + username=_clean(value.get("username") or credentials.get("username")), + password_env=_clean(value.get("password_env") or credentials.get("password_env")), + token_env=_clean(value.get("token_env") or credentials.get("token_env")), + secret_ref=_clean(value.get("secret_ref") or credentials.get("secret_ref")), + password_value=_clean(value.get("password") or credentials.get("password")), + token_value=_clean(value.get("token") or credentials.get("token")), + has_inline_secret=any(_clean(value.get(field) or credentials.get(field)) for field in _INLINE_SECRET_FIELDS), + capabilities=tuple(_string_list(value.get("capabilities"))), + metadata=_public_metadata(value.get("metadata")), + ) + + def _raw_profiles_payload(settings: object | None) -> object: for key in _ENV_JSON_KEYS: value = os.environ.get(key) diff --git a/src/govoplan_files/backend/storage/connector_providers.py b/src/govoplan_files/backend/storage/connector_providers.py index 58c101a..c7b9ca2 100644 --- a/src/govoplan_files/backend/storage/connector_providers.py +++ b/src/govoplan_files/backend/storage/connector_providers.py @@ -112,6 +112,51 @@ def connector_provider_descriptors() -> tuple[ConnectorProviderDescriptor, ...]: audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), notes="Requires the optional smbprotocol dependency and an smb://server[:port]/share[/path] profile endpoint.", ), + ConnectorProviderDescriptor( + provider="s3", + label="S3-compatible object storage", + protocol="s3-api", + implemented=True, + browse_supported=True, + import_supported=True, + optional_dependency="boto3", + permission_model=governed_permissions, + sync_strategy="On-demand bucket/prefix browse and object import/sync; sync updates managed versions and never mutates the remote bucket.", + conflict_strategy="Managed import/sync uses existing files conflict_strategy handling after object download.", + preview_strategy="Previews are generated from the frozen managed file after import or sync, not directly from the bucket.", + audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), + notes="Configure endpoint_url for MinIO or other S3-compatible stores; use profile metadata for bucket, region, addressing_style, verify_tls, and ca_bundle.", + ), + ConnectorProviderDescriptor( + provider="sharepoint", + label="SharePoint", + protocol="microsoft-graph/sharepoint", + implemented=False, + browse_supported=False, + import_supported=False, + optional_dependency=None, + permission_model=governed_permissions, + sync_strategy="Planned read-only browse/import through deployment-managed Microsoft Graph or SharePoint credentials.", + conflict_strategy=freeze_model, + preview_strategy="Will preview from frozen managed file after import, not directly from SharePoint.", + audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), + notes="Provider/profile key is reserved; live browsing/import still needs Graph/SharePoint authentication and paging implementation.", + ), + ConnectorProviderDescriptor( + provider="onedrive", + label="OneDrive", + protocol="microsoft-graph/onedrive", + implemented=False, + browse_supported=False, + import_supported=False, + optional_dependency=None, + permission_model=governed_permissions, + sync_strategy="Planned read-only browse/import through deployment-managed Microsoft Graph credentials.", + conflict_strategy=freeze_model, + preview_strategy="Will preview from frozen managed file after import, not directly from OneDrive.", + audit_events=("files.connector.imported", "files.connector.synced", "files.connector.accessed"), + notes="Provider/profile key is reserved; live browsing/import still needs Graph drive selection, OAuth/app registration, and paging implementation.", + ), ConnectorProviderDescriptor( provider="nfs", label="NFS", diff --git a/src/govoplan_files/backend/storage/transfers.py b/src/govoplan_files/backend/storage/transfers.py index 77e285f..860acf7 100644 --- a/src/govoplan_files/backend/storage/transfers.py +++ b/src/govoplan_files/backend/storage/transfers.py @@ -1,5 +1,6 @@ from __future__ import annotations +from dataclasses import dataclass from pathlib import PurePosixPath from typing import Iterable @@ -26,6 +27,26 @@ from govoplan_files.backend.storage.folders import _active_folder_exists, _ensur from govoplan_files.backend.storage.paths import filename_from_path, join_folder_filename, normalize_folder, normalize_logical_path +@dataclass(frozen=True, slots=True) +class _NormalizedTransferRequest: + operation: str + source_owner_type: str + target_owner_type: str + source_folder_paths: list[str] + target_folder: str + conflict_strategy: str + conflict_resolution_map: dict[str, FileConflictResolution] + + +@dataclass(frozen=True, slots=True) +class _NormalizedRenameRequest: + mode: str + selected_file_ids: list[str] + selected_folder_roots: list[str] + owner_type: str | None + owner_id: str | None + + def transfer_selection( session: Session, *, @@ -51,23 +72,36 @@ def transfer_selection( create new file assets/versions that reference the existing immutable blob. """ - operation = operation.lower().strip() - if operation not in {"move", "copy"}: - raise FileStorageError("Unsupported transfer operation") - source_owner_type = source_owner_type.lower().strip() - target_owner_type = target_owner_type.lower().strip() - source_folder_paths = [normalize_folder(path) for path in folder_paths if normalize_folder(path)] - target_folder = normalize_folder(target_folder) - conflict_strategy = _normalize_conflict_strategy(conflict_strategy) - conflict_resolution_map = _resolution_by_path(conflict_resolutions) + normalized = _normalize_transfer_request( + operation=operation, + source_owner_type=source_owner_type, + target_owner_type=target_owner_type, + folder_paths=folder_paths, + target_folder=target_folder, + conflict_strategy=conflict_strategy, + conflict_resolutions=conflict_resolutions, + ) + operation = normalized.operation + source_owner_type = normalized.source_owner_type + target_owner_type = normalized.target_owner_type + source_folder_paths = normalized.source_folder_paths + target_folder = normalized.target_folder + conflict_strategy = normalized.conflict_strategy + conflict_resolution_map = normalized.conflict_resolution_map - ensure_owner_access(session, tenant_id=tenant_id, owner_type=source_owner_type, owner_id=source_owner_id, user_id=user_id, is_admin=is_admin) - ensure_owner_access(session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, user_id=user_id, is_admin=is_admin) - - if operation == "move" and source_owner_type == target_owner_type and source_owner_id == target_owner_id: - for folder_path in source_folder_paths: - if target_folder == folder_path or target_folder.startswith(f"{folder_path}/"): - raise FileStorageError("Cannot move a folder into itself or one of its child folders") + _ensure_transfer_access_and_shape( + session, + tenant_id=tenant_id, + user_id=user_id, + source_owner_type=source_owner_type, + source_owner_id=source_owner_id, + target_owner_type=target_owner_type, + target_owner_id=target_owner_id, + source_folder_paths=source_folder_paths, + target_folder=target_folder, + operation=operation, + is_admin=is_admin, + ) assets_by_id = _selected_assets_for_owner( session, @@ -77,114 +111,327 @@ def transfer_selection( file_ids=file_ids, ) - folder_asset_targets: dict[str, str] = {} - folder_target_paths: dict[str, str] = {} - for folder_path in source_folder_paths: - folder_basename = PurePosixPath(folder_path).name - target_prefix = normalize_folder(f"{target_folder}/{folder_basename}" if target_folder else folder_basename) - folder_target_paths[folder_path] = target_prefix - if operation == "copy" or not (source_owner_type == target_owner_type and source_owner_id == target_owner_id and target_prefix == folder_path): - if _active_folder_exists(session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, path=target_prefix): - resolution = conflict_resolution_map.get(target_prefix) - action = resolution.action if resolution else conflict_strategy - if action == "skip": - folder_target_paths.pop(folder_path, None) - continue - if action == "rename": - # Rename the selected folder root while preserving its contents below it. - counter = 1 - candidate = target_prefix - while _active_folder_exists(session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, path=candidate): - candidate = _candidate_renamed_path(target_prefix, counter) - counter += 1 - target_prefix = normalize_folder(candidate) - folder_target_paths[folder_path] = target_prefix - elif action == "reject": - raise FileStorageError(f"Target folder already exists: {target_prefix}") - # overwrite on folders means merge into the existing folder; conflicting files are still handled below. - source_assets = _asset_query_for_owner(session, tenant_id=tenant_id, owner_type=source_owner_type, owner_id=source_owner_id).filter( - FileAsset.deleted_at.is_(None), - FileAsset.display_path.like(f"{folder_path}/%"), - ).all() - for asset in source_assets: - relative = asset.display_path[len(folder_path) + 1 :] - folder_asset_targets[asset.id] = normalize_logical_path(f"{target_prefix}/{relative}") - assets_by_id[asset.id] = asset - - direct_file_targets: dict[str, str] = {} - for file_id, asset in assets_by_id.items(): - if file_id in folder_asset_targets: - continue - direct_file_targets[file_id] = normalize_logical_path(join_folder_filename(target_folder, filename_from_path(asset.display_path))) - - all_targets = {**folder_asset_targets, **direct_file_targets} + folder_target_paths, all_targets = _plan_transfer_targets( + session, + tenant_id=tenant_id, + assets_by_id=assets_by_id, + operation=operation, + source_owner_type=source_owner_type, + source_owner_id=source_owner_id, + target_owner_type=target_owner_type, + target_owner_id=target_owner_id, + source_folder_paths=source_folder_paths, + target_folder=target_folder, + conflict_strategy=conflict_strategy, + conflict_resolution_map=conflict_resolution_map, + ) if not all_targets and not source_folder_paths: return 0, 0 + all_targets = _resolve_transfer_targets( + session, + tenant_id=tenant_id, + assets_by_id=assets_by_id, + all_targets=all_targets, + operation=operation, + target_owner_type=target_owner_type, + target_owner_id=target_owner_id, + conflict_strategy=conflict_strategy, + conflict_resolution_map=conflict_resolution_map, + ) + copied_or_moved_folders = _ensure_transfer_target_hierarchy( + session, + tenant_id=tenant_id, + user_id=user_id, + operation=operation, + source_owner_type=source_owner_type, + source_owner_id=source_owner_id, + target_owner_type=target_owner_type, + target_owner_id=target_owner_id, + folder_target_paths=folder_target_paths, + all_targets=all_targets, + ) + copied_or_moved_files = _apply_transfer_assets( + session, + tenant_id=tenant_id, + user_id=user_id, + operation=operation, + assets_by_id=assets_by_id, + all_targets=all_targets, + target_owner_type=target_owner_type, + target_owner_id=target_owner_id, + ) + _apply_transfer_folder_records( + session, + tenant_id=tenant_id, + user_id=user_id, + operation=operation, + source_owner_type=source_owner_type, + source_owner_id=source_owner_id, + target_owner_type=target_owner_type, + target_owner_id=target_owner_id, + folder_target_paths=folder_target_paths, + ) + return copied_or_moved_files, copied_or_moved_folders + + +def _plan_transfer_targets( + session: Session, + *, + tenant_id: str, + assets_by_id: dict[str, FileAsset], + operation: str, + source_owner_type: str, + source_owner_id: str, + target_owner_type: str, + target_owner_id: str, + source_folder_paths: list[str], + target_folder: str, + conflict_strategy: str, + conflict_resolution_map: dict[str, FileConflictResolution], +) -> tuple[dict[str, str], dict[str, str]]: + folder_target_paths, folder_asset_targets = _plan_folder_transfer_targets( + session, + tenant_id=tenant_id, + assets_by_id=assets_by_id, + operation=operation, + source_owner_type=source_owner_type, + source_owner_id=source_owner_id, + target_owner_type=target_owner_type, + target_owner_id=target_owner_id, + source_folder_paths=source_folder_paths, + target_folder=target_folder, + conflict_strategy=conflict_strategy, + conflict_resolution_map=conflict_resolution_map, + ) + direct_file_targets = _direct_file_transfer_targets( + assets_by_id, + folder_asset_targets=folder_asset_targets, + target_folder=target_folder, + ) + return folder_target_paths, {**folder_asset_targets, **direct_file_targets} + + +def _plan_folder_transfer_targets( + session: Session, + *, + tenant_id: str, + assets_by_id: dict[str, FileAsset], + operation: str, + source_owner_type: str, + source_owner_id: str, + target_owner_type: str, + target_owner_id: str, + source_folder_paths: list[str], + target_folder: str, + conflict_strategy: str, + conflict_resolution_map: dict[str, FileConflictResolution], +) -> tuple[dict[str, str], dict[str, str]]: + folder_asset_targets: dict[str, str] = {} + folder_target_paths: dict[str, str] = {} + for folder_path in source_folder_paths: + target_prefix = _resolved_folder_target_prefix( + session, + tenant_id=tenant_id, + operation=operation, + source_owner_type=source_owner_type, + source_owner_id=source_owner_id, + target_owner_type=target_owner_type, + target_owner_id=target_owner_id, + folder_path=folder_path, + target_folder=target_folder, + conflict_strategy=conflict_strategy, + conflict_resolution_map=conflict_resolution_map, + ) + if target_prefix is None: + continue + folder_target_paths[folder_path] = target_prefix + for asset in _assets_under_folder(session, tenant_id=tenant_id, owner_type=source_owner_type, owner_id=source_owner_id, folder_path=folder_path): + relative = asset.display_path[len(folder_path) + 1 :] + folder_asset_targets[asset.id] = normalize_logical_path(f"{target_prefix}/{relative}") + assets_by_id[asset.id] = asset + return folder_target_paths, folder_asset_targets + + +def _resolved_folder_target_prefix( + session: Session, + *, + tenant_id: str, + operation: str, + source_owner_type: str, + source_owner_id: str, + target_owner_type: str, + target_owner_id: str, + folder_path: str, + target_folder: str, + conflict_strategy: str, + conflict_resolution_map: dict[str, FileConflictResolution], +) -> str | None: + folder_basename = PurePosixPath(folder_path).name + target_prefix = normalize_folder(f"{target_folder}/{folder_basename}" if target_folder else folder_basename) + same_owner_same_path = source_owner_type == target_owner_type and source_owner_id == target_owner_id and target_prefix == folder_path + if operation == "move" and same_owner_same_path: + return target_prefix + if not _active_folder_exists(session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, path=target_prefix): + return target_prefix + resolution = conflict_resolution_map.get(target_prefix) + action = resolution.action if resolution else conflict_strategy + if action == "skip": + return None + if action == "reject": + raise FileStorageError(f"Target folder already exists: {target_prefix}") + if action == "rename": + return _next_available_folder_path(session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, desired_path=target_prefix) + return target_prefix + + +def _next_available_folder_path( + session: Session, + *, + tenant_id: str, + owner_type: str, + owner_id: str, + desired_path: str, +) -> str: + counter = 1 + candidate = desired_path + while _active_folder_exists(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, path=candidate): + candidate = _candidate_renamed_path(desired_path, counter) + counter += 1 + return normalize_folder(candidate) + + +def _assets_under_folder( + session: Session, + *, + tenant_id: str, + owner_type: str, + owner_id: str, + folder_path: str, +) -> list[FileAsset]: + return _asset_query_for_owner(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id).filter( + FileAsset.deleted_at.is_(None), + FileAsset.display_path.like(f"{folder_path}/%"), + ).all() + + +def _direct_file_transfer_targets( + assets_by_id: dict[str, FileAsset], + *, + folder_asset_targets: dict[str, str], + target_folder: str, +) -> dict[str, str]: + return { + file_id: normalize_logical_path(join_folder_filename(target_folder, filename_from_path(asset.display_path))) + for file_id, asset in assets_by_id.items() + if file_id not in folder_asset_targets + } + + +def _resolve_transfer_targets( + session: Session, + *, + tenant_id: str, + assets_by_id: dict[str, FileAsset], + all_targets: dict[str, str], + operation: str, + target_owner_type: str, + target_owner_id: str, + conflict_strategy: str, + conflict_resolution_map: dict[str, FileConflictResolution], +) -> dict[str, str]: resolved_targets: dict[str, str] = {} - skipped_asset_ids: set[str] = set() reserved_targets: set[str] = set() for asset_id, target_path in all_targets.items(): - source_asset = assets_by_id[asset_id] - same_owner = (source_asset.owner_type == target_owner_type and _asset_owner_id(source_asset) == target_owner_id) - exclude = source_asset.id if operation == "move" and same_owner else None - normalized_target = normalize_logical_path(target_path) - resolution = conflict_resolution_map.get(normalized_target) - action = resolution.action if resolution else conflict_strategy - if resolution and resolution.new_path and action == "rename": - normalized_target = normalize_logical_path(resolution.new_path) - conflict = _active_asset_exists( + normalized_target = _resolve_transfer_asset_target( + session, + tenant_id=tenant_id, + source_asset=assets_by_id[asset_id], + target_path=target_path, + operation=operation, + target_owner_type=target_owner_type, + target_owner_id=target_owner_id, + conflict_strategy=conflict_strategy, + conflict_resolution_map=conflict_resolution_map, + reserved_targets=reserved_targets, + ) + if normalized_target is None: + continue + reserved_targets.add(normalized_target) + resolved_targets[asset_id] = normalized_target + return resolved_targets + + +def _resolve_transfer_asset_target( + session: Session, + *, + tenant_id: str, + source_asset: FileAsset, + target_path: str, + operation: str, + target_owner_type: str, + target_owner_id: str, + conflict_strategy: str, + conflict_resolution_map: dict[str, FileConflictResolution], + reserved_targets: set[str], +) -> str | None: + same_owner = source_asset.owner_type == target_owner_type and _asset_owner_id(source_asset) == target_owner_id + exclude = source_asset.id if operation == "move" and same_owner else None + normalized_target = normalize_logical_path(target_path) + resolution = conflict_resolution_map.get(normalized_target) + action = resolution.action if resolution else conflict_strategy + if resolution and resolution.new_path and action == "rename": + normalized_target = normalize_logical_path(resolution.new_path) + conflict = _active_asset_exists( + session, + tenant_id=tenant_id, + owner_type=target_owner_type, + owner_id=target_owner_id, + path=normalized_target, + exclude_asset_id=exclude, + ) or normalized_target in reserved_targets + if not conflict and action != "rename": + return normalized_target + if conflict and action == "reject": + raise FileStorageError(f"Target file already exists: {normalized_target}") + if conflict and action == "skip": + return None + if conflict and action == "overwrite": + _soft_delete_conflicting_asset( session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, path=normalized_target, exclude_asset_id=exclude, - ) or normalized_target in reserved_targets - if conflict: - if action == "reject": - raise FileStorageError(f"Target file already exists: {normalized_target}") - if action == "skip": - skipped_asset_ids.add(asset_id) - continue - if action == "overwrite": - _soft_delete_conflicting_asset( - session, - tenant_id=tenant_id, - owner_type=target_owner_type, - owner_id=target_owner_id, - path=normalized_target, - exclude_asset_id=exclude, - ) - elif action == "rename": - normalized_target = _next_available_logical_path( - session, - tenant_id=tenant_id, - owner_type=target_owner_type, - owner_id=target_owner_id, - desired_path=normalized_target, - reserved_paths=reserved_targets, - exclude_asset_id=exclude, - ) - elif action == "rename": - normalized_target = _next_available_logical_path( - session, - tenant_id=tenant_id, - owner_type=target_owner_type, - owner_id=target_owner_id, - desired_path=normalized_target, - reserved_paths=reserved_targets, - exclude_asset_id=exclude, - ) - reserved_targets.add(normalized_target) - resolved_targets[asset_id] = normalized_target - all_targets = resolved_targets + ) + return normalized_target + if action == "rename": + return _next_available_logical_path( + session, + tenant_id=tenant_id, + owner_type=target_owner_type, + owner_id=target_owner_id, + desired_path=normalized_target, + reserved_paths=reserved_targets, + exclude_asset_id=exclude, + ) + return normalized_target - copied_or_moved_files = 0 - copied_or_moved_folders = 0 - # Create target folder hierarchy first, including selected folder roots and - # any nested folders/files parents. +def _ensure_transfer_target_hierarchy( + session: Session, + *, + tenant_id: str, + user_id: str, + operation: str, + source_owner_type: str, + source_owner_id: str, + target_owner_type: str, + target_owner_id: str, + folder_target_paths: dict[str, str], + all_targets: dict[str, str], +) -> int: same_owner_transfer = source_owner_type == target_owner_type and source_owner_id == target_owner_id for target_path in folder_target_paths.values(): path_to_create = target_path @@ -193,19 +440,51 @@ def transfer_selection( path_to_create = "" if parent == "." else parent if path_to_create: _ensure_target_folder_hierarchy(session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, user_id=user_id, path=path_to_create) - copied_or_moved_folders += 1 for target_path in all_targets.values(): parent = str(PurePosixPath(target_path).parent) if parent and parent != ".": _ensure_target_folder_hierarchy(session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, user_id=user_id, path=parent) + return len(folder_target_paths) + +def _apply_transfer_assets( + session: Session, + *, + tenant_id: str, + user_id: str, + operation: str, + assets_by_id: dict[str, FileAsset], + all_targets: dict[str, str], + target_owner_type: str, + target_owner_id: str, +) -> int: if operation == "copy": for asset_id, target_path in all_targets.items(): _copy_asset_to_path(session, assets_by_id[asset_id], tenant_id=tenant_id, target_owner_type=target_owner_type, target_owner_id=target_owner_id, target_path=target_path, user_id=user_id) - copied_or_moved_files += 1 - return copied_or_moved_files, copied_or_moved_folders + return len(all_targets) + return _move_transfer_assets( + session, + tenant_id=tenant_id, + user_id=user_id, + assets_by_id=assets_by_id, + all_targets=all_targets, + target_owner_type=target_owner_type, + target_owner_id=target_owner_id, + ) + +def _move_transfer_assets( + session: Session, + *, + tenant_id: str, + user_id: str, + assets_by_id: dict[str, FileAsset], + all_targets: dict[str, str], + target_owner_type: str, + target_owner_id: str, +) -> int: now = utcnow() + moved = 0 for asset_id, target_path in all_targets.items(): asset = assets_by_id[asset_id] same_owner = asset.owner_type == target_owner_type and _asset_owner_id(asset) == target_owner_id @@ -218,28 +497,101 @@ def transfer_selection( _copy_asset_to_path(session, asset, tenant_id=tenant_id, target_owner_type=target_owner_type, target_owner_id=target_owner_id, target_path=target_path, user_id=user_id) asset.deleted_at = now session.add(asset) - copied_or_moved_files += 1 + moved += 1 + return moved - # Move/copy persisted folder records after files. For cross-owner moves, create - # target records and soft-delete source records. For same-owner moves, rename - # them in place. + +def _apply_transfer_folder_records( + session: Session, + *, + tenant_id: str, + user_id: str, + operation: str, + source_owner_type: str, + source_owner_id: str, + target_owner_type: str, + target_owner_id: str, + folder_target_paths: dict[str, str], +) -> None: + now = utcnow() + same_owner_transfer = operation == "move" and source_owner_type == target_owner_type and source_owner_id == target_owner_id for source_path, target_prefix in folder_target_paths.items(): - folder_rows = _folder_query_for_owner(session, tenant_id=tenant_id, owner_type=source_owner_type, owner_id=source_owner_id).filter( - FileFolder.deleted_at.is_(None), - or_(FileFolder.path == source_path, FileFolder.path.like(f"{source_path}/%")), - ).all() - for folder in folder_rows: - relative = "" if folder.path == source_path else folder.path[len(source_path) + 1 :] - new_path = normalize_folder(f"{target_prefix}/{relative}" if relative else target_prefix) - if operation == "move" and source_owner_type == target_owner_type and source_owner_id == target_owner_id: + for folder in _folder_rows_under_path(session, tenant_id=tenant_id, owner_type=source_owner_type, owner_id=source_owner_id, source_path=source_path): + new_path = _folder_transfer_target_path(folder.path, source_path=source_path, target_prefix=target_prefix) + if same_owner_transfer: folder.path = new_path session.add(folder) - else: - _ensure_target_folder_hierarchy(session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, user_id=user_id, path=new_path) - if operation == "move": - folder.deleted_at = now - session.add(folder) - return copied_or_moved_files, copied_or_moved_folders + continue + _ensure_target_folder_hierarchy(session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, user_id=user_id, path=new_path) + if operation == "move": + folder.deleted_at = now + session.add(folder) + + +def _folder_rows_under_path( + session: Session, + *, + tenant_id: str, + owner_type: str, + owner_id: str, + source_path: str, +) -> list[FileFolder]: + return _folder_query_for_owner(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id).filter( + FileFolder.deleted_at.is_(None), + or_(FileFolder.path == source_path, FileFolder.path.like(f"{source_path}/%")), + ).all() + + +def _folder_transfer_target_path(path: str, *, source_path: str, target_prefix: str) -> str: + relative = "" if path == source_path else path[len(source_path) + 1 :] + return normalize_folder(f"{target_prefix}/{relative}" if relative else target_prefix) + + +def _normalize_transfer_request( + *, + operation: str, + source_owner_type: str, + target_owner_type: str, + folder_paths: list[str], + target_folder: str, + conflict_strategy: str, + conflict_resolutions: Iterable[FileConflictResolution] | None, +) -> _NormalizedTransferRequest: + operation = operation.lower().strip() + if operation not in {"move", "copy"}: + raise FileStorageError("Unsupported transfer operation") + return _NormalizedTransferRequest( + operation=operation, + source_owner_type=source_owner_type.lower().strip(), + target_owner_type=target_owner_type.lower().strip(), + source_folder_paths=[normalized for path in folder_paths if (normalized := normalize_folder(path))], + target_folder=normalize_folder(target_folder), + conflict_strategy=_normalize_conflict_strategy(conflict_strategy), + conflict_resolution_map=_resolution_by_path(conflict_resolutions), + ) + + +def _ensure_transfer_access_and_shape( + session: Session, + *, + tenant_id: str, + user_id: str, + source_owner_type: str, + source_owner_id: str, + target_owner_type: str, + target_owner_id: str, + source_folder_paths: list[str], + target_folder: str, + operation: str, + is_admin: bool, +) -> None: + ensure_owner_access(session, tenant_id=tenant_id, owner_type=source_owner_type, owner_id=source_owner_id, user_id=user_id, is_admin=is_admin) + ensure_owner_access(session, tenant_id=tenant_id, owner_type=target_owner_type, owner_id=target_owner_id, user_id=user_id, is_admin=is_admin) + if operation != "move" or source_owner_type != target_owner_type or source_owner_id != target_owner_id: + return + for folder_path in source_folder_paths: + if target_folder == folder_path or target_folder.startswith(f"{folder_path}/"): + raise FileStorageError("Cannot move a folder into itself or one of its child folders") def _rename_basename(name: str, *, mode: str, new_name: str | None = None, find: str | None = None, replacement: str = "", prefix: str = "", suffix: str = "") -> str: @@ -361,119 +713,287 @@ def _rename_selection( is_admin: bool = False, allow_legacy_without_owner_context: bool = False, ) -> list[RenamePlanItem]: - mode = mode.lower().strip() - if mode not in {"direct", "prefix", "suffix", "replace"}: - raise FileStorageError("Unsupported rename mode") - selected_file_ids = list(dict.fromkeys(file_ids)) - selected_folder_roots = _collapse_folder_roots(folder_paths) - if not selected_file_ids and not selected_folder_roots: - raise FileStorageError("No files or folders selected") - if selected_folder_roots and (not owner_type or not owner_id): - raise FileStorageError("Folder rename requires an owner file space") - if mode == "direct" and (len(selected_file_ids) + len(selected_folder_roots)) != 1: - raise FileStorageError("Direct rename requires exactly one selected item") + normalized = _normalize_rename_request( + file_ids=file_ids, + folder_paths=folder_paths, + owner_type=owner_type, + owner_id=owner_id, + mode=mode, + ) + mode = normalized.mode + selected_file_ids = normalized.selected_file_ids + selected_folder_roots = normalized.selected_folder_roots + owner_type_norm = normalized.owner_type + owner_id_norm = normalized.owner_id + assets = _collect_rename_assets( + session, + tenant_id=tenant_id, + user_id=user_id, + selected_file_ids=selected_file_ids, + owner_type=owner_type_norm, + owner_id=owner_id_norm, + is_admin=is_admin, + allow_legacy_without_owner_context=allow_legacy_without_owner_context, + ) + folder_rows_by_path, affected_folder_paths = _collect_rename_folder_scope( + session, + tenant_id=tenant_id, + user_id=user_id, + assets=assets, + selected_folder_roots=selected_folder_roots, + owner_type=owner_type_norm, + owner_id=owner_id_norm, + is_admin=is_admin, + ) + folder_plan, root_target_paths = _build_folder_rename_plan( + selected_folder_roots, + affected_folder_paths=affected_folder_paths, + mode=mode, + new_name=new_name, + find=find, + replacement=replacement, + prefix=prefix, + suffix=suffix, + recursive=recursive, + ) + file_plan = _build_file_rename_plan( + assets, + selected_folder_roots=selected_folder_roots, + root_target_paths=root_target_paths, + mode=mode, + new_name=new_name, + find=find, + replacement=replacement, + prefix=prefix, + suffix=suffix, + recursive=recursive, + ) + _validate_rename_plans( + session, + tenant_id=tenant_id, + assets=assets, + file_plan=file_plan, + folder_plan=folder_plan, + selected_folder_roots=selected_folder_roots, + owner_type=owner_type_norm, + owner_id=owner_id_norm, + ) + plan = _rename_plan_items(assets=assets, file_plan=file_plan, folder_plan=folder_plan) - owner_type_norm = owner_type.lower().strip() if owner_type else None - owner_id_norm = owner_id + if dry_run: + return plan + + _apply_rename_plans(session, assets=assets, folder_rows_by_path=folder_rows_by_path, file_plan=file_plan, folder_plan=folder_plan) + return plan + + +def _collect_rename_assets( + session: Session, + *, + tenant_id: str, + user_id: str, + selected_file_ids: list[str], + owner_type: str | None, + owner_id: str | None, + is_admin: bool, + allow_legacy_without_owner_context: bool, +) -> dict[str, FileAsset]: + if not selected_file_ids: + return {} + if owner_type and owner_id: + ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin) + return _selected_assets_for_owner(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, file_ids=selected_file_ids) + if not allow_legacy_without_owner_context: + raise FileStorageError("Bulk rename requires an owner file space") assets: dict[str, FileAsset] = {} - if selected_file_ids: - if owner_type_norm and owner_id_norm: - ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type_norm, owner_id=owner_id_norm, user_id=user_id, is_admin=is_admin) - assets.update( - _selected_assets_for_owner( - session, - tenant_id=tenant_id, - owner_type=owner_type_norm, - owner_id=owner_id_norm, - file_ids=selected_file_ids, - ) - ) - elif allow_legacy_without_owner_context: - for file_id in selected_file_ids: - asset = get_asset_for_user(session, tenant_id=tenant_id, user_id=user_id, asset_id=file_id, require_write=True, is_admin=is_admin) - assets[asset.id] = asset - else: - raise FileStorageError("Bulk rename requires an owner file space") + for file_id in selected_file_ids: + asset = get_asset_for_user(session, tenant_id=tenant_id, user_id=user_id, asset_id=file_id, require_write=True, is_admin=is_admin) + assets[asset.id] = asset + return assets + +def _collect_rename_folder_scope( + session: Session, + *, + tenant_id: str, + user_id: str, + assets: dict[str, FileAsset], + selected_folder_roots: list[str], + owner_type: str | None, + owner_id: str | None, + is_admin: bool, +) -> tuple[dict[str, FileFolder], set[str]]: + if not selected_folder_roots or not owner_type or not owner_id: + return {}, set() + ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin) folder_rows_by_path: dict[str, FileFolder] = {} affected_folder_paths: set[str] = set() - if selected_folder_roots and owner_type_norm and owner_id_norm: - ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type_norm, owner_id=owner_id_norm, user_id=user_id, is_admin=is_admin) - for root in selected_folder_roots: - rows = _folder_query_for_owner(session, tenant_id=tenant_id, owner_type=owner_type_norm, owner_id=owner_id_norm).filter( - FileFolder.deleted_at.is_(None), - or_(FileFolder.path == root, FileFolder.path.like(f"{root}/%")), - ).all() - if not rows and not _asset_query_for_owner(session, tenant_id=tenant_id, owner_type=owner_type_norm, owner_id=owner_id_norm).filter(FileAsset.deleted_at.is_(None), FileAsset.display_path.like(f"{root}/%")).first(): - raise FileStorageError(f"Folder not found: {root}") - for row in rows: - folder_rows_by_path[row.path] = row - affected_folder_paths.add(row.path) - for asset in _asset_query_for_owner(session, tenant_id=tenant_id, owner_type=owner_type_norm, owner_id=owner_id_norm).filter(FileAsset.deleted_at.is_(None), FileAsset.display_path.like(f"{root}/%")).all(): - assets[asset.id] = asset + for root in selected_folder_roots: + rows = _folder_rows_under_path(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, source_path=root) + if not rows and not _folder_has_assets(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, root=root): + raise FileStorageError(f"Folder not found: {root}") + for row in rows: + folder_rows_by_path[row.path] = row + affected_folder_paths.add(row.path) + for asset in _assets_under_folder(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, folder_path=root): + assets[asset.id] = asset + return folder_rows_by_path, affected_folder_paths + +def _folder_has_assets(session: Session, *, tenant_id: str, owner_type: str, owner_id: str, root: str) -> bool: + return bool( + _asset_query_for_owner(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id) + .filter(FileAsset.deleted_at.is_(None), FileAsset.display_path.like(f"{root}/%")) + .first() + ) + + +def _build_folder_rename_plan( + selected_folder_roots: list[str], + *, + affected_folder_paths: set[str], + mode: str, + new_name: str | None, + find: str | None, + replacement: str, + prefix: str, + suffix: str, + recursive: bool, +) -> tuple[dict[str, str], dict[str, str]]: folder_plan: dict[str, str] = {} root_target_paths: dict[str, str] = {} for root in selected_folder_roots: - if mode == "direct": - root_parent = "" if str(PurePosixPath(root).parent) == "." else str(PurePosixPath(root).parent) - root_new_name = _rename_basename(PurePosixPath(root).name, mode=mode, new_name=new_name) - new_root = normalize_folder(f"{root_parent}/{root_new_name}" if root_parent else root_new_name) - else: - new_root = normalize_folder(_rename_path(root, mode=mode, find=find, replacement=replacement, prefix=prefix, suffix=suffix)) + new_root = _rename_root_target(root, mode=mode, new_name=new_name, find=find, replacement=replacement, prefix=prefix, suffix=suffix) root_target_paths[root] = new_root if root in affected_folder_paths: folder_plan[root] = new_root for path in sorted(affected_folder_paths, key=lambda item: item.count("/")): if path != root and _path_under_root(path, root): folder_plan[path] = _folder_new_path_for_root(path, root, new_root, recursive=recursive, mode=mode, find=find, replacement=replacement, prefix=prefix, suffix=suffix) + return folder_plan, root_target_paths + +def _rename_root_target( + root: str, + *, + mode: str, + new_name: str | None, + find: str | None, + replacement: str, + prefix: str, + suffix: str, +) -> str: + if mode != "direct": + return normalize_folder(_rename_path(root, mode=mode, find=find, replacement=replacement, prefix=prefix, suffix=suffix)) + root_parent = "" if str(PurePosixPath(root).parent) == "." else str(PurePosixPath(root).parent) + root_new_name = _rename_basename(PurePosixPath(root).name, mode=mode, new_name=new_name) + return normalize_folder(f"{root_parent}/{root_new_name}" if root_parent else root_new_name) + + +def _build_file_rename_plan( + assets: dict[str, FileAsset], + *, + selected_folder_roots: list[str], + root_target_paths: dict[str, str], + mode: str, + new_name: str | None, + find: str | None, + replacement: str, + prefix: str, + suffix: str, + recursive: bool, +) -> dict[str, str]: file_plan: dict[str, str] = {} + folder_roots_by_length = sorted(selected_folder_roots, key=len, reverse=True) for asset in assets.values(): - matched_root = next((root for root in sorted(selected_folder_roots, key=len, reverse=True) if _path_under_root(asset.display_path, root)), None) + matched_root = next((root for root in folder_roots_by_length if _path_under_root(asset.display_path, root)), None) if matched_root: root_new = root_target_paths.get(matched_root) - if not root_new: - continue - file_plan[asset.id] = _file_new_path_for_root(asset.display_path, matched_root, root_new, recursive=recursive, mode=mode, find=find, replacement=replacement, prefix=prefix, suffix=suffix) + if root_new: + file_plan[asset.id] = _file_new_path_for_root(asset.display_path, matched_root, root_new, recursive=recursive, mode=mode, find=find, replacement=replacement, prefix=prefix, suffix=suffix) else: file_plan[asset.id] = _rename_path(asset.display_path, mode=mode, new_name=new_name, find=find, replacement=replacement, prefix=prefix, suffix=suffix) + return file_plan - # Detect duplicate targets and existing active target conflicts before applying. + +def _validate_rename_plans( + session: Session, + *, + tenant_id: str, + assets: dict[str, FileAsset], + file_plan: dict[str, str], + folder_plan: dict[str, str], + selected_folder_roots: list[str], + owner_type: str | None, + owner_id: str | None, +) -> None: + _validate_file_rename_targets(session, tenant_id=tenant_id, assets=assets, file_plan=file_plan) + if selected_folder_roots and owner_type and owner_id: + _validate_folder_rename_targets(session, tenant_id=tenant_id, folder_plan=folder_plan, owner_type=owner_type, owner_id=owner_id) + + +def _validate_file_rename_targets( + session: Session, + *, + tenant_id: str, + assets: dict[str, FileAsset], + file_plan: dict[str, str], +) -> None: target_files: set[str] = set() - target_folders: set[str] = set() - affected_file_ids = set(file_plan) for asset_id, target in file_plan.items(): normalized = normalize_logical_path(target) if normalized in target_files: raise FileStorageError(f"Rename would create duplicate file path: {normalized}") target_files.add(normalized) asset = assets[asset_id] - if _active_asset_exists(session, tenant_id=tenant_id, owner_type=asset.owner_type, owner_id=_asset_owner_id(asset), path=normalized, exclude_asset_id=asset.id): + owner_id = _asset_owner_id(asset) + if _active_asset_exists(session, tenant_id=tenant_id, owner_type=asset.owner_type, owner_id=owner_id, path=normalized, exclude_asset_id=asset.id): raise FileStorageError(f"Target file already exists: {normalized}") - if _active_folder_exists(session, tenant_id=tenant_id, owner_type=asset.owner_type, owner_id=_asset_owner_id(asset), path=normalized): + if _active_folder_exists(session, tenant_id=tenant_id, owner_type=asset.owner_type, owner_id=owner_id, path=normalized): raise FileStorageError(f"Target path is already a folder: {normalized}") - if selected_folder_roots and owner_type_norm and owner_id_norm: - for old_path, target in folder_plan.items(): - normalized = normalize_folder(target) - if normalized in target_folders: - raise FileStorageError(f"Rename would create duplicate folder path: {normalized}") - target_folders.add(normalized) - if _active_folder_exists(session, tenant_id=tenant_id, owner_type=owner_type_norm, owner_id=owner_id_norm, path=normalized, exclude_paths=set(folder_plan.keys())): - raise FileStorageError(f"Target folder already exists: {normalized}") - if _active_asset_exists(session, tenant_id=tenant_id, owner_type=owner_type_norm, owner_id=owner_id_norm, path=normalized): - raise FileStorageError(f"Target path is already a file: {normalized}") - plan: list[RenamePlanItem] = [] - for old_path, new_path in sorted(folder_plan.items()): - plan.append(RenamePlanItem(kind="folder", id=old_path, old_path=old_path, new_path=new_path)) + +def _validate_folder_rename_targets( + session: Session, + *, + tenant_id: str, + folder_plan: dict[str, str], + owner_type: str, + owner_id: str, +) -> None: + target_folders: set[str] = set() + for target in folder_plan.values(): + normalized = normalize_folder(target) + if normalized in target_folders: + raise FileStorageError(f"Rename would create duplicate folder path: {normalized}") + target_folders.add(normalized) + if _active_folder_exists(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, path=normalized, exclude_paths=set(folder_plan.keys())): + raise FileStorageError(f"Target folder already exists: {normalized}") + if _active_asset_exists(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, path=normalized): + raise FileStorageError(f"Target path is already a file: {normalized}") + + +def _rename_plan_items( + *, + assets: dict[str, FileAsset], + file_plan: dict[str, str], + folder_plan: dict[str, str], +) -> list[RenamePlanItem]: + plan = [RenamePlanItem(kind="folder", id=old_path, old_path=old_path, new_path=new_path) for old_path, new_path in sorted(folder_plan.items())] for asset_id, new_path in file_plan.items(): asset = assets[asset_id] plan.append(RenamePlanItem(kind="file", id=asset.id, old_path=asset.display_path, new_path=new_path)) + return plan - if dry_run: - return plan +def _apply_rename_plans( + session: Session, + *, + assets: dict[str, FileAsset], + folder_rows_by_path: dict[str, FileFolder], + file_plan: dict[str, str], + folder_plan: dict[str, str], +) -> None: for old_path, new_path in folder_plan.items(): folder = folder_rows_by_path.get(old_path) if folder: @@ -482,7 +1002,34 @@ def _rename_selection( for asset_id, new_path in file_plan.items(): rename_asset(assets[asset_id], new_path=new_path) session.add(assets[asset_id]) - return plan + + +def _normalize_rename_request( + *, + file_ids: list[str], + folder_paths: list[str], + owner_type: str | None, + owner_id: str | None, + mode: str, +) -> _NormalizedRenameRequest: + clean_mode = mode.lower().strip() + if clean_mode not in {"direct", "prefix", "suffix", "replace"}: + raise FileStorageError("Unsupported rename mode") + selected_file_ids = list(dict.fromkeys(file_ids)) + selected_folder_roots = _collapse_folder_roots(folder_paths) + if not selected_file_ids and not selected_folder_roots: + raise FileStorageError("No files or folders selected") + if selected_folder_roots and (not owner_type or not owner_id): + raise FileStorageError("Folder rename requires an owner file space") + if clean_mode == "direct" and (len(selected_file_ids) + len(selected_folder_roots)) != 1: + raise FileStorageError("Direct rename requires exactly one selected item") + return _NormalizedRenameRequest( + mode=clean_mode, + selected_file_ids=selected_file_ids, + selected_folder_roots=selected_folder_roots, + owner_type=owner_type.lower().strip() if owner_type else None, + owner_id=owner_id, + ) def rename_selection( diff --git a/tests/test_connector_providers.py b/tests/test_connector_providers.py new file mode 100644 index 0000000..e354bab --- /dev/null +++ b/tests/test_connector_providers.py @@ -0,0 +1,157 @@ +from __future__ import annotations + +import unittest +from datetime import UTC, datetime +from unittest.mock import patch + +from govoplan_files.backend.storage.connector_browse import browse_connector_profile +from govoplan_files.backend.storage.connector_browse import ConnectorBrowseUnsupported +from govoplan_files.backend.storage.connector_imports import read_connector_file +from govoplan_files.backend.storage.connector_profiles import ConnectorProfile, connector_profiles_from_payload +from govoplan_files.backend.storage.connector_providers import connector_provider_descriptors + + +class FakeBody: + def __init__(self, data: bytes) -> None: + self.data = data + + def read(self, _limit: int) -> bytes: + return self.data + + +class FakeS3Client: + def __init__(self) -> None: + self.list_objects_request: dict[str, object] | None = None + self.head_request: dict[str, object] | None = None + self.get_request: dict[str, object] | None = None + + def list_objects_v2(self, **kwargs: object) -> dict[str, object]: + self.list_objects_request = dict(kwargs) + return { + "CommonPrefixes": [{"Prefix": "root/reports/"}], + "Contents": [ + { + "Key": "root/report.xlsx", + "Size": 123, + "LastModified": datetime(2026, 7, 12, 10, 0, tzinfo=UTC), + "ETag": '"etag-1"', + "StorageClass": "STANDARD", + } + ], + "NextContinuationToken": "next-page", + } + + def list_buckets(self) -> dict[str, object]: + return {"Buckets": [{"Name": "archive", "CreationDate": datetime(2026, 7, 12, 9, 0, tzinfo=UTC)}]} + + def head_object(self, **kwargs: object) -> dict[str, object]: + self.head_request = dict(kwargs) + return { + "ContentLength": 13, + "ContentType": "text/plain", + "ETag": '"etag-2"', + "VersionId": "version-1", + "ChecksumSHA256": "checksum", + } + + def get_object(self, **kwargs: object) -> dict[str, object]: + self.get_request = dict(kwargs) + return {"Body": FakeBody(b"hello s3\n"), "ContentType": "text/plain", "ETag": '"etag-2"'} + + +def s3_profile(**overrides: object) -> ConnectorProfile: + values = { + "id": "dev-s3", + "label": "Dev S3", + "provider": "s3", + "endpoint_url": "http://127.0.0.1:9000", + "base_path": "root", + "credential_mode": "basic", + "username": "access-key", + "password_value": "secret-key", + "metadata": {"bucket": "govoplan", "region": "eu-central-1", "path_style": True}, + } + values.update(overrides) + return ConnectorProfile(**values) + + +class ConnectorProviderTests(unittest.TestCase): + def test_provider_descriptors_include_s3_and_reserved_microsoft_providers(self) -> None: + descriptors = {descriptor.provider: descriptor for descriptor in connector_provider_descriptors()} + + self.assertTrue(descriptors["s3"].implemented) + self.assertTrue(descriptors["s3"].browse_supported) + self.assertEqual("boto3", descriptors["s3"].optional_dependency) + self.assertFalse(descriptors["sharepoint"].implemented) + self.assertFalse(descriptors["onedrive"].implemented) + + def test_profile_payload_accepts_new_providers_and_redacts_secret_metadata(self) -> None: + profiles = connector_profiles_from_payload( + { + "profiles": [ + { + "id": "dev-s3", + "label": "Dev S3", + "provider": "s3", + "username": "access-key", + "password": "secret-key", + "metadata": {"bucket": "govoplan", "secret_access_key": "do-not-return"}, + }, + {"id": "sharepoint", "provider": "sharepoint"}, + {"id": "onedrive", "provider": "onedrive"}, + ] + } + ) + + by_id = {profile.id: profile for profile in profiles} + self.assertEqual("s3", by_id["dev-s3"].provider) + self.assertTrue(by_id["dev-s3"].credentials_configured) + self.assertEqual({"bucket": "govoplan"}, dict(by_id["dev-s3"].metadata)) + self.assertEqual("sharepoint", by_id["sharepoint"].provider) + self.assertEqual("onedrive", by_id["onedrive"].provider) + + def test_s3_browse_lists_prefixes_and_objects_with_provenance_metadata(self) -> None: + client = FakeS3Client() + + with patch("govoplan_files.backend.storage.connector_browse._s3_client", return_value=client): + items = browse_connector_profile(s3_profile(), path="") + + self.assertEqual({"Bucket": "govoplan", "Prefix": "root/", "Delimiter": "/", "MaxKeys": 1000}, client.list_objects_request) + self.assertEqual(["folder", "file"], [item.kind for item in items]) + self.assertEqual("reports", items[0].path) + self.assertEqual("report.xlsx", items[1].path) + self.assertEqual("govoplan:root/report.xlsx", items[1].external_id) + self.assertEqual("root/report.xlsx", items[1].metadata["key"]) + self.assertEqual("next-page", items[1].metadata["next_continuation_token"]) + + def test_s3_browse_lists_buckets_when_profile_has_no_bucket(self) -> None: + client = FakeS3Client() + + with patch("govoplan_files.backend.storage.connector_browse._s3_client", return_value=client): + items = browse_connector_profile(s3_profile(metadata={}), path="") + + self.assertEqual(["archive"], [item.path for item in items]) + + def test_s3_browse_reports_missing_optional_dependency(self) -> None: + with patch("govoplan_files.backend.storage.connector_browse.import_module", side_effect=ImportError): + with self.assertRaisesRegex(ConnectorBrowseUnsupported, "boto3"): + browse_connector_profile(s3_profile(), path="") + + def test_s3_import_downloads_object_and_preserves_remote_identity(self) -> None: + client = FakeS3Client() + + with patch("govoplan_files.backend.storage.connector_imports._s3_client", return_value=client): + downloaded = read_connector_file(s3_profile(), library_id="", path="report.txt", max_bytes=1024) + + self.assertEqual({"Bucket": "govoplan", "Key": "root/report.txt"}, client.head_request) + self.assertEqual({"Bucket": "govoplan", "Key": "root/report.txt", "VersionId": "version-1"}, client.get_request) + self.assertEqual("report.txt", downloaded.filename) + self.assertEqual(b"hello s3\n", downloaded.data) + self.assertEqual("version-1", downloaded.revision) + self.assertEqual("govoplan:root/report.txt", downloaded.external_id) + self.assertEqual("s3://govoplan/root/report.txt", downloaded.external_url) + self.assertEqual("checksum", downloaded.metadata["checksum_sha256"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_router_contract.py b/tests/test_router_contract.py new file mode 100644 index 0000000..7e119d8 --- /dev/null +++ b/tests/test_router_contract.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import unittest + +from govoplan_files.backend.router import router + + +class FilesRouterContractTests(unittest.TestCase): + def test_connector_routes_keep_existing_api_paths(self) -> None: + routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes} + + expected = { + (("GET",), "/files/connectors/providers"), + (("GET",), "/files/connectors/settings/delta"), + (("GET",), "/files/connectors/profiles"), + (("POST",), "/files/connectors/profiles"), + (("GET",), "/files/connectors/profiles/{profile_id}/browse"), + (("POST",), "/files/connectors/profiles/{profile_id}/import"), + (("POST",), "/files/connectors/profiles/{profile_id}/sync"), + (("GET",), "/files/connectors/credentials"), + (("POST",), "/files/connectors/credentials"), + (("GET",), "/files/connector-spaces"), + (("POST",), "/files/connector-spaces"), + } + + self.assertTrue(expected.issubset(routes)) + + def test_bulk_organize_routes_keep_existing_api_paths(self) -> None: + routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes} + + self.assertIn((("POST",), "/files/bulk-rename"), routes) + self.assertIn((("POST",), "/files/transfer"), routes) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_transfer_helpers.py b/tests/test_transfer_helpers.py new file mode 100644 index 0000000..720463d --- /dev/null +++ b/tests/test_transfer_helpers.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import unittest + +from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError +from govoplan_files.backend.storage.transfers import _normalize_rename_request, _normalize_transfer_request + + +class TransferHelperTests(unittest.TestCase): + def test_transfer_normalization_deduplicates_folder_paths_and_conflicts(self) -> None: + normalized = _normalize_transfer_request( + operation=" COPY ", + source_owner_type=" USER ", + target_owner_type=" GROUP ", + folder_paths=["Reports", "Reports/2026", "", "./Archive"], + target_folder=" Target ", + conflict_strategy="rename", + conflict_resolutions=[FileConflictResolution(target_path="Target/a.txt", action="skip")], + ) + + self.assertEqual("copy", normalized.operation) + self.assertEqual("user", normalized.source_owner_type) + self.assertEqual("group", normalized.target_owner_type) + self.assertEqual(["Reports", "Reports/2026", "Archive"], normalized.source_folder_paths) + self.assertEqual("Target", normalized.target_folder) + self.assertEqual("rename", normalized.conflict_strategy) + self.assertEqual("skip", normalized.conflict_resolution_map["Target/a.txt"].action) + + def test_transfer_normalization_rejects_unknown_operation(self) -> None: + with self.assertRaisesRegex(FileStorageError, "Unsupported transfer operation"): + _normalize_transfer_request( + operation="link", + source_owner_type="user", + target_owner_type="group", + folder_paths=[], + target_folder="", + conflict_strategy="reject", + conflict_resolutions=None, + ) + + def test_rename_normalization_collapses_nested_folder_roots(self) -> None: + normalized = _normalize_rename_request( + file_ids=["file-1", "file-1", "file-2"], + folder_paths=["Reports", "Reports/2026", "Archive"], + owner_type=" USER ", + owner_id="owner-1", + mode=" prefix ", + ) + + self.assertEqual("prefix", normalized.mode) + self.assertEqual(["file-1", "file-2"], normalized.selected_file_ids) + self.assertEqual(["Archive", "Reports"], normalized.selected_folder_roots) + self.assertEqual("user", normalized.owner_type) + + def test_rename_normalization_rejects_invalid_direct_multi_select(self) -> None: + with self.assertRaisesRegex(FileStorageError, "Direct rename requires exactly one selected item"): + _normalize_rename_request( + file_ids=["file-1", "file-2"], + folder_paths=[], + owner_type="user", + owner_id="owner-1", + mode="direct", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/src/api/files.ts b/webui/src/api/files.ts index 8db6750..98f45b6 100644 --- a/webui/src/api/files.ts +++ b/webui/src/api/files.ts @@ -150,7 +150,7 @@ export type FileConnectorDiscoveryResponse = { export type FileConnectorProfilePayload = { id: string; label: string; - provider: "seafile" | "nextcloud" | "webdav" | "smb" | "nfs" | "dms" | "generic"; + provider: "seafile" | "nextcloud" | "webdav" | "smb" | "s3" | "sharepoint" | "onedrive" | "nfs" | "dms" | "generic"; endpoint_url?: string | null; base_path?: string | null; enabled?: boolean; @@ -172,7 +172,7 @@ export type FileConnectorProfileUpdatePayload = Partial { const search = new URLSearchParams(); if (params.path) search.set("path", params.path); if (params.library_id) search.set("library_id", params.library_id); + if (params.continuation_token) search.set("continuation_token", params.continuation_token); if (params.campaign_id) search.set("campaign_id", params.campaign_id); const suffix = search.toString() ? `?${search.toString()}` : ""; return apiFetch(settings, `/api/v1/files/connectors/profiles/${encodeURIComponent(profileId)}/browse${suffix}`); diff --git a/webui/src/features/files/FileConnectorSettingsPanel.tsx b/webui/src/features/files/FileConnectorSettingsPanel.tsx index fb55d79..aa3d2f0 100644 --- a/webui/src/features/files/FileConnectorSettingsPanel.tsx +++ b/webui/src/features/files/FileConnectorSettingsPanel.tsx @@ -5,6 +5,8 @@ import { Card, ConnectionTree, ConfirmDialog, + CredentialPanel, + DisabledActionTooltip, DismissibleAlert, Dialog, FormField, @@ -123,6 +125,9 @@ const PROVIDERS: Array<{id: Provider;label: string;}> = [ { id: "nextcloud", label: "i18n:govoplan-files.nextcloud.aab73a3d" }, { id: "seafile", label: "i18n:govoplan-files.seafile.600192cc" }, { id: "smb", label: "i18n:govoplan-files.smb.515d2f88" }, +{ id: "s3", label: "S3" }, +{ id: "sharepoint", label: "SharePoint" }, +{ id: "onedrive", label: "OneDrive" }, { id: "nfs", label: "i18n:govoplan-files.nfs.db121865" }, { id: "dms", label: "i18n:govoplan-files.dms.477e5652" }, { id: "generic", label: "i18n:govoplan-files.generic.ff7613e5" }]; @@ -133,6 +138,9 @@ const ENDPOINT_PLACEHOLDERS: Record = { nextcloud: "http://127.0.0.1:9081/remote.php/dav/files/admin/", seafile: "http://127.0.0.1:9082/", smb: "smb://127.0.0.1:1445/files", + s3: "http://127.0.0.1:9000", + sharepoint: "https://graph.microsoft.com/v1.0/sites/{site-id}/drives/{drive-id}", + onedrive: "https://graph.microsoft.com/v1.0/drives/{drive-id}", nfs: "nfs://127.0.0.1/export", dms: "https://dms.example.test", generic: "https://files.example.test" @@ -340,6 +348,17 @@ export default function FileConnectorSettingsPanel({ setCredentialDraft((current) => current ? { ...current, ...patch } : current); } + function patchCredentialValues(patch: {username?: string | null;password?: string | null;}) { + const next: Partial = {}; + if (patch.username !== undefined) next.username = String(patch.username ?? ""); + if (patch.password !== undefined) next.password = String(patch.password ?? ""); + patchCredentialDraft(next); + } + + function patchCredentialToken(patch: {password?: string | null;}) { + if (patch.password !== undefined) patchCredentialDraft({ token: String(patch.password ?? "") }); + } + function patchPolicyDraft(patch: Partial) { setPolicyDraft((current) => ({ ...current, ...patch })); } @@ -906,11 +925,11 @@ export default function FileConnectorSettingsPanel({ footer={credentialDraft ? <> - + - + : null}> @@ -983,35 +1002,47 @@ export default function FileConnectorSettingsPanel({ actor: "Connector administrator", target: "Credential mode" }} /> : -
- {(credentialDraft.credentialMode === "basic" || credentialDraft.credentialMode === "secret_ref") && - - patchCredentialDraft({ username: event.target.value })} autoComplete="username" /> - - } +
{credentialDraft.credentialMode === "basic" && - - patchCredentialDraft({ password: event.target.value })} type="password" autoComplete="new-password" placeholder={editingCredentialId ? "i18n:govoplan-files.leave_empty_to_keep_saved_password.6ec39f5e" : ""} /> - - } - {credentialDraft.credentialMode === "token" && - - patchCredentialDraft({ token: event.target.value })} type="password" placeholder={editingCredentialId ? "i18n:govoplan-files.leave_empty_to_keep_saved_token.e725857b" : ""} /> - + } {credentialDraft.credentialMode === "secret_ref" && - - patchCredentialDraft({ secretRef: event.target.value })} /> - + <> + +
+ + patchCredentialDraft({ secretRef: event.target.value })} /> + +
+ + } + {credentialDraft.credentialMode === "token" && + }
}
- + - +
{credentialLoginResult && @@ -1072,11 +1103,11 @@ export default function FileConnectorSettingsPanel({ footer={draft ? <> - + - + : null}> diff --git a/webui/src/features/files/FilesPage.tsx b/webui/src/features/files/FilesPage.tsx index c919dd0..5a5aed5 100644 --- a/webui/src/features/files/FilesPage.tsx +++ b/webui/src/features/files/FilesPage.tsx @@ -182,6 +182,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut }, [visibleFiles, folders, currentFolder, searchActive, sortColumn, sortDirection]); const fileListViewportRef = useRef(null); const fileListMeasureRowRef = useRef(null); + const managedSpaceLoadsInFlightRef = useRef>(new Set()); const [fileListViewportHeight, setFileListViewportHeight] = useState(0); const [fileListScrollTop, setFileListScrollTop] = useState(0); const [fileListRowHeight, setFileListRowHeight] = useState(DEFAULT_FILE_LIST_ROW_HEIGHT); @@ -291,8 +292,9 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut try { const response = await listFileSpaces(settings); setSpaces(response.spaces); - setActiveSpaceId((current) => current || response.spaces[0]?.id || ""); - await Promise.all(response.spaces.filter((space) => !isConnectorSpace(space)).map((space) => loadSpaceContents(space, { silent: true }))); + const nextActiveSpace = response.spaces.find((space) => space.id === activeSpaceId) ?? response.spaces[0] ?? null; + setActiveSpaceId(nextActiveSpace?.id || ""); + if (nextActiveSpace && !isConnectorSpace(nextActiveSpace)) await loadSpaceContents(nextActiveSpace, { silent: true }); } catch (err) { setError(err instanceof Error ? err.message : String(err)); } finally { @@ -331,6 +333,8 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut await loadConnectorSpaceContents(space, { silent: options.silent, folderPath: "" }); return; } + if (managedSpaceLoadsInFlightRef.current.has(space.id)) return; + managedSpaceLoadsInFlightRef.current.add(space.id); if (!options.silent) { setBusy(true); setError(""); @@ -372,6 +376,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut } catch (err) { setError(err instanceof Error ? err.message : String(err)); } finally { + managedSpaceLoadsInFlightRef.current.delete(space.id); if (!options.silent) setBusy(false); } } @@ -411,6 +416,8 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut const nextSpace = spaces.find((space) => space.id === activeSpaceId); if (nextSpace && isConnectorSpace(nextSpace)) { void loadConnectorSpaceContents(nextSpace, { folderPath: "", silent: true }); + } else if (nextSpace && filesBySpace[nextSpace.id] === undefined && foldersBySpace[nextSpace.id] === undefined) { + void loadSpaceContents(nextSpace, { silent: true }); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [activeSpaceId, spaces]); diff --git a/webui/src/styles/file-manager.css b/webui/src/styles/file-manager.css index 23e0d85..ac1f73c 100644 --- a/webui/src/styles/file-manager.css +++ b/webui/src/styles/file-manager.css @@ -14,7 +14,7 @@ gap: 8px; flex-wrap: wrap; padding: 10px 14px; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); background: var(--panel-header); } @@ -29,7 +29,7 @@ grid-template-columns: minmax(240px, 300px) minmax(0, 1fr); min-height: 0; height: 100%; - border: 1px solid var(--line); + border: var(--border-line); border-radius: 0; overflow: hidden; background: var(--panel); @@ -46,7 +46,7 @@ .file-list-sticky { flex: 0 0 auto; z-index: 2; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); background: var(--panel-soft); } @@ -97,7 +97,7 @@ gap: 12px; flex-wrap: wrap; padding: 8px 14px; - border-top: 1px solid var(--line); + border-top: var(--border-line); color: var(--muted); font-size: 12px; } @@ -130,7 +130,7 @@ .file-list-table-head { padding: 10px 14px; - border-top: 1px solid var(--line); + border-top: var(--border-line); background: var(--panel); color: var(--muted); font-size: 12px; @@ -161,7 +161,7 @@ .file-list-row { min-height: 58px; padding: 9px 14px; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); cursor: default; user-select: none; } @@ -244,7 +244,7 @@ } .folder-row .file-row-icon { - color: #b6791d; + color: var(--warning-deep); } .file-row-tail { @@ -274,7 +274,7 @@ display: grid; min-width: 190px; overflow: hidden; - border: 1px solid var(--line-dark); + border: var(--border-line-dark); border-radius: 12px; background: var(--panel); box-shadow: var(--shadow-popover); @@ -299,7 +299,7 @@ .file-context-menu button:hover, .file-context-menu button:focus-visible { - background: rgba(13, 110, 253, .08); + background: var(--primary-soft); outline: none; } @@ -315,14 +315,14 @@ display: grid; place-items: center; padding: 22px; - background: rgba(28, 25, 22, .38); + background: var(--dialog-backdrop); } .file-dialog { width: min(620px, 100%); max-height: min(720px, calc(100vh - 44px)); overflow: auto; - border: 1px solid var(--line-dark); + border: var(--border-line-dark); border-radius: 16px; background: var(--panel); box-shadow: var(--shadow-popover); @@ -334,7 +334,7 @@ justify-content: space-between; gap: 12px; padding: 15px 18px; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); background: var(--panel-soft); } @@ -397,7 +397,7 @@ overflow-y: auto; scrollbar-gutter: stable; padding: 8px; - border: 1px solid var(--line); + border: var(--border-line); border-radius: 12px; background: var(--panel-soft); } @@ -428,13 +428,13 @@ .file-folder-selector-node:hover:not(:disabled), .file-folder-selector-node:focus-visible, .file-folder-selector-node.is-selected { - background: rgba(13, 110, 253, .09); + background: var(--primary-soft); outline: none; } .file-folder-selector-node.is-selected { color: var(--text-strong); - box-shadow: inset 0 0 0 1px rgba(13, 110, 253, .25); + box-shadow: var(--primary-inset-ring); } .file-folder-selector-empty { @@ -462,7 +462,7 @@ grid-template-rows: auto auto minmax(0, 1fr); min-height: 320px; overflow: hidden; - border: 1px solid var(--line); + border: var(--border-line); border-radius: 12px; background: var(--panel); } @@ -473,7 +473,7 @@ gap: 10px; align-items: center; padding: 10px; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); background: var(--panel-soft); } @@ -577,7 +577,7 @@ gap: 3px; min-width: 0; padding: 10px 12px; - border: 1px solid var(--line); + border: var(--border-line); border-radius: 10px; background: var(--panel-soft); } @@ -625,7 +625,7 @@ gap: 12px; align-items: center; padding: 10px 12px; - border: 1px solid var(--line); + border: var(--border-line); border-radius: 8px; background: var(--panel-soft); } @@ -698,6 +698,11 @@ white-space: nowrap; } +.file-connector-secret-fields { + display: grid; + gap: 18px; +} + @media (max-width: 760px) { .file-connector-profile-row { grid-template-columns: 1fr; @@ -726,7 +731,7 @@ .file-tree-panel { max-height: 260px; border-right: 0; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); } .file-list-table-head { @@ -760,7 +765,7 @@ z-index: 35; display: grid; place-items: center; - background: rgba(255, 255, 255, .12); + background: var(--panel-glass); backdrop-filter: blur(1px); } @@ -768,7 +773,7 @@ display: grid; gap: 3px; padding: 12px 14px; - border: 1px solid var(--line); + border: var(--border-line); border-radius: 12px; background: var(--panel-soft); } @@ -791,7 +796,7 @@ gap: 10px; align-items: center; padding: 10px; - border: 1px solid var(--line); + border: var(--border-line); border-radius: 12px; background: var(--panel-soft); } @@ -842,8 +847,8 @@ .rename-preview-more:hover, .rename-preview-more:focus-visible { - border-color: rgba(13, 110, 253, .45); - background: rgba(13, 110, 253, .08); + border-color: var(--primary-border); + background: var(--primary-soft); color: var(--text-strong); outline: none; } @@ -860,7 +865,7 @@ .managed-file-chooser-dialog > .dialog-header { padding: 16px 18px; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); background: var(--panel); } @@ -891,9 +896,9 @@ .managed-file-chooser-footer { flex: 0 0 auto; padding: 12px 18px; - border-top: 1px solid var(--line); + border-top: var(--border-line); background: var(--panel); - box-shadow: 0 -8px 24px rgba(15, 23, 42, .06); + box-shadow: var(--shadow-footer); } .managed-file-chooser-layout { @@ -911,7 +916,7 @@ .managed-file-chooser-spaces { min-width: 0; padding: 16px; - border-right: 1px solid var(--line); + border-right: var(--border-line); background: var(--panel-soft); overflow: auto; } @@ -1014,7 +1019,7 @@ gap: 12px; align-items: center; padding: 12px 14px; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); } .managed-file-breadcrumb { @@ -1046,7 +1051,7 @@ display: grid; gap: 8px; padding: 12px 14px; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); background: var(--panel-soft); } @@ -1075,7 +1080,7 @@ align-items: center; min-height: 38px; padding: 0 20px; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); background: var(--panel-soft); color: var(--muted); font-size: 12px; @@ -1206,7 +1211,7 @@ .managed-file-chooser-note { margin: 0; padding: 10px 14px; - border-top: 1px solid var(--line); + border-top: var(--border-line); background: var(--panel-soft); } @@ -1313,7 +1318,7 @@ .managed-file-chooser-spaces { border-right: 0; - border-bottom: 1px solid var(--line); + border-bottom: var(--border-line); max-height: 160px; }