intermittent commit

src/govoplan_files/backend/router.py

@@ -105,6 +105,7 @@ from govoplan_files.backend.storage.access import ensure_group_access, group_ref
 from govoplan_files.backend.storage.archives import create_zip_file, extract_zip_upload
 from govoplan_files.backend.storage.common import FileStorageError
 from govoplan_files.backend.storage.connector_credential_store import (
+    ConnectorCredential,
     connector_credential_from_row,
     create_connector_credential_row,
     deactivate_connector_credential_row,
@@ -802,6 +803,14 @@ def _download_connector_payload(
     return source_path, downloaded, metadata or {}
 
 
+def _connector_browse_next_token(items: list[Any]) -> str | None:
+    for item in items:
+        token = item.metadata.get("next_continuation_token") if hasattr(item, "metadata") else None
+        if token:
+            return str(token)
+    return None
+
+
 def _connector_audit_details(asset: FileAsset, version: FileVersion, blob: FileBlob, *, operation: str) -> dict[str, object] | None:
     metadata = asset.metadata_ or {}
     provenance = source_provenance_from_metadata(metadata)
@@ -2148,6 +2157,148 @@ def _full_file_connector_settings_delta_response(
     )
 
 
+def _changed_file_connector_setting_ids(entries: list[ChangeSequenceEntry]) -> tuple[set[str], set[str], set[str], bool]:
+    changed_profile_ids = {
+        entry.resource_id
+        for entry in entries
+        if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
+    }
+    changed_credential_ids = {
+        entry.resource_id
+        for entry in entries
+        if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
+    }
+    changed_space_ids = {
+        entry.resource_id
+        for entry in entries
+        if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
+    }
+    policy_changed = any(entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries)
+    return changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed
+
+
+def _file_connector_settings_changed_sections(
+    *,
+    changed_profile_ids: set[str],
+    changed_credential_ids: set[str],
+    changed_space_ids: set[str],
+    policy_changed: bool,
+    profiles: list[ConnectorProfile],
+) -> list[str]:
+    changed_sections = []
+    if changed_profile_ids or any(profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids for profile in profiles):
+        changed_sections.append("profiles")
+    if changed_credential_ids:
+        changed_sections.append("credentials")
+    if changed_space_ids:
+        changed_sections.append("spaces")
+    if policy_changed:
+        changed_sections.append("policy")
+    return changed_sections
+
+
+def _file_connector_settings_deleted_items(
+    entries: list[ChangeSequenceEntry],
+    *,
+    visible_profiles: dict[str, ConnectorProfile],
+    visible_credentials: dict[str, ConnectorCredential],
+    visible_spaces: dict[str, FileConnectorSpace],
+) -> list[DeltaDeletedItem]:
+    return [
+        _connector_deleted_item(entry)
+        for entry in entries
+        if (
+            entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
+            and entry.resource_id not in visible_profiles
+        )
+        or (
+            entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
+            and entry.resource_id not in visible_credentials
+        )
+        or (
+            entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
+            and entry.resource_id not in visible_spaces
+        )
+    ]
+
+
+def _incremental_file_connector_settings_delta_response(
+    session: Session,
+    principal: ApiPrincipal,
+    *,
+    entries: list[ChangeSequenceEntry],
+    has_more: bool,
+    scope_type: str,
+    scope_id: str | None,
+    provider: str | None,
+    campaign_id: str | None,
+    include_disabled: bool,
+    include_inactive: bool,
+    owner_type: Literal["user", "group"] | None,
+    owner_id: str | None,
+) -> FileConnectorSettingsDeltaResponse:
+    changed_profile_ids, changed_credential_ids, changed_space_ids, policy_changed = _changed_file_connector_setting_ids(entries)
+    profiles = _visible_connector_profiles(
+        session,
+        principal,
+        provider=provider,
+        campaign_id=campaign_id,
+        include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal),
+        include_admin_scopes=_can_read_disabled_connector_profiles(principal),
+        include_effective_policy=False,
+    )
+    visible_profiles = {
+        profile.id: profile
+        for profile in profiles
+        if profile.id in changed_profile_ids or (profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids)
+    }
+    credentials = _visible_connector_credentials(
+        session,
+        principal,
+        provider=provider,
+        include_disabled=include_disabled,
+    )
+    visible_credentials = {
+        credential.id: credential
+        for credential in credentials
+        if credential.id in changed_credential_ids
+    }
+    spaces = _visible_connector_spaces(
+        session,
+        principal,
+        owner_type=owner_type,
+        owner_id=owner_id,
+        include_inactive=include_inactive,
+    )
+    visible_spaces = {
+        space.id: space
+        for space in spaces
+        if space.id in changed_space_ids
+    }
+    return FileConnectorSettingsDeltaResponse(
+        profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in visible_profiles.values()],
+        credentials=[FileConnectorCredentialResponse(**credential.to_response()) for credential in visible_credentials.values()],
+        spaces=[_connector_space_response(space) for space in visible_spaces.values()],
+        policy=FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) if policy_changed else None,
+        changed_sections=_file_connector_settings_changed_sections(
+            changed_profile_ids=changed_profile_ids,
+            changed_credential_ids=changed_credential_ids,
+            changed_space_ids=changed_space_ids,
+            policy_changed=policy_changed,
+            profiles=profiles,
+        ),
+        deleted=_file_connector_settings_deleted_items(
+            entries,
+            visible_profiles=visible_profiles,
+            visible_credentials=visible_credentials,
+            visible_spaces=visible_spaces,
+        ),
+        watermark=_file_connector_settings_response_watermark(session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more),
+        has_more=has_more,
+        full=False,
+    )
+
+
 @router.get("/connectors/settings/delta", response_model=FileConnectorSettingsDeltaResponse)
 def connector_settings_delta(
     scope_type: str = Query(default="tenant"),
@@ -2193,94 +2344,19 @@ def connector_settings_delta(
                 owner_type=owner_type,
                 owner_id=owner_id,
             )
-        changed_profile_ids = {
-            entry.resource_id
-            for entry in entries
-            if entry.collection == FILES_CONNECTOR_PROFILES_COLLECTION and entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
-        }
-        changed_credential_ids = {
-            entry.resource_id
-            for entry in entries
-            if entry.collection == FILES_CONNECTOR_CREDENTIALS_COLLECTION and entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
-        }
-        changed_space_ids = {
-            entry.resource_id
-            for entry in entries
-            if entry.collection == FILES_CONNECTOR_SPACES_COLLECTION and entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
-        }
-        policy_changed = any(entry.collection == FILES_CONNECTOR_POLICIES_COLLECTION for entry in entries)
-        profiles = _visible_connector_profiles(
+        return _incremental_file_connector_settings_delta_response(
             session,
             principal,
+            entries=entries,
+            has_more=has_more,
+            scope_type=scope_type,
+            scope_id=scope_id,
             provider=provider,
             campaign_id=campaign_id,
-            include_disabled=include_disabled and _can_read_disabled_connector_profiles(principal),
-            include_admin_scopes=_can_read_disabled_connector_profiles(principal),
-            include_effective_policy=False,
-        )
-        visible_profiles = {
-            profile.id: profile
-            for profile in profiles
-            if profile.id in changed_profile_ids or (profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids)
-        }
-        credentials = _visible_connector_credentials(
-            session,
-            principal,
-            provider=provider,
             include_disabled=include_disabled,
-        )
-        visible_credentials = {
-            credential.id: credential
-            for credential in credentials
-            if credential.id in changed_credential_ids
-        }
-        spaces = _visible_connector_spaces(
-            session,
-            principal,
+            include_inactive=include_inactive,
             owner_type=owner_type,
             owner_id=owner_id,
-            include_inactive=include_inactive,
-        )
-        visible_spaces = {
-            space.id: space
-            for space in spaces
-            if space.id in changed_space_ids
-        }
-        changed_sections = []
-        if changed_profile_ids or any(profile.credential_profile_id and profile.credential_profile_id in changed_credential_ids for profile in profiles):
-            changed_sections.append("profiles")
-        if changed_credential_ids:
-            changed_sections.append("credentials")
-        if changed_space_ids:
-            changed_sections.append("spaces")
-        if policy_changed:
-            changed_sections.append("policy")
-        deleted = [
-            _connector_deleted_item(entry)
-            for entry in entries
-            if (
-                entry.resource_type == FILES_CONNECTOR_PROFILE_RESOURCE
-                and entry.resource_id not in visible_profiles
-            )
-            or (
-                entry.resource_type == FILES_CONNECTOR_CREDENTIAL_RESOURCE
-                and entry.resource_id not in visible_credentials
-            )
-            or (
-                entry.resource_type == FILES_CONNECTOR_SPACE_RESOURCE
-                and entry.resource_id not in visible_spaces
-            )
-        ]
-        return FileConnectorSettingsDeltaResponse(
-            profiles=[FileConnectorProfileResponse(**profile.to_response()) for profile in visible_profiles.values()],
-            credentials=[FileConnectorCredentialResponse(**credential.to_response()) for credential in visible_credentials.values()],
-            spaces=[_connector_space_response(space) for space in visible_spaces.values()],
-            policy=FileConnectorPolicyResponse(**connector_policy_response(session, tenant_id=principal.tenant_id, scope_type=scope_type, scope_id=scope_id)) if policy_changed else None,
-            changed_sections=changed_sections,
-            deleted=deleted,
-            watermark=_file_connector_settings_response_watermark(session, tenant_id=principal.tenant_id, entries=entries, has_more=has_more),
-            has_more=has_more,
-            full=False,
         )
     except (FileStorageError, ValueError, json.JSONDecodeError) as exc:
         raise _http_error(exc) from exc
@@ -2819,6 +2895,7 @@ def browse_connector_profile_items(
     profile_id: str,
     path: str | None = None,
     library_id: str | None = None,
+    continuation_token: str | None = None,
     campaign_id: str | None = None,
     session: Session = Depends(get_session),
     principal: ApiPrincipal = Depends(require_any_scope("files:file:read", "files:file:upload", "files:file:download", "files:file:admin", "system:settings:read", "admin:settings:read")),
@@ -2839,7 +2916,7 @@ def browse_connector_profile_items(
         )
         if not decision.allowed:
             raise ConnectorPolicyDenied(decision)
-        items = browse_connector_profile(profile, path=browse_path, library_id=library_id)
+        items = browse_connector_profile(profile, path=browse_path, library_id=library_id, continuation_token=continuation_token)
     except ConnectorPolicyDenied as exc:
         raise _connector_policy_error(exc) from exc
     except ConnectorBrowseUnsupported as exc:
@@ -2851,6 +2928,8 @@ def browse_connector_profile_items(
         provider=profile.provider,
         path=browse_path,
         library_id=library_id,
+        next_continuation_token=_connector_browse_next_token(items),
+        has_more=any(bool(item.metadata.get("listing_truncated")) for item in items),
         decision=decision.to_dict(),
         items=[FileConnectorBrowseItem(**item.to_response()) for item in items],
     )
@@ -3230,7 +3309,7 @@ def download_archive(
             get_asset_for_user(session, tenant_id=principal.tenant_id, user_id=principal.user.id, asset_id=file_id, is_admin=_is_admin(principal))
             for file_id in payload.file_ids
         ]
-        tmp = tempfile.NamedTemporaryFile(prefix="multimailer-files-", suffix=".zip", delete=False)
+        tmp = tempfile.NamedTemporaryFile(prefix="govoplan-files-", suffix=".zip", delete=False)
         tmp_path = tmp.name
         tmp.close()
         try: