diff --git a/docs/CONNECTOR_SPACES.md b/docs/CONNECTOR_SPACES.md index a859d76..568ee93 100644 --- a/docs/CONNECTOR_SPACES.md +++ b/docs/CONNECTOR_SPACES.md @@ -61,11 +61,10 @@ Linked connector space: - read-only flag from provider/policy - active/deleted state -The linked space should be addressable as a normal file space in the files UI. -For the first implementation slice, actions may use the existing connector -browse/sync APIs and managed file storage. A linked space can therefore browse -the remote folder and sync selected files into managed storage. Later slices can -add a native remote listing view or background sync jobs. +The linked space is addressable as a normal file space in the Files UI. It can +browse the remote folder, synchronize one selected file, or run a bounded manual +folder synchronization into managed storage. Background sync jobs remain a +separate scheduling concern. ## Policy Semantics @@ -105,6 +104,9 @@ Implemented: - Files UI sync dialog for choosing a profile, browsing a remote folder, and syncing a selected file into a managed destination folder - Files UI connector-space view with provider/read-only/manual-sync state +- bounded folder-level manual sync with recursive traversal, pagination, + per-item review outcomes, source-provenance matching, and one summary audit + event - Docker dev stack smoke checks for WebDAV, Nextcloud, and SMB Missing: @@ -137,9 +139,14 @@ Missing: - Remaining: edit/manage actions for existing linked spaces. 4. Add sync orchestration. - - Manual sync selected file is already available. - - Add folder-level manual sync. - - Add optional scheduled/background sync with conflict reporting. + - Manual selected-file and folder-level synchronization are available. + - Folder sync preserves remote relative paths, bounds file count and depth, + and reports created, updated, unchanged, skipped, conflict, policy-denied, + and failed items without rolling back successful siblings. + - The default conflict strategy is `skip`, so unrelated managed files are + left untouched. `rename`, `reject`, and `overwrite` remain explicit choices. + - Add optional scheduled/background sync separately; it must reuse the same + policy, provenance, result, and audit contracts. 5. Add provider-specific expansion. - OAuth/secret-store credentials. diff --git a/docs/FILES_HANDBOOK.md b/docs/FILES_HANDBOOK.md index 4b01819..2116940 100644 --- a/docs/FILES_HANDBOOK.md +++ b/docs/FILES_HANDBOOK.md @@ -107,8 +107,11 @@ Files resources when their owner is specified. Linked connector spaces appear beside managed spaces when they are active and visible to the user. A connector space is read-only by default. It is a view of an approved remote -location and a starting point for importing or synchronizing selected files -into managed storage. An administrator may opt an S3 space into two-way mode +location and a starting point for importing a selected file or synchronizing a +selected file or bounded folder tree into managed storage. Folder sync preserves +relative remote paths in the linked owner's managed space and returns a +reviewable result for every discovered or deliberately skipped item. An +administrator may opt an S3 space into two-way mode only after enabling the profile's `write` capability. Even then, Files exposes explicit, conditional file write-back—not a mounted filesystem. Automatic remote delete, rename, move, and ACL propagation stay disabled. @@ -281,6 +284,19 @@ inside the chosen owner space: - identical checksum and size updates provenance and returns `unchanged`; - changed bytes append a version and return `updated`. +Folder-level manual sync uses +`POST /api/v1/files/connector-spaces/{space_id}/sync`. The interactive request +is bounded by file count and traversal depth, follows provider pagination, and +can include or exclude subfolders. Every file is isolated in a database +savepoint, so one conflict, policy denial, invalid provider result, or transport +failure remains reviewable without discarding successful siblings. Results are +classified as `created`, `updated`, `unchanged`, `skipped`, `conflict`, +`policy_denied`, or `failed`. The UI defaults unrelated target collisions to +`skip`; `rename`, `reject`, and `overwrite` require an explicit operator choice. +The response marks a run as truncated when its bound is reached so the operator +can continue with a narrower remote folder. This is interactive orchestration, +not background scheduling. + Browse, import, and inbound sync never mutate the remote source. An S3 space in explicit two-way mode can write a selected managed file to one remote object path through `POST /api/v1/files/connector-spaces/{space_id}/write-back`. @@ -394,7 +410,7 @@ those relationships first. | --- | --- | | `files:file:read` | List and inspect accessible files, folders, spaces, and visible connectors | | `files:file:download` | Download an accessible current version or ZIP archive | -| `files:file:upload` | Upload managed assets and import/sync selected connector files | +| `files:file:upload` | Upload managed assets and import/sync selected connector files or bounded connector folders | | `files:file:organize` | Create folders, rename, move/copy, and manage linked connector spaces | | `files:file:share` | Create or update file shares | | `files:file:delete` | Soft-delete accessible writable files and folders | @@ -813,7 +829,7 @@ All routes below are under `/api/v1/files`. | File access | `GET /{file_id}`, `GET /{file_id}/download`, `DELETE /{file_id}`, `POST /bulk-delete` | | Organization | `POST /bulk-rename`, `POST /transfer`, `POST /archive.zip`, `POST /resolve-patterns` | | Sharing | `POST /{file_id}/shares`, `POST /bulk-shares` | -| Connector spaces | `GET/POST /connector-spaces`, `PATCH/DELETE /connector-spaces/{space_id}`, `POST /connector-spaces/{space_id}/restore`, `POST /connector-spaces/{space_id}/write-back` | +| Connector spaces | `GET/POST /connector-spaces`, `PATCH/DELETE /connector-spaces/{space_id}`, `POST /connector-spaces/{space_id}/restore`, `POST /connector-spaces/{space_id}/sync`, `POST /connector-spaces/{space_id}/write-back` | | Connector catalog/discovery | `GET /connectors/providers`, `POST /connectors/discover` | | Connector profiles | `GET/POST /connectors/profiles`, `GET/PATCH/DELETE /connectors/profiles/{profile_id}` | | Browse/import/sync | `GET /connectors/profiles/{profile_id}/browse`, `POST /connectors/profiles/{profile_id}/import`, `POST /connectors/profiles/{profile_id}/sync` | @@ -916,7 +932,7 @@ events. External metadata is provider/user input and is not a digital signature. Files records canonical audit events for: - connector discovery attempts, before the attempted external I/O; -- connector imports and manual syncs; +- connector imports, selected-file syncs, and bounded folder sync summaries; - download/archive access to connector-originated managed files; - immediate connector profile and credential deletion/scrubbing; - credential scrubbing during destructive module retirement. @@ -924,6 +940,10 @@ Files records canonical audit events for: Connector audit details include the managed asset/version/blob, checksum, size, operation, source revision, and provenance where applicable. Deletion audit details name secret/reference kinds but never the secret values. +`files.connector.folder_synced` additionally records the connector profile, +provider, library/root, managed target, traversal mode, truncation state, typed +counts, and bounded per-item source/target/action references. It does not embed +downloaded bytes, credentials, or transport content. Assets, folders, shares, profiles, credentials, policies, and connector spaces also feed Core's incremental change sequence for UI synchronization. A change @@ -1021,6 +1041,13 @@ must carry source identity, revision when available, current version, checksum, and size. Re-syncing identical bytes must return `unchanged`; changed bytes must create a higher version while preserving the previous version. +Given a permitted linked connector folder contains changed, unchanged, +colliding, and policy-denied files, a manual folder sync must preserve remote +relative paths, commit the allowed siblings, return one typed outcome for every +discovered/skipped item, and emit a matching summary audit event. Reaching the +configured file bound must mark the run truncated rather than imply complete +coverage. + ### Campaign freeze Given a campaign snapshot selected version V1, when the managed asset later @@ -1053,10 +1080,10 @@ returning different content or credentials. | Deletion/retention | Soft-delete and restore assets/folders/spaces; optimistic retention and legal-hold controls; preview-bound, approval-referenced hard purge; reference-checked blob GC; immediate audited connector-secret scrubbing | Automatic time-based purge scheduling and richer lifecycle administration UI | | Privacy requests | Tenant-scoped bounded DSAR metadata search; retained/manual/revoke/detach planning; idempotent share revocation and mutable actor-reference detachment; explicit separation from byte purge | Content-specific automated redaction and policy-specific approval remain manual or belong to the owning process | | Connector governance | Scoped profiles/credentials/policies, effective source explanation, separate credentials, linked user/group spaces | Provider-owned external secret lifecycle; API `secret_ref` remains rejected | -| HTTP connectors | Pinned, bounded, no-redirect Seafile and WebDAV/Nextcloud browse/import/manual sync | Background/folder sync, remote mutation, long-running transfer workers | +| HTTP connectors | Pinned, bounded, no-redirect Seafile and WebDAV/Nextcloud browse/import, selected-file sync, and folder sync | Background sync, remote mutation, long-running transfer workers | | SMB and S3 connectors | Provider descriptors, browse/import/manual sync, pinned SDK transports, redirect/retry/referral transport-contract tests, and explicit conditional S3 write-back with Core-ledger recovery | Live topology smoke evidence, provider-specific OAuth, additional provider writes, and background indexing remain separate deployment or connector-module concerns | | Other providers | Reserved SharePoint/OneDrive keys and NFS/local descriptors | Graph/OAuth/provider paging, NFS deployment integration, DMS connectors | -| Connector spaces | User/group link, browse, manual selected-file sync, edit/disable/delete/restore, read-only default, and opt-in S3 two-way mode | Background sync and automatic remote delete/rename/move/ACL propagation | +| Connector spaces | User/group link, browse, bounded manual file/folder sync, edit/disable/delete/restore, read-only default, and opt-in S3 two-way mode | Background sync and automatic remote delete/rename/move/ACL propagation | | Profile capabilities | Stored, displayed, and enforced for explicit connector write-back | Broader provider-specific capability negotiation | | Audit | Connector discovery/import/sync/access and connector deletion; campaign exact-use evidence | Dedicated canonical audit events for every ordinary Files mutation | | Preview | File metadata and attachment download | Dedicated safe content-preview service | diff --git a/package.json b/package.json index 0889e46..be48104 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/files-webui", - "version": "0.1.19", + "version": "0.1.20", "private": true, "type": "module", "main": "webui/src/index.ts", diff --git a/pyproject.toml b/pyproject.toml index 97f73e3..2287a49 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta" [project] name = "govoplan-files" -version = "0.1.19" +version = "0.1.20" description = "GovOPlaN files module with backend and WebUI integration." readme = "README.md" requires-python = ">=3.12" license = { file = "LICENSE" } authors = [{ name = "GovOPlaN" }] dependencies = [ - "govoplan-core>=0.1.19", + "govoplan-core>=0.1.20", "defusedxml>=0.7,<1", "pyzipper>=0.3.6,<1", "python-multipart>=0.0.31,<1", diff --git a/src/govoplan_files/backend/manifest.py b/src/govoplan_files/backend/manifest.py index aa7ad1d..3f46e14 100644 --- a/src/govoplan_files/backend/manifest.py +++ b/src/govoplan_files/backend/manifest.py @@ -452,7 +452,7 @@ def _dsar_provider(context: ModuleContext) -> object: manifest = ModuleManifest( id="files", name="Files", - version="0.1.19", + version="0.1.20", required_capabilities=( CAPABILITY_AUTH_PRINCIPAL_RESOLVER, CAPABILITY_AUTH_PERMISSION_EVALUATOR, @@ -1299,12 +1299,14 @@ manifest = ModuleManifest( ), DocumentationTopic( id="files.governed-connectors-and-provenance", - title="Govern file connections and credential deletion", - summary="Keep endpoint profiles, reusable credentials, and inherited connector policy separate, and understand what DELETE removes immediately.", + title="Govern file connections, folder sync, and credential deletion", + summary="Keep endpoints, credentials, inherited policy, and bounded manual synchronization separate, with reviewable outcomes and provenance.", body=( "System, tenant, and one user/group/campaign leaf form the effective policy chain: deny rules win and every configured allow rule must match. " "Responses redact secret values and deployment references. Deleting a database-managed credential or profile immediately scrubs Files-owned encrypted material and private metadata in the same transaction as a non-secret audit event; dependent profiles are disabled, while legacy non-owned references are only detached and audited. " - "Removing a connector space is a separate owner-authorized operation: it retires only the local virtual-space link and leaves provider content, imported managed files and shares, profiles, credentials, and remote references untouched. Intrinsic user and group managed spaces cannot be removed. Connector spaces remain read-only by default. An administrator may explicitly enable two-way mode only for an S3 profile carrying the write capability; each write requires separate authority, inherited path policy, a conditional create or expected revision, and durable recovery evidence. Automatic remote deletion, rename, and move propagation remain disabled." + "Removing a connector space is a separate owner-authorized operation: it retires only the local virtual-space link and leaves provider content, imported managed files and shares, profiles, credentials, and remote references untouched. Intrinsic user and group managed spaces cannot be removed. " + "A manual connector-folder sync traverses provider pagination with explicit file-count and depth bounds, preserves remote relative paths in the linked owner's managed space, and evaluates policy for the root and every downloaded item. Matching source identities produce unchanged or version-appending updates; new identities create managed assets. Unrelated target collisions default to skip, while rename, reject, and overwrite are explicit choices. Per-file savepoints keep successful siblings while conflicts, skips, policy denials, and transport failures remain visible in the response. The files.connector.folder_synced audit event records provenance, typed counts, truncation, and bounded result references without content or credentials. Scheduling remains separate. " + "Connector spaces remain read-only by default. An administrator may explicitly enable two-way mode only for an S3 profile carrying the write capability; each write requires separate authority, inherited path policy, a conditional create or expected revision, and durable recovery evidence. Automatic remote deletion, rename, and move propagation remain disabled." ), layer="configured", documentation_types=("admin",), @@ -1343,12 +1345,30 @@ manifest = ModuleManifest( href="/api/v1/files/connectors/credentials", kind="api", ), + DocumentationLink( + label="Manual connector-folder sync API", + href="/api/v1/files/connector-spaces/{space_id}/sync", + kind="api", + ), DocumentationLink( label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository", ), ), + translations={ + "de": { + "title": "Dateiverbindungen, Ordnersynchronisierung und das Löschen von Zugangsdaten steuern", + "summary": "Endpunkte, Zugangsdaten, vererbte Richtlinien und begrenzte manuelle Synchronisierung getrennt und mit prüfbaren Ergebnissen sowie Herkunftsnachweisen verwalten.", + "body": ( + "System, Mandant und genau eine Benutzer-, Gruppen- oder Kampagnenebene bilden die wirksame Richtlinienkette: Ablehnungsregeln haben Vorrang und jede konfigurierte Erlaubnisregel muss zutreffen. " + "Antworten blenden Geheimwerte und Bereitstellungsverweise aus. Beim Löschen datenbankverwalteter Zugangsdaten oder Profile entfernt Files eigenes verschlüsseltes Material und private Metadaten in derselben Transaktion wie das nicht geheime Audit-Ereignis. Abhängige Profile werden deaktiviert; ältere, nicht Files gehörende Verweise werden nur getrennt und auditiert. " + "Das Entfernen eines Connector-Bereichs ist ein eigener, eigentümerberechtigter Vorgang: Nur die lokale Verknüpfung des virtuellen Bereichs wird außer Kraft gesetzt. Inhalte beim Anbieter, importierte verwaltete Dateien und Freigaben, Profile, Zugangsdaten und Remote-Verweise bleiben erhalten. Intrinsische Benutzer- und Gruppenbereiche können nicht entfernt werden. " + "Eine manuelle Connector-Ordnersynchronisierung durchläuft die Anbieter-Paginierung innerhalb ausdrücklicher Grenzen für Dateizahl und Tiefe, erhält relative Remote-Pfade im verwalteten Bereich des verknüpften Eigentümers und prüft die Richtlinie für die Wurzel sowie jede heruntergeladene Datei. Übereinstimmende Quellidentitäten bleiben unverändert oder erhalten eine neue Version; neue Identitäten erzeugen verwaltete Dateien. Nicht zugehörige Zielkonflikte werden standardmäßig übersprungen; Umbenennen, Ablehnen und Ersetzen sind ausdrückliche Entscheidungen. Savepoints je Datei bewahren erfolgreiche Geschwister, während Konflikte, Überspringungen, Richtlinienablehnungen und Transportfehler in der Antwort prüfbar bleiben. Das Audit-Ereignis files.connector.folder_synced hält Herkunft, typisierte Zähler, Abbruch am Grenzwert und begrenzte Ergebnisverweise ohne Inhalte oder Zugangsdaten fest. Zeitplanung bleibt getrennt. " + "Connector-Bereiche bleiben standardmäßig schreibgeschützt. Eine Administration kann den Zweiwege-Modus nur für ein S3-Profil mit Schreib-Capability ausdrücklich aktivieren. Jeder Schreibvorgang benötigt eigene Berechtigung, die vererbte Pfadrichtlinie, eine bedingte Neuanlage oder erwartete Revision sowie dauerhafte Wiederherstellungsnachweise. Automatisches Löschen, Umbenennen und Verschieben auf dem Remote-System bleibt deaktiviert." + ), + } + }, related_modules=("access", "audit", "mail"), unlocks=( "Scoped, explainable external-file access without exposing credentials to consuming modules.", @@ -1369,12 +1389,34 @@ manifest = ModuleManifest( "files.connectors", "files.connector.credentials", "files.connector.policy", + "files.connector-folder-sync.remote-path", + "files.connector-folder-sync.target-folder", + "files.connector-folder-sync.conflict-strategy", + "files.connector-folder-sync.max-files", + "files.connector-folder-sync.recursive", ], + "prerequisites": [ + "A visible active connector space uses manual sync and links to an accessible managed user or group space.", + "The operator has files:file:upload and the effective connector policy permits the requested root and source paths.", + ], + "steps": [ + "Open the linked connector folder, choose Sync folder, and review the managed destination.", + "Keep the safe skip default or explicitly choose rename, reject, or overwrite for unrelated target collisions.", + "Set the bounded file count and subfolder choice, run the sync, and review every typed item outcome and any truncation warning.", + "Continue a truncated run from a narrower remote folder; configure scheduling separately if background operation is required.", + ], + "limitations": [ + "Manual folder sync does not schedule future runs or mutate, delete, rename, move, or change ACLs on remote content.", + "A successful partial run is not complete coverage when the response is truncated or contains review outcomes.", + ], + "outcome": "Permitted connector files become governed managed assets or versions while every exception remains explicit and auditable.", + "verification": "Compare the response summary and per-item outcomes with the files.connector.folder_synced audit event, then inspect provenance on representative created, updated, and unchanged assets.", "security_invariants": [ "New API-managed external secret references fail closed until Files can prove ownership and provider-side deletion.", "Deletion and destructive retirement scrub Files-owned encrypted connector material before completion and emit non-secret audit evidence.", "Legacy non-owned external references are detached and audited, never sent to an arbitrary provider delete operation.", "Connector-space removal is local and soft; it never claims to delete remote or previously imported managed content.", + "Folder sync is bounded, path-contained, policy-checked per item, and never includes remote bytes or credentials in audit details.", "Two-way connector mode is explicit and S3-only; it never enables automatic remote delete, rename, move, or ACL propagation.", ], "related_topic_ids": [ diff --git a/src/govoplan_files/backend/routes/connector_io.py b/src/govoplan_files/backend/routes/connector_io.py index a7190dc..6eb8122 100644 --- a/src/govoplan_files/backend/routes/connector_io.py +++ b/src/govoplan_files/backend/routes/connector_io.py @@ -9,6 +9,10 @@ from govoplan_core.audit.logging import audit_from_principal from govoplan_files.backend.schemas import ( FileConnectorBrowseItem, FileConnectorBrowseResponse, + FileConnectorFolderSyncItemResponse, + FileConnectorFolderSyncRequest, + FileConnectorFolderSyncResponse, + FileConnectorFolderSyncSummary, FileConnectorImportRequest, FileConnectorSyncResponse, FileConnectorWriteRequest, @@ -28,6 +32,11 @@ from govoplan_files.backend.storage.connector_imports import ( ConnectorImportError, ConnectorImportUnsupported, ) +from govoplan_files.backend.storage.connector_folder_sync import ( + connector_relative_path, + discover_connector_folder, + join_connector_path, +) from govoplan_files.backend.storage.connector_deployment import ( connector_effective_endpoint_url, ) @@ -43,6 +52,7 @@ from govoplan_files.backend.storage.files import ( sync_file_asset_from_source, ) from govoplan_files.backend.storage.connector_spaces import ( + connector_space_owner_id, get_connector_space_for_user, ) from govoplan_files.backend.storage.connector_writes import write_connector_file @@ -54,6 +64,7 @@ from govoplan_files.backend.route_support import ( _audit_connector_sync, _connector_browse_next_token, _connector_policy_error, + _connector_space_policy_decision, _download_connector_payload, _ensure_campaign_file_access, _http_error, @@ -64,6 +75,259 @@ from govoplan_files.backend.route_support import ( router = APIRouter(prefix="/files", tags=["files"]) +@router.post( + "/connector-spaces/{space_id}/sync", + response_model=FileConnectorFolderSyncResponse, +) +def sync_connector_space_folder( + space_id: str, + payload: FileConnectorFolderSyncRequest, + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("files:file:upload")), +): + try: + space = get_connector_space_for_user( + session, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + space_id=space_id, + is_admin=_is_admin(principal), + ) + except FileStorageError as exc: + session.rollback() + raise _http_error(exc, not_found=True) from exc + + try: + if space.sync_mode != "manual": + raise FileStorageError("This connector space is not configured for manual sync") + profile = _visible_connector_profile( + session, principal, space.connector_profile_id + ) + remote_path = join_connector_path(space.remote_path, payload.path) + target_folder = join_connector_path(payload.target_folder) + decision = _connector_space_policy_decision( + profile, + library_id=space.library_id, + remote_path=remote_path, + operation="sync", + ) + if not decision.allowed: + raise ConnectorPolicyDenied(decision) + discovery = discover_connector_folder( + profile, + path=remote_path, + library_id=space.library_id, + recursive=payload.recursive, + max_files=payload.max_files, + max_depth=payload.max_depth, + ) + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc + except ConnectorBrowseUnsupported as exc: + session.rollback() + raise HTTPException( + status_code=status.HTTP_501_NOT_IMPLEMENTED, detail=str(exc) + ) from exc + except (ConnectorBrowseError, FileStorageError, ValueError) as exc: + session.rollback() + raise _http_error(exc) from exc + + owner_id = connector_space_owner_id(space) + items: list[FileConnectorFolderSyncItemResponse] = [ + FileConnectorFolderSyncItemResponse( + source_path=skipped.path, + action="skipped", + detail=skipped.reason, + ) + for skipped in discovery.skipped + ] + counts = { + "created": 0, + "updated": 0, + "unchanged": 0, + "skipped": len(discovery.skipped), + "conflicts": 0, + "policy_denied": 0, + "failed": 0, + } + + for source in discovery.files: + source_path = normalize_connector_browse_path(source.path) + try: + relative_path = connector_relative_path( + space_root=space.remote_path, + item_path=source_path, + ) + target_path = join_connector_path(target_folder, relative_path) + except (ConnectorBrowseError, ValueError) as exc: + counts["failed"] += 1 + items.append( + FileConnectorFolderSyncItemResponse( + source_path=source_path, + action="failed", + detail=str(exc), + ) + ) + continue + + try: + with session.begin_nested(): + file_payload = FileConnectorImportRequest( + library_id=space.library_id or "", + path=source_path, + owner_type=space.owner_type, + owner_id=owner_id, + target_path=target_path, + conflict_strategy=payload.conflict_strategy, + source_revision=source.etag, + metadata={ + **dict(source.metadata), + **payload.metadata, + "connector_space_id": space.id, + "browse_name": source.name, + "browse_path": source_path, + "browse_modified_at": source.modified_at, + "browse_etag": source.etag, + "folder_sync": True, + }, + ) + _source_path, downloaded, metadata = _download_connector_payload( + profile, file_payload, operation="sync" + ) + stored, sync_action, previous_version_id = ( + sync_file_asset_from_source( + session, + tenant_id=principal.tenant_id, + owner_type=space.owner_type, + owner_id=owner_id, + user_id=principal.user.id, + filename=downloaded.filename, + data=downloaded.data, + display_path=target_path, + content_type=downloaded.content_type, + metadata=metadata, + conflict_strategy=payload.conflict_strategy, + is_admin=_is_admin(principal), + ) + ) + _audit_connector_sync( + session, + principal, + stored.asset, + sync_action=sync_action, + previous_version_id=previous_version_id, + ) + file_response = _asset_response( + session, stored.asset, include_shares=True + ) + counts[sync_action] += 1 + items.append( + FileConnectorFolderSyncItemResponse( + source_path=source_path, + target_path=target_path, + action=sync_action, + file=file_response, + previous_version_id=previous_version_id, + current_version_id=stored.version.id, + source_revision=file_response.source_revision, + ) + ) + except ConnectorPolicyDenied as exc: + counts["policy_denied"] += 1 + items.append( + FileConnectorFolderSyncItemResponse( + source_path=source_path, + target_path=target_path, + action="policy_denied", + source_revision=source.etag, + detail=str(exc), + policy_decision=exc.decision.to_dict(), + ) + ) + except FileStorageError as exc: + detail = str(exc) + if detail.startswith("Skipped upload target:"): + action = "skipped" + counts["skipped"] += 1 + elif detail.startswith("Target file already exists:"): + action = "conflict" + counts["conflicts"] += 1 + else: + action = "failed" + counts["failed"] += 1 + items.append( + FileConnectorFolderSyncItemResponse( + source_path=source_path, + target_path=target_path, + action=action, + source_revision=source.etag, + detail=detail, + ) + ) + except ( + ConnectorImportError, + UnsafeFilePathError, + OSError, + ValueError, + json.JSONDecodeError, + ) as exc: + counts["failed"] += 1 + items.append( + FileConnectorFolderSyncItemResponse( + source_path=source_path, + target_path=target_path, + action="failed", + source_revision=source.etag, + detail=str(exc), + ) + ) + + summary = FileConnectorFolderSyncSummary( + discovered=len(discovery.files), + **counts, + ) + audit_from_principal( + session, + principal, + action="files.connector.folder_synced", + object_type="file_connector_space", + object_id=space.id, + details={ + "connector_profile_id": profile.id, + "provider": profile.provider, + "library_id": space.library_id, + "remote_path": remote_path, + "target_folder": target_folder, + "recursive": payload.recursive, + "truncated": discovery.truncated, + "summary": summary.model_dump(), + "results": [ + { + "source_path": item.source_path, + "target_path": item.target_path, + "action": item.action, + "file_id": item.file.id if item.file else None, + "current_version_id": item.current_version_id, + } + for item in items + ], + }, + ) + session.commit() + return FileConnectorFolderSyncResponse( + connector_space_id=space.id, + connector_profile_id=profile.id, + provider=profile.provider, + remote_path=remote_path, + target_folder=target_folder, + recursive=payload.recursive, + truncated=discovery.truncated, + summary=summary, + items=items, + ) + + @router.post( "/connector-spaces/{space_id}/write-back", response_model=FileConnectorWriteResponse, diff --git a/src/govoplan_files/backend/schemas.py b/src/govoplan_files/backend/schemas.py index b52fe78..decb42f 100644 --- a/src/govoplan_files/backend/schemas.py +++ b/src/govoplan_files/backend/schemas.py @@ -530,7 +530,7 @@ class FileConnectorImportRequest(BaseModel): target_folder: str | None = None target_path: str | None = None campaign_id: str | None = None - conflict_strategy: Literal["reject", "overwrite", "rename"] = "reject" + conflict_strategy: Literal["reject", "overwrite", "rename", "skip"] = "reject" source_revision: str | None = None metadata: dict[str, Any] = Field(default_factory=dict) @@ -542,6 +542,59 @@ class FileConnectorSyncResponse(BaseModel): current_version_id: str +class FileConnectorFolderSyncRequest(BaseModel): + path: str = "" + target_folder: str | None = None + recursive: bool = True + conflict_strategy: Literal["reject", "overwrite", "rename", "skip"] = "skip" + max_files: int = Field(default=100, ge=1, le=500) + max_depth: int = Field(default=12, ge=0, le=50) + metadata: dict[str, Any] = Field(default_factory=dict) + + +class FileConnectorFolderSyncItemResponse(BaseModel): + source_path: str + target_path: str | None = None + action: Literal[ + "created", + "updated", + "unchanged", + "skipped", + "conflict", + "policy_denied", + "failed", + ] + file: FileAssetResponse | None = None + previous_version_id: str | None = None + current_version_id: str | None = None + source_revision: str | None = None + detail: str | None = None + policy_decision: dict[str, Any] | None = None + + +class FileConnectorFolderSyncSummary(BaseModel): + discovered: int = 0 + created: int = 0 + updated: int = 0 + unchanged: int = 0 + skipped: int = 0 + conflicts: int = 0 + policy_denied: int = 0 + failed: int = 0 + + +class FileConnectorFolderSyncResponse(BaseModel): + connector_space_id: str + connector_profile_id: str + provider: str + remote_path: str + target_folder: str + recursive: bool + truncated: bool = False + summary: FileConnectorFolderSyncSummary + items: list[FileConnectorFolderSyncItemResponse] = Field(default_factory=list) + + class FileConnectorWriteRequest(BaseModel): file_id: str remote_path: str diff --git a/src/govoplan_files/backend/storage/connector_folder_sync.py b/src/govoplan_files/backend/storage/connector_folder_sync.py new file mode 100644 index 0000000..564ce66 --- /dev/null +++ b/src/govoplan_files/backend/storage/connector_folder_sync.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field + +from govoplan_files.backend.storage.connector_browse import ( + ConnectorBrowseItem, + browse_connector_profile, + normalize_connector_browse_path, +) +from govoplan_files.backend.storage.connector_profiles import ConnectorProfile + + +@dataclass(frozen=True, slots=True) +class ConnectorFolderSkip: + path: str + reason: str + + +@dataclass(slots=True) +class ConnectorFolderDiscovery: + files: list[ConnectorBrowseItem] = field(default_factory=list) + skipped: list[ConnectorFolderSkip] = field(default_factory=list) + truncated: bool = False + + +def discover_connector_folder( + profile: ConnectorProfile, + *, + path: str, + library_id: str | None, + recursive: bool, + max_files: int, + max_depth: int, +) -> ConnectorFolderDiscovery: + """Discover a bounded, deterministic set of files below one connector folder.""" + + root_path = normalize_connector_browse_path(path) + queue: deque[tuple[str, int]] = deque([(root_path, 0)]) + visited_folders: set[str] = set() + seen_files: set[tuple[str, str]] = set() + result = ConnectorFolderDiscovery() + + while queue: + folder_path, depth = queue.popleft() + if folder_path in visited_folders: + continue + visited_folders.add(folder_path) + + continuation_token: str | None = None + seen_tokens: set[str] = set() + while True: + items = browse_connector_profile( + profile, + path=folder_path, + library_id=library_id, + continuation_token=continuation_token, + ) + for item in items: + item_path = normalize_connector_browse_path(item.path) + if item.kind == "file": + identity = (str(item.external_id or ""), item_path) + if identity in seen_files: + continue + if len(result.files) >= max_files: + result.truncated = True + result.skipped.append( + ConnectorFolderSkip( + path=item_path or folder_path, + reason=f"The manual sync limit of {max_files} files was reached.", + ) + ) + return result + seen_files.add(identity) + result.files.append(item) + continue + + if item.kind == "folder": + if not recursive: + result.skipped.append( + ConnectorFolderSkip( + path=item_path, + reason="Subfolder skipped because recursive sync is disabled.", + ) + ) + elif depth >= max_depth: + result.skipped.append( + ConnectorFolderSkip( + path=item_path, + reason=f"Subfolder skipped at the configured depth limit of {max_depth}.", + ) + ) + else: + queue.append((item_path, depth + 1)) + continue + + result.skipped.append( + ConnectorFolderSkip( + path=item_path, + reason=f"Connector item kind {item.kind!r} cannot be synchronized as a file.", + ) + ) + + next_token = _next_continuation_token(items) + if not next_token or next_token in seen_tokens: + break + seen_tokens.add(next_token) + continuation_token = next_token + + result.files.sort(key=lambda item: normalize_connector_browse_path(item.path).casefold()) + result.skipped.sort(key=lambda item: (item.path.casefold(), item.reason)) + return result + + +def connector_relative_path(*, space_root: str, item_path: str) -> str: + root = normalize_connector_browse_path(space_root) + item = normalize_connector_browse_path(item_path) + if not root: + return item + if item == root: + return item.rsplit("/", 1)[-1] + prefix = f"{root}/" + if not item.startswith(prefix): + raise ValueError("Connector returned a file outside the linked connector-space root") + return item[len(prefix) :] + + +def join_connector_path(*parts: str | None) -> str: + return normalize_connector_browse_path( + "/".join(str(part or "").strip("/\\") for part in parts if str(part or "").strip("/\\")) + ) + + +def _next_continuation_token(items: list[ConnectorBrowseItem]) -> str | None: + for item in reversed(items): + value = item.metadata.get("next_continuation_token") + if value is not None and str(value).strip(): + return str(value).strip() + return None diff --git a/src/govoplan_files/backend/storage/files.py b/src/govoplan_files/backend/storage/files.py index a5db9d0..99b7c25 100644 --- a/src/govoplan_files/backend/storage/files.py +++ b/src/govoplan_files/backend/storage/files.py @@ -1108,7 +1108,7 @@ def _resolution_by_path(conflict_resolutions: Iterable[FileConflictResolution] | def _normalize_conflict_strategy(strategy: str | None) -> str: normalized = (strategy or "reject").lower().strip() - if normalized not in {"reject", "overwrite", "rename"}: + if normalized not in {"reject", "overwrite", "rename", "skip"}: raise FileStorageError("Unsupported conflict strategy") return normalized diff --git a/tests/test_connector_folder_sync.py b/tests/test_connector_folder_sync.py new file mode 100644 index 0000000..218c63a --- /dev/null +++ b/tests/test_connector_folder_sync.py @@ -0,0 +1,303 @@ +from __future__ import annotations + +import unittest +from datetime import UTC, datetime +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock, call, patch + +from govoplan_files.backend.routes.connector_io import sync_connector_space_folder +from govoplan_files.backend.schemas import ( + FileAssetResponse, + FileConnectorFolderSyncRequest, +) +from govoplan_files.backend.storage.common import FileStorageError, UploadedStoredFile +from govoplan_files.backend.storage.connector_browse import ConnectorBrowseItem +from govoplan_files.backend.storage.connector_folder_sync import ( + connector_relative_path, + discover_connector_folder, +) +from govoplan_files.backend.storage.connector_imports import ConnectorDownloadedFile +from govoplan_files.backend.storage.connector_policy import ConnectorPolicyDenied +from govoplan_files.backend.storage.connector_profiles import ConnectorProfile + + +def _item(kind: str, path: str, *, token: str | None = None) -> ConnectorBrowseItem: + return ConnectorBrowseItem( + kind=kind, + name=path.rsplit("/", 1)[-1], + path=path, + external_id=f"external:{path}", + etag=f"etag:{path}", + metadata={"next_continuation_token": token} if token else {}, + ) + + +class ConnectorFolderDiscoveryTests(unittest.TestCase): + def setUp(self) -> None: + self.profile = ConnectorProfile( + id="profile-1", + label="Documents", + provider="webdav", + capabilities=("browse", "sync"), + ) + + @patch("govoplan_files.backend.storage.connector_folder_sync.browse_connector_profile") + def test_recursively_discovers_files_across_pages_in_deterministic_order( + self, browse: Mock + ) -> None: + browse.side_effect = [ + [_item("file", "root/z.txt", token="page-2")], + [_item("folder", "root/nested"), _item("file", "root/a.txt")], + [_item("file", "root/nested/b.txt")], + ] + + result = discover_connector_folder( + self.profile, + path="root", + library_id="library-1", + recursive=True, + max_files=10, + max_depth=3, + ) + + self.assertEqual( + ["root/a.txt", "root/nested/b.txt", "root/z.txt"], + [item.path for item in result.files], + ) + self.assertFalse(result.truncated) + self.assertEqual([], result.skipped) + self.assertEqual( + [ + call( + self.profile, + path="root", + library_id="library-1", + continuation_token=None, + ), + call( + self.profile, + path="root", + library_id="library-1", + continuation_token="page-2", + ), + call( + self.profile, + path="root/nested", + library_id="library-1", + continuation_token=None, + ), + ], + browse.call_args_list, + ) + + @patch("govoplan_files.backend.storage.connector_folder_sync.browse_connector_profile") + def test_non_recursive_and_file_limit_skips_are_explicit(self, browse: Mock) -> None: + browse.return_value = [ + _item("folder", "root/nested"), + _item("file", "root/a.txt"), + _item("file", "root/b.txt"), + ] + + result = discover_connector_folder( + self.profile, + path="root", + library_id=None, + recursive=False, + max_files=1, + max_depth=0, + ) + + self.assertEqual(["root/a.txt"], [item.path for item in result.files]) + self.assertTrue(result.truncated) + self.assertEqual( + ["root/nested", "root/b.txt"], + [item.path for item in result.skipped], + ) + + def test_relative_path_rejects_provider_results_outside_linked_root(self) -> None: + self.assertEqual( + "nested/report.csv", + connector_relative_path( + space_root="linked/root", item_path="linked/root/nested/report.csv" + ), + ) + with self.assertRaisesRegex(ValueError, "outside"): + connector_relative_path( + space_root="linked/root", item_path="another/root/report.csv" + ) + + +class ConnectorFolderSyncRouteTests(unittest.TestCase): + def setUp(self) -> None: + self.session = MagicMock() + self.space = SimpleNamespace( + id="space-1", + tenant_id="tenant-1", + owner_type="user", + owner_user_id="user-1", + owner_group_id=None, + connector_profile_id="profile-1", + provider="webdav", + library_id="library-1", + remote_path="linked/root", + sync_mode="manual", + ) + self.profile = ConnectorProfile( + id="profile-1", + label="Documents", + provider="webdav", + capabilities=("browse", "sync"), + ) + self.principal = SimpleNamespace( + tenant_id="tenant-1", user=SimpleNamespace(id="user-1") + ) + + @staticmethod + def _stored(index: int) -> UploadedStoredFile: + return UploadedStoredFile( + asset=SimpleNamespace(id=f"asset-{index}"), + version=SimpleNamespace(id=f"version-{index}"), + blob=SimpleNamespace(id=f"blob-{index}"), + ) + + @staticmethod + def _response(index: int, action: str) -> FileAssetResponse: + now = datetime(2026, 8, 21, tzinfo=UTC).isoformat() + return FileAssetResponse( + id=f"asset-{index}", + tenant_id="tenant-1", + owner_type="user", + owner_id="user-1", + display_path=f"Imports/file-{index}.txt", + filename=f"file-{index}.txt", + size_bytes=index, + checksum_sha256=str(index) * 64, + version_id=f"version-{index}", + created_at=now, + updated_at=now, + source_revision=f"revision-{action}", + ) + + @patch("govoplan_files.backend.routes.connector_io.audit_from_principal") + @patch("govoplan_files.backend.routes.connector_io._asset_response") + @patch("govoplan_files.backend.routes.connector_io._audit_connector_sync") + @patch("govoplan_files.backend.routes.connector_io.sync_file_asset_from_source") + @patch("govoplan_files.backend.routes.connector_io._download_connector_payload") + @patch("govoplan_files.backend.routes.connector_io.discover_connector_folder") + @patch("govoplan_files.backend.routes.connector_io._connector_space_policy_decision") + @patch("govoplan_files.backend.routes.connector_io._visible_connector_profile") + @patch("govoplan_files.backend.routes.connector_io.get_connector_space_for_user") + @patch("govoplan_files.backend.routes.connector_io._is_admin", return_value=False) + def test_returns_reviewable_per_item_results_and_audits_the_summary( + self, + _is_admin: Mock, + get_space: Mock, + visible_profile: Mock, + policy_decision: Mock, + discover: Mock, + download: Mock, + sync_file: Mock, + audit_sync: Mock, + asset_response: Mock, + audit: Mock, + ) -> None: + get_space.return_value = self.space + visible_profile.return_value = self.profile + policy_decision.return_value = SimpleNamespace(allowed=True) + sources = [_item("file", f"linked/root/file-{index}.txt") for index in range(1, 6)] + discover.return_value = SimpleNamespace( + files=sources, + skipped=[SimpleNamespace(path="linked/root/ignored", reason="Unsupported item")], + truncated=False, + ) + downloads = [ + ( + source.path, + ConnectorDownloadedFile( + filename=source.name, + data=source.name.encode(), + revision=source.etag, + ), + {"source_provenance": {"connector_id": "profile-1"}}, + ) + for source in sources + ] + denied_decision = SimpleNamespace( + reason="Path denied", + to_dict=Mock(return_value={"allowed": False, "reason": "Path denied"}), + ) + downloads[-1] = ConnectorPolicyDenied(denied_decision) # type: ignore[assignment] + download.side_effect = downloads + sync_file.side_effect = [ + (self._stored(1), "created", None), + (self._stored(2), "updated", "previous-2"), + (self._stored(3), "unchanged", "previous-3"), + FileStorageError("Target file already exists: Imports/file-4.txt"), + ] + asset_response.side_effect = [ + self._response(1, "created"), + self._response(2, "updated"), + self._response(3, "unchanged"), + ] + + response = sync_connector_space_folder( + "space-1", + FileConnectorFolderSyncRequest(target_folder="Imports"), + session=self.session, + principal=self.principal, # type: ignore[arg-type] + ) + + self.assertEqual( + { + "discovered": 5, + "created": 1, + "updated": 1, + "unchanged": 1, + "skipped": 1, + "conflicts": 1, + "policy_denied": 1, + "failed": 0, + }, + response.summary.model_dump(), + ) + self.assertEqual( + [ + "skipped", + "created", + "updated", + "unchanged", + "conflict", + "policy_denied", + ], + [item.action for item in response.items], + ) + self.assertEqual("Imports/file-1.txt", response.items[1].target_path) + self.assertEqual( + {"allowed": False, "reason": "Path denied"}, + response.items[-1].policy_decision, + ) + self.session.commit.assert_called_once_with() + audit_sync.assert_has_calls( + [ + call( + self.session, + self.principal, + self._stored(index).asset, + sync_action=action, + previous_version_id=previous, + ) + for index, action, previous in ( + (1, "created", None), + (2, "updated", "previous-2"), + (3, "unchanged", "previous-3"), + ) + ] + ) + summary_details = audit.call_args.kwargs["details"] + self.assertEqual(response.summary.model_dump(), summary_details["summary"]) + self.assertEqual(6, len(summary_details["results"])) + self.assertNotIn("detail", summary_details["results"][-1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_manifest_documentation.py b/tests/test_manifest_documentation.py index 43cf87d..5e6661d 100644 --- a/tests/test_manifest_documentation.py +++ b/tests/test_manifest_documentation.py @@ -154,6 +154,11 @@ class FilesManifestDocumentationTests(unittest.TestCase): "files.connectors", "files.connector.credentials", "files.connector.policy", + "files.connector-folder-sync.remote-path", + "files.connector-folder-sync.target-folder", + "files.connector-folder-sync.conflict-strategy", + "files.connector-folder-sync.max-files", + "files.connector-folder-sync.recursive", }, set(topic.metadata["help_contexts"]), ) @@ -162,6 +167,9 @@ class FilesManifestDocumentationTests(unittest.TestCase): self.assertIn("same transaction", topic.body) self.assertIn("read-only by default", topic.body) self.assertIn("S3", topic.body) + self.assertIn("manual connector-folder sync", topic.body) + self.assertIn("Per-file savepoints", topic.body) + self.assertIn("files.connector.folder_synced", topic.body) self.assertIn("Automatic remote deletion", topic.body) self.assertTrue( any( @@ -176,6 +184,11 @@ class FilesManifestDocumentationTests(unittest.TestCase): self.assertIn( "/api/v1/files/connectors/credentials", {link.href for link in topic.links} ) + self.assertIn( + "/api/v1/files/connector-spaces/{space_id}/sync", + {link.href for link in topic.links}, + ) + self.assertIn("de", topic.translations) def test_operator_topic_covers_recovery_and_pinned_s3_smb(self) -> None: topic = self.topic( diff --git a/tests/test_router_contract.py b/tests/test_router_contract.py index 3618319..2f3a79c 100644 --- a/tests/test_router_contract.py +++ b/tests/test_router_contract.py @@ -57,7 +57,7 @@ class FilesRouterContractTests(unittest.TestCase): actual = self._operation_keys(router) self.assertEqual(expected, actual) - self.assertEqual(62, len(actual)) + self.assertEqual(63, len(actual)) self.assertFalse( [operation for operation, count in Counter(actual).items() if count > 1] ) @@ -83,6 +83,7 @@ class FilesRouterContractTests(unittest.TestCase): (("GET",), "/files/connectors/profiles/{profile_id}/browse"), (("POST",), "/files/connectors/profiles/{profile_id}/import"), (("POST",), "/files/connectors/profiles/{profile_id}/sync"), + (("POST",), "/files/connector-spaces/{space_id}/sync"), (("POST",), "/files/connector-spaces/{space_id}/write-back"), (("GET",), "/files/connectors/credentials"), (("POST",), "/files/connectors/credentials"), diff --git a/webui/package.json b/webui/package.json index b154b03..5e996e1 100644 --- a/webui/package.json +++ b/webui/package.json @@ -1,6 +1,6 @@ { "name": "@govoplan/files-webui", - "version": "0.1.19", + "version": "0.1.20", "private": true, "type": "module", "main": "src/index.ts", @@ -14,6 +14,7 @@ "./styles/file-manager.css": "./src/styles/file-manager.css" }, "scripts": { + "test:connector-folder-sync": "node scripts/test-connector-folder-sync-structure.mjs", "test:file-drop-target": "node scripts/test-file-drop-target-structure.mjs", "test:file-property-filters": "node scripts/test-file-property-filters-structure.mjs", "test:connector-space-removal": "node scripts/test-connector-space-removal-structure.mjs", diff --git a/webui/scripts/test-connector-folder-sync-structure.mjs b/webui/scripts/test-connector-folder-sync-structure.mjs new file mode 100644 index 0000000..e28449f --- /dev/null +++ b/webui/scripts/test-connector-folder-sync-structure.mjs @@ -0,0 +1,27 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; + +const page = readFileSync(new URL("../src/features/files/FilesPage.tsx", import.meta.url), "utf8"); +const api = readFileSync(new URL("../src/api/files.ts", import.meta.url), "utf8"); + +assert.match(api, /connector-spaces\/\$\{encodeURIComponent\(spaceId\)\}\/sync/); +assert.match(api, /conflict_strategy: "skip", max_files: 100, max_depth: 12/); +assert.match(page, /syncFileConnectorSpaceFolder\(settings, space\.connector_space_id/); +assert.match(page, /recursive: folderSyncRecursive/); +assert.match(page, /conflict_strategy: folderSyncConflictStrategy/); +assert.match(page, /folderSyncResult\.items\.map/); +assert.match(page, /folderSyncActionLabel\(item\.action\)/); + +for (const contextId of [ + "files.connector-folder-sync.remote-path", + "files.connector-folder-sync.target-folder", + "files.connector-folder-sync.conflict-strategy", + "files.connector-folder-sync.max-files", + "files.connector-folder-sync.recursive" +]) { + assert.ok(page.includes(`helpContextId="${contextId}"`), `missing contextual help for ${contextId}`); +} + +assert.doesNotMatch(page, /window\.confirm\([\s\S]*connector-folder-sync/); + +console.log("Connector folder-sync structure checks passed."); diff --git a/webui/src/api/files.ts b/webui/src/api/files.ts index b349184..26fae94 100644 --- a/webui/src/api/files.ts +++ b/webui/src/api/files.ts @@ -426,6 +426,48 @@ export type FileConnectorSyncResponse = { previous_version_id?: string | null; current_version_id: string; }; +export type FileConnectorFolderSyncAction = "created" | "updated" | "unchanged" | "skipped" | "conflict" | "policy_denied" | "failed"; +export type FileConnectorFolderSyncItem = { + source_path: string; + target_path?: string | null; + action: FileConnectorFolderSyncAction; + file?: ManagedFile | null; + previous_version_id?: string | null; + current_version_id?: string | null; + source_revision?: string | null; + detail?: string | null; + policy_decision?: FileConnectorPolicyDecision | null; +}; +export type FileConnectorFolderSyncSummary = { + discovered: number; + created: number; + updated: number; + unchanged: number; + skipped: number; + conflicts: number; + policy_denied: number; + failed: number; +}; +export type FileConnectorFolderSyncResponse = { + connector_space_id: string; + connector_profile_id: string; + provider: string; + remote_path: string; + target_folder: string; + recursive: boolean; + truncated: boolean; + summary: FileConnectorFolderSyncSummary; + items: FileConnectorFolderSyncItem[]; +}; +export type FileConnectorFolderSyncPayload = { + path?: string; + target_folder?: string | null; + recursive?: boolean; + conflict_strategy?: "reject" | "overwrite" | "rename" | "skip"; + max_files?: number; + max_depth?: number; + metadata?: Record; +}; export type FileConnectorWritePayload = { file_id: string; remote_path: string; @@ -502,7 +544,7 @@ export type FileBlobGcResponse = { export type RenameResponse = {dry_run: boolean;items: {kind: "file" | "folder";id: string;file_id?: string | null;folder_path?: string | null;old_path: string;new_path: string;}[];}; export type TransferResponse = {operation: "move" | "copy";files: number;folders: number;}; export type ConflictAction = "overwrite" | "rename" | "skip"; -export type ConflictStrategy = "reject" | "overwrite" | "rename"; +export type ConflictStrategy = "reject" | "overwrite" | "rename" | "skip"; export type ConflictResolution = {target_path: string;action: ConflictAction;new_path?: string;}; export type PatternResolveResponse = { patterns: {pattern: string;matches: ManagedFile[];}[]; @@ -1176,6 +1218,17 @@ payload: FileConnectorFilePayload) }); } +export function syncFileConnectorSpaceFolder( +settings: ApiSettings, +spaceId: string, +payload: FileConnectorFolderSyncPayload) +: Promise { + return apiFetch(settings, `/api/v1/files/connector-spaces/${encodeURIComponent(spaceId)}/sync`, { + method: "POST", + body: JSON.stringify({ path: "", target_folder: "", recursive: true, conflict_strategy: "skip", max_files: 100, max_depth: 12, metadata: {}, ...payload }) + }); +} + export function shareFilesWithCampaign(settings: ApiSettings, fileIds: string[], campaignId: string): Promise { return shareFilesWithTarget(settings, fileIds, { type: "campaign", id: campaignId, label: "campaign" }); } diff --git a/webui/src/features/files/FilesPage.tsx b/webui/src/features/files/FilesPage.tsx index 639532c..b863d7d 100644 --- a/webui/src/features/files/FilesPage.tsx +++ b/webui/src/features/files/FilesPage.tsx @@ -13,6 +13,7 @@ import { FormGrid, ActionToolbar, ResourceAccessExplanation, ToggleSwitch, hasScope, + usePlatformLanguage, type ApiSettings, type AuthInfo, i18nMessage } from "@govoplan/core-webui"; @@ -37,6 +38,7 @@ import { listManagedFileSnapshot, previewArchiveUpload, resolveFilePatterns, + syncFileConnectorSpaceFolder, syncFileConnectorFile, transferFiles, uploadFiles, @@ -46,6 +48,7 @@ import { type ConflictResolution, type ConflictStrategy, type FileConnectorBrowseItem, + type FileConnectorFolderSyncResponse, type FileConnectorProfile, type FileDeltaResponse, type FileCampaignUsageFilter, @@ -120,6 +123,7 @@ const FILES_WORKFLOW_DOCUMENTATION = { export default function FilesPage({ settings, auth }: {settings: ApiSettings;auth: AuthInfo;}) { const location = useLocation(); const navigate = useNavigate(); + const { translateText } = usePlatformLanguage(); const canDownload = hasScope(auth, "files:download"); const canUpload = hasScope(auth, "files:upload"); const canOrganize = hasScope(auth, "files:organize"); @@ -175,6 +179,10 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut const [connectorSpaceRemovalTarget, setConnectorSpaceRemovalTarget] = useState(null); const [connectorSpaceLoading, setConnectorSpaceLoading] = useState(false); const [connectorSpaceError, setConnectorSpaceError] = useState(""); + const [folderSyncRecursive, setFolderSyncRecursive] = useState(true); + const [folderSyncConflictStrategy, setFolderSyncConflictStrategy] = useState<"skip" | "rename" | "reject" | "overwrite">("skip"); + const [folderSyncMaxFiles, setFolderSyncMaxFiles] = useState(100); + const [folderSyncResult, setFolderSyncResult] = useState(null); const [message, setMessage] = useState(""); const [error, setError] = useState(""); const [accessExplanationTarget, setAccessExplanationTarget] = useState(null); @@ -555,10 +563,10 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut } function openDialog(kind: DialogKind, target: FileActionTarget | null = null) { - if ((kind === "upload" || kind === "connector-sync") && !canUpload) return; + if ((kind === "upload" || kind === "connector-sync" || kind === "connector-folder-sync") && !canUpload) return; if (kind === "connector-space" && !canOrganize) return; if (kind && ["create-folder", "rename", "single-rename", "transfer"].includes(kind) && !canOrganize) return; - if (target && isConnectorSpace(findSpace(target.spaceId)) && kind !== "connector-sync") return; + if (target && isConnectorSpace(findSpace(target.spaceId)) && kind !== "connector-sync" && kind !== "connector-folder-sync") return; if (kind === "create-folder") { setNewFolderName(""); setNewFolderError(""); @@ -586,6 +594,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut setConnectorSelectedItem(null); setConnectorSpaceLabel(""); setConnectorSpaceReadOnly(true); + setFolderSyncResult(null); resetArchiveUploadState(); } @@ -739,6 +748,56 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut } } + function openConnectorFolderSyncDialog() { + if (!canUpload || !activeSpace || !isConnectorSpace(activeSpace) || !activeSpace.connector_space_id) return; + setFolderSyncRecursive(true); + setFolderSyncConflictStrategy("skip"); + setFolderSyncMaxFiles(100); + setFolderSyncResult(null); + setDialog("connector-folder-sync"); + } + + async function syncActiveConnectorFolder() { + const space = activeSpace; + if (!canUpload || !space || !isConnectorSpace(space) || !space.connector_space_id) return; + setBusy(true); + setError(""); + setMessage(""); + setConnectorSpaceError(""); + try { + const response = await syncFileConnectorSpaceFolder(settings, space.connector_space_id, { + path: currentFolder, + target_folder: "", + recursive: folderSyncRecursive, + conflict_strategy: folderSyncConflictStrategy, + max_files: folderSyncMaxFiles, + max_depth: 12, + metadata: { + initiated_from: "files_connector_space", + connector_space_label: space.label + } + }); + setFolderSyncResult(response); + const summary = response.summary; + setMessage(i18nMessage("i18n:govoplan-files.folder_sync.finished", { + value0: summary.created, + value1: summary.updated, + value2: summary.unchanged, + value3: summary.skipped, + value4: summary.conflicts + summary.policy_denied + summary.failed + })); + const ownerSpace = spaces.find((item) => item.space_type !== "connector" && item.owner_type === space.owner_type && item.owner_id === space.owner_id); + if (ownerSpace) await loadSpaceContents(ownerSpace, { silent: true }); + await loadConnectorSpaceContents(space, { folderPath: currentFolder, silent: true }); + } catch (err) { + const detail = err instanceof Error ? err.message : String(err); + setError(detail); + setConnectorSpaceError(detail); + } finally { + setBusy(false); + } + } + async function loadArchivePreview( @@ -2197,6 +2256,17 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut const accessExplanationBlocker = workingBlocker || (activeSpaceIsConnector ? "Access explanations apply to managed files and folders." : "") || (!canExplainResourceAccess ? "Permission to inspect resource access is required." : "") || (!accessExplainableTarget ? "Select one file or folder first." : ""); const deleteBlocker = workingBlocker || managedSpaceBlocker || (!canDelete ? "File deletion permission is required." : "") || selectionBlocker; const syncBlocker = workingBlocker || (!activeSpace ? "Select a file space first." : "") || (!canUpload ? "File upload permission is required to import synchronized content." : "") || (activeSpaceIsConnector && connectorSpaceSelectedItem?.kind !== "file" ? "Select one remote file to synchronize." : ""); + const folderSyncBlocker = workingBlocker || (!activeSpaceIsConnector || !activeSpace?.connector_space_id ? "i18n:govoplan-files.folder_sync.blocker.select_space" : "") || (!canUpload ? "i18n:govoplan-files.folder_sync.blocker.permission" : "") || (connectorSpaceLoading ? "i18n:govoplan-files.folder_sync.blocker.loading" : ""); + const folderSyncSummaryRows = folderSyncResult ? [ + ["i18n:govoplan-files.folder_sync.summary.discovered", folderSyncResult.summary.discovered], + ["i18n:govoplan-files.folder_sync.summary.created", folderSyncResult.summary.created], + ["i18n:govoplan-files.folder_sync.summary.updated", folderSyncResult.summary.updated], + ["i18n:govoplan-files.folder_sync.summary.unchanged", folderSyncResult.summary.unchanged], + ["i18n:govoplan-files.folder_sync.summary.skipped", folderSyncResult.summary.skipped], + ["i18n:govoplan-files.folder_sync.summary.conflicts", folderSyncResult.summary.conflicts], + ["i18n:govoplan-files.folder_sync.summary.policy_denied", folderSyncResult.summary.policy_denied], + ["i18n:govoplan-files.folder_sync.summary.failed", folderSyncResult.summary.failed] + ] as const : []; const toolbar = @@ -2208,6 +2278,11 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut