From b752dea6106dc19e21846c89ac4f2c95a004c162 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Fri, 31 Jul 2026 02:48:56 +0200 Subject: [PATCH] Add safe archive preview and extraction workflows --- README.md | 21 +- docs/FILES_HANDBOOK.md | 75 +- pyproject.toml | 1 + src/govoplan_files/backend/documentation.py | 100 ++- src/govoplan_files/backend/manifest.py | 15 +- src/govoplan_files/backend/routes/uploads.py | 277 ++++++- src/govoplan_files/backend/schemas.py | 21 + .../backend/storage/archives.py | 679 ++++++++++++++++-- .../backend/storage/backends.py | 51 +- .../backend/storage/campaign_attachments.py | 1 - tests/test_archive_preview_token.py | 101 +++ tests/test_archives.py | 170 +++++ tests/test_documentation.py | 10 +- tests/test_router_contract.py | 11 +- tests/test_storage_backends.py | 53 +- webui/package.json | 10 +- webui/src/api/files.ts | 89 ++- webui/src/features/files/FileSpacesWidget.tsx | 2 +- webui/src/features/files/FilesPage.tsx | 326 ++++++++- webui/src/styles/file-manager.css | 130 ++++ 20 files changed, 1961 insertions(+), 182 deletions(-) create mode 100644 tests/test_archive_preview_token.py create mode 100644 tests/test_archives.py diff --git a/README.md b/README.md index dcbde56..ff471e4 100644 --- a/README.md +++ b/README.md @@ -119,11 +119,14 @@ access fails closed in both public-only and private-network deployments. It will remain disabled until every initial connection and referral target can be policy-validated and pinned. -The S3 browse/import implementation and S3 managed-storage backend currently +S3 browse/import and arbitrary external S3 managed-storage endpoints currently fail closed before creating a `boto3` client. Botocore does not yet use the -GovOPlaN pinned HTTP transport and may manage redirects itself, so neither an -explicit endpoint nor SDK endpoint discovery is allowed until both peer pinning -and redirect revalidation are enforced. +GovOPlaN pinned HTTP transport and may manage redirects itself. The only +exception is the deployment-owned Garage service installed at the exact +`http://garage:3900` endpoint with +`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`; this flag cannot authorize another +host. External endpoints remain disabled until peer pinning and redirect +revalidation are enforced. Destructive Files-module retirement applies the same credential lifecycle before dropping tables. Every remaining Files-owned encrypted connector secret is @@ -140,10 +143,12 @@ Connector and collaboration ownership boundaries are documented in The role-adaptive user, administration, integration, and operator guide is the [Files handbook](docs/FILES_HANDBOOK.md). -ZIP uploads are processed without buffering the whole archive in memory. The API -spools incoming ZIP request bodies to a bounded temporary file, then extracts -members with per-file and total extracted-size limits before storing managed -files. +Archive imports use a two-phase preview and confirmation flow for ZIP, TAR, +TAR.GZ, TAR.BZ2, and TAR.XZ. Requests are spooled to bounded temporary files; +the server validates paths, entry count, expanded size, and expansion ratio, +then returns a 30-minute tenant/user-bound preview token. Confirmation reuploads +the original archive and stores only the selected members. Password-protected +ZIP passwords remain request-only and are never included in the preview token. Bulk rename and transfer APIs are owner-scoped: callers must provide the active user or group file space with `owner_type` and `owner_id`. The storage layer diff --git a/docs/FILES_HANDBOOK.md b/docs/FILES_HANDBOOK.md index 8af3b62..052dea1 100644 --- a/docs/FILES_HANDBOOK.md +++ b/docs/FILES_HANDBOOK.md @@ -94,21 +94,29 @@ An ordinary upload does not append a new version to an existing asset. The `overwrite` strategy retires the old asset at that path. Version-preserving updates currently belong to connector sync. -### Upload and unpack ZIP files +### Preview and unpack archives -The UI can unpack a ZIP upload. The request body is spooled to a bounded -temporary file rather than buffered wholly in memory. The defaults are: +The UI previews ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ before writing managed +files. Users may select individual files or complete folders. The request is +spooled to a bounded temporary file rather than buffered wholly in memory. The +defaults are: -- 250 MiB compressed request and total extracted data; +- 250 MiB compressed request; - 50 MiB per extracted member; -- 1,000 non-directory members; -- encrypted archives are rejected; -- member paths are normalized and `..` traversal is rejected; -- the actual bytes read are counted, not only the ZIP header declarations. +- 2 GiB total expanded data; +- 10,000 declared entries; +- 100:1 maximum expansion ratio; +- a 30-minute preview token bound to the tenant, user, archive digest, and + destination; +- password-protected ZIP support with request-only password handling; +- traversal, duplicate paths, links, devices, and other special entries are + rejected; +- actual bytes read are counted, not only archive header declarations. -The byte limits use `FILE_UPLOAD_ZIP_MAX_BYTES` and -`FILE_UPLOAD_MAX_BYTES`. The 1,000-member limit is currently fixed in the -extraction service. +The browser retains the selected archive and password until confirmation. +Confirmation reuploads the archive, verifies the token and digest, repeats all +safety checks, and commits only the selected files. No preview archive or +password is retained server-side. ### Organize files and folders @@ -399,8 +407,13 @@ the table above says live access is disabled. | `FILE_STORAGE_LOCAL_ROOT` | `runtime/files` | Primary local write/read root | | `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Comma-separated older read-only roots checked after the primary root | | `FILE_STORAGE_S3_ENDPOINT_URL` and related `FILE_STORAGE_S3_*` values | deployment-specific | S3-compatible endpoint, region, credentials, and bucket | -| `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and ZIP-member maximum | -| `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | ZIP request and extracted-total maximum | +| `FILE_STORAGE_S3_DEPLOYMENT_MANAGED` | `false` | Installer-only trust marker for the exact `http://garage:3900` service; never use it for another endpoint | +| `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and extracted archive-member maximum | +| `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | Compressed archive request maximum (legacy name retained for compatibility) | +| `FILE_ARCHIVE_MAX_ENTRIES` | 10,000 | Maximum declared archive entries | +| `FILE_ARCHIVE_MAX_EXPANDED_BYTES` | 2 GiB | Maximum expanded archive bytes | +| `FILE_ARCHIVE_MAX_EXPANSION_RATIO` | 100 | Maximum expanded-to-compressed ratio | +| `FILE_ARCHIVE_PREVIEW_TTL_SECONDS` | 1,800 | Lifetime of the sealed archive preview token | | `MASTER_KEY_B64` | development fallback only | Encrypts database-managed connector secrets | The local backend is the operational baseline. It resolves every storage key @@ -408,10 +421,14 @@ under the configured root and rejects escape attempts. Fallback roots support a controlled storage-root migration: new writes go to the primary root while reads can still find older objects. -The S3 managed-storage adapter currently fails closed before creating a boto3 -client because the SDK cannot yet guarantee connection-time DNS/IP pinning and -redirect revalidation. Do not select `FILE_STORAGE_BACKEND=s3` for a live Files -deployment until that boundary is implemented and the status above changes. +The S3 managed-storage adapter fails closed before creating a boto3 client for +arbitrary external endpoints because the SDK cannot yet guarantee +connection-time DNS/IP pinning and redirect revalidation. The supported +installer may provision a deployment-owned Garage service at the exact +`http://garage:3900` endpoint and set +`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`. Files accepts only that exact +service-discovery endpoint and forces path-style S3 addressing. The marker is +deployment authority, not a general private-network bypass. Multiple API replicas require the same durable blob namespace. Separate local container filesystems will produce incomplete reads. Until a pinned shared @@ -558,7 +575,7 @@ All routes below are under `/api/v1/files`. | Area | Routes | | --- | --- | | Spaces and content | `GET /spaces`, `GET /`, `GET /folders`, `GET /delta` | -| Upload and folders | `POST /upload`, `POST /upload-zip`, `POST /folders`, `POST /folders/delete` | +| Upload and folders | `POST /upload`, `POST /upload-zip` (compatibility), `POST /archive-preview`, `POST /archive-confirm`, `POST /folders`, `POST /folders/delete` | | File access | `GET /{file_id}`, `GET /{file_id}/download`, `DELETE /{file_id}`, `POST /bulk-delete` | | Organization | `POST /bulk-rename`, `POST /transfer`, `POST /archive.zip`, `POST /resolve-patterns` | | Sharing | `POST /{file_id}/shares`, `POST /bulk-shares` | @@ -603,8 +620,9 @@ Files baseline indiscriminately. - Owner and share checks protect individual resources after operation-scope checks. - Logical paths reject traversal and storage keys cannot escape the local root. -- Upload, ZIP extraction, connector response, and S3 stream code use bounded - reads; encrypted ZIP archives are rejected. +- Upload, archive extraction, connector response, and S3 stream code use bounded + reads. Archive previews are sealed and short-lived; ZIP passwords are + request-only. - Connector HTTP sockets use connection-time DNS/IP validation and pinning, redirects are refused, and unsafe SDK transports fail before client creation. - Database-managed connector passwords/tokens are encrypted; responses redact @@ -699,12 +717,14 @@ her personal or Finance space, then the asset has the selected owner and tenant. Given Bob is outside Finance and has no share or Files admin permission, the same asset ID must not make the asset readable to Bob. -### Safe ZIP import +### Safe archive import -Given a ZIP member contains `../../secret.txt`, is encrypted, exceeds 50 MiB, -or pushes actual extracted bytes above 250 MiB, unpacking must fail without a -committed managed asset. A valid archive must preserve normalized relative paths -below the chosen logical folder. +Given an archive member contains `../../secret.txt`, is a link or special +filesystem object, exceeds 50 MiB, pushes actual expanded bytes above 2 GiB, or +exceeds the 100:1 expansion ratio, preview or confirmation must fail without a +committed managed asset. A password-protected ZIP requires the correct +request-only password. A valid confirmation must match its unexpired preview +token and preserve normalized relative paths below the chosen logical folder. ### Explicit conflicts @@ -760,7 +780,7 @@ returning different content or credentials. | Area | Implemented now | Planned or explicitly outside the current boundary | | --- | --- | --- | | Managed storage | Local durable-root backend, fallback read roots, tenant blob deduplication, checksums, bounded resumable integrity scans, quarantine, and dry-run-first orphan cleanup | Operational S3 after pinned SDK transport ([#34](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/34)); scheduled scan execution | -| Upload | Bounded direct upload, drag-and-drop UI, ZIP spool/extract limits, explicit conflicts | Malware scanning, quotas, type policy, resumable/chunked upload | +| Upload | Bounded direct upload, drag-and-drop UI, archive preview/selective extraction, password-protected ZIP support, explicit conflicts | Malware scanning, quotas, type policy, resumable/chunked upload | | Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution | General file-history UI and user-driven append-version/restore | | Sharing | User/group/tenant/campaign grants, expiry, idempotent revocation, searchable share-management UI, and campaign linkage display | Richer policy-driven share lifecycles | | Deletion/retention | Soft-delete assets/folders/spaces; immediate audited connector-secret scrubbing | File restore API, hard purge, retention policy, legal hold, and blob GC ([#38](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/38)) | @@ -788,7 +808,8 @@ Before releasing Files: 5. Exercise an allowed and denied owner/share path. 6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion. 7. Exercise connector policy explanation and one pinned HTTP provider where - configured; verify S3/SMB still fail closed. + configured; verify managed Garage if selected, and verify arbitrary external + S3 and SMB still fail closed. 8. Verify credential deletion scrubs dependents and produces audit evidence. 9. Verify a campaign attachment snapshot still identifies its exact version and checksum after the current file changes. diff --git a/pyproject.toml b/pyproject.toml index ae9d2f3..8c4b741 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ authors = [{ name = "GovOPlaN" }] dependencies = [ "govoplan-core>=0.1.9", "defusedxml>=0.7,<1", + "pyzipper>=0.3.6,<1", "python-multipart>=0.0.31,<1", ] diff --git a/src/govoplan_files/backend/documentation.py b/src/govoplan_files/backend/documentation.py index eaa0f48..695017c 100644 --- a/src/govoplan_files/backend/documentation.py +++ b/src/govoplan_files/backend/documentation.py @@ -12,7 +12,7 @@ from govoplan_core.core.modules import ( DocumentationLink, DocumentationTopic, ) -from govoplan_files.backend.storage.archives import ZIP_UPLOAD_MAX_FILES +from govoplan_files.backend.storage.archives import ARCHIVE_UPLOAD_MAX_ENTRIES from govoplan_files.backend.storage.access import user_group_ids from govoplan_files.backend.storage.connector_visibility import ( connector_profile_usable_for_import, @@ -22,6 +22,9 @@ from govoplan_files.backend.storage.connector_visibility import ( _DEFAULT_UPLOAD_MAX_BYTES = 50 * 1024 * 1024 _DEFAULT_ZIP_MAX_BYTES = 250 * 1024 * 1024 +_DEFAULT_ARCHIVE_MAX_EXPANDED_BYTES = 2 * 1024 * 1024 * 1024 +_DEFAULT_ARCHIVE_MAX_EXPANSION_RATIO = 100 +_DEFAULT_ARCHIVE_PREVIEW_TTL_SECONDS = 30 * 60 _FILES_READ_SCOPE = "files:file:read" _FILES_UPLOAD_SCOPE = "files:file:upload" @@ -42,11 +45,47 @@ def documentation_topics( "file_upload_zip_max_bytes", default=_DEFAULT_ZIP_MAX_BYTES, ) + archive_expanded_limit = _configured_positive_int( + context.settings, + "file_archive_max_expanded_bytes", + default=_DEFAULT_ARCHIVE_MAX_EXPANDED_BYTES, + ) + archive_entry_limit = _configured_positive_int( + context.settings, + "file_archive_max_entries", + default=ARCHIVE_UPLOAD_MAX_ENTRIES, + ) + archive_ratio_limit = _configured_positive_int( + context.settings, + "file_archive_max_expansion_ratio", + default=_DEFAULT_ARCHIVE_MAX_EXPANSION_RATIO, + ) + archive_preview_ttl = _configured_positive_int( + context.settings, + "file_archive_preview_ttl_seconds", + default=_DEFAULT_ARCHIVE_PREVIEW_TTL_SECONDS, + ) topics: list[DocumentationTopic] = [] if upload_limit is not None: topics.append(_upload_topic(upload_limit)) - if upload_limit is not None and zip_limit is not None: - topics.append(_zip_topic(upload_limit, zip_limit)) + if ( + upload_limit is not None + and zip_limit is not None + and archive_expanded_limit is not None + and archive_entry_limit is not None + and archive_ratio_limit is not None + and archive_preview_ttl is not None + ): + topics.append( + _archive_topic( + upload_limit, + zip_limit, + archive_expanded_limit, + archive_entry_limit, + archive_ratio_limit, + archive_preview_ttl, + ) + ) topics.append(_connector_import_topic(context)) return tuple(topics) @@ -118,19 +157,29 @@ def _upload_topic(max_bytes: int) -> DocumentationTopic: ) -def _zip_topic(max_file_bytes: int, max_zip_bytes: int) -> DocumentationTopic: +def _archive_topic( + max_file_bytes: int, + max_archive_request_bytes: int, + max_expanded_bytes: int, + max_entries: int, + max_expansion_ratio: int, + preview_ttl_seconds: int, +) -> DocumentationTopic: member_limit = _format_byte_limit(max_file_bytes) - archive_limit = _format_byte_limit(max_zip_bytes) + request_limit = _format_byte_limit(max_archive_request_bytes) + expanded_limit = _format_byte_limit(max_expanded_bytes) + preview_minutes = max(1, preview_ttl_seconds // 60) return DocumentationTopic( id="files.workflow.upload-and-unpack-zip", - title="Upload and unpack a ZIP archive", + title="Preview and unpack an archive", summary=( - f"Safely unpack up to {ZIP_UPLOAD_MAX_FILES:,} files from a ZIP whose request and extracted total are each limited to {archive_limit}." + f"Review and selectively unpack up to {max_entries:,} entries from ZIP or TAR archives before any managed file is created." ), body=( - f"The current deployment limits the ZIP request and actual extracted total to {archive_limit}, each member to {member_limit}, " - f"and the archive to {ZIP_UPLOAD_MAX_FILES:,} non-directory members. Encrypted archives and unsafe member paths are rejected. " - "Actual extracted bytes are counted instead of trusting ZIP headers." + f"The deployment accepts ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ requests up to {request_limit}, limits actual expanded data to {expanded_limit}, " + f"each member to {member_limit}, the archive to {max_entries:,} entries, and expansion to {max_expansion_ratio}:1. " + f"The server-issued preview expires after {preview_minutes} minutes. Password-protected ZIP archives are supported; passwords remain request-only. " + "Unsafe paths and special filesystem entries are rejected. Actual extracted bytes are counted instead of trusting archive headers." ), layer="configured", documentation_types=("user",), @@ -161,35 +210,42 @@ def _zip_topic(max_file_bytes: int, max_zip_bytes: int) -> DocumentationTopic: "help_contexts": ["files.list"], "prerequisites": [ "You may view and upload managed files.", - "The archive is not encrypted and fits the current configured limits.", + "The archive fits the configured request, expansion, size, and entry limits.", "The destination personal or group space grants this account write access.", ], "steps": [ "Open Files and choose the managed destination space and folder.", - "Enable Unpack ZIP uploads, then choose or drag the ZIP archive.", + "Enable Preview and unpack archive, then choose or drag one supported archive.", + "Review the discovered entries, supply a ZIP password when required, and select the files or folders to import.", "Resolve every destination conflict explicitly.", - "Wait for extraction and finalization to finish before leaving the page.", + "Confirm the selection, then wait for extraction and finalization to finish before leaving the page.", ], "outcome": "Accepted archive members are stored as separate governed managed assets below the selected folder.", "verification": "Confirm the expected member paths and inspect representative file sizes, checksums, and versions.", "constraints": [ { - "id": "zip-request-and-total", - "label": "Maximum ZIP request and extracted total", - "description": f"Both the compressed request and the actual extracted total are limited to {archive_limit}.", - "values": [archive_limit], + "id": "archive-request-and-total", + "label": "Maximum archive request and expanded total", + "description": f"The compressed request is limited to {request_limit}; actual expanded data is limited to {expanded_limit}.", + "values": [request_limit, expanded_limit], }, { - "id": "zip-member-size", + "id": "archive-member-size", "label": "Maximum extracted member size", "description": f"Each extracted file is limited to {member_limit}.", "values": [member_limit], }, { - "id": "zip-member-count", - "label": "Maximum file count", - "description": f"A ZIP may contain at most {ZIP_UPLOAD_MAX_FILES:,} non-directory members.", - "values": [f"{ZIP_UPLOAD_MAX_FILES:,} files"], + "id": "archive-entry-count", + "label": "Maximum entry count", + "description": f"An archive may contain at most {max_entries:,} declared entries.", + "values": [f"{max_entries:,} entries"], + }, + { + "id": "archive-expansion-ratio", + "label": "Maximum expansion ratio", + "description": f"Declared and actual output may not exceed {max_expansion_ratio} times the compressed request size.", + "values": [f"{max_expansion_ratio}:1"], }, ], "related_topic_ids": [ diff --git a/src/govoplan_files/backend/manifest.py b/src/govoplan_files/backend/manifest.py index d219643..48345c1 100644 --- a/src/govoplan_files/backend/manifest.py +++ b/src/govoplan_files/backend/manifest.py @@ -509,7 +509,7 @@ manifest = ModuleManifest( summary="Back up database evidence, blob bytes, and the encryption key as one recovery unit, and keep unsupported SDK transports fail-closed.", body=( "Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the original master key, then run the bounded resumable integrity scan and verify representative access paths. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. " - "Live S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Destructive module retirement drops database tables but does not remove backend blob objects." + "Arbitrary external S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Installer-owned Garage storage is supported only at the exact deployment service endpoint with its explicit trust marker. Destructive module retirement drops database tables but does not remove backend blob objects." ), layer="configured", documentation_types=("admin",), @@ -538,8 +538,13 @@ manifest = ModuleManifest( "FILE_STORAGE_BACKEND", "FILE_STORAGE_LOCAL_ROOT", "FILE_STORAGE_LOCAL_FALLBACK_ROOTS", + "FILE_STORAGE_S3_DEPLOYMENT_MANAGED", "FILE_UPLOAD_MAX_BYTES", "FILE_UPLOAD_ZIP_MAX_BYTES", + "FILE_ARCHIVE_MAX_ENTRIES", + "FILE_ARCHIVE_MAX_EXPANDED_BYTES", + "FILE_ARCHIVE_MAX_EXPANSION_RATIO", + "FILE_ARCHIVE_PREVIEW_TTL_SECONDS", "MASTER_KEY_B64", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", "GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES", @@ -634,6 +639,10 @@ manifest = ModuleManifest( "FILE_STORAGE_LOCAL_ROOT", "FILE_UPLOAD_MAX_BYTES", "FILE_UPLOAD_ZIP_MAX_BYTES", + "FILE_ARCHIVE_MAX_ENTRIES", + "FILE_ARCHIVE_MAX_EXPANDED_BYTES", + "FILE_ARCHIVE_MAX_EXPANSION_RATIO", + "FILE_ARCHIVE_PREVIEW_TTL_SECONDS", "MASTER_KEY_B64", "GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS", ), @@ -651,8 +660,8 @@ manifest = ModuleManifest( "Compare the process requirements with the handbook's implemented/planned boundary and record every unsupported requirement or compensating control.", "Confirm version alignment, apply migrations on clean and upgrade databases, and run the repository plus meta-repository security and static-analysis gates.", "Exercise allowed and denied personal, group, and share access with representative accounts.", - "Exercise bounded upload and ZIP handling, every required conflict strategy, organization, download, and soft deletion.", - "Where connectors are configured, verify policy explanation and one pinned HTTP provider; verify live S3 and SMB access still fails closed.", + "Exercise bounded archive preview and confirmation, every required conflict strategy, organization, download, and soft deletion.", + "Where connectors are configured, verify policy explanation and one pinned HTTP provider; verify installer-owned Garage when selected, and verify arbitrary external S3 and SMB access still fails closed.", "Restore a coordinated database/blob/key backup and compare representative downloaded bytes with their recorded SHA-256 checksums.", "Record the tested versions, results, known limitations, evidence locations, residual risks, owner, and approval decision.", ], diff --git a/src/govoplan_files/backend/routes/uploads.py b/src/govoplan_files/backend/routes/uploads.py index 1e9d98e..93dacfa 100644 --- a/src/govoplan_files/backend/routes/uploads.py +++ b/src/govoplan_files/backend/routes/uploads.py @@ -1,12 +1,22 @@ from __future__ import annotations +import hashlib +import hmac import json +from datetime import UTC, datetime, timedelta from typing import Literal from fastapi import APIRouter, Depends, File as FastAPIFile, Form, UploadFile from sqlalchemy.orm import Session from govoplan_core.auth import ApiPrincipal, require_scope +from govoplan_core.security.secrets import ( + TransientPayloadError, + open_transient_payload, + seal_transient_payload, +) from govoplan_files.backend.schemas import ( + ArchiveEntryResponse, + ArchivePreviewResponse, ConflictResolutionRequest, FileUploadResponse, _conflict_resolutions, @@ -14,8 +24,16 @@ from govoplan_files.backend.schemas import ( from govoplan_files.backend.db.models import FileAsset from govoplan_core.db.session import get_session from govoplan_files.backend.runtime import settings -from govoplan_files.backend.storage.paths import UnsafeFilePathError -from govoplan_files.backend.storage.archives import extract_zip_upload +from govoplan_files.backend.storage.paths import ( + UnsafeFilePathError, + normalize_folder, +) +from govoplan_files.backend.storage.archives import ( + archive_format_for_filename, + extract_archive_upload, + extract_zip_upload, + inspect_archive, +) from govoplan_files.backend.storage.common import FileStorageError from govoplan_files.backend.storage.connector_policy import ( ConnectorPolicyDenied, @@ -39,6 +57,261 @@ from govoplan_files.backend.route_support import ( ) router = APIRouter(prefix="/files", tags=["files"]) +_ARCHIVE_PREVIEW_PURPOSE = "files.archive-preview.v1" + + +def _archive_suffix(filename: str) -> str: + lowered = filename.casefold() + for suffix in (".tar.bz2", ".tar.gz", ".tar.xz", ".tbz2", ".tgz", ".txz", ".tar", ".zip"): + if lowered.endswith(suffix): + return suffix + return ".archive" + + +def _archive_sha256(path: str) -> str: + digest = hashlib.sha256() + with open(path, "rb") as source: + while chunk := source.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _archive_selected_paths(value: str) -> list[str]: + parsed = json.loads(value) + if not isinstance(parsed, list) or not parsed: + raise FileStorageError("Select at least one archive file or folder") + if len(parsed) > settings.file_archive_max_entries: + raise FileStorageError( + "Archive selection exceeds the configured entry limit" + ) + selected: list[str] = [] + for item in parsed: + if not isinstance(item, str) or not item.strip(): + raise FileStorageError("Archive selection contains an invalid path") + if len(item) > 4096: + raise FileStorageError("Archive selection path is too long") + selected.append(item) + return selected + + +def _validate_archive_preview_token( + token: str, + *, + tenant_id: str, + user_id: str, + owner_type: str, + owner_id: str, + path: str, + campaign_id: str | None, + archive_format: str, + archive_sha256: str, +) -> None: + payload = open_transient_payload( + token, + ttl_seconds=settings.file_archive_preview_ttl_seconds, + ) + expected = { + "purpose": _ARCHIVE_PREVIEW_PURPOSE, + "tenant_id": tenant_id, + "user_id": user_id, + "owner_type": owner_type, + "owner_id": owner_id, + "path": path, + "campaign_id": campaign_id or "", + "archive_format": archive_format, + } + if any(payload.get(key) != value for key, value in expected.items()): + raise FileStorageError( + "Archive preview does not match this upload destination" + ) + token_digest = payload.get("archive_sha256") + if not isinstance(token_digest, str) or not hmac.compare_digest( + token_digest, archive_sha256 + ): + raise FileStorageError( + "Archive contents changed after preview; preview it again" + ) + + +@router.post("/archive-preview", response_model=ArchivePreviewResponse) +def preview_archive_upload( + file: UploadFile = FastAPIFile(...), + owner_type: Literal["user", "group"] = Form(default="user"), + owner_id: str | None = Form(default=None), + path: str = Form(default=""), + campaign_id: str | None = Form(default=None), + password: str | None = Form(default=None), + principal: ApiPrincipal = Depends(require_scope("files:file:upload")), +): + target_owner = owner_id or principal.user.id + archive_path: str | None = None + filename = file.filename or "archive" + try: + archive_format = archive_format_for_filename(filename) + archive_path = _spool_limited_upload_to_temp( + file, + max_bytes=settings.file_upload_zip_max_bytes, + suffix=_archive_suffix(filename), + ) + inspection = inspect_archive( + archive_path, + filename=filename, + password=password, + max_entries=settings.file_archive_max_entries, + max_expanded_bytes=settings.file_archive_max_expanded_bytes, + max_expansion_ratio=settings.file_archive_max_expansion_ratio, + ) + digest = _archive_sha256(archive_path) + normalized_path = normalize_folder(path) + preview_token = seal_transient_payload( + { + "purpose": _ARCHIVE_PREVIEW_PURPOSE, + "tenant_id": principal.tenant_id, + "user_id": principal.user.id, + "owner_type": owner_type, + "owner_id": target_owner, + "path": normalized_path, + "campaign_id": campaign_id or "", + "archive_format": archive_format, + "archive_sha256": digest, + } + ) + expires_at = datetime.now(UTC) + timedelta( + seconds=settings.file_archive_preview_ttl_seconds + ) + return ArchivePreviewResponse( + preview_token=preview_token, + archive_format=inspection.archive_format, + entries=[ + ArchiveEntryResponse( + path=entry.path, + kind=entry.kind, + size_bytes=entry.size_bytes, + compressed_size_bytes=entry.compressed_size_bytes, + encrypted=entry.encrypted, + ) + for entry in inspection.entries + ], + file_count=inspection.file_count, + directory_count=inspection.directory_count, + expanded_size_bytes=inspection.expanded_size_bytes, + compressed_size_bytes=inspection.compressed_size_bytes, + requires_password=inspection.requires_password, + password_verified=inspection.password_verified, + expires_at=expires_at.isoformat(), + ) + except (FileStorageError, UnsafeFilePathError, ValueError) as exc: + raise _http_error(exc) from exc + finally: + if archive_path: + _cleanup_temp_file(archive_path) + + +@router.post("/archive-confirm", response_model=FileUploadResponse) +def confirm_archive_upload( + file: UploadFile = FastAPIFile(...), + preview_token: str = Form(...), + selected_paths_json: str = Form(...), + owner_type: Literal["user", "group"] = Form(default="user"), + owner_id: str | None = Form(default=None), + path: str = Form(default=""), + campaign_id: str | None = Form(default=None), + password: str | None = Form(default=None), + conflict_strategy: Literal["reject", "overwrite", "rename"] = Form( + default="reject" + ), + conflict_resolutions_json: str | None = Form(default=None), + source_provenance_json: str | None = Form(default=None), + source_revision: str | None = Form(default=None), + connector_policy_json: str | None = Form(default=None), + session: Session = Depends(get_session), + principal: ApiPrincipal = Depends(require_scope("files:file:upload")), +): + target_owner = owner_id or principal.user.id + archive_path: str | None = None + filename = file.filename or "archive" + try: + raw_resolutions = ( + json.loads(conflict_resolutions_json) + if conflict_resolutions_json + else [] + ) + upload_resolutions = _conflict_resolutions( + [ConflictResolutionRequest(**item) for item in raw_resolutions] + ) + selected_paths = _archive_selected_paths(selected_paths_json) + _enforce_connector_policy( + source_provenance_json, + connector_policy_json, + operation="import", + ) + metadata = _source_metadata_from_form( + source_provenance_json, source_revision + ) + archive_format = archive_format_for_filename(filename) + normalized_path = normalize_folder(path) + archive_path = _spool_limited_upload_to_temp( + file, + max_bytes=settings.file_upload_zip_max_bytes, + suffix=_archive_suffix(filename), + ) + digest = _archive_sha256(archive_path) + _validate_archive_preview_token( + preview_token, + tenant_id=principal.tenant_id, + user_id=principal.user.id, + owner_type=owner_type, + owner_id=target_owner, + path=normalized_path, + campaign_id=campaign_id, + archive_format=archive_format, + archive_sha256=digest, + ) + extracted = extract_archive_upload( + session, + tenant_id=principal.tenant_id, + owner_type=owner_type, + owner_id=target_owner, + user_id=principal.user.id, + archive_data=archive_path, + filename=filename, + folder=normalized_path, + campaign_id=campaign_id, + selected_paths=selected_paths, + password=password, + conflict_strategy=conflict_strategy, + conflict_resolutions=upload_resolutions, + metadata=metadata, + is_admin=_is_admin(principal), + max_entries=settings.file_archive_max_entries, + max_file_bytes=settings.file_upload_max_bytes, + max_expanded_bytes=settings.file_archive_max_expanded_bytes, + max_expansion_ratio=settings.file_archive_max_expansion_ratio, + ) + uploaded_assets = [item.asset for item in extracted] + _audit_connector_imports(session, principal, uploaded_assets) + session.commit() + return FileUploadResponse( + files=[ + _asset_response(session, asset, include_shares=True) + for asset in uploaded_assets + ] + ) + except ConnectorPolicyDenied as exc: + session.rollback() + raise _connector_policy_error(exc) from exc + except ( + FileStorageError, + TransientPayloadError, + UnsafeFilePathError, + ValueError, + json.JSONDecodeError, + ) as exc: + session.rollback() + raise _http_error(exc) from exc + finally: + if archive_path: + _cleanup_temp_file(archive_path) @router.post("/upload", response_model=FileUploadResponse) diff --git a/src/govoplan_files/backend/schemas.py b/src/govoplan_files/backend/schemas.py index 2bea6ee..1701a2f 100644 --- a/src/govoplan_files/backend/schemas.py +++ b/src/govoplan_files/backend/schemas.py @@ -249,6 +249,27 @@ class FileUploadResponse(BaseModel): files: list[FileAssetResponse] +class ArchiveEntryResponse(BaseModel): + path: str + kind: Literal["file", "directory"] + size_bytes: int + compressed_size_bytes: int | None = None + encrypted: bool = False + + +class ArchivePreviewResponse(BaseModel): + preview_token: str + archive_format: str + entries: list[ArchiveEntryResponse] = Field(default_factory=list) + file_count: int + directory_count: int + expanded_size_bytes: int + compressed_size_bytes: int + requires_password: bool + password_verified: bool + expires_at: str + + class FileConnectorPolicySource(BaseModel): scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system" scope_id: str | None = None diff --git a/src/govoplan_files/backend/storage/archives.py b/src/govoplan_files/backend/storage/archives.py index 7658d8c..194ace1 100644 --- a/src/govoplan_files/backend/storage/archives.py +++ b/src/govoplan_files/backend/storage/archives.py @@ -1,55 +1,91 @@ from __future__ import annotations import mimetypes +import re +import stat +import tarfile import zipfile +from dataclasses import dataclass from io import BytesIO from os import PathLike -from pathlib import Path -from typing import Any, Iterable +from pathlib import Path, PurePosixPath +from typing import Any, BinaryIO, Iterable, Literal +import pyzipper from sqlalchemy.orm import Session from govoplan_files.backend.db.models import FileAsset -from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile -from govoplan_files.backend.storage.backends import StorageBackendError, get_storage_backend -from govoplan_files.backend.storage.files import create_file_asset, current_versions_and_blobs -from govoplan_files.backend.storage.paths import filename_from_path, normalize_folder, normalize_logical_path +from govoplan_files.backend.storage.backends import ( + StorageBackendError, + get_storage_backend, +) +from govoplan_files.backend.storage.common import ( + FileConflictResolution, + FileStorageError, + UploadedStoredFile, +) +from govoplan_files.backend.storage.files import ( + create_file_asset, + current_versions_and_blobs, +) +from govoplan_files.backend.storage.paths import ( + filename_from_path, + normalize_folder, + normalize_logical_path, +) -_ZIP_READ_CHUNK_SIZE = 1024 * 1024 -ZIP_UPLOAD_MAX_FILES = 1000 +_ARCHIVE_READ_CHUNK_SIZE = 1024 * 1024 +_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:") +ARCHIVE_UPLOAD_MAX_ENTRIES = 10_000 +# Kept for callers and documentation using the former ZIP-specific name. +ZIP_UPLOAD_MAX_FILES = ARCHIVE_UPLOAD_MAX_ENTRIES +SUPPORTED_ARCHIVE_SUFFIXES = ( + ".tar.bz2", + ".tar.gz", + ".tar.xz", + ".tbz2", + ".tgz", + ".txz", + ".tar", + ".zip", +) -def _read_zip_member( - archive: zipfile.ZipFile, - info: zipfile.ZipInfo, - *, - max_file_bytes: int, - max_total_bytes: int, - current_total: int, -) -> tuple[bytes, int]: - parts: list[bytes] = [] - actual_size = 0 - with archive.open(info) as source: - while True: - read_size = min(_ZIP_READ_CHUNK_SIZE, max_file_bytes + 1 - actual_size) - chunk = source.read(read_size) - if not chunk: - break - actual_size += len(chunk) - if actual_size > max_file_bytes: - raise FileStorageError(f"ZIP member {info.filename!r} exceeds per-file limit") - if current_total + actual_size > max_total_bytes: - raise FileStorageError("ZIP is too large after extraction") - parts.append(chunk) - return b"".join(parts), current_total + actual_size +class ArchivePasswordError(FileStorageError): + pass -def create_zip_file(session: Session, assets: Iterable[FileAsset], output_path: str | Path) -> None: +@dataclass(frozen=True, slots=True) +class ArchiveEntry: + path: str + kind: Literal["file", "directory"] + size_bytes: int + compressed_size_bytes: int | None = None + encrypted: bool = False + + +@dataclass(frozen=True, slots=True) +class ArchiveInspection: + archive_format: str + entries: tuple[ArchiveEntry, ...] + file_count: int + directory_count: int + expanded_size_bytes: int + compressed_size_bytes: int + requires_password: bool + password_verified: bool + + +def create_zip_file( + session: Session, assets: Iterable[FileAsset], output_path: str | Path +) -> None: backend = get_storage_backend() asset_list = list(assets) version_blobs = current_versions_and_blobs(session, asset_list) - with zipfile.ZipFile(output_path, mode="w", compression=zipfile.ZIP_DEFLATED) as archive: + with zipfile.ZipFile( + output_path, mode="w", compression=zipfile.ZIP_DEFLATED + ) as archive: for asset in asset_list: _version, blob = version_blobs[asset.id] info = zipfile.ZipInfo(asset.display_path) @@ -63,6 +99,145 @@ def create_zip_file(session: Session, assets: Iterable[FileAsset], output_path: raise FileStorageError(str(exc)) from exc +def archive_format_for_filename(filename: str) -> str: + lowered = filename.strip().casefold() + if lowered.endswith(".zip"): + return "zip" + if lowered.endswith((".tar.gz", ".tgz")): + return "tar.gz" + if lowered.endswith((".tar.bz2", ".tbz2")): + return "tar.bz2" + if lowered.endswith((".tar.xz", ".txz")): + return "tar.xz" + if lowered.endswith(".tar"): + return "tar" + raise FileStorageError( + "Unsupported archive format. Use ZIP, TAR, TAR.GZ, TAR.BZ2, or TAR.XZ." + ) + + +def is_supported_archive_filename(filename: str) -> bool: + try: + archive_format_for_filename(filename) + except FileStorageError: + return False + return True + + +def inspect_archive( + archive_data: bytes | str | PathLike[str], + *, + filename: str, + password: str | None = None, + max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES, + max_expanded_bytes: int = 2 * 1024 * 1024 * 1024, + max_expansion_ratio: int = 100, +) -> ArchiveInspection: + archive_format = archive_format_for_filename(filename) + compressed_size = _archive_size(archive_data) + if archive_format == "zip": + entries, requires_password, password_verified = _inspect_zip( + archive_data, + password=password, + ) + else: + entries = _inspect_tar(archive_data) + requires_password = False + password_verified = True + _validate_archive_limits( + entries, + compressed_size=compressed_size, + max_entries=max_entries, + max_expanded_bytes=max_expanded_bytes, + max_expansion_ratio=max_expansion_ratio, + ) + complete_entries = _with_derived_directories(entries) + return ArchiveInspection( + archive_format=archive_format, + entries=tuple(complete_entries), + file_count=sum(entry.kind == "file" for entry in complete_entries), + directory_count=sum( + entry.kind == "directory" for entry in complete_entries + ), + expanded_size_bytes=sum( + entry.size_bytes for entry in entries if entry.kind == "file" + ), + compressed_size_bytes=compressed_size, + requires_password=requires_password, + password_verified=password_verified, + ) + + +def extract_archive_upload( + session: Session, + *, + tenant_id: str, + owner_type: str, + owner_id: str, + user_id: str, + archive_data: bytes | str | PathLike[str], + filename: str, + folder: str | None, + campaign_id: str | None, + selected_paths: Iterable[str] | None = None, + password: str | None = None, + conflict_strategy: str = "reject", + conflict_resolutions: Iterable[FileConflictResolution] | None = None, + metadata: dict[str, Any] | None = None, + is_admin: bool = False, + max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES, + max_file_bytes: int = 50 * 1024 * 1024, + max_expanded_bytes: int = 2 * 1024 * 1024 * 1024, + max_expansion_ratio: int = 100, +) -> list[UploadedStoredFile]: + inspection = inspect_archive( + archive_data, + filename=filename, + password=password, + max_entries=max_entries, + max_expanded_bytes=max_expanded_bytes, + max_expansion_ratio=max_expansion_ratio, + ) + if inspection.requires_password and not inspection.password_verified: + raise ArchivePasswordError("Archive password is required") + selected_files = _selected_file_paths(inspection.entries, selected_paths) + if not selected_files: + raise FileStorageError("Select at least one archive file to import") + actual_total_limit = min( + max_expanded_bytes, + inspection.compressed_size_bytes * max_expansion_ratio, + ) + if inspection.archive_format == "zip": + members = _read_selected_zip_members( + archive_data, + selected_files=selected_files, + password=password, + max_file_bytes=max_file_bytes, + max_total_bytes=actual_total_limit, + ) + else: + members = _read_selected_tar_members( + archive_data, + selected_files=selected_files, + max_file_bytes=max_file_bytes, + max_total_bytes=actual_total_limit, + ) + return _store_archive_members( + session, + members=members, + tenant_id=tenant_id, + owner_type=owner_type, + owner_id=owner_id, + user_id=user_id, + folder=folder, + campaign_id=campaign_id, + conflict_strategy=conflict_strategy, + conflict_resolutions=conflict_resolutions, + metadata=metadata, + is_admin=is_admin, + ) + + def extract_zip_upload( session: Session, *, @@ -81,51 +256,401 @@ def extract_zip_upload( max_file_bytes: int = 50 * 1024 * 1024, max_total_bytes: int = 250 * 1024 * 1024, ) -> list[UploadedStoredFile]: - uploaded: list[UploadedStoredFile] = [] - total = 0 - base_folder = normalize_folder(folder) + """Backward-compatible wrapper for the original immediate ZIP endpoint.""" + + return extract_archive_upload( + session, + tenant_id=tenant_id, + owner_type=owner_type, + owner_id=owner_id, + user_id=user_id, + archive_data=zip_data, + filename="archive.zip", + folder=folder, + campaign_id=campaign_id, + conflict_strategy=conflict_strategy, + conflict_resolutions=conflict_resolutions, + metadata=metadata, + is_admin=is_admin, + max_entries=max_files, + max_file_bytes=max_file_bytes, + max_expanded_bytes=max_total_bytes, + ) + + +def _inspect_zip( + archive_data: bytes | str | PathLike[str], + *, + password: str | None, +) -> tuple[list[ArchiveEntry], bool, bool]: try: - source = BytesIO(zip_data) if isinstance(zip_data, bytes) else zip_data - with zipfile.ZipFile(source) as archive: - infos = [info for info in archive.infolist() if not info.is_dir()] - if len(infos) > max_files: - raise FileStorageError(f"ZIP contains too many files (limit {max_files})") - for info in infos: - if info.flag_bits & 0x1: - raise FileStorageError("Encrypted ZIP uploads are not supported") - if info.file_size < 0: - raise FileStorageError("Invalid ZIP member") - if info.file_size > max_file_bytes: - raise FileStorageError(f"ZIP member {info.filename!r} exceeds per-file limit") - if total + info.file_size > max_total_bytes: - raise FileStorageError("ZIP is too large after extraction") - inner_path = normalize_logical_path(info.filename) - target_path = f"{base_folder}/{inner_path}" if base_folder else inner_path - data, total = _read_zip_member( - archive, - info, - max_file_bytes=max_file_bytes, - max_total_bytes=max_total_bytes, - current_total=total, - ) - uploaded.append( - create_file_asset( - session, - tenant_id=tenant_id, - owner_type=owner_type, - owner_id=owner_id, - user_id=user_id, - filename=filename_from_path(inner_path), - data=data, - display_path=target_path, - content_type=mimetypes.guess_type(inner_path)[0] or "application/octet-stream", - metadata=metadata, - campaign_id=campaign_id, - conflict_strategy=conflict_strategy, - conflict_resolutions=conflict_resolutions, - is_admin=is_admin, + with pyzipper.AESZipFile(_archive_source(archive_data)) as archive: + infos = archive.infolist() + entries = [_zip_entry(info) for info in infos] + encrypted_info = next( + ( + info + for info in infos + if not info.is_dir() and bool(info.flag_bits & 0x1) + ), + None, + ) + if encrypted_info is None: + return entries, False, True + if not password: + return entries, True, False + try: + with archive.open( + encrypted_info, pwd=password.encode("utf-8") + ) as member: + member.read(1) + except (RuntimeError, ValueError, zipfile.BadZipFile) as exc: + raise ArchivePasswordError("Archive password is incorrect") from exc + return entries, True, True + except ArchivePasswordError: + raise + except (OSError, ValueError, zipfile.BadZipFile) as exc: + raise FileStorageError("Invalid ZIP upload") from exc + + +def _inspect_tar( + archive_data: bytes | str | PathLike[str], +) -> list[ArchiveEntry]: + try: + with _open_tar(archive_data) as archive: + entries: list[ArchiveEntry] = [] + for member in archive.getmembers(): + path = _safe_member_path(member.name) + if member.isdir(): + entries.append( + ArchiveEntry(path=path, kind="directory", size_bytes=0) + ) + continue + if not member.isfile(): + raise FileStorageError( + f"Archive member {member.name!r} is not a regular file or directory" + ) + entries.append( + ArchiveEntry( + path=path, + kind="file", + size_bytes=max(0, int(member.size)), ) ) - except zipfile.BadZipFile as exc: - raise FileStorageError("Invalid ZIP upload") from exc + return entries + except FileStorageError: + raise + except (OSError, tarfile.TarError) as exc: + raise FileStorageError("Invalid TAR upload") from exc + + +def _zip_entry(info: zipfile.ZipInfo) -> ArchiveEntry: + path = _safe_member_path(info.filename) + unix_mode = (info.external_attr >> 16) & 0xFFFF + file_type = stat.S_IFMT(unix_mode) + if file_type and not ( + stat.S_ISREG(unix_mode) or stat.S_ISDIR(unix_mode) + ): + raise FileStorageError( + f"Archive member {info.filename!r} is not a regular file or directory" + ) + if info.file_size < 0 or info.compress_size < 0: + raise FileStorageError(f"Archive member {info.filename!r} has invalid size metadata") + return ArchiveEntry( + path=path, + kind="directory" if info.is_dir() else "file", + size_bytes=0 if info.is_dir() else int(info.file_size), + compressed_size_bytes=( + None if info.is_dir() else int(info.compress_size) + ), + encrypted=bool(info.flag_bits & 0x1), + ) + + +def _validate_archive_limits( + entries: list[ArchiveEntry], + *, + compressed_size: int, + max_entries: int, + max_expanded_bytes: int, + max_expansion_ratio: int, +) -> None: + if len(entries) > max_entries: + raise FileStorageError( + f"Archive contains too many entries (limit {max_entries})" + ) + seen: dict[str, str] = {} + expanded_size = 0 + for entry in entries: + previous_kind = seen.get(entry.path) + if previous_kind is not None: + raise FileStorageError( + f"Archive contains duplicate path {entry.path!r}" + ) + seen[entry.path] = entry.kind + if entry.kind == "file": + expanded_size += entry.size_bytes + if expanded_size > max_expanded_bytes: + raise FileStorageError( + "Archive is too large after extraction " + f"(limit {max_expanded_bytes} bytes)" + ) + if expanded_size and ( + compressed_size <= 0 + or expanded_size > compressed_size * max_expansion_ratio + ): + raise FileStorageError( + "Archive expansion ratio exceeds " + f"{max_expansion_ratio}:1" + ) + + +def _with_derived_directories( + entries: list[ArchiveEntry], +) -> list[ArchiveEntry]: + by_path = {entry.path: entry for entry in entries} + for entry in entries: + parent = PurePosixPath(entry.path).parent + while str(parent) not in {"", "."}: + path = str(parent) + existing = by_path.get(path) + if existing and existing.kind != "directory": + raise FileStorageError( + f"Archive path {path!r} is both a file and a directory" + ) + by_path.setdefault( + path, + ArchiveEntry(path=path, kind="directory", size_bytes=0), + ) + parent = parent.parent + return sorted( + by_path.values(), + key=lambda entry: ( + tuple(entry.path.casefold().split("/")), + entry.kind != "directory", + ), + ) + + +def _selected_file_paths( + entries: tuple[ArchiveEntry, ...], + selected_paths: Iterable[str] | None, +) -> set[str]: + file_paths = {entry.path for entry in entries if entry.kind == "file"} + if selected_paths is None: + return file_paths + known = {entry.path: entry for entry in entries} + normalized = {_safe_member_path(path) for path in selected_paths} + unknown = normalized - known.keys() + if unknown: + raise FileStorageError( + f"Archive selection contains unknown path {sorted(unknown)[0]!r}" + ) + selected_files: set[str] = set() + for path in normalized: + entry = known[path] + if entry.kind == "file": + selected_files.add(path) + continue + prefix = f"{path}/" + selected_files.update( + file_path + for file_path in file_paths + if file_path.startswith(prefix) + ) + return selected_files + + +def _read_selected_zip_members( + archive_data: bytes | str | PathLike[str], + *, + selected_files: set[str], + password: str | None, + max_file_bytes: int, + max_total_bytes: int, +) -> list[tuple[str, bytes]]: + result: list[tuple[str, bytes]] = [] + total = 0 + try: + with pyzipper.AESZipFile(_archive_source(archive_data)) as archive: + for info in archive.infolist(): + if info.is_dir(): + continue + path = _safe_member_path(info.filename) + if path not in selected_files: + continue + pwd = password.encode("utf-8") if password else None + try: + with archive.open(info, pwd=pwd) as source: + data, total = _read_member( + source, + path=path, + max_file_bytes=max_file_bytes, + max_total_bytes=max_total_bytes, + current_total=total, + ) + except (RuntimeError, ValueError, zipfile.BadZipFile) as exc: + if info.flag_bits & 0x1: + raise ArchivePasswordError( + "Archive password is incorrect" + ) from exc + raise + result.append((path, data)) + except (FileStorageError, ArchivePasswordError): + raise + except (OSError, ValueError, zipfile.BadZipFile) as exc: + raise FileStorageError("ZIP extraction failed") from exc + return result + + +def _read_selected_tar_members( + archive_data: bytes | str | PathLike[str], + *, + selected_files: set[str], + max_file_bytes: int, + max_total_bytes: int, +) -> list[tuple[str, bytes]]: + result: list[tuple[str, bytes]] = [] + total = 0 + try: + with _open_tar(archive_data) as archive: + for member in archive.getmembers(): + if not member.isfile(): + continue + path = _safe_member_path(member.name) + if path not in selected_files: + continue + source = archive.extractfile(member) + if source is None: + raise FileStorageError( + f"Archive member {path!r} could not be read" + ) + with source: + data, total = _read_member( + source, + path=path, + max_file_bytes=max_file_bytes, + max_total_bytes=max_total_bytes, + current_total=total, + ) + result.append((path, data)) + except FileStorageError: + raise + except (OSError, tarfile.TarError) as exc: + raise FileStorageError("TAR extraction failed") from exc + return result + + +def _read_member( + source: BinaryIO, + *, + path: str, + max_file_bytes: int, + max_total_bytes: int, + current_total: int, +) -> tuple[bytes, int]: + parts: list[bytes] = [] + actual_size = 0 + while True: + read_size = min( + _ARCHIVE_READ_CHUNK_SIZE, + max_file_bytes + 1 - actual_size, + ) + chunk = source.read(read_size) + if not chunk: + break + actual_size += len(chunk) + if actual_size > max_file_bytes: + raise FileStorageError( + f"Archive member {path!r} exceeds per-file limit" + ) + if current_total + actual_size > max_total_bytes: + raise FileStorageError("Archive is too large after extraction") + parts.append(chunk) + return b"".join(parts), current_total + actual_size + + +def _store_archive_members( + session: Session, + *, + members: Iterable[tuple[str, bytes]], + tenant_id: str, + owner_type: str, + owner_id: str, + user_id: str, + folder: str | None, + campaign_id: str | None, + conflict_strategy: str, + conflict_resolutions: Iterable[FileConflictResolution] | None, + metadata: dict[str, Any] | None, + is_admin: bool, +) -> list[UploadedStoredFile]: + uploaded: list[UploadedStoredFile] = [] + base_folder = normalize_folder(folder) + for inner_path, data in members: + target_path = ( + f"{base_folder}/{inner_path}" if base_folder else inner_path + ) + uploaded.append( + create_file_asset( + session, + tenant_id=tenant_id, + owner_type=owner_type, + owner_id=owner_id, + user_id=user_id, + filename=filename_from_path(inner_path), + data=data, + display_path=target_path, + content_type=mimetypes.guess_type(inner_path)[0] + or "application/octet-stream", + metadata=metadata, + campaign_id=campaign_id, + conflict_strategy=conflict_strategy, + conflict_resolutions=conflict_resolutions, + is_admin=is_admin, + ) + ) return uploaded + + +def _safe_member_path(value: str) -> str: + raw = str(value or "").replace("\\", "/").strip() + if ( + not raw + or "\x00" in raw + or raw.startswith("/") + or _WINDOWS_DRIVE_RE.match(raw) + ): + raise FileStorageError(f"Unsafe archive member path {value!r}") + if any(part == ".." for part in raw.split("/")): + raise FileStorageError(f"Unsafe archive member path {value!r}") + try: + return normalize_logical_path(raw.rstrip("/")) + except ValueError as exc: + raise FileStorageError(f"Unsafe archive member path {value!r}") from exc + + +def _archive_size(archive_data: bytes | str | PathLike[str]) -> int: + if isinstance(archive_data, bytes): + return len(archive_data) + try: + return Path(archive_data).stat().st_size + except OSError as exc: + raise FileStorageError("Archive upload could not be read") from exc + + +def _archive_source( + archive_data: bytes | str | PathLike[str], +) -> BytesIO | str | PathLike[str]: + return BytesIO(archive_data) if isinstance(archive_data, bytes) else archive_data + + +def _open_tar( + archive_data: bytes | str | PathLike[str], +) -> tarfile.TarFile: + if isinstance(archive_data, bytes): + return tarfile.open(fileobj=BytesIO(archive_data), mode="r:*") + try: + return tarfile.open(name=archive_data, mode="r:*") + except OSError as exc: + raise FileStorageError("Archive upload could not be read") from exc diff --git a/src/govoplan_files/backend/storage/backends.py b/src/govoplan_files/backend/storage/backends.py index 21bd98e..cf599ea 100644 --- a/src/govoplan_files/backend/storage/backends.py +++ b/src/govoplan_files/backend/storage/backends.py @@ -145,28 +145,35 @@ class S3StorageBackend: region_name: str access_key_id: str secret_access_key: str + deployment_managed: bool = False name: str = "s3" @property def client(self): - try: - endpoint_url = validate_unpinned_sdk_http_url( - self.endpoint_url, - label="File storage S3 endpoint", - ) - except OutboundHttpError as exc: - raise StorageBackendError(str(exc)) from exc + if self.deployment_managed: + endpoint_url = _deployment_managed_garage_endpoint(self.endpoint_url) + else: + try: + endpoint_url = validate_unpinned_sdk_http_url( + self.endpoint_url, + label="File storage S3 endpoint", + ) + except OutboundHttpError as exc: + raise StorageBackendError(str(exc)) from exc try: import boto3 + from botocore.config import Config except ModuleNotFoundError as exc: raise StorageBackendError("boto3 is required for the S3 storage backend") from exc - return boto3.client( - "s3", - endpoint_url=endpoint_url, - region_name=self.region_name, - aws_access_key_id=self.access_key_id, - aws_secret_access_key=self.secret_access_key, - ) + options: dict[str, object] = { + "endpoint_url": endpoint_url, + "region_name": self.region_name, + "aws_access_key_id": self.access_key_id, + "aws_secret_access_key": self.secret_access_key, + } + if self.deployment_managed: + options["config"] = Config(s3={"addressing_style": "path"}) + return boto3.client("s3", **options) def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: max_bytes = response_limit("file") @@ -303,6 +310,15 @@ def _s3_missing_error(exc: Exception) -> bool: return code in {"404", "NoSuchKey", "NotFound"} or status_code == 404 +def _deployment_managed_garage_endpoint(value: str) -> str: + endpoint = str(value or "").strip() + if endpoint != "http://garage:3900": + raise StorageBackendError( + "Deployment-managed S3 trust is restricted to http://garage:3900" + ) + return endpoint + + def _fallback_roots() -> tuple[Path, ...]: raw = getattr(settings, "file_storage_local_fallback_roots", "") or "" return tuple(Path(item.strip()) for item in str(raw).split(",") if item.strip()) @@ -319,5 +335,12 @@ def get_storage_backend() -> StorageBackend: region_name=settings.file_storage_s3_region or settings.s3_region, access_key_id=settings.file_storage_s3_access_key_id or settings.s3_access_key_id, secret_access_key=settings.file_storage_s3_secret_access_key or settings.s3_secret_access_key, + deployment_managed=bool( + getattr( + settings, + "file_storage_s3_deployment_managed", + False, + ) + ), ) raise StorageBackendError(f"Unsupported file storage backend: {settings.file_storage_backend}") diff --git a/src/govoplan_files/backend/storage/campaign_attachments.py b/src/govoplan_files/backend/storage/campaign_attachments.py index 0c81814..6eaef3d 100644 --- a/src/govoplan_files/backend/storage/campaign_attachments.py +++ b/src/govoplan_files/backend/storage/campaign_attachments.py @@ -14,7 +14,6 @@ from sqlalchemy.orm import Session from govoplan_files.backend.db.models import FileAsset, FileBlob, FileShare, FileVersion from govoplan_files.backend.storage.backends import StorageBackend, get_storage_backend -from govoplan_files.backend.storage.common import FileStorageError from govoplan_files.backend.storage.files import current_versions_and_blobs, get_asset_for_user, list_assets_for_user, share_files from govoplan_files.backend.storage.access import ensure_owner_access from govoplan_files.backend.storage.paths import normalize_folder, normalize_logical_path, safe_storage_component diff --git a/tests/test_archive_preview_token.py b/tests/test_archive_preview_token.py new file mode 100644 index 0000000..cdb1688 --- /dev/null +++ b/tests/test_archive_preview_token.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import io +import hashlib +import unittest +import zipfile +from types import SimpleNamespace +from unittest.mock import patch + +from fastapi import UploadFile + +from govoplan_files.backend.routes.uploads import ( + _validate_archive_preview_token, + preview_archive_upload, +) +from govoplan_files.backend.storage.common import FileStorageError + + +def _archive_bytes() -> bytes: + target = io.BytesIO() + with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("folder/report.txt", b"report") + return target.getvalue() + + +class ArchivePreviewTokenTests(unittest.TestCase): + def setUp(self) -> None: + self.settings = SimpleNamespace( + file_upload_zip_max_bytes=10 * 1024 * 1024, + file_archive_max_entries=10_000, + file_archive_max_expanded_bytes=2 * 1024 * 1024 * 1024, + file_archive_max_expansion_ratio=100, + file_archive_preview_ttl_seconds=30 * 60, + ) + self.principal = SimpleNamespace( + tenant_id="tenant-1", + user=SimpleNamespace(id="user-1"), + ) + + def test_preview_token_is_bound_to_actor_archive_and_destination(self) -> None: + archive_data = _archive_bytes() + upload = UploadFile( + file=io.BytesIO(archive_data), + filename="monthly.zip", + ) + with patch( + "govoplan_files.backend.routes.uploads.settings", + self.settings, + ): + preview = preview_archive_upload( + file=upload, + owner_type="user", + owner_id="user-1", + path="incoming", + campaign_id=None, + password=None, + principal=self.principal, + ) + _validate_archive_preview_token( + preview.preview_token, + tenant_id="tenant-1", + user_id="user-1", + owner_type="user", + owner_id="user-1", + path="incoming", + campaign_id=None, + archive_format="zip", + archive_sha256=hashlib.sha256(archive_data).hexdigest(), + ) + with self.assertRaisesRegex( + FileStorageError, "does not match this upload destination" + ): + _validate_archive_preview_token( + preview.preview_token, + tenant_id="tenant-1", + user_id="user-1", + owner_type="user", + owner_id="user-1", + path="elsewhere", + campaign_id=None, + archive_format="zip", + archive_sha256=hashlib.sha256(archive_data).hexdigest(), + ) + with self.assertRaisesRegex( + FileStorageError, "contents changed" + ): + _validate_archive_preview_token( + preview.preview_token, + tenant_id="tenant-1", + user_id="user-1", + owner_type="user", + owner_id="user-1", + path="incoming", + campaign_id=None, + archive_format="zip", + archive_sha256="0" * 64, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_archives.py b/tests/test_archives.py new file mode 100644 index 0000000..4242888 --- /dev/null +++ b/tests/test_archives.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import io +import tarfile +import unittest +import zipfile +from types import SimpleNamespace +from unittest.mock import patch + +import pyzipper + +from govoplan_files.backend.storage.archives import ( + ArchivePasswordError, + extract_archive_upload, + inspect_archive, +) +from govoplan_files.backend.storage.common import FileStorageError + + +def _zip_bytes(entries: dict[str, bytes]) -> bytes: + target = io.BytesIO() + with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path, data in entries.items(): + archive.writestr(path, data) + return target.getvalue() + + +def _tar_bytes(entries: dict[str, bytes], mode: str = "w:gz") -> bytes: + target = io.BytesIO() + with tarfile.open(fileobj=target, mode=mode) as archive: + for path, data in entries.items(): + info = tarfile.TarInfo(path) + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + return target.getvalue() + + +def _encrypted_zip_bytes(password: str) -> bytes: + target = io.BytesIO() + with pyzipper.AESZipFile( + target, + "w", + compression=zipfile.ZIP_DEFLATED, + encryption=pyzipper.WZ_AES, + ) as archive: + archive.setpassword(password.encode("utf-8")) + archive.writestr("secure/report.txt", b"classified") + return target.getvalue() + + +class ArchiveInspectionTests(unittest.TestCase): + def test_zip_preview_derives_folders_and_sizes(self) -> None: + inspection = inspect_archive( + _zip_bytes( + { + "department/one.txt": b"one", + "department/nested/two.csv": b"two", + } + ), + filename="monthly.zip", + ) + + self.assertEqual("zip", inspection.archive_format) + self.assertEqual(2, inspection.file_count) + self.assertEqual(2, inspection.directory_count) + self.assertEqual(6, inspection.expanded_size_bytes) + self.assertEqual( + [ + "department", + "department/nested", + "department/nested/two.csv", + "department/one.txt", + ], + [entry.path for entry in inspection.entries], + ) + + def test_tar_compression_variants_are_inspected(self) -> None: + for filename, mode in ( + ("monthly.tar", "w"), + ("monthly.tar.gz", "w:gz"), + ("monthly.tar.bz2", "w:bz2"), + ("monthly.tar.xz", "w:xz"), + ): + with self.subTest(filename=filename): + inspection = inspect_archive( + _tar_bytes({"report.txt": b"report"}, mode=mode), + filename=filename, + ) + self.assertEqual(1, inspection.file_count) + self.assertEqual(filename.removeprefix("monthly."), inspection.archive_format) + + def test_unsafe_member_path_is_rejected(self) -> None: + with self.assertRaisesRegex(FileStorageError, "Unsafe archive member"): + inspect_archive( + _zip_bytes({"../escape.txt": b"no"}), + filename="unsafe.zip", + ) + + def test_archive_bomb_ratio_is_rejected(self) -> None: + with self.assertRaisesRegex(FileStorageError, "expansion ratio"): + inspect_archive( + _zip_bytes({"zeros.bin": b"\0" * 50_000}), + filename="bomb.zip", + max_expansion_ratio=2, + ) + + def test_encrypted_zip_reports_and_verifies_password(self) -> None: + archive = _encrypted_zip_bytes("correct horse") + + missing = inspect_archive(archive, filename="secure.zip") + self.assertTrue(missing.requires_password) + self.assertFalse(missing.password_verified) + + verified = inspect_archive( + archive, + filename="secure.zip", + password="correct horse", + ) + self.assertTrue(verified.requires_password) + self.assertTrue(verified.password_verified) + + with self.assertRaisesRegex(ArchivePasswordError, "incorrect"): + inspect_archive( + archive, + filename="secure.zip", + password="wrong", + ) + + def test_folder_selection_extracts_only_descendants(self) -> None: + archive = _zip_bytes( + { + "selected/one.txt": b"one", + "selected/two.txt": b"two", + "other/three.txt": b"three", + } + ) + stored = SimpleNamespace( + asset=SimpleNamespace(id="asset"), + version=SimpleNamespace(id="version"), + blob=SimpleNamespace(id="blob"), + ) + with patch( + "govoplan_files.backend.storage.archives.create_file_asset", + return_value=stored, + ) as create_asset: + result = extract_archive_upload( + object(), # type: ignore[arg-type] + tenant_id="tenant-1", + owner_type="user", + owner_id="user-1", + user_id="user-1", + archive_data=archive, + filename="selection.zip", + folder="imports", + campaign_id=None, + selected_paths=["selected"], + ) + + self.assertEqual(2, len(result)) + self.assertEqual( + {"imports/selected/one.txt", "imports/selected/two.txt"}, + { + call.kwargs["display_path"] + for call in create_asset.call_args_list + }, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_documentation.py b/tests/test_documentation.py index 78d3944..2d33a64 100644 --- a/tests/test_documentation.py +++ b/tests/test_documentation.py @@ -49,6 +49,10 @@ class FilesRuntimeDocumentationTests(unittest.TestCase): settings = SimpleNamespace( file_upload_max_bytes=7 * 1024 * 1024, file_upload_zip_max_bytes=19 * 1024 * 1024, + file_archive_max_expanded_bytes=31 * 1024 * 1024, + file_archive_max_entries=321, + file_archive_max_expansion_ratio=42, + file_archive_preview_ttl_seconds=15 * 60, ) with patch( "govoplan_files.backend.documentation.visible_connector_profiles_for_actor", @@ -73,8 +77,12 @@ class FilesRuntimeDocumentationTests(unittest.TestCase): archive = topics["files.workflow.upload-and-unpack-zip"] self.assertIn("19 MiB (19,922,944 bytes)", archive.body) + self.assertIn("31 MiB (32,505,856 bytes)", archive.body) self.assertIn("7 MiB (7,340,032 bytes)", archive.body) - self.assertIn("1,000", archive.body) + self.assertIn("321", archive.body) + self.assertIn("42:1", archive.body) + self.assertIn("15 minutes", archive.body) + self.assertIn("passwords remain request-only", archive.body) self.assertIn("Actual extracted bytes are counted", archive.body) def test_connector_task_requires_authority_and_a_visible_usable_profile( diff --git a/tests/test_router_contract.py b/tests/test_router_contract.py index b06e500..5e66ca7 100644 --- a/tests/test_router_contract.py +++ b/tests/test_router_contract.py @@ -49,11 +49,20 @@ class FilesRouterContractTests(unittest.TestCase): actual = self._operation_keys(router) self.assertEqual(expected, actual) - self.assertEqual(50, len(actual)) + self.assertEqual(52, len(actual)) self.assertFalse( [operation for operation, count in Counter(actual).items() if count > 1] ) + def test_archive_preview_and_confirmation_routes_are_exposed(self) -> None: + routes = { + (tuple(sorted(route.methods or ())), route.path) + for route in router.routes + } + + self.assertIn((("POST",), "/files/archive-preview"), routes) + self.assertIn((("POST",), "/files/archive-confirm"), routes) + def test_connector_routes_keep_existing_api_paths(self) -> None: routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes} diff --git a/tests/test_storage_backends.py b/tests/test_storage_backends.py index 2a27357..a6038d7 100644 --- a/tests/test_storage_backends.py +++ b/tests/test_storage_backends.py @@ -1,19 +1,26 @@ from __future__ import annotations import io +from types import ModuleType +import sys import unittest from unittest.mock import MagicMock, PropertyMock, patch from govoplan_files.backend.storage.backends import S3StorageBackend, StorageBackendError -def _backend() -> S3StorageBackend: +def _backend( + *, + endpoint_url: str = "https://objects.example.test", + deployment_managed: bool = False, +) -> S3StorageBackend: return S3StorageBackend( bucket="files", - endpoint_url="https://objects.example.test", + endpoint_url=endpoint_url, region_name="test", access_key_id="access", secret_access_key="secret", + deployment_managed=deployment_managed, ) @@ -32,6 +39,48 @@ class S3StorageBackendTests(unittest.TestCase): ), self.assertRaisesRegex(StorageBackendError, "until that transport supports.*DNS/IP pinning"): _backend().client + def test_installer_managed_garage_uses_exact_endpoint_and_path_style(self) -> None: + boto3 = ModuleType("boto3") + boto3.client = MagicMock(return_value=object()) + botocore = ModuleType("botocore") + botocore.__path__ = [] + botocore_config = ModuleType("botocore.config") + + class Config: + def __init__(self, **values): + self.values = values + + botocore_config.Config = Config + with patch.dict( + sys.modules, + { + "boto3": boto3, + "botocore": botocore, + "botocore.config": botocore_config, + }, + ): + _backend( + endpoint_url="http://garage:3900", + deployment_managed=True, + ).client + + _args, kwargs = boto3.client.call_args + self.assertEqual("http://garage:3900", kwargs["endpoint_url"]) + self.assertEqual( + {"s3": {"addressing_style": "path"}}, + kwargs["config"].values, + ) + + def test_installer_managed_garage_rejects_any_other_endpoint(self) -> None: + with self.assertRaisesRegex( + StorageBackendError, + "restricted to http://garage:3900", + ): + _backend( + endpoint_url="http://other-s3:3900", + deployment_managed=True, + ).client + def test_get_bytes_rejects_declared_oversize_object_without_reading(self) -> None: body = MagicMock() client = MagicMock() diff --git a/webui/package.json b/webui/package.json index 91e1902..cf733ea 100644 --- a/webui/package.json +++ b/webui/package.json @@ -18,12 +18,12 @@ "test:file-property-filters": "node scripts/test-file-property-filters-structure.mjs" }, "peerDependencies": { - "@vitejs/plugin-react": "^4.3.4", - "vite": "^6.0.6", + "@vitejs/plugin-react": "^5.2.0", + "vite": "^7.3.6", "typescript": "^5.7.2", - "react": "^19.0.0", - "react-dom": "^19.0.0", - "react-router-dom": ">=7.18.2 <8", + "react": ">=19.2.7 <20", + "react-dom": ">=19.2.7 <20", + "react-router": ">=8.3.0 <9", "lucide-react": "^1.23.0", "@govoplan/core-webui": "^0.1.9" }, diff --git a/webui/src/api/files.ts b/webui/src/api/files.ts index f129985..91b0fc0 100644 --- a/webui/src/api/files.ts +++ b/webui/src/api/files.ts @@ -328,6 +328,25 @@ export type ManagedFileSnapshotResponse = {files: ManagedFile[];folders: FileFol export type FileSpacesResponse = {spaces: FileSpace[];}; export type FileUploadResponse = {files: ManagedFile[];}; export type FileUploadProgress = {loaded: number;total?: number;percentage: number | null;}; +export type ArchivePreviewEntry = { + path: string; + kind: "file" | "directory"; + size_bytes: number; + compressed_size_bytes?: number | null; + encrypted: boolean; +}; +export type ArchivePreviewResponse = { + preview_token: string; + archive_format: string; + entries: ArchivePreviewEntry[]; + file_count: number; + directory_count: number; + expanded_size_bytes: number; + compressed_size_bytes: number; + requires_password: boolean; + password_verified: boolean; + expires_at: string; +}; export type FileConnectorFilePayload = { library_id: string; path: string; @@ -556,14 +575,80 @@ options: { return apiFetch(settings, "/api/v1/files/upload", { method: "POST", body: form }); } +export function previewArchiveUpload( +settings: ApiSettings, +file: File, +options: { + owner_type: "user" | "group"; + owner_id: string; + path?: string; + campaign_id?: string; + password?: string; +}) +: Promise { + const form = new FormData(); + form.append("file", file); + form.append("owner_type", options.owner_type); + form.append("owner_id", options.owner_id); + form.append("path", options.path ?? ""); + if (options.campaign_id) form.append("campaign_id", options.campaign_id); + if (options.password) form.append("password", options.password); + return apiFetch(settings, "/api/v1/files/archive-preview", { method: "POST", body: form }); +} + +export function confirmArchiveUpload( +settings: ApiSettings, +file: File, +options: { + preview_token: string; + selected_paths: string[]; + owner_type: "user" | "group"; + owner_id: string; + path?: string; + campaign_id?: string; + password?: string; + conflict_strategy?: ConflictStrategy; + conflict_resolutions?: ConflictResolution[]; + source_provenance?: FileSourceProvenance; + source_revision?: string; + connector_policy_sources?: FileConnectorPolicySource[]; + onProgress?: (progress: FileUploadProgress) => void; +}) +: Promise { + const form = new FormData(); + form.append("file", file); + form.append("preview_token", options.preview_token); + form.append("selected_paths_json", JSON.stringify(options.selected_paths)); + form.append("owner_type", options.owner_type); + form.append("owner_id", options.owner_id); + form.append("path", options.path ?? ""); + if (options.campaign_id) form.append("campaign_id", options.campaign_id); + if (options.password) form.append("password", options.password); + if (options.conflict_strategy) form.append("conflict_strategy", options.conflict_strategy); + if (options.conflict_resolutions?.length) form.append("conflict_resolutions_json", JSON.stringify(options.conflict_resolutions)); + if (options.source_provenance) form.append("source_provenance_json", JSON.stringify(options.source_provenance)); + if (options.source_revision) form.append("source_revision", options.source_revision); + if (options.connector_policy_sources?.length) form.append("connector_policy_json", JSON.stringify({ sources: options.connector_policy_sources })); + if (options.onProgress) { + return uploadFilesWithProgress( + settings, + form, + options.onProgress, + "/api/v1/files/archive-confirm" + ); + } + return apiFetch(settings, "/api/v1/files/archive-confirm", { method: "POST", body: form }); +} + function uploadFilesWithProgress( settings: ApiSettings, form: FormData, -onProgress: (progress: FileUploadProgress) => void) +onProgress: (progress: FileUploadProgress) => void, +endpoint = "/api/v1/files/upload") : Promise { return new Promise((resolve, reject) => { const xhr = new XMLHttpRequest(); - xhr.open("POST", apiUrl(settings, "/api/v1/files/upload")); + xhr.open("POST", apiUrl(settings, endpoint)); xhr.withCredentials = true; for (const [key, value] of authHeaders(settings)) xhr.setRequestHeader(key, value); const csrf = csrfToken(); diff --git a/webui/src/features/files/FileSpacesWidget.tsx b/webui/src/features/files/FileSpacesWidget.tsx index 23c48cd..e9184cd 100644 --- a/webui/src/features/files/FileSpacesWidget.tsx +++ b/webui/src/features/files/FileSpacesWidget.tsx @@ -1,6 +1,6 @@ import { useCallback } from "react"; import { Folder, Network } from "lucide-react"; -import { Link } from "react-router-dom"; +import { Link } from "react-router"; import { DashboardWidgetList, DismissibleAlert, diff --git a/webui/src/features/files/FilesPage.tsx b/webui/src/features/files/FilesPage.tsx index 682f07a..5dbc8b9 100644 --- a/webui/src/features/files/FilesPage.tsx +++ b/webui/src/features/files/FilesPage.tsx @@ -8,6 +8,7 @@ import { FileDropZone, FormField, LoadingIndicator, + PasswordField, ResourceAccessExplanation, ToggleSwitch, hasScope, @@ -20,6 +21,7 @@ import { browseFileConnectorProfile, createFileConnectorSpace, createFolder, + confirmArchiveUpload, deleteFolder, downloadFile, downloadFilesAsZip, @@ -29,11 +31,14 @@ import { listFileConnectorProfiles, listFileSpaces, listManagedFileSnapshot, + previewArchiveUpload, resolveFilePatterns, syncFileConnectorFile, transferFiles, uploadFiles, virtualFolderResourceId, + type ArchivePreviewEntry, + type ArchivePreviewResponse, type ConflictResolution, type ConflictStrategy, type FileConnectorBrowseItem, @@ -101,6 +106,7 @@ type FileAccessExplanationTarget = { const DEFAULT_FILE_LIST_ROW_HEIGHT = 58; const FILE_LIST_OVERSCAN_ROWS = 8; +const ARCHIVE_FILENAME_PATTERN = /\.(?:zip|tar|tar\.gz|tgz|tar\.bz2|tbz2|tar\.xz|txz)$/i; export default function FilesPage({ settings, auth }: {settings: ApiSettings;auth: AuthInfo;}) { const canDownload = hasScope(auth, "files:download"); @@ -132,6 +138,10 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut const [sortDirection, setSortDirection] = useState("asc"); const [unmatchedCount, setUnmatchedCount] = useState(null); const [unpackZip, setUnpackZip] = useState(false); + const [archiveFile, setArchiveFile] = useState(null); + const [archivePreview, setArchivePreview] = useState(null); + const [archivePassword, setArchivePassword] = useState(""); + const [selectedArchivePaths, setSelectedArchivePaths] = useState>(new Set()); const [busy, setBusy] = useState(false); const [uploadActive, setUploadActive] = useState(false); const [uploadPhase, setUploadPhase] = useState("idle"); @@ -501,9 +511,19 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut setNewFolderName(""); setNewFolderError(""); } + if (kind === "upload") { + resetArchiveUploadState(); + } openRawDialog(kind, target); } + function resetArchiveUploadState() { + setArchiveFile(null); + setArchivePreview(null); + setArchivePassword(""); + setSelectedArchivePaths(new Set()); + } + function closeDialog() { closeRawDialog(); setNewFolderError(""); @@ -513,6 +533,7 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut setConnectorError(""); setConnectorSelectedItem(null); setConnectorSpaceLabel(""); + resetArchiveUploadState(); } function updateActiveDialogFolder(spaceId: string, folderPath: string) { @@ -667,6 +688,156 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut + async function loadArchivePreview( + file: File, + target: FileActionTarget, + options: { preserveSelection?: boolean } = {} + ) { + const targetSpace = findSpace(target.spaceId); + if (!targetSpace || isConnectorSpace(targetSpace)) { + setError(uploadRejectedReason(target)); + return; + } + setArchiveFile(file); + setBusy(true); + setUploadActive(true); + setUploadPhase("uploading"); + setUploadProgress(null); + setError(""); + setMessage(""); + try { + const response = await previewArchiveUpload(settings, file, { + owner_type: targetSpace.owner_type, + owner_id: targetSpace.owner_id, + path: target.folderPath, + password: archivePassword || undefined + }); + const availableFiles = new Set( + response.entries + .filter((entry) => entry.kind === "file") + .map((entry) => entry.path) + ); + setArchivePreview(response); + setSelectedArchivePaths((current) => { + if (!options.preserveSelection) return availableFiles; + const retained = new Set( + Array.from(current).filter((path) => availableFiles.has(path)) + ); + return retained.size > 0 ? retained : availableFiles; + }); + } catch (err) { + setArchivePreview(null); + setSelectedArchivePaths(new Set()); + setError(err instanceof Error ? err.message : String(err)); + } finally { + setBusy(false); + setUploadActive(false); + setUploadPhase("idle"); + setUploadProgress(null); + } + } + + function archiveFilesForEntry(entry: ArchivePreviewEntry): string[] { + if (!archivePreview) return []; + if (entry.kind === "file") return [entry.path]; + const prefix = `${entry.path}/`; + return archivePreview.entries + .filter((candidate) => candidate.kind === "file" && candidate.path.startsWith(prefix)) + .map((candidate) => candidate.path); + } + + function archiveEntryIsSelected(entry: ArchivePreviewEntry): boolean { + const paths = archiveFilesForEntry(entry); + return paths.length > 0 && paths.every((path) => selectedArchivePaths.has(path)); + } + + function toggleArchiveEntry(entry: ArchivePreviewEntry, selected: boolean) { + const paths = archiveFilesForEntry(entry); + setSelectedArchivePaths((current) => { + const next = new Set(current); + paths.forEach((path) => selected ? next.add(path) : next.delete(path)); + return next; + }); + } + + function toggleAllArchiveFiles(selected: boolean) { + setSelectedArchivePaths( + selected && archivePreview + ? new Set( + archivePreview.entries + .filter((entry) => entry.kind === "file") + .map((entry) => entry.path) + ) + : new Set() + ); + } + + async function confirmCurrentArchive() { + const target = currentActionTarget(); + const targetSpace = target ? findSpace(target.spaceId) : null; + if ( + busy + || !archiveFile + || !archivePreview + || !target + || !targetSpace + || isConnectorSpace(targetSpace) + ) { + return; + } + if (selectedArchivePaths.size === 0) { + setError("Select at least one archive file to import."); + return; + } + if (archivePreview.requires_password && !archivePassword) { + setError("Enter the archive password before importing."); + return; + } + setBusy(true); + setUploadActive(true); + setUploadPhase("uploading"); + setUploadProgress(0); + setError(""); + setMessage("Uploading the archive for confirmed extraction."); + try { + const response = await confirmArchiveUpload(settings, archiveFile, { + preview_token: archivePreview.preview_token, + selected_paths: Array.from(selectedArchivePaths), + owner_type: targetSpace.owner_type, + owner_id: targetSpace.owner_id, + path: target.folderPath, + password: archivePassword || undefined, + conflict_strategy: "reject", + onProgress: ({ percentage }) => { + setUploadProgress(percentage); + if (percentage !== null && percentage >= 100) { + setUploadPhase("unpacking"); + setMessage("Extracting the selected archive files."); + } + } + }); + setUploadPhase("finalizing"); + setUploadProgress(100); + const uploadedCount = response.files.length; + const destination = target.folderPath || "i18n:govoplan-files.root.e96857c5"; + closeDialog(); + setMessage(i18nMessage("i18n:govoplan-files.uploaded_value_file_s_into_value.d0fa052b", { + value0: uploadedCount, + value1: destination + })); + resetTransientState(); + await loadSpaceContents(targetSpace, { silent: targetSpace.id !== activeSpaceId }); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + setMessage(""); + } finally { + setBusy(false); + setUploadActive(false); + setUploadPhase("idle"); + setUploadProgress(null); + } + } + async function handleFilesUpload(fileList: FileList | File[], options: {conflictStrategy?: ConflictStrategy;conflictResolutions?: ConflictResolution[];bypassConflictDialog?: boolean;target?: FileActionTarget;} = {}) { const target = options.target ?? currentActionTarget(); const targetSpace = target ? findSpace(target.spaceId) : null; @@ -676,6 +847,14 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut } const selected = Array.from(fileList); if (selected.length === 0) return; + if (unpackZip) { + if (selected.length !== 1 || !ARCHIVE_FILENAME_PATTERN.test(selected[0].name)) { + setError("Choose one ZIP, TAR, TAR.GZ, TAR.BZ2, or TAR.XZ archive to preview and unpack."); + return; + } + await loadArchivePreview(selected[0], target); + return; + } if (!options.bypassConflictDialog && !unpackZip) { const conflicts = uploadConflicts(selected, target); if (conflicts.length > 0) { @@ -691,7 +870,6 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut return; } } - const uploadsZipArchive = unpackZip && selected.some((file) => file.name.toLowerCase().endsWith(".zip")); setBusy(true); setUploadActive(true); setUploadPhase("uploading"); @@ -703,14 +881,13 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut owner_type: targetSpace.owner_type, owner_id: targetSpace.owner_id, path: target.folderPath, - unpack_zip: unpackZip, conflict_strategy: options.conflictStrategy ?? "reject", conflict_resolutions: options.conflictResolutions, onProgress: ({ percentage }) => { setUploadProgress(percentage); if (percentage !== null && percentage >= 100) { - setUploadPhase(uploadsZipArchive ? "unpacking" : "finalizing"); - setMessage(uploadsZipArchive ? "i18n:govoplan-files.unpacking_zip_upload.35019691" : "i18n:govoplan-files.finalizing_upload.bcce936d"); + setUploadPhase("finalizing"); + setMessage("i18n:govoplan-files.finalizing_upload.bcce936d"); } } }); @@ -2332,6 +2509,29 @@ export default function FilesPage({ settings, auth }: {settings: ApiSettings;aut {dialog === "upload" && + {!archivePreview && + <> + + + {unpackZip && +

+ Supports ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ. Nothing is written until you review and confirm the archive contents. +

+ } +
+ i18n:govoplan-files.destination_folder.f8ccb63b + activeDialogTarget && updateActiveDialogFolder(activeDialogTarget.spaceId, folderPath)} /> + +
setError(reason === "disabled" ? uploadRejectedReason(activeDialogTarget) : "The browser did not provide readable file data for this drop. Use the file picker, or drag local files from a file manager that exposes file contents to the browser.")} onFiles={(files) => handleFilesUpload(files, { target: activeDialogTarget || undefined })} /> - - -
- i18n:govoplan-files.destination_folder.f8ccb63b - activeDialogTarget && updateActiveDialogFolder(activeDialogTarget.spaceId, folderPath)} /> - -
+ + } + {archivePreview && archiveFile && +
+
+
+ {archiveFile.name} + {archivePreview.archive_format.toUpperCase()} · {formatBytes(archivePreview.compressed_size_bytes)} compressed +
+
+ {archivePreview.file_count} files + {formatBytes(archivePreview.expanded_size_bytes)} expanded · {archivePreview.directory_count} folders +
+
+ Destination + {activeDialogTarget?.folderPath || "i18n:govoplan-files.root.e96857c5"} +
+
+ + {archivePreview.requires_password && + + + + + } + +
+ + {selectedArchivePaths.size} of {archivePreview.file_count} selected +
+ +
+ {archivePreview.entries.map((entry) => { + const selectableFiles = archiveFilesForEntry(entry); + const depth = Math.max(0, entry.path.split("/").length - 1); + return ( + ); + })} +
+ +

+ Preview expires {formatDate(archivePreview.expires_at)}. The original archive is uploaded again only when you confirm. +

+ +
+ + {archivePreview.requires_password && + + } + + + +
+
+ } } diff --git a/webui/src/styles/file-manager.css b/webui/src/styles/file-manager.css index 2f030f1..4869037 100644 --- a/webui/src/styles/file-manager.css +++ b/webui/src/styles/file-manager.css @@ -288,6 +288,132 @@ width: min(820px, 100%); } +.file-dialog:has(.archive-preview) { + width: min(860px, 100%); +} + +.archive-upload-help { + margin: -8px 0 0; +} + +.archive-preview { + display: grid; + min-width: 0; + gap: 16px; +} + +.archive-preview-summary { + display: grid; + grid-template-columns: minmax(0, 1.3fr) minmax(0, 1fr) minmax(0, 1fr); + gap: 1px; + overflow: hidden; + border: var(--border-line); + border-radius: 6px; + background: var(--line); +} + +.archive-preview-summary > div { + display: grid; + min-width: 0; + gap: 4px; + padding: 11px 12px; + background: var(--panel-soft); +} + +.archive-preview-summary strong, +.archive-preview-summary span { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.archive-preview-summary span { + color: var(--muted); + font-size: var(--font-size-sm); +} + +.archive-selection-toolbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.archive-selection-toolbar label, +.archive-entry-row { + display: flex; + align-items: center; + gap: 9px; +} + +.archive-selection-toolbar label { + font-weight: 700; +} + +.archive-selection-toolbar > span, +.archive-entry-size { + color: var(--muted); + font-size: var(--font-size-sm); +} + +.archive-entry-list { + display: grid; + align-content: start; + max-height: min(380px, 42vh); + overflow: auto; + scrollbar-gutter: stable; + border: var(--border-line); + border-radius: 6px; + background: var(--surface); +} + +.archive-entry-row { + min-width: 0; + min-height: 38px; + padding-top: 7px; + padding-right: 12px; + padding-bottom: 7px; + border-bottom: var(--border-line); + cursor: pointer; +} + +.archive-entry-row:last-child { + border-bottom: 0; +} + +.archive-entry-row:hover { + background: var(--hover-tint); +} + +.archive-entry-row.is-directory { + background: var(--panel-soft); + font-weight: 700; +} + +.archive-entry-row.is-directory:hover { + background: var(--primary-soft); +} + +.archive-entry-path { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.archive-entry-size { + flex: 0 0 auto; + margin-left: auto; +} + +.archive-preview-actions { + align-items: center; +} + +.archive-preview-action-spacer { + flex: 1 1 auto; +} + .file-dialog:has(.file-share-dialog-content) { width: min(1080px, 100%); } @@ -312,6 +438,10 @@ } @media (max-width: 620px) { + .archive-preview-summary { + grid-template-columns: minmax(0, 1fr); + } + .file-share-form { grid-template-columns: minmax(0, 1fr); }