Add safe archive preview and extraction workflows

This commit is contained in:
2026-07-31 02:48:56 +02:00
parent 159a012833
commit b752dea610
20 changed files with 1961 additions and 182 deletions
+78 -22
View File
@@ -12,7 +12,7 @@ from govoplan_core.core.modules import (
DocumentationLink,
DocumentationTopic,
)
from govoplan_files.backend.storage.archives import ZIP_UPLOAD_MAX_FILES
from govoplan_files.backend.storage.archives import ARCHIVE_UPLOAD_MAX_ENTRIES
from govoplan_files.backend.storage.access import user_group_ids
from govoplan_files.backend.storage.connector_visibility import (
connector_profile_usable_for_import,
@@ -22,6 +22,9 @@ from govoplan_files.backend.storage.connector_visibility import (
_DEFAULT_UPLOAD_MAX_BYTES = 50 * 1024 * 1024
_DEFAULT_ZIP_MAX_BYTES = 250 * 1024 * 1024
_DEFAULT_ARCHIVE_MAX_EXPANDED_BYTES = 2 * 1024 * 1024 * 1024
_DEFAULT_ARCHIVE_MAX_EXPANSION_RATIO = 100
_DEFAULT_ARCHIVE_PREVIEW_TTL_SECONDS = 30 * 60
_FILES_READ_SCOPE = "files:file:read"
_FILES_UPLOAD_SCOPE = "files:file:upload"
@@ -42,11 +45,47 @@ def documentation_topics(
"file_upload_zip_max_bytes",
default=_DEFAULT_ZIP_MAX_BYTES,
)
archive_expanded_limit = _configured_positive_int(
context.settings,
"file_archive_max_expanded_bytes",
default=_DEFAULT_ARCHIVE_MAX_EXPANDED_BYTES,
)
archive_entry_limit = _configured_positive_int(
context.settings,
"file_archive_max_entries",
default=ARCHIVE_UPLOAD_MAX_ENTRIES,
)
archive_ratio_limit = _configured_positive_int(
context.settings,
"file_archive_max_expansion_ratio",
default=_DEFAULT_ARCHIVE_MAX_EXPANSION_RATIO,
)
archive_preview_ttl = _configured_positive_int(
context.settings,
"file_archive_preview_ttl_seconds",
default=_DEFAULT_ARCHIVE_PREVIEW_TTL_SECONDS,
)
topics: list[DocumentationTopic] = []
if upload_limit is not None:
topics.append(_upload_topic(upload_limit))
if upload_limit is not None and zip_limit is not None:
topics.append(_zip_topic(upload_limit, zip_limit))
if (
upload_limit is not None
and zip_limit is not None
and archive_expanded_limit is not None
and archive_entry_limit is not None
and archive_ratio_limit is not None
and archive_preview_ttl is not None
):
topics.append(
_archive_topic(
upload_limit,
zip_limit,
archive_expanded_limit,
archive_entry_limit,
archive_ratio_limit,
archive_preview_ttl,
)
)
topics.append(_connector_import_topic(context))
return tuple(topics)
@@ -118,19 +157,29 @@ def _upload_topic(max_bytes: int) -> DocumentationTopic:
)
def _zip_topic(max_file_bytes: int, max_zip_bytes: int) -> DocumentationTopic:
def _archive_topic(
max_file_bytes: int,
max_archive_request_bytes: int,
max_expanded_bytes: int,
max_entries: int,
max_expansion_ratio: int,
preview_ttl_seconds: int,
) -> DocumentationTopic:
member_limit = _format_byte_limit(max_file_bytes)
archive_limit = _format_byte_limit(max_zip_bytes)
request_limit = _format_byte_limit(max_archive_request_bytes)
expanded_limit = _format_byte_limit(max_expanded_bytes)
preview_minutes = max(1, preview_ttl_seconds // 60)
return DocumentationTopic(
id="files.workflow.upload-and-unpack-zip",
title="Upload and unpack a ZIP archive",
title="Preview and unpack an archive",
summary=(
f"Safely unpack up to {ZIP_UPLOAD_MAX_FILES:,} files from a ZIP whose request and extracted total are each limited to {archive_limit}."
f"Review and selectively unpack up to {max_entries:,} entries from ZIP or TAR archives before any managed file is created."
),
body=(
f"The current deployment limits the ZIP request and actual extracted total to {archive_limit}, each member to {member_limit}, "
f"and the archive to {ZIP_UPLOAD_MAX_FILES:,} non-directory members. Encrypted archives and unsafe member paths are rejected. "
"Actual extracted bytes are counted instead of trusting ZIP headers."
f"The deployment accepts ZIP, TAR, TAR.GZ, TAR.BZ2, and TAR.XZ requests up to {request_limit}, limits actual expanded data to {expanded_limit}, "
f"each member to {member_limit}, the archive to {max_entries:,} entries, and expansion to {max_expansion_ratio}:1. "
f"The server-issued preview expires after {preview_minutes} minutes. Password-protected ZIP archives are supported; passwords remain request-only. "
"Unsafe paths and special filesystem entries are rejected. Actual extracted bytes are counted instead of trusting archive headers."
),
layer="configured",
documentation_types=("user",),
@@ -161,35 +210,42 @@ def _zip_topic(max_file_bytes: int, max_zip_bytes: int) -> DocumentationTopic:
"help_contexts": ["files.list"],
"prerequisites": [
"You may view and upload managed files.",
"The archive is not encrypted and fits the current configured limits.",
"The archive fits the configured request, expansion, size, and entry limits.",
"The destination personal or group space grants this account write access.",
],
"steps": [
"Open Files and choose the managed destination space and folder.",
"Enable Unpack ZIP uploads, then choose or drag the ZIP archive.",
"Enable Preview and unpack archive, then choose or drag one supported archive.",
"Review the discovered entries, supply a ZIP password when required, and select the files or folders to import.",
"Resolve every destination conflict explicitly.",
"Wait for extraction and finalization to finish before leaving the page.",
"Confirm the selection, then wait for extraction and finalization to finish before leaving the page.",
],
"outcome": "Accepted archive members are stored as separate governed managed assets below the selected folder.",
"verification": "Confirm the expected member paths and inspect representative file sizes, checksums, and versions.",
"constraints": [
{
"id": "zip-request-and-total",
"label": "Maximum ZIP request and extracted total",
"description": f"Both the compressed request and the actual extracted total are limited to {archive_limit}.",
"values": [archive_limit],
"id": "archive-request-and-total",
"label": "Maximum archive request and expanded total",
"description": f"The compressed request is limited to {request_limit}; actual expanded data is limited to {expanded_limit}.",
"values": [request_limit, expanded_limit],
},
{
"id": "zip-member-size",
"id": "archive-member-size",
"label": "Maximum extracted member size",
"description": f"Each extracted file is limited to {member_limit}.",
"values": [member_limit],
},
{
"id": "zip-member-count",
"label": "Maximum file count",
"description": f"A ZIP may contain at most {ZIP_UPLOAD_MAX_FILES:,} non-directory members.",
"values": [f"{ZIP_UPLOAD_MAX_FILES:,} files"],
"id": "archive-entry-count",
"label": "Maximum entry count",
"description": f"An archive may contain at most {max_entries:,} declared entries.",
"values": [f"{max_entries:,} entries"],
},
{
"id": "archive-expansion-ratio",
"label": "Maximum expansion ratio",
"description": f"Declared and actual output may not exceed {max_expansion_ratio} times the compressed request size.",
"values": [f"{max_expansion_ratio}:1"],
},
],
"related_topic_ids": [
+12 -3
View File
@@ -509,7 +509,7 @@ manifest = ModuleManifest(
summary="Back up database evidence, blob bytes, and the encryption key as one recovery unit, and keep unsupported SDK transports fail-closed.",
body=(
"Local durable storage is the operational baseline. Recover Files from a coordinated database/blob snapshot with the original master key, then run the bounded resumable integrity scan and verify representative access paths. Missing or mismatched blobs are quarantined; orphan objects are reported before dry-run-first, explicitly authorized cleanup. "
"Live S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Destructive module retirement drops database tables but does not remove backend blob objects."
"Arbitrary external S3 managed storage/connectors and SMB connectors fail closed until botocore redirects/endpoint discovery and SMB initial connections/DFS referrals support connection-time DNS/IP pinning. Installer-owned Garage storage is supported only at the exact deployment service endpoint with its explicit trust marker. Destructive module retirement drops database tables but does not remove backend blob objects."
),
layer="configured",
documentation_types=("admin",),
@@ -538,8 +538,13 @@ manifest = ModuleManifest(
"FILE_STORAGE_BACKEND",
"FILE_STORAGE_LOCAL_ROOT",
"FILE_STORAGE_LOCAL_FALLBACK_ROOTS",
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
"FILE_UPLOAD_MAX_BYTES",
"FILE_UPLOAD_ZIP_MAX_BYTES",
"FILE_ARCHIVE_MAX_ENTRIES",
"FILE_ARCHIVE_MAX_EXPANDED_BYTES",
"FILE_ARCHIVE_MAX_EXPANSION_RATIO",
"FILE_ARCHIVE_PREVIEW_TTL_SECONDS",
"MASTER_KEY_B64",
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
"GOVOPLAN_CONNECTOR_MAX_STRUCTURED_RESPONSE_BYTES",
@@ -634,6 +639,10 @@ manifest = ModuleManifest(
"FILE_STORAGE_LOCAL_ROOT",
"FILE_UPLOAD_MAX_BYTES",
"FILE_UPLOAD_ZIP_MAX_BYTES",
"FILE_ARCHIVE_MAX_ENTRIES",
"FILE_ARCHIVE_MAX_EXPANDED_BYTES",
"FILE_ARCHIVE_MAX_EXPANSION_RATIO",
"FILE_ARCHIVE_PREVIEW_TTL_SECONDS",
"MASTER_KEY_B64",
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS",
),
@@ -651,8 +660,8 @@ manifest = ModuleManifest(
"Compare the process requirements with the handbook's implemented/planned boundary and record every unsupported requirement or compensating control.",
"Confirm version alignment, apply migrations on clean and upgrade databases, and run the repository plus meta-repository security and static-analysis gates.",
"Exercise allowed and denied personal, group, and share access with representative accounts.",
"Exercise bounded upload and ZIP handling, every required conflict strategy, organization, download, and soft deletion.",
"Where connectors are configured, verify policy explanation and one pinned HTTP provider; verify live S3 and SMB access still fails closed.",
"Exercise bounded archive preview and confirmation, every required conflict strategy, organization, download, and soft deletion.",
"Where connectors are configured, verify policy explanation and one pinned HTTP provider; verify installer-owned Garage when selected, and verify arbitrary external S3 and SMB access still fails closed.",
"Restore a coordinated database/blob/key backup and compare representative downloaded bytes with their recorded SHA-256 checksums.",
"Record the tested versions, results, known limitations, evidence locations, residual risks, owner, and approval decision.",
],
+275 -2
View File
@@ -1,12 +1,22 @@
from __future__ import annotations
import hashlib
import hmac
import json
from datetime import UTC, datetime, timedelta
from typing import Literal
from fastapi import APIRouter, Depends, File as FastAPIFile, Form, UploadFile
from sqlalchemy.orm import Session
from govoplan_core.auth import ApiPrincipal, require_scope
from govoplan_core.security.secrets import (
TransientPayloadError,
open_transient_payload,
seal_transient_payload,
)
from govoplan_files.backend.schemas import (
ArchiveEntryResponse,
ArchivePreviewResponse,
ConflictResolutionRequest,
FileUploadResponse,
_conflict_resolutions,
@@ -14,8 +24,16 @@ from govoplan_files.backend.schemas import (
from govoplan_files.backend.db.models import FileAsset
from govoplan_core.db.session import get_session
from govoplan_files.backend.runtime import settings
from govoplan_files.backend.storage.paths import UnsafeFilePathError
from govoplan_files.backend.storage.archives import extract_zip_upload
from govoplan_files.backend.storage.paths import (
UnsafeFilePathError,
normalize_folder,
)
from govoplan_files.backend.storage.archives import (
archive_format_for_filename,
extract_archive_upload,
extract_zip_upload,
inspect_archive,
)
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.connector_policy import (
ConnectorPolicyDenied,
@@ -39,6 +57,261 @@ from govoplan_files.backend.route_support import (
)
router = APIRouter(prefix="/files", tags=["files"])
_ARCHIVE_PREVIEW_PURPOSE = "files.archive-preview.v1"
def _archive_suffix(filename: str) -> str:
lowered = filename.casefold()
for suffix in (".tar.bz2", ".tar.gz", ".tar.xz", ".tbz2", ".tgz", ".txz", ".tar", ".zip"):
if lowered.endswith(suffix):
return suffix
return ".archive"
def _archive_sha256(path: str) -> str:
digest = hashlib.sha256()
with open(path, "rb") as source:
while chunk := source.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def _archive_selected_paths(value: str) -> list[str]:
parsed = json.loads(value)
if not isinstance(parsed, list) or not parsed:
raise FileStorageError("Select at least one archive file or folder")
if len(parsed) > settings.file_archive_max_entries:
raise FileStorageError(
"Archive selection exceeds the configured entry limit"
)
selected: list[str] = []
for item in parsed:
if not isinstance(item, str) or not item.strip():
raise FileStorageError("Archive selection contains an invalid path")
if len(item) > 4096:
raise FileStorageError("Archive selection path is too long")
selected.append(item)
return selected
def _validate_archive_preview_token(
token: str,
*,
tenant_id: str,
user_id: str,
owner_type: str,
owner_id: str,
path: str,
campaign_id: str | None,
archive_format: str,
archive_sha256: str,
) -> None:
payload = open_transient_payload(
token,
ttl_seconds=settings.file_archive_preview_ttl_seconds,
)
expected = {
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
"tenant_id": tenant_id,
"user_id": user_id,
"owner_type": owner_type,
"owner_id": owner_id,
"path": path,
"campaign_id": campaign_id or "",
"archive_format": archive_format,
}
if any(payload.get(key) != value for key, value in expected.items()):
raise FileStorageError(
"Archive preview does not match this upload destination"
)
token_digest = payload.get("archive_sha256")
if not isinstance(token_digest, str) or not hmac.compare_digest(
token_digest, archive_sha256
):
raise FileStorageError(
"Archive contents changed after preview; preview it again"
)
@router.post("/archive-preview", response_model=ArchivePreviewResponse)
def preview_archive_upload(
file: UploadFile = FastAPIFile(...),
owner_type: Literal["user", "group"] = Form(default="user"),
owner_id: str | None = Form(default=None),
path: str = Form(default=""),
campaign_id: str | None = Form(default=None),
password: str | None = Form(default=None),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
target_owner = owner_id or principal.user.id
archive_path: str | None = None
filename = file.filename or "archive"
try:
archive_format = archive_format_for_filename(filename)
archive_path = _spool_limited_upload_to_temp(
file,
max_bytes=settings.file_upload_zip_max_bytes,
suffix=_archive_suffix(filename),
)
inspection = inspect_archive(
archive_path,
filename=filename,
password=password,
max_entries=settings.file_archive_max_entries,
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
)
digest = _archive_sha256(archive_path)
normalized_path = normalize_folder(path)
preview_token = seal_transient_payload(
{
"purpose": _ARCHIVE_PREVIEW_PURPOSE,
"tenant_id": principal.tenant_id,
"user_id": principal.user.id,
"owner_type": owner_type,
"owner_id": target_owner,
"path": normalized_path,
"campaign_id": campaign_id or "",
"archive_format": archive_format,
"archive_sha256": digest,
}
)
expires_at = datetime.now(UTC) + timedelta(
seconds=settings.file_archive_preview_ttl_seconds
)
return ArchivePreviewResponse(
preview_token=preview_token,
archive_format=inspection.archive_format,
entries=[
ArchiveEntryResponse(
path=entry.path,
kind=entry.kind,
size_bytes=entry.size_bytes,
compressed_size_bytes=entry.compressed_size_bytes,
encrypted=entry.encrypted,
)
for entry in inspection.entries
],
file_count=inspection.file_count,
directory_count=inspection.directory_count,
expanded_size_bytes=inspection.expanded_size_bytes,
compressed_size_bytes=inspection.compressed_size_bytes,
requires_password=inspection.requires_password,
password_verified=inspection.password_verified,
expires_at=expires_at.isoformat(),
)
except (FileStorageError, UnsafeFilePathError, ValueError) as exc:
raise _http_error(exc) from exc
finally:
if archive_path:
_cleanup_temp_file(archive_path)
@router.post("/archive-confirm", response_model=FileUploadResponse)
def confirm_archive_upload(
file: UploadFile = FastAPIFile(...),
preview_token: str = Form(...),
selected_paths_json: str = Form(...),
owner_type: Literal["user", "group"] = Form(default="user"),
owner_id: str | None = Form(default=None),
path: str = Form(default=""),
campaign_id: str | None = Form(default=None),
password: str | None = Form(default=None),
conflict_strategy: Literal["reject", "overwrite", "rename"] = Form(
default="reject"
),
conflict_resolutions_json: str | None = Form(default=None),
source_provenance_json: str | None = Form(default=None),
source_revision: str | None = Form(default=None),
connector_policy_json: str | None = Form(default=None),
session: Session = Depends(get_session),
principal: ApiPrincipal = Depends(require_scope("files:file:upload")),
):
target_owner = owner_id or principal.user.id
archive_path: str | None = None
filename = file.filename or "archive"
try:
raw_resolutions = (
json.loads(conflict_resolutions_json)
if conflict_resolutions_json
else []
)
upload_resolutions = _conflict_resolutions(
[ConflictResolutionRequest(**item) for item in raw_resolutions]
)
selected_paths = _archive_selected_paths(selected_paths_json)
_enforce_connector_policy(
source_provenance_json,
connector_policy_json,
operation="import",
)
metadata = _source_metadata_from_form(
source_provenance_json, source_revision
)
archive_format = archive_format_for_filename(filename)
normalized_path = normalize_folder(path)
archive_path = _spool_limited_upload_to_temp(
file,
max_bytes=settings.file_upload_zip_max_bytes,
suffix=_archive_suffix(filename),
)
digest = _archive_sha256(archive_path)
_validate_archive_preview_token(
preview_token,
tenant_id=principal.tenant_id,
user_id=principal.user.id,
owner_type=owner_type,
owner_id=target_owner,
path=normalized_path,
campaign_id=campaign_id,
archive_format=archive_format,
archive_sha256=digest,
)
extracted = extract_archive_upload(
session,
tenant_id=principal.tenant_id,
owner_type=owner_type,
owner_id=target_owner,
user_id=principal.user.id,
archive_data=archive_path,
filename=filename,
folder=normalized_path,
campaign_id=campaign_id,
selected_paths=selected_paths,
password=password,
conflict_strategy=conflict_strategy,
conflict_resolutions=upload_resolutions,
metadata=metadata,
is_admin=_is_admin(principal),
max_entries=settings.file_archive_max_entries,
max_file_bytes=settings.file_upload_max_bytes,
max_expanded_bytes=settings.file_archive_max_expanded_bytes,
max_expansion_ratio=settings.file_archive_max_expansion_ratio,
)
uploaded_assets = [item.asset for item in extracted]
_audit_connector_imports(session, principal, uploaded_assets)
session.commit()
return FileUploadResponse(
files=[
_asset_response(session, asset, include_shares=True)
for asset in uploaded_assets
]
)
except ConnectorPolicyDenied as exc:
session.rollback()
raise _connector_policy_error(exc) from exc
except (
FileStorageError,
TransientPayloadError,
UnsafeFilePathError,
ValueError,
json.JSONDecodeError,
) as exc:
session.rollback()
raise _http_error(exc) from exc
finally:
if archive_path:
_cleanup_temp_file(archive_path)
@router.post("/upload", response_model=FileUploadResponse)
+21
View File
@@ -249,6 +249,27 @@ class FileUploadResponse(BaseModel):
files: list[FileAssetResponse]
class ArchiveEntryResponse(BaseModel):
path: str
kind: Literal["file", "directory"]
size_bytes: int
compressed_size_bytes: int | None = None
encrypted: bool = False
class ArchivePreviewResponse(BaseModel):
preview_token: str
archive_format: str
entries: list[ArchiveEntryResponse] = Field(default_factory=list)
file_count: int
directory_count: int
expanded_size_bytes: int
compressed_size_bytes: int
requires_password: bool
password_verified: bool
expires_at: str
class FileConnectorPolicySource(BaseModel):
scope_type: Literal["system", "tenant", "user", "group", "campaign"] = "system"
scope_id: str | None = None
+602 -77
View File
@@ -1,55 +1,91 @@
from __future__ import annotations
import mimetypes
import re
import stat
import tarfile
import zipfile
from dataclasses import dataclass
from io import BytesIO
from os import PathLike
from pathlib import Path
from typing import Any, Iterable
from pathlib import Path, PurePosixPath
from typing import Any, BinaryIO, Iterable, Literal
import pyzipper
from sqlalchemy.orm import Session
from govoplan_files.backend.db.models import FileAsset
from govoplan_files.backend.storage.common import FileConflictResolution, FileStorageError, UploadedStoredFile
from govoplan_files.backend.storage.backends import StorageBackendError, get_storage_backend
from govoplan_files.backend.storage.files import create_file_asset, current_versions_and_blobs
from govoplan_files.backend.storage.paths import filename_from_path, normalize_folder, normalize_logical_path
from govoplan_files.backend.storage.backends import (
StorageBackendError,
get_storage_backend,
)
from govoplan_files.backend.storage.common import (
FileConflictResolution,
FileStorageError,
UploadedStoredFile,
)
from govoplan_files.backend.storage.files import (
create_file_asset,
current_versions_and_blobs,
)
from govoplan_files.backend.storage.paths import (
filename_from_path,
normalize_folder,
normalize_logical_path,
)
_ZIP_READ_CHUNK_SIZE = 1024 * 1024
ZIP_UPLOAD_MAX_FILES = 1000
_ARCHIVE_READ_CHUNK_SIZE = 1024 * 1024
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:")
ARCHIVE_UPLOAD_MAX_ENTRIES = 10_000
# Kept for callers and documentation using the former ZIP-specific name.
ZIP_UPLOAD_MAX_FILES = ARCHIVE_UPLOAD_MAX_ENTRIES
SUPPORTED_ARCHIVE_SUFFIXES = (
".tar.bz2",
".tar.gz",
".tar.xz",
".tbz2",
".tgz",
".txz",
".tar",
".zip",
)
def _read_zip_member(
archive: zipfile.ZipFile,
info: zipfile.ZipInfo,
*,
max_file_bytes: int,
max_total_bytes: int,
current_total: int,
) -> tuple[bytes, int]:
parts: list[bytes] = []
actual_size = 0
with archive.open(info) as source:
while True:
read_size = min(_ZIP_READ_CHUNK_SIZE, max_file_bytes + 1 - actual_size)
chunk = source.read(read_size)
if not chunk:
break
actual_size += len(chunk)
if actual_size > max_file_bytes:
raise FileStorageError(f"ZIP member {info.filename!r} exceeds per-file limit")
if current_total + actual_size > max_total_bytes:
raise FileStorageError("ZIP is too large after extraction")
parts.append(chunk)
return b"".join(parts), current_total + actual_size
class ArchivePasswordError(FileStorageError):
pass
def create_zip_file(session: Session, assets: Iterable[FileAsset], output_path: str | Path) -> None:
@dataclass(frozen=True, slots=True)
class ArchiveEntry:
path: str
kind: Literal["file", "directory"]
size_bytes: int
compressed_size_bytes: int | None = None
encrypted: bool = False
@dataclass(frozen=True, slots=True)
class ArchiveInspection:
archive_format: str
entries: tuple[ArchiveEntry, ...]
file_count: int
directory_count: int
expanded_size_bytes: int
compressed_size_bytes: int
requires_password: bool
password_verified: bool
def create_zip_file(
session: Session, assets: Iterable[FileAsset], output_path: str | Path
) -> None:
backend = get_storage_backend()
asset_list = list(assets)
version_blobs = current_versions_and_blobs(session, asset_list)
with zipfile.ZipFile(output_path, mode="w", compression=zipfile.ZIP_DEFLATED) as archive:
with zipfile.ZipFile(
output_path, mode="w", compression=zipfile.ZIP_DEFLATED
) as archive:
for asset in asset_list:
_version, blob = version_blobs[asset.id]
info = zipfile.ZipInfo(asset.display_path)
@@ -63,6 +99,145 @@ def create_zip_file(session: Session, assets: Iterable[FileAsset], output_path:
raise FileStorageError(str(exc)) from exc
def archive_format_for_filename(filename: str) -> str:
lowered = filename.strip().casefold()
if lowered.endswith(".zip"):
return "zip"
if lowered.endswith((".tar.gz", ".tgz")):
return "tar.gz"
if lowered.endswith((".tar.bz2", ".tbz2")):
return "tar.bz2"
if lowered.endswith((".tar.xz", ".txz")):
return "tar.xz"
if lowered.endswith(".tar"):
return "tar"
raise FileStorageError(
"Unsupported archive format. Use ZIP, TAR, TAR.GZ, TAR.BZ2, or TAR.XZ."
)
def is_supported_archive_filename(filename: str) -> bool:
try:
archive_format_for_filename(filename)
except FileStorageError:
return False
return True
def inspect_archive(
archive_data: bytes | str | PathLike[str],
*,
filename: str,
password: str | None = None,
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
max_expansion_ratio: int = 100,
) -> ArchiveInspection:
archive_format = archive_format_for_filename(filename)
compressed_size = _archive_size(archive_data)
if archive_format == "zip":
entries, requires_password, password_verified = _inspect_zip(
archive_data,
password=password,
)
else:
entries = _inspect_tar(archive_data)
requires_password = False
password_verified = True
_validate_archive_limits(
entries,
compressed_size=compressed_size,
max_entries=max_entries,
max_expanded_bytes=max_expanded_bytes,
max_expansion_ratio=max_expansion_ratio,
)
complete_entries = _with_derived_directories(entries)
return ArchiveInspection(
archive_format=archive_format,
entries=tuple(complete_entries),
file_count=sum(entry.kind == "file" for entry in complete_entries),
directory_count=sum(
entry.kind == "directory" for entry in complete_entries
),
expanded_size_bytes=sum(
entry.size_bytes for entry in entries if entry.kind == "file"
),
compressed_size_bytes=compressed_size,
requires_password=requires_password,
password_verified=password_verified,
)
def extract_archive_upload(
session: Session,
*,
tenant_id: str,
owner_type: str,
owner_id: str,
user_id: str,
archive_data: bytes | str | PathLike[str],
filename: str,
folder: str | None,
campaign_id: str | None,
selected_paths: Iterable[str] | None = None,
password: str | None = None,
conflict_strategy: str = "reject",
conflict_resolutions: Iterable[FileConflictResolution] | None = None,
metadata: dict[str, Any] | None = None,
is_admin: bool = False,
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
max_file_bytes: int = 50 * 1024 * 1024,
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
max_expansion_ratio: int = 100,
) -> list[UploadedStoredFile]:
inspection = inspect_archive(
archive_data,
filename=filename,
password=password,
max_entries=max_entries,
max_expanded_bytes=max_expanded_bytes,
max_expansion_ratio=max_expansion_ratio,
)
if inspection.requires_password and not inspection.password_verified:
raise ArchivePasswordError("Archive password is required")
selected_files = _selected_file_paths(inspection.entries, selected_paths)
if not selected_files:
raise FileStorageError("Select at least one archive file to import")
actual_total_limit = min(
max_expanded_bytes,
inspection.compressed_size_bytes * max_expansion_ratio,
)
if inspection.archive_format == "zip":
members = _read_selected_zip_members(
archive_data,
selected_files=selected_files,
password=password,
max_file_bytes=max_file_bytes,
max_total_bytes=actual_total_limit,
)
else:
members = _read_selected_tar_members(
archive_data,
selected_files=selected_files,
max_file_bytes=max_file_bytes,
max_total_bytes=actual_total_limit,
)
return _store_archive_members(
session,
members=members,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
folder=folder,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
)
def extract_zip_upload(
session: Session,
*,
@@ -81,51 +256,401 @@ def extract_zip_upload(
max_file_bytes: int = 50 * 1024 * 1024,
max_total_bytes: int = 250 * 1024 * 1024,
) -> list[UploadedStoredFile]:
uploaded: list[UploadedStoredFile] = []
total = 0
base_folder = normalize_folder(folder)
"""Backward-compatible wrapper for the original immediate ZIP endpoint."""
return extract_archive_upload(
session,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
archive_data=zip_data,
filename="archive.zip",
folder=folder,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
max_entries=max_files,
max_file_bytes=max_file_bytes,
max_expanded_bytes=max_total_bytes,
)
def _inspect_zip(
archive_data: bytes | str | PathLike[str],
*,
password: str | None,
) -> tuple[list[ArchiveEntry], bool, bool]:
try:
source = BytesIO(zip_data) if isinstance(zip_data, bytes) else zip_data
with zipfile.ZipFile(source) as archive:
infos = [info for info in archive.infolist() if not info.is_dir()]
if len(infos) > max_files:
raise FileStorageError(f"ZIP contains too many files (limit {max_files})")
for info in infos:
if info.flag_bits & 0x1:
raise FileStorageError("Encrypted ZIP uploads are not supported")
if info.file_size < 0:
raise FileStorageError("Invalid ZIP member")
if info.file_size > max_file_bytes:
raise FileStorageError(f"ZIP member {info.filename!r} exceeds per-file limit")
if total + info.file_size > max_total_bytes:
raise FileStorageError("ZIP is too large after extraction")
inner_path = normalize_logical_path(info.filename)
target_path = f"{base_folder}/{inner_path}" if base_folder else inner_path
data, total = _read_zip_member(
archive,
info,
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
current_total=total,
)
uploaded.append(
create_file_asset(
session,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
filename=filename_from_path(inner_path),
data=data,
display_path=target_path,
content_type=mimetypes.guess_type(inner_path)[0] or "application/octet-stream",
metadata=metadata,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
is_admin=is_admin,
with pyzipper.AESZipFile(_archive_source(archive_data)) as archive:
infos = archive.infolist()
entries = [_zip_entry(info) for info in infos]
encrypted_info = next(
(
info
for info in infos
if not info.is_dir() and bool(info.flag_bits & 0x1)
),
None,
)
if encrypted_info is None:
return entries, False, True
if not password:
return entries, True, False
try:
with archive.open(
encrypted_info, pwd=password.encode("utf-8")
) as member:
member.read(1)
except (RuntimeError, ValueError, zipfile.BadZipFile) as exc:
raise ArchivePasswordError("Archive password is incorrect") from exc
return entries, True, True
except ArchivePasswordError:
raise
except (OSError, ValueError, zipfile.BadZipFile) as exc:
raise FileStorageError("Invalid ZIP upload") from exc
def _inspect_tar(
archive_data: bytes | str | PathLike[str],
) -> list[ArchiveEntry]:
try:
with _open_tar(archive_data) as archive:
entries: list[ArchiveEntry] = []
for member in archive.getmembers():
path = _safe_member_path(member.name)
if member.isdir():
entries.append(
ArchiveEntry(path=path, kind="directory", size_bytes=0)
)
continue
if not member.isfile():
raise FileStorageError(
f"Archive member {member.name!r} is not a regular file or directory"
)
entries.append(
ArchiveEntry(
path=path,
kind="file",
size_bytes=max(0, int(member.size)),
)
)
except zipfile.BadZipFile as exc:
raise FileStorageError("Invalid ZIP upload") from exc
return entries
except FileStorageError:
raise
except (OSError, tarfile.TarError) as exc:
raise FileStorageError("Invalid TAR upload") from exc
def _zip_entry(info: zipfile.ZipInfo) -> ArchiveEntry:
path = _safe_member_path(info.filename)
unix_mode = (info.external_attr >> 16) & 0xFFFF
file_type = stat.S_IFMT(unix_mode)
if file_type and not (
stat.S_ISREG(unix_mode) or stat.S_ISDIR(unix_mode)
):
raise FileStorageError(
f"Archive member {info.filename!r} is not a regular file or directory"
)
if info.file_size < 0 or info.compress_size < 0:
raise FileStorageError(f"Archive member {info.filename!r} has invalid size metadata")
return ArchiveEntry(
path=path,
kind="directory" if info.is_dir() else "file",
size_bytes=0 if info.is_dir() else int(info.file_size),
compressed_size_bytes=(
None if info.is_dir() else int(info.compress_size)
),
encrypted=bool(info.flag_bits & 0x1),
)
def _validate_archive_limits(
entries: list[ArchiveEntry],
*,
compressed_size: int,
max_entries: int,
max_expanded_bytes: int,
max_expansion_ratio: int,
) -> None:
if len(entries) > max_entries:
raise FileStorageError(
f"Archive contains too many entries (limit {max_entries})"
)
seen: dict[str, str] = {}
expanded_size = 0
for entry in entries:
previous_kind = seen.get(entry.path)
if previous_kind is not None:
raise FileStorageError(
f"Archive contains duplicate path {entry.path!r}"
)
seen[entry.path] = entry.kind
if entry.kind == "file":
expanded_size += entry.size_bytes
if expanded_size > max_expanded_bytes:
raise FileStorageError(
"Archive is too large after extraction "
f"(limit {max_expanded_bytes} bytes)"
)
if expanded_size and (
compressed_size <= 0
or expanded_size > compressed_size * max_expansion_ratio
):
raise FileStorageError(
"Archive expansion ratio exceeds "
f"{max_expansion_ratio}:1"
)
def _with_derived_directories(
entries: list[ArchiveEntry],
) -> list[ArchiveEntry]:
by_path = {entry.path: entry for entry in entries}
for entry in entries:
parent = PurePosixPath(entry.path).parent
while str(parent) not in {"", "."}:
path = str(parent)
existing = by_path.get(path)
if existing and existing.kind != "directory":
raise FileStorageError(
f"Archive path {path!r} is both a file and a directory"
)
by_path.setdefault(
path,
ArchiveEntry(path=path, kind="directory", size_bytes=0),
)
parent = parent.parent
return sorted(
by_path.values(),
key=lambda entry: (
tuple(entry.path.casefold().split("/")),
entry.kind != "directory",
),
)
def _selected_file_paths(
entries: tuple[ArchiveEntry, ...],
selected_paths: Iterable[str] | None,
) -> set[str]:
file_paths = {entry.path for entry in entries if entry.kind == "file"}
if selected_paths is None:
return file_paths
known = {entry.path: entry for entry in entries}
normalized = {_safe_member_path(path) for path in selected_paths}
unknown = normalized - known.keys()
if unknown:
raise FileStorageError(
f"Archive selection contains unknown path {sorted(unknown)[0]!r}"
)
selected_files: set[str] = set()
for path in normalized:
entry = known[path]
if entry.kind == "file":
selected_files.add(path)
continue
prefix = f"{path}/"
selected_files.update(
file_path
for file_path in file_paths
if file_path.startswith(prefix)
)
return selected_files
def _read_selected_zip_members(
archive_data: bytes | str | PathLike[str],
*,
selected_files: set[str],
password: str | None,
max_file_bytes: int,
max_total_bytes: int,
) -> list[tuple[str, bytes]]:
result: list[tuple[str, bytes]] = []
total = 0
try:
with pyzipper.AESZipFile(_archive_source(archive_data)) as archive:
for info in archive.infolist():
if info.is_dir():
continue
path = _safe_member_path(info.filename)
if path not in selected_files:
continue
pwd = password.encode("utf-8") if password else None
try:
with archive.open(info, pwd=pwd) as source:
data, total = _read_member(
source,
path=path,
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
current_total=total,
)
except (RuntimeError, ValueError, zipfile.BadZipFile) as exc:
if info.flag_bits & 0x1:
raise ArchivePasswordError(
"Archive password is incorrect"
) from exc
raise
result.append((path, data))
except (FileStorageError, ArchivePasswordError):
raise
except (OSError, ValueError, zipfile.BadZipFile) as exc:
raise FileStorageError("ZIP extraction failed") from exc
return result
def _read_selected_tar_members(
archive_data: bytes | str | PathLike[str],
*,
selected_files: set[str],
max_file_bytes: int,
max_total_bytes: int,
) -> list[tuple[str, bytes]]:
result: list[tuple[str, bytes]] = []
total = 0
try:
with _open_tar(archive_data) as archive:
for member in archive.getmembers():
if not member.isfile():
continue
path = _safe_member_path(member.name)
if path not in selected_files:
continue
source = archive.extractfile(member)
if source is None:
raise FileStorageError(
f"Archive member {path!r} could not be read"
)
with source:
data, total = _read_member(
source,
path=path,
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
current_total=total,
)
result.append((path, data))
except FileStorageError:
raise
except (OSError, tarfile.TarError) as exc:
raise FileStorageError("TAR extraction failed") from exc
return result
def _read_member(
source: BinaryIO,
*,
path: str,
max_file_bytes: int,
max_total_bytes: int,
current_total: int,
) -> tuple[bytes, int]:
parts: list[bytes] = []
actual_size = 0
while True:
read_size = min(
_ARCHIVE_READ_CHUNK_SIZE,
max_file_bytes + 1 - actual_size,
)
chunk = source.read(read_size)
if not chunk:
break
actual_size += len(chunk)
if actual_size > max_file_bytes:
raise FileStorageError(
f"Archive member {path!r} exceeds per-file limit"
)
if current_total + actual_size > max_total_bytes:
raise FileStorageError("Archive is too large after extraction")
parts.append(chunk)
return b"".join(parts), current_total + actual_size
def _store_archive_members(
session: Session,
*,
members: Iterable[tuple[str, bytes]],
tenant_id: str,
owner_type: str,
owner_id: str,
user_id: str,
folder: str | None,
campaign_id: str | None,
conflict_strategy: str,
conflict_resolutions: Iterable[FileConflictResolution] | None,
metadata: dict[str, Any] | None,
is_admin: bool,
) -> list[UploadedStoredFile]:
uploaded: list[UploadedStoredFile] = []
base_folder = normalize_folder(folder)
for inner_path, data in members:
target_path = (
f"{base_folder}/{inner_path}" if base_folder else inner_path
)
uploaded.append(
create_file_asset(
session,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
filename=filename_from_path(inner_path),
data=data,
display_path=target_path,
content_type=mimetypes.guess_type(inner_path)[0]
or "application/octet-stream",
metadata=metadata,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
is_admin=is_admin,
)
)
return uploaded
def _safe_member_path(value: str) -> str:
raw = str(value or "").replace("\\", "/").strip()
if (
not raw
or "\x00" in raw
or raw.startswith("/")
or _WINDOWS_DRIVE_RE.match(raw)
):
raise FileStorageError(f"Unsafe archive member path {value!r}")
if any(part == ".." for part in raw.split("/")):
raise FileStorageError(f"Unsafe archive member path {value!r}")
try:
return normalize_logical_path(raw.rstrip("/"))
except ValueError as exc:
raise FileStorageError(f"Unsafe archive member path {value!r}") from exc
def _archive_size(archive_data: bytes | str | PathLike[str]) -> int:
if isinstance(archive_data, bytes):
return len(archive_data)
try:
return Path(archive_data).stat().st_size
except OSError as exc:
raise FileStorageError("Archive upload could not be read") from exc
def _archive_source(
archive_data: bytes | str | PathLike[str],
) -> BytesIO | str | PathLike[str]:
return BytesIO(archive_data) if isinstance(archive_data, bytes) else archive_data
def _open_tar(
archive_data: bytes | str | PathLike[str],
) -> tarfile.TarFile:
if isinstance(archive_data, bytes):
return tarfile.open(fileobj=BytesIO(archive_data), mode="r:*")
try:
return tarfile.open(name=archive_data, mode="r:*")
except OSError as exc:
raise FileStorageError("Archive upload could not be read") from exc
+37 -14
View File
@@ -145,28 +145,35 @@ class S3StorageBackend:
region_name: str
access_key_id: str
secret_access_key: str
deployment_managed: bool = False
name: str = "s3"
@property
def client(self):
try:
endpoint_url = validate_unpinned_sdk_http_url(
self.endpoint_url,
label="File storage S3 endpoint",
)
except OutboundHttpError as exc:
raise StorageBackendError(str(exc)) from exc
if self.deployment_managed:
endpoint_url = _deployment_managed_garage_endpoint(self.endpoint_url)
else:
try:
endpoint_url = validate_unpinned_sdk_http_url(
self.endpoint_url,
label="File storage S3 endpoint",
)
except OutboundHttpError as exc:
raise StorageBackendError(str(exc)) from exc
try:
import boto3
from botocore.config import Config
except ModuleNotFoundError as exc:
raise StorageBackendError("boto3 is required for the S3 storage backend") from exc
return boto3.client(
"s3",
endpoint_url=endpoint_url,
region_name=self.region_name,
aws_access_key_id=self.access_key_id,
aws_secret_access_key=self.secret_access_key,
)
options: dict[str, object] = {
"endpoint_url": endpoint_url,
"region_name": self.region_name,
"aws_access_key_id": self.access_key_id,
"aws_secret_access_key": self.secret_access_key,
}
if self.deployment_managed:
options["config"] = Config(s3={"addressing_style": "path"})
return boto3.client("s3", **options)
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
max_bytes = response_limit("file")
@@ -303,6 +310,15 @@ def _s3_missing_error(exc: Exception) -> bool:
return code in {"404", "NoSuchKey", "NotFound"} or status_code == 404
def _deployment_managed_garage_endpoint(value: str) -> str:
endpoint = str(value or "").strip()
if endpoint != "http://garage:3900":
raise StorageBackendError(
"Deployment-managed S3 trust is restricted to http://garage:3900"
)
return endpoint
def _fallback_roots() -> tuple[Path, ...]:
raw = getattr(settings, "file_storage_local_fallback_roots", "") or ""
return tuple(Path(item.strip()) for item in str(raw).split(",") if item.strip())
@@ -319,5 +335,12 @@ def get_storage_backend() -> StorageBackend:
region_name=settings.file_storage_s3_region or settings.s3_region,
access_key_id=settings.file_storage_s3_access_key_id or settings.s3_access_key_id,
secret_access_key=settings.file_storage_s3_secret_access_key or settings.s3_secret_access_key,
deployment_managed=bool(
getattr(
settings,
"file_storage_s3_deployment_managed",
False,
)
),
)
raise StorageBackendError(f"Unsupported file storage backend: {settings.file_storage_backend}")
@@ -14,7 +14,6 @@ from sqlalchemy.orm import Session
from govoplan_files.backend.db.models import FileAsset, FileBlob, FileShare, FileVersion
from govoplan_files.backend.storage.backends import StorageBackend, get_storage_backend
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.files import current_versions_and_blobs, get_asset_for_user, list_assets_for_user, share_files
from govoplan_files.backend.storage.access import ensure_owner_access
from govoplan_files.backend.storage.paths import normalize_folder, normalize_logical_path, safe_storage_component