GovOPlaN/govoplan-files
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Release govoplan-files v0.1.26: speed archive workflows and unify file tools
Module Package Release / publish-packages (push) Successful in 12s
Details

Browse Source
This commit is contained in:
zemion
2026-09-08 01:32:41 +02:00
parent 2baa8f2657
commit ff84812f7f
35 changed files with 4331 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/govoplan_files/backend/storage/archives.py
+208 -69
View File
@@ -6,10 +6,11 @@ import stat
import tarfile
import zipfile
from dataclasses import dataclass
from contextlib import closing
from io import BytesIO
from os import PathLike
from pathlib import Path, PurePosixPath
from typing import Any, BinaryIO, Iterable, Literal
from typing import Any, BinaryIO, Callable, Iterable, Iterator, Literal
import pyzipper
from sqlalchemy.orm import Session
@@ -25,6 +26,7 @@ from govoplan_files.backend.storage.common import (
UploadedStoredFile,
)
from govoplan_files.backend.storage.files import (
archive_storage_backend_scope,
create_file_asset,
current_versions_and_blobs,
)
@@ -33,11 +35,17 @@ from govoplan_files.backend.storage.paths import (
normalize_folder,
normalize_logical_path,
)
from govoplan_files.backend.storage.native_zip import (
native_zip_library,
read_native_zip_members,
)
_ARCHIVE_READ_CHUNK_SIZE = 1024 * 1024
_WINDOWS_DRIVE_RE = re.compile(r"^[A-Za-z]:")
ARCHIVE_UPLOAD_MAX_ENTRIES = 10_000
ARCHIVE_MAX_PATH_BYTES = 4096
ARCHIVE_MAX_PATH_DEPTH = 128
# Kept for callers and documentation using the former ZIP-specific name.
ZIP_UPLOAD_MAX_FILES = ARCHIVE_UPLOAD_MAX_ENTRIES
SUPPORTED_ARCHIVE_SUFFIXES = (
@@ -51,6 +59,8 @@ SUPPORTED_ARCHIVE_SUFFIXES = (
".zip",
)
ArchiveProgress = Callable[[str, int, int, int, int], None]
class ArchivePasswordError(FileStorageError):
pass
@@ -141,7 +151,13 @@ def inspect_archive(
password=password,
)
else:
entries = _inspect_tar(archive_data)
entries = _inspect_tar(
archive_data,
compressed_size=compressed_size,
max_entries=max_entries,
max_expanded_bytes=max_expanded_bytes,
max_expansion_ratio=max_expansion_ratio,
)
requires_password = False
password_verified = True
_validate_archive_limits(
@@ -151,14 +167,12 @@ def inspect_archive(
max_expanded_bytes=max_expanded_bytes,
max_expansion_ratio=max_expansion_ratio,
)
complete_entries = _with_derived_directories(entries)
complete_entries = _with_derived_directories(entries, max_entries=max_entries)
return ArchiveInspection(
archive_format=archive_format,
entries=tuple(complete_entries),
file_count=sum(entry.kind == "file" for entry in complete_entries),
directory_count=sum(
entry.kind == "directory" for entry in complete_entries
),
directory_count=sum(entry.kind == "directory" for entry in complete_entries),
expanded_size_bytes=sum(
entry.size_bytes for entry in entries if entry.kind == "file"
),
@@ -190,7 +204,10 @@ def extract_archive_upload(
max_file_bytes: int = 50 * 1024 * 1024,
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
max_expansion_ratio: int = 100,
progress: ArchiveProgress | None = None,
) -> list[UploadedStoredFile]:
if progress:
progress("inspecting", 0, 0, 0, 0)
inspection = inspect_archive(
archive_data,
filename=filename,
@@ -204,6 +221,9 @@ def extract_archive_upload(
selected_files = _selected_file_paths(inspection.entries, selected_paths)
if not selected_files:
raise FileStorageError("Select at least one archive file to import")
selected_total_bytes = sum(
entry.size_bytes for entry in inspection.entries if entry.path in selected_files
)
actual_total_limit = min(
max_expanded_bytes,
inspection.compressed_size_bytes * max_expansion_ratio,
@@ -215,6 +235,8 @@ def extract_archive_upload(
password=password,
max_file_bytes=max_file_bytes,
max_total_bytes=actual_total_limit,
progress=progress,
total_bytes=selected_total_bytes,
)
else:
members = _read_selected_tar_members(
@@ -222,22 +244,30 @@ def extract_archive_upload(
selected_files=selected_files,
max_file_bytes=max_file_bytes,
max_total_bytes=actual_total_limit,
progress=progress,
total_bytes=selected_total_bytes,
)
# Release Python/native archive handles immediately if storage or a
# callback fails while the member iterator is suspended at a yield.
with closing(members), archive_storage_backend_scope():
return _store_archive_members(
session,
members=members,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
folder=folder,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
progress=progress,
total_files=len(selected_files),
total_bytes=selected_total_bytes,
)
return _store_archive_members(
session,
members=members,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
folder=folder,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
)
def extract_zip_upload(
@@ -319,11 +349,23 @@ def _inspect_zip(
def _inspect_tar(
archive_data: bytes | str | PathLike[str],
*,
compressed_size: int,
max_entries: int,
max_expanded_bytes: int,
max_expansion_ratio: int,
) -> list[ArchiveEntry]:
try:
with _open_tar(archive_data) as archive:
entries: list[ArchiveEntry] = []
for member in archive.getmembers():
expanded_size = 0
for member in archive:
# Check each header before advancing across its payload. In a
# compressed TAR getmembers() would inflate everything first.
if len(entries) >= max_entries:
raise FileStorageError(
f"Archive contains too many entries (limit {max_entries})"
)
path = _safe_member_path(member.name)
if member.isdir():
entries.append(
@@ -334,6 +376,21 @@ def _inspect_tar(
raise FileStorageError(
f"Archive member {member.name!r} is not a regular file or directory"
)
if member.size < 0:
raise FileStorageError("Archive member has invalid size metadata")
expanded_size += member.size
if expanded_size > max_expanded_bytes:
raise FileStorageError(
"Archive is too large after extraction "
f"(limit {max_expanded_bytes} bytes)"
)
if expanded_size and (
compressed_size <= 0
or expanded_size > compressed_size * max_expansion_ratio
):
raise FileStorageError(
f"Archive expansion ratio exceeds {max_expansion_ratio}:1"
)
entries.append(
ArchiveEntry(
path=path,
@@ -352,21 +409,19 @@ def _zip_entry(info: zipfile.ZipInfo) -> ArchiveEntry:
path = _safe_member_path(info.filename)
unix_mode = (info.external_attr >> 16) & 0xFFFF
file_type = stat.S_IFMT(unix_mode)
if file_type and not (
stat.S_ISREG(unix_mode) or stat.S_ISDIR(unix_mode)
):
if file_type and not (stat.S_ISREG(unix_mode) or stat.S_ISDIR(unix_mode)):
raise FileStorageError(
f"Archive member {info.filename!r} is not a regular file or directory"
)
if info.file_size < 0 or info.compress_size < 0:
raise FileStorageError(f"Archive member {info.filename!r} has invalid size metadata")
raise FileStorageError(
f"Archive member {info.filename!r} has invalid size metadata"
)
return ArchiveEntry(
path=path,
kind="directory" if info.is_dir() else "file",
size_bytes=0 if info.is_dir() else int(info.file_size),
compressed_size_bytes=(
None if info.is_dir() else int(info.compress_size)
),
compressed_size_bytes=(None if info.is_dir() else int(info.compress_size)),
encrypted=bool(info.flag_bits & 0x1),
)
@@ -388,9 +443,7 @@ def _validate_archive_limits(
for entry in entries:
previous_kind = seen.get(entry.path)
if previous_kind is not None:
raise FileStorageError(
f"Archive contains duplicate path {entry.path!r}"
)
raise FileStorageError(f"Archive contains duplicate path {entry.path!r}")
seen[entry.path] = entry.kind
if entry.kind == "file":
expanded_size += entry.size_bytes
@@ -400,17 +453,17 @@ def _validate_archive_limits(
f"(limit {max_expanded_bytes} bytes)"
)
if expanded_size and (
compressed_size <= 0
or expanded_size > compressed_size * max_expansion_ratio
compressed_size <= 0 or expanded_size > compressed_size * max_expansion_ratio
):
raise FileStorageError(
"Archive expansion ratio exceeds "
f"{max_expansion_ratio}:1"
f"Archive expansion ratio exceeds {max_expansion_ratio}:1"
)
def _with_derived_directories(
entries: list[ArchiveEntry],
*,
max_entries: int,
) -> list[ArchiveEntry]:
by_path = {entry.path: entry for entry in entries}
for entry in entries:
@@ -422,10 +475,13 @@ def _with_derived_directories(
raise FileStorageError(
f"Archive path {path!r} is both a file and a directory"
)
by_path.setdefault(
path,
ArchiveEntry(path=path, kind="directory", size_bytes=0),
)
if path not in by_path:
if len(by_path) >= max_entries:
raise FileStorageError(
"Archive contains too many entries including parent "
f"directories (limit {max_entries})"
)
by_path[path] = ArchiveEntry(path=path, kind="directory", size_bytes=0)
parent = parent.parent
return sorted(
by_path.values(),
@@ -458,9 +514,7 @@ def _selected_file_paths(
continue
prefix = f"{path}/"
selected_files.update(
file_path
for file_path in file_paths
if file_path.startswith(prefix)
file_path for file_path in file_paths if file_path.startswith(prefix)
)
return selected_files
@@ -472,12 +526,45 @@ def _read_selected_zip_members(
password: str | None,
max_file_bytes: int,
max_total_bytes: int,
) -> list[tuple[str, bytes]]:
result: list[tuple[str, bytes]] = []
progress: ArchiveProgress | None = None,
total_bytes: int = 0,
) -> Iterator[tuple[str, bytes]]:
total = 0
completed = 0
try:
with pyzipper.AESZipFile(_archive_source(archive_data)) as archive:
for info in archive.infolist():
infos = archive.infolist()
# Accelerate only classic ZIPCrypto using established public C
# decoding APIs; AES and other formats retain their existing path.
native_compatible = (
bool(password)
and "\x00" not in password
and any(info.flag_bits & 1 for info in infos)
and all(
not getattr(info, "wz_aes_version", None)
and info.compress_type in {zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED}
and (info.filename.isascii() or bool(info.flag_bits & 0x800))
for info in infos
)
)
native = native_zip_library() if native_compatible else None
if native is not None:
yield from read_native_zip_members(
native,
archive_data,
infos_by_path={
_safe_member_path(info.filename): info for info in infos
},
selected_files=selected_files,
normalize_name=_safe_member_path,
password=password,
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
progress=progress,
total_bytes=total_bytes,
)
return
for info in infos:
if info.is_dir():
continue
path = _safe_member_path(info.filename)
@@ -492,6 +579,17 @@ def _read_selected_zip_members(
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
current_total=total,
on_bytes=(
lambda count: progress(
"extracting",
completed,
len(selected_files),
count,
total_bytes,
)
)
if progress
else None,
)
except (RuntimeError, ValueError, zipfile.BadZipFile) as exc:
if info.flag_bits & 0x1:
@@ -499,12 +597,17 @@ def _read_selected_zip_members(
"Archive password is incorrect"
) from exc
raise
result.append((path, data))
completed += 1
if progress:
progress(
"extracting", completed, len(selected_files), total, total_bytes
)
yield path, data
del data
except (FileStorageError, ArchivePasswordError):
raise
except (OSError, ValueError, zipfile.BadZipFile) as exc:
raise FileStorageError("ZIP extraction failed") from exc
return result
def _read_selected_tar_members(
@@ -513,12 +616,16 @@ def _read_selected_tar_members(
selected_files: set[str],
max_file_bytes: int,
max_total_bytes: int,
) -> list[tuple[str, bytes]]:
result: list[tuple[str, bytes]] = []
progress: ArchiveProgress | None = None,
total_bytes: int = 0,
) -> Iterator[tuple[str, bytes]]:
total = 0
completed = 0
try:
with _open_tar(archive_data) as archive:
for member in archive.getmembers():
# Iterate as headers are read instead of inflating the entire TAR
# with getmembers(), then seeking backwards to inflate it again.
for member in archive:
if not member.isfile():
continue
path = _safe_member_path(member.name)
@@ -526,9 +633,7 @@ def _read_selected_tar_members(
continue
source = archive.extractfile(member)
if source is None:
raise FileStorageError(
f"Archive member {path!r} could not be read"
)
raise FileStorageError(f"Archive member {path!r} could not be read")
with source:
data, total = _read_member(
source,
@@ -536,13 +641,29 @@ def _read_selected_tar_members(
max_file_bytes=max_file_bytes,
max_total_bytes=max_total_bytes,
current_total=total,
on_bytes=(
lambda count: progress(
"extracting",
completed,
len(selected_files),
count,
total_bytes,
)
)
if progress
else None,
)
result.append((path, data))
completed += 1
if progress:
progress(
"extracting", completed, len(selected_files), total, total_bytes
)
yield path, data
del data
except FileStorageError:
raise
except (OSError, tarfile.TarError) as exc:
raise FileStorageError("TAR extraction failed") from exc
return result
def _read_member(
@@ -552,6 +673,7 @@ def _read_member(
max_file_bytes: int,
max_total_bytes: int,
current_total: int,
on_bytes: Callable[[int], None] | None = None,
) -> tuple[bytes, int]:
parts: list[bytes] = []
actual_size = 0
@@ -565,12 +687,12 @@ def _read_member(
break
actual_size += len(chunk)
if actual_size > max_file_bytes:
raise FileStorageError(
f"Archive member {path!r} exceeds per-file limit"
)
raise FileStorageError(f"Archive member {path!r} exceeds per-file limit")
if current_total + actual_size > max_total_bytes:
raise FileStorageError("Archive is too large after extraction")
parts.append(chunk)
if on_bytes:
on_bytes(current_total + actual_size)
return b"".join(parts), current_total + actual_size
@@ -589,13 +711,15 @@ def _store_archive_members(
metadata: dict[str, Any] | None,
is_admin: bool,
encryption_vault_id: str | None,
progress: ArchiveProgress | None = None,
total_files: int = 0,
total_bytes: int = 0,
) -> list[UploadedStoredFile]:
uploaded: list[UploadedStoredFile] = []
base_folder = normalize_folder(folder)
stored_bytes = 0
for inner_path, data in members:
target_path = (
f"{base_folder}/{inner_path}" if base_folder else inner_path
)
target_path = f"{base_folder}/{inner_path}" if base_folder else inner_path
uploaded.append(
create_file_asset(
session,
@@ -616,24 +740,39 @@ def _store_archive_members(
encryption_vault_id=encryption_vault_id,
)
)
stored_bytes += len(data)
if progress:
progress("storing", len(uploaded), total_files, stored_bytes, total_bytes)
del data
return uploaded
def _safe_member_path(value: str) -> str:
raw = str(value or "").replace("\\", "/").strip()
if (
not raw
or "\x00" in raw
or raw.startswith("/")
or _WINDOWS_DRIVE_RE.match(raw)
):
try:
oversized = (
len(raw) > ARCHIVE_MAX_PATH_BYTES
or len(raw.encode("utf-8")) > ARCHIVE_MAX_PATH_BYTES
)
except UnicodeEncodeError as exc:
raise FileStorageError("Archive member path is not valid Unicode") from exc
if oversized:
raise FileStorageError(
f"Archive member path exceeds {ARCHIVE_MAX_PATH_BYTES} UTF-8 bytes"
)
if not raw or "\x00" in raw or raw.startswith("/") or _WINDOWS_DRIVE_RE.match(raw):
raise FileStorageError(f"Unsafe archive member path {value!r}")
if any(part == ".." for part in raw.split("/")):
raise FileStorageError(f"Unsafe archive member path {value!r}")
try:
return normalize_logical_path(raw.rstrip("/"))
normalized = normalize_logical_path(raw.rstrip("/"))
except ValueError as exc:
raise FileStorageError(f"Unsafe archive member path {value!r}") from exc
if normalized.count("/") + 1 > ARCHIVE_MAX_PATH_DEPTH:
raise FileStorageError(
f"Archive member path exceeds {ARCHIVE_MAX_PATH_DEPTH} components"
)
return normalized
def _archive_size(archive_data: bytes | str | PathLike[str]) -> int: