fix(files): isolate archive workers and make handoff event driven

This commit is contained in:
2026-09-08 07:47:18 +02:00
parent ff84812f7f
commit 13433514b9
11 changed files with 945 additions and 86 deletions
+27 -63
View File
@@ -6,7 +6,6 @@ import stat
import tarfile
import zipfile
from dataclasses import dataclass
from contextlib import closing
from io import BytesIO
from os import PathLike
from pathlib import Path, PurePosixPath
@@ -26,7 +25,6 @@ from govoplan_files.backend.storage.common import (
UploadedStoredFile,
)
from govoplan_files.backend.storage.files import (
archive_storage_backend_scope,
create_file_asset,
current_versions_and_blobs,
)
@@ -142,6 +140,22 @@ def inspect_archive(
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
max_expansion_ratio: int = 100,
) -> ArchiveInspection:
from govoplan_files.backend.storage.archive_workers import inspect_archive_isolated
return inspect_archive_isolated(
archive_data, filename=filename, password=password, max_entries=max_entries,
max_expanded_bytes=max_expanded_bytes, max_expansion_ratio=max_expansion_ratio,
)
def _inspect_archive_content(
archive_data: bytes | str | PathLike[str],
*,
filename: str,
password: str | None = None,
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
max_expansion_ratio: int = 100,
) -> ArchiveInspection:
archive_format = archive_format_for_filename(filename)
compressed_size = _archive_size(archive_data)
@@ -206,68 +220,18 @@ def extract_archive_upload(
max_expansion_ratio: int = 100,
progress: ArchiveProgress | None = None,
) -> list[UploadedStoredFile]:
if progress:
progress("inspecting", 0, 0, 0, 0)
inspection = inspect_archive(
archive_data,
filename=filename,
password=password,
max_entries=max_entries,
max_expanded_bytes=max_expanded_bytes,
max_expansion_ratio=max_expansion_ratio,
from govoplan_files.backend.storage.archive_workers import extract_archive_isolated
return extract_archive_isolated(
session, archive_data=archive_data, filename=filename, password=password,
selected_paths=selected_paths, max_entries=max_entries, max_file_bytes=max_file_bytes,
max_expanded_bytes=max_expanded_bytes, max_expansion_ratio=max_expansion_ratio,
progress=progress, store_options={
"tenant_id": tenant_id, "owner_type": owner_type, "owner_id": owner_id,
"user_id": user_id, "folder": folder, "campaign_id": campaign_id,
"conflict_strategy": conflict_strategy, "conflict_resolutions": conflict_resolutions,
"metadata": metadata, "is_admin": is_admin, "encryption_vault_id": encryption_vault_id,
},
)
if inspection.requires_password and not inspection.password_verified:
raise ArchivePasswordError("Archive password is required")
selected_files = _selected_file_paths(inspection.entries, selected_paths)
if not selected_files:
raise FileStorageError("Select at least one archive file to import")
selected_total_bytes = sum(
entry.size_bytes for entry in inspection.entries if entry.path in selected_files
)
actual_total_limit = min(
max_expanded_bytes,
inspection.compressed_size_bytes * max_expansion_ratio,
)
if inspection.archive_format == "zip":
members = _read_selected_zip_members(
archive_data,
selected_files=selected_files,
password=password,
max_file_bytes=max_file_bytes,
max_total_bytes=actual_total_limit,
progress=progress,
total_bytes=selected_total_bytes,
)
else:
members = _read_selected_tar_members(
archive_data,
selected_files=selected_files,
max_file_bytes=max_file_bytes,
max_total_bytes=actual_total_limit,
progress=progress,
total_bytes=selected_total_bytes,
)
# Release Python/native archive handles immediately if storage or a
# callback fails while the member iterator is suspended at a yield.
with closing(members), archive_storage_backend_scope():
return _store_archive_members(
session,
members=members,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
folder=folder,
campaign_id=campaign_id,
conflict_strategy=conflict_strategy,
conflict_resolutions=conflict_resolutions,
metadata=metadata,
is_admin=is_admin,
encryption_vault_id=encryption_vault_id,
progress=progress,
total_files=len(selected_files),
total_bytes=selected_total_bytes,
)
def extract_zip_upload(