fix(files): isolate archive workers and make handoff event driven
This commit is contained in:
@@ -6,7 +6,6 @@ import stat
|
||||
import tarfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass
|
||||
from contextlib import closing
|
||||
from io import BytesIO
|
||||
from os import PathLike
|
||||
from pathlib import Path, PurePosixPath
|
||||
@@ -26,7 +25,6 @@ from govoplan_files.backend.storage.common import (
|
||||
UploadedStoredFile,
|
||||
)
|
||||
from govoplan_files.backend.storage.files import (
|
||||
archive_storage_backend_scope,
|
||||
create_file_asset,
|
||||
current_versions_and_blobs,
|
||||
)
|
||||
@@ -142,6 +140,22 @@ def inspect_archive(
|
||||
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
|
||||
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
|
||||
max_expansion_ratio: int = 100,
|
||||
) -> ArchiveInspection:
|
||||
from govoplan_files.backend.storage.archive_workers import inspect_archive_isolated
|
||||
return inspect_archive_isolated(
|
||||
archive_data, filename=filename, password=password, max_entries=max_entries,
|
||||
max_expanded_bytes=max_expanded_bytes, max_expansion_ratio=max_expansion_ratio,
|
||||
)
|
||||
|
||||
|
||||
def _inspect_archive_content(
|
||||
archive_data: bytes | str | PathLike[str],
|
||||
*,
|
||||
filename: str,
|
||||
password: str | None = None,
|
||||
max_entries: int = ARCHIVE_UPLOAD_MAX_ENTRIES,
|
||||
max_expanded_bytes: int = 2 * 1024 * 1024 * 1024,
|
||||
max_expansion_ratio: int = 100,
|
||||
) -> ArchiveInspection:
|
||||
archive_format = archive_format_for_filename(filename)
|
||||
compressed_size = _archive_size(archive_data)
|
||||
@@ -206,68 +220,18 @@ def extract_archive_upload(
|
||||
max_expansion_ratio: int = 100,
|
||||
progress: ArchiveProgress | None = None,
|
||||
) -> list[UploadedStoredFile]:
|
||||
if progress:
|
||||
progress("inspecting", 0, 0, 0, 0)
|
||||
inspection = inspect_archive(
|
||||
archive_data,
|
||||
filename=filename,
|
||||
password=password,
|
||||
max_entries=max_entries,
|
||||
max_expanded_bytes=max_expanded_bytes,
|
||||
max_expansion_ratio=max_expansion_ratio,
|
||||
from govoplan_files.backend.storage.archive_workers import extract_archive_isolated
|
||||
return extract_archive_isolated(
|
||||
session, archive_data=archive_data, filename=filename, password=password,
|
||||
selected_paths=selected_paths, max_entries=max_entries, max_file_bytes=max_file_bytes,
|
||||
max_expanded_bytes=max_expanded_bytes, max_expansion_ratio=max_expansion_ratio,
|
||||
progress=progress, store_options={
|
||||
"tenant_id": tenant_id, "owner_type": owner_type, "owner_id": owner_id,
|
||||
"user_id": user_id, "folder": folder, "campaign_id": campaign_id,
|
||||
"conflict_strategy": conflict_strategy, "conflict_resolutions": conflict_resolutions,
|
||||
"metadata": metadata, "is_admin": is_admin, "encryption_vault_id": encryption_vault_id,
|
||||
},
|
||||
)
|
||||
if inspection.requires_password and not inspection.password_verified:
|
||||
raise ArchivePasswordError("Archive password is required")
|
||||
selected_files = _selected_file_paths(inspection.entries, selected_paths)
|
||||
if not selected_files:
|
||||
raise FileStorageError("Select at least one archive file to import")
|
||||
selected_total_bytes = sum(
|
||||
entry.size_bytes for entry in inspection.entries if entry.path in selected_files
|
||||
)
|
||||
actual_total_limit = min(
|
||||
max_expanded_bytes,
|
||||
inspection.compressed_size_bytes * max_expansion_ratio,
|
||||
)
|
||||
if inspection.archive_format == "zip":
|
||||
members = _read_selected_zip_members(
|
||||
archive_data,
|
||||
selected_files=selected_files,
|
||||
password=password,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_total_bytes=actual_total_limit,
|
||||
progress=progress,
|
||||
total_bytes=selected_total_bytes,
|
||||
)
|
||||
else:
|
||||
members = _read_selected_tar_members(
|
||||
archive_data,
|
||||
selected_files=selected_files,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_total_bytes=actual_total_limit,
|
||||
progress=progress,
|
||||
total_bytes=selected_total_bytes,
|
||||
)
|
||||
# Release Python/native archive handles immediately if storage or a
|
||||
# callback fails while the member iterator is suspended at a yield.
|
||||
with closing(members), archive_storage_backend_scope():
|
||||
return _store_archive_members(
|
||||
session,
|
||||
members=members,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
user_id=user_id,
|
||||
folder=folder,
|
||||
campaign_id=campaign_id,
|
||||
conflict_strategy=conflict_strategy,
|
||||
conflict_resolutions=conflict_resolutions,
|
||||
metadata=metadata,
|
||||
is_admin=is_admin,
|
||||
encryption_vault_id=encryption_vault_id,
|
||||
progress=progress,
|
||||
total_files=len(selected_files),
|
||||
total_bytes=selected_total_bytes,
|
||||
)
|
||||
|
||||
|
||||
def extract_zip_upload(
|
||||
|
||||
Reference in New Issue
Block a user