GovOPlaN/govoplan-files
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

Add safe archive preview and extraction workflows

Browse Source
This commit is contained in:
zemion
2026-07-31 02:48:56 +02:00
parent 159a012833
commit b752dea610
20 changed files with 1961 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_archive_preview_token.py
+101
View File
@@ -0,0 +1,101 @@
from __future__ import annotations
import io
import hashlib
import unittest
import zipfile
from types import SimpleNamespace
from unittest.mock import patch
from fastapi import UploadFile
from govoplan_files.backend.routes.uploads import (
_validate_archive_preview_token,
preview_archive_upload,
)
from govoplan_files.backend.storage.common import FileStorageError
def _archive_bytes() -> bytes:
target = io.BytesIO()
with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive:
archive.writestr("folder/report.txt", b"report")
return target.getvalue()
class ArchivePreviewTokenTests(unittest.TestCase):
def setUp(self) -> None:
self.settings = SimpleNamespace(
file_upload_zip_max_bytes=10 * 1024 * 1024,
file_archive_max_entries=10_000,
file_archive_max_expanded_bytes=2 * 1024 * 1024 * 1024,
file_archive_max_expansion_ratio=100,
file_archive_preview_ttl_seconds=30 * 60,
)
self.principal = SimpleNamespace(
tenant_id="tenant-1",
user=SimpleNamespace(id="user-1"),
)
def test_preview_token_is_bound_to_actor_archive_and_destination(self) -> None:
archive_data = _archive_bytes()
upload = UploadFile(
file=io.BytesIO(archive_data),
filename="monthly.zip",
)
with patch(
"govoplan_files.backend.routes.uploads.settings",
self.settings,
):
preview = preview_archive_upload(
file=upload,
owner_type="user",
owner_id="user-1",
path="incoming",
campaign_id=None,
password=None,
principal=self.principal,
)
_validate_archive_preview_token(
preview.preview_token,
tenant_id="tenant-1",
user_id="user-1",
owner_type="user",
owner_id="user-1",
path="incoming",
campaign_id=None,
archive_format="zip",
archive_sha256=hashlib.sha256(archive_data).hexdigest(),
)
with self.assertRaisesRegex(
FileStorageError, "does not match this upload destination"
):
_validate_archive_preview_token(
preview.preview_token,
tenant_id="tenant-1",
user_id="user-1",
owner_type="user",
owner_id="user-1",
path="elsewhere",
campaign_id=None,
archive_format="zip",
archive_sha256=hashlib.sha256(archive_data).hexdigest(),
)
with self.assertRaisesRegex(
FileStorageError, "contents changed"
):
_validate_archive_preview_token(
preview.preview_token,
tenant_id="tenant-1",
user_id="user-1",
owner_type="user",
owner_id="user-1",
path="incoming",
campaign_id=None,
archive_format="zip",
archive_sha256="0" * 64,
)
if __name__ == "__main__":
unittest.main()
tests/test_archives.py
+170
View File
@@ -0,0 +1,170 @@
from __future__ import annotations
import io
import tarfile
import unittest
import zipfile
from types import SimpleNamespace
from unittest.mock import patch
import pyzipper
from govoplan_files.backend.storage.archives import (
ArchivePasswordError,
extract_archive_upload,
inspect_archive,
)
from govoplan_files.backend.storage.common import FileStorageError
def _zip_bytes(entries: dict[str, bytes]) -> bytes:
target = io.BytesIO()
with zipfile.ZipFile(target, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for path, data in entries.items():
archive.writestr(path, data)
return target.getvalue()
def _tar_bytes(entries: dict[str, bytes], mode: str = "w:gz") -> bytes:
target = io.BytesIO()
with tarfile.open(fileobj=target, mode=mode) as archive:
for path, data in entries.items():
info = tarfile.TarInfo(path)
info.size = len(data)
archive.addfile(info, io.BytesIO(data))
return target.getvalue()
def _encrypted_zip_bytes(password: str) -> bytes:
target = io.BytesIO()
with pyzipper.AESZipFile(
target,
"w",
compression=zipfile.ZIP_DEFLATED,
encryption=pyzipper.WZ_AES,
) as archive:
archive.setpassword(password.encode("utf-8"))
archive.writestr("secure/report.txt", b"classified")
return target.getvalue()
class ArchiveInspectionTests(unittest.TestCase):
def test_zip_preview_derives_folders_and_sizes(self) -> None:
inspection = inspect_archive(
_zip_bytes(
{
"department/one.txt": b"one",
"department/nested/two.csv": b"two",
}
),
filename="monthly.zip",
)
self.assertEqual("zip", inspection.archive_format)
self.assertEqual(2, inspection.file_count)
self.assertEqual(2, inspection.directory_count)
self.assertEqual(6, inspection.expanded_size_bytes)
self.assertEqual(
[
"department",
"department/nested",
"department/nested/two.csv",
"department/one.txt",
],
[entry.path for entry in inspection.entries],
)
def test_tar_compression_variants_are_inspected(self) -> None:
for filename, mode in (
("monthly.tar", "w"),
("monthly.tar.gz", "w:gz"),
("monthly.tar.bz2", "w:bz2"),
("monthly.tar.xz", "w:xz"),
):
with self.subTest(filename=filename):
inspection = inspect_archive(
_tar_bytes({"report.txt": b"report"}, mode=mode),
filename=filename,
)
self.assertEqual(1, inspection.file_count)
self.assertEqual(filename.removeprefix("monthly."), inspection.archive_format)
def test_unsafe_member_path_is_rejected(self) -> None:
with self.assertRaisesRegex(FileStorageError, "Unsafe archive member"):
inspect_archive(
_zip_bytes({"../escape.txt": b"no"}),
filename="unsafe.zip",
)
def test_archive_bomb_ratio_is_rejected(self) -> None:
with self.assertRaisesRegex(FileStorageError, "expansion ratio"):
inspect_archive(
_zip_bytes({"zeros.bin": b"\0" * 50_000}),
filename="bomb.zip",
max_expansion_ratio=2,
)
def test_encrypted_zip_reports_and_verifies_password(self) -> None:
archive = _encrypted_zip_bytes("correct horse")
missing = inspect_archive(archive, filename="secure.zip")
self.assertTrue(missing.requires_password)
self.assertFalse(missing.password_verified)
verified = inspect_archive(
archive,
filename="secure.zip",
password="correct horse",
)
self.assertTrue(verified.requires_password)
self.assertTrue(verified.password_verified)
with self.assertRaisesRegex(ArchivePasswordError, "incorrect"):
inspect_archive(
archive,
filename="secure.zip",
password="wrong",
)
def test_folder_selection_extracts_only_descendants(self) -> None:
archive = _zip_bytes(
{
"selected/one.txt": b"one",
"selected/two.txt": b"two",
"other/three.txt": b"three",
}
)
stored = SimpleNamespace(
asset=SimpleNamespace(id="asset"),
version=SimpleNamespace(id="version"),
blob=SimpleNamespace(id="blob"),
)
with patch(
"govoplan_files.backend.storage.archives.create_file_asset",
return_value=stored,
) as create_asset:
result = extract_archive_upload(
object(), # type: ignore[arg-type]
tenant_id="tenant-1",
owner_type="user",
owner_id="user-1",
user_id="user-1",
archive_data=archive,
filename="selection.zip",
folder="imports",
campaign_id=None,
selected_paths=["selected"],
)
self.assertEqual(2, len(result))
self.assertEqual(
{"imports/selected/one.txt", "imports/selected/two.txt"},
{
call.kwargs["display_path"]
for call in create_asset.call_args_list
},
)
if __name__ == "__main__":
unittest.main()
tests/test_documentation.py
+9 -1
View File
@@ -49,6 +49,10 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
settings = SimpleNamespace(
file_upload_max_bytes=7 * 1024 * 1024,
file_upload_zip_max_bytes=19 * 1024 * 1024,
file_archive_max_expanded_bytes=31 * 1024 * 1024,
file_archive_max_entries=321,
file_archive_max_expansion_ratio=42,
file_archive_preview_ttl_seconds=15 * 60,
)
with patch(
"govoplan_files.backend.documentation.visible_connector_profiles_for_actor",
@@ -73,8 +77,12 @@ class FilesRuntimeDocumentationTests(unittest.TestCase):
archive = topics["files.workflow.upload-and-unpack-zip"]
self.assertIn("19 MiB (19,922,944 bytes)", archive.body)
self.assertIn("31 MiB (32,505,856 bytes)", archive.body)
self.assertIn("7 MiB (7,340,032 bytes)", archive.body)
self.assertIn("1,000", archive.body)
self.assertIn("321", archive.body)
self.assertIn("42:1", archive.body)
self.assertIn("15 minutes", archive.body)
self.assertIn("passwords remain request-only", archive.body)
self.assertIn("Actual extracted bytes are counted", archive.body)
def test_connector_task_requires_authority_and_a_visible_usable_profile(
tests/test_router_contract.py
+10 -1
View File
@@ -49,11 +49,20 @@ class FilesRouterContractTests(unittest.TestCase):
actual = self._operation_keys(router)
self.assertEqual(expected, actual)
self.assertEqual(50, len(actual))
self.assertEqual(52, len(actual))
self.assertFalse(
[operation for operation, count in Counter(actual).items() if count > 1]
)
def test_archive_preview_and_confirmation_routes_are_exposed(self) -> None:
routes = {
(tuple(sorted(route.methods or ())), route.path)
for route in router.routes
}
self.assertIn((("POST",), "/files/archive-preview"), routes)
self.assertIn((("POST",), "/files/archive-confirm"), routes)
def test_connector_routes_keep_existing_api_paths(self) -> None:
routes = {(tuple(sorted(route.methods or ())), route.path) for route in router.routes}
tests/test_storage_backends.py
+51 -2
View File
@@ -1,19 +1,26 @@
from __future__ import annotations
import io
from types import ModuleType
import sys
import unittest
from unittest.mock import MagicMock, PropertyMock, patch
from govoplan_files.backend.storage.backends import S3StorageBackend, StorageBackendError
def _backend() -> S3StorageBackend:
def _backend(
*,
endpoint_url: str = "https://objects.example.test",
deployment_managed: bool = False,
) -> S3StorageBackend:
return S3StorageBackend(
bucket="files",
endpoint_url="https://objects.example.test",
endpoint_url=endpoint_url,
region_name="test",
access_key_id="access",
secret_access_key="secret",
deployment_managed=deployment_managed,
)
@@ -32,6 +39,48 @@ class S3StorageBackendTests(unittest.TestCase):
), self.assertRaisesRegex(StorageBackendError, "until that transport supports.*DNS/IP pinning"):
_backend().client
def test_installer_managed_garage_uses_exact_endpoint_and_path_style(self) -> None:
boto3 = ModuleType("boto3")
boto3.client = MagicMock(return_value=object())
botocore = ModuleType("botocore")
botocore.__path__ = []
botocore_config = ModuleType("botocore.config")
class Config:
def __init__(self, **values):
self.values = values
botocore_config.Config = Config
with patch.dict(
sys.modules,
{
"boto3": boto3,
"botocore": botocore,
"botocore.config": botocore_config,
},
):
_backend(
endpoint_url="http://garage:3900",
deployment_managed=True,
).client
_args, kwargs = boto3.client.call_args
self.assertEqual("http://garage:3900", kwargs["endpoint_url"])
self.assertEqual(
{"s3": {"addressing_style": "path"}},
kwargs["config"].values,
)
def test_installer_managed_garage_rejects_any_other_endpoint(self) -> None:
with self.assertRaisesRegex(
StorageBackendError,
"restricted to http://garage:3900",
):
_backend(
endpoint_url="http://other-s3:3900",
deployment_managed=True,
).client
def test_get_bytes_rejects_declared_oversize_object_without_reading(self) -> None:
body = MagicMock()
client = MagicMock()