GovOPlaN/govoplan-files
2
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Release govoplan-files v0.1.26: speed archive workflows and unify file tools
Module Package Release / publish-packages (push) Successful in 12s
Details

Browse Source
This commit is contained in:
zemion
2026-09-08 01:32:41 +02:00
parent 2baa8f2657
commit ff84812f7f
35 changed files with 4331 additions and 275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/benchmark_archive_storage.py
+228
View File
@@ -0,0 +1,228 @@
"""Isolated synthetic archive benchmark; never reads the configured Files store.
Run from this repository: python tests/benchmark_archive_storage.py.
Uses the existing temporary SQLite/local-storage recovery fixture so durable
ledger, checksum verification, commit settlement and compensation stay enabled.
"""
from __future__ import annotations
import argparse
import cProfile
from contextlib import ExitStack
from collections import Counter
from io import BytesIO
import json
from pathlib import Path
import pstats
import random
import re
import shutil
import subprocess
import tempfile
import time
from unittest.mock import patch
import zipfile
import pyzipper
from sqlalchemy import event
from govoplan_files.backend.change_tracking import register_files_change_tracking
from govoplan_files.backend.storage.archives import (
extract_archive_upload,
_read_selected_zip_members,
)
from govoplan_files.backend.storage.native_zip import native_zip_library
from test_storage_recovery import StorageRecoveryTests
def storage_probe(
member_count: int,
member_size: int,
profile: bool,
baseline_revision: str | None = None,
):
case = StorageRecoveryTests()
case.setUp()
try:
register_files_change_tracking()
data = BytesIO()
rng = random.Random(20260907)
with zipfile.ZipFile(data, "w", compression=zipfile.ZIP_DEFLATED) as archive:
for index in range(member_count):
archive.writestr(
f"documents/{index:04d}.dat", rng.randbytes(member_size)
)
queries = []
event.listen(
case.engine,
"before_cursor_execute",
lambda conn, cursor, statement, parameters, context, executemany: (
queries.append(statement)
),
)
profiler = cProfile.Profile()
if profile:
profiler.enable()
with ExitStack() as overrides:
overrides.enter_context(
patch(
"govoplan_files.backend.storage.files.get_storage_backend",
return_value=case.backend,
)
)
if baseline_revision:
# Read an earlier local revision into this isolated process;
# never replace shared worktree files or connect to live data.
source = subprocess.run(
[
"git",
"show",
f"{baseline_revision}:src/govoplan_files/backend/storage/files.py",
],
check=True,
capture_output=True,
text=True,
).stdout
namespace = {"__name__": "govoplan_files.backend.storage.files"}
exec(
compile(source, f"{baseline_revision}:files.py", "exec"), namespace
)
namespace.update(
get_storage_backend=lambda: case.backend,
_storage_backend_name=lambda: case.backend.name,
_storage_bucket_name=lambda: "",
)
overrides.enter_context(
patch(
"govoplan_files.backend.storage.archives.create_file_asset",
namespace["create_file_asset"],
)
)
started = time.perf_counter()
result = extract_archive_upload(
case.session,
tenant_id="tenant-1",
owner_type="user",
owner_id="user-1",
user_id="user-1",
archive_data=data.getvalue(),
filename="synthetic.zip",
folder="imported",
campaign_id=None,
)
storage_done = time.perf_counter()
storage_queries = len(queries)
case.session.commit()
completed = time.perf_counter()
if profile:
profiler.disable()
print(
json.dumps(
{
"probe": "durable-storage",
"storage_revision": baseline_revision or "working-tree",
"members": len(result),
"member_bytes": member_size,
"extract_store_seconds": round(storage_done - started, 4),
"settlement_seconds": round(completed - storage_done, 4),
"total_seconds": round(completed - started, 4),
"store_queries": storage_queries,
"total_queries": len(queries),
"queries_by_table": dict(
Counter(
(
match.group(1)
if (
match := re.search(
r"(?:FROM|INTO|UPDATE)\s+([a-z_]+)", statement
)
)
else "other"
)
for statement in queries
)
),
}
)
)
if profile:
pstats.Stats(profiler).strip_dirs().sort_stats("cumulative").print_stats(24)
finally:
case.doCleanups()
def zipcrypto_probe(size: int):
binary = shutil.which("zip")
if not binary:
print(
json.dumps(
{"probe": "zipcrypto", "skipped": "fixture ZIP writer unavailable"}
)
)
return
# Public fixture password only: never pass a user credential to a process.
with tempfile.TemporaryDirectory(prefix="files-synthetic-zipcrypto-") as temporary:
path = Path(temporary)
content = random.Random(20260907).randbytes(size)
(path / "synthetic.bin").write_bytes(content)
subprocess.run(
[binary, "-q", "-P", "fixture-only", "synthetic.zip", "synthetic.bin"],
cwd=path,
check=True,
)
for implementation in (pyzipper.AESZipFile, zipfile.ZipFile):
started = time.perf_counter()
with implementation(path / "synthetic.zip") as archive:
result = archive.read("synthetic.bin", pwd=b"fixture-only")
assert result == content
print(
json.dumps(
{
"probe": "zipcrypto",
"reader": implementation.__module__,
"bytes": size,
"seconds": round(time.perf_counter() - started, 4),
}
)
)
if native_zip_library() is not None:
started = time.perf_counter()
result = list(
_read_selected_zip_members(
path / "synthetic.zip",
selected_files={"synthetic.bin"},
password="fixture-only",
max_file_bytes=size,
max_total_bytes=size,
)
)
assert result == [("synthetic.bin", content)]
print(
json.dumps(
{
"probe": "zipcrypto",
"reader": "libarchive-with-independent-crc",
"bytes": size,
"seconds": round(time.perf_counter() - started, 4),
}
)
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--members", type=int, default=100)
parser.add_argument("--member-size", type=int, default=4096)
parser.add_argument("--zipcrypto-bytes", type=int, default=4 * 1024 * 1024)
parser.add_argument("--profile", action="store_true")
parser.add_argument(
"--baseline-storage-revision",
help="Read a trusted earlier local Git revision for isolated storage comparison, without changing the worktree",
)
args = parser.parse_args()
storage_probe(
args.members, args.member_size, args.profile, args.baseline_storage_revision
)
if args.zipcrypto_bytes:
zipcrypto_probe(args.zipcrypto_bytes)
tests/test_archive_inspection_bounds.py
+77
View File
@@ -0,0 +1,77 @@
from __future__ import annotations
import tarfile
import unittest
from unittest.mock import patch
from govoplan_files.backend.storage.archives import _safe_member_path, inspect_archive
from govoplan_files.backend.storage.common import FileStorageError
from test_archives import _tar_bytes, _zip_bytes
class _HeadersOnly:
def __init__(self, count: int, size: int = 0):
self.count = count
self.size = size
self.seen = 0
def __enter__(self):
return self
def __exit__(self, *args):
return False
def getmembers(self):
raise AssertionError("TAR preview must not inflate every member before checking limits")
def __iter__(self):
for index in range(self.count):
member = tarfile.TarInfo(f"file-{index}.txt")
member.size = self.size
self.seen += 1
yield member
raise AssertionError("Inspection advanced beyond the rejecting header")
class ArchiveInspectionBoundTests(unittest.TestCase):
def test_tar_entry_limit_stops_at_first_excess_header(self):
headers = _HeadersOnly(4)
with patch("govoplan_files.backend.storage.archives._open_tar", return_value=headers):
with self.assertRaisesRegex(FileStorageError, "too many entries"):
inspect_archive(b"fixture", filename="fixture.tar.gz", max_entries=2)
self.assertEqual(3, headers.seen)
def test_tar_expanded_size_rejected_before_payload_decompression(self):
headers = _HeadersOnly(1, size=100)
with patch("govoplan_files.backend.storage.archives._open_tar", return_value=headers):
with self.assertRaisesRegex(FileStorageError, "too large after extraction"):
inspect_archive(b"fixture", filename="fixture.tar.gz", max_expanded_bytes=10)
self.assertEqual(1, headers.seen)
def test_tar_ratio_rejected_before_payload_decompression(self):
headers = _HeadersOnly(1, size=100)
with patch("govoplan_files.backend.storage.archives._open_tar", return_value=headers):
with self.assertRaisesRegex(FileStorageError, "expansion ratio"):
inspect_archive(b"fixture", filename="fixture.tar.gz", max_expansion_ratio=2)
def test_derived_directories_are_included_in_entry_limit(self):
for filename, payload in (("fixture.zip", _zip_bytes({"a/b/c/file.txt": b"x"})), ("fixture.tar.gz", _tar_bytes({"a/b/c/file.txt": b"x"}))):
with self.subTest(filename=filename), self.assertRaisesRegex(FileStorageError, "including parent directories"):
inspect_archive(payload, filename=filename, max_entries=3)
def test_extreme_path_depth_and_byte_lengths_fail_before_deriving_directories(self):
for path, error in (("a/" * 128 + "file.txt", "components"), ("ü" * 2049, "UTF-8 bytes")):
with self.subTest(path_length=len(path)), self.assertRaisesRegex(FileStorageError, error):
inspect_archive(_zip_bytes({path: b"x"}), filename="fixture.zip")
def test_directory_at_exact_depth_limit_accepts_trailing_separator(self):
path = "a/" * 128
self.assertEqual(path.rstrip("/"), _safe_member_path(path))
def test_invalid_unicode_member_name_has_controlled_validation_error(self):
with self.assertRaisesRegex(FileStorageError, "not valid Unicode"):
_safe_member_path("invalid-\udcff.txt")
if __name__ == "__main__":
unittest.main()
tests/test_archive_performance.py
+452
View File
@@ -0,0 +1,452 @@
from __future__ import annotations
from io import BytesIO
from pathlib import Path
import random
import shutil
import subprocess
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
import pyzipper
from sqlalchemy import event
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_files.backend.change_tracking import register_files_change_tracking
from govoplan_files.backend.storage.archives import (
_read_selected_zip_members,
_safe_member_path,
extract_archive_upload,
)
from govoplan_files.backend.storage.common import FileStorageError
from govoplan_files.backend.storage.files import (
archive_storage_backend_scope,
_archive_write_backend,
)
from govoplan_files.backend.storage.native_zip import (
native_zip_library,
read_native_zip_members,
)
from test_archives import _zip_bytes, _tar_bytes, _encrypted_zip_bytes
import test_storage_recovery as recovery_fixture
def legacy_archive(entries: dict[str, bytes]) -> bytes:
# Fixture-only known password, never an application secret.
with tempfile.TemporaryDirectory(prefix="files-legacy-fixture-") as temporary:
root = Path(temporary)
for name, content in entries.items():
path = root / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(content)
subprocess.run(
[shutil.which("zip"), "-q", "-P", "fixture-only", "fixture.zip", *entries],
cwd=root,
check=True,
)
return (root / "fixture.zip").read_bytes()
@unittest.skipUnless(
shutil.which("zip") and native_zip_library() is not None,
"optional libarchive and synthetic ZIP writer required",
)
class NativeArchivePerformanceTests(unittest.TestCase):
def setUp(self):
self.contents = {
"first.dat": random.Random(123).randbytes(8192),
"folder/second.txt": b"second",
}
self.archive = legacy_archive(self.contents)
def read(self, **kwargs):
return list(
_read_selected_zip_members(
self.archive,
selected_files={"first.dat"},
password="fixture-only",
max_file_bytes=100_000,
max_total_bytes=100_000,
**kwargs,
)
)
def test_native_selection_and_missing_library_python_fallback_match(self):
with patch(
"govoplan_files.backend.storage.archives._read_member",
side_effect=AssertionError("Python decoder must not run"),
):
native = self.read()
with patch(
"govoplan_files.backend.storage.archives.native_zip_library",
return_value=None,
):
fallback = self.read()
self.assertEqual(native, fallback)
self.assertEqual([("first.dat", self.contents["first.dat"])], native)
def test_utf8_flagged_german_names_use_native_reader(self):
# Info-ZIP stores UTF-8 names without the language flag on some hosts.
# Set the standard UTF-8 flag in local and central headers for this
# fixture; encryption header verification is unaffected by bit 11.
encoded_name = "Grüße.txt".encode("utf-8")
placeholder = "x" * len(encoded_name)
data = bytearray(
legacy_archive({placeholder: b"german"}).replace(
placeholder.encode(), encoded_name
)
)
for signature, offset in ((b"PK\x03\x04", 6), (b"PK\x01\x02", 8)):
position = data.index(signature) + offset
flags = int.from_bytes(data[position : position + 2], "little") | 0x800
data[position : position + 2] = flags.to_bytes(2, "little")
self.archive = bytes(data)
with patch(
"govoplan_files.backend.storage.archives._read_member",
side_effect=AssertionError("Python decoder must not run"),
):
result = list(
_read_selected_zip_members(
self.archive,
selected_files={"Grüße.txt"},
password="fixture-only",
max_file_bytes=100,
max_total_bytes=100,
)
)
self.assertEqual([("Grüße.txt", b"german")], result)
def test_wrong_password_and_corruption_never_fall_back(self):
with patch(
"govoplan_files.backend.storage.archives._read_member",
side_effect=AssertionError("No fallback after native error"),
):
with self.assertRaisesRegex(FileStorageError, "Native ZIP decoding failed"):
list(
_read_selected_zip_members(
self.archive,
selected_files={"first.dat"},
password="wrong",
max_file_bytes=100_000,
max_total_bytes=100_000,
)
)
corrupted = bytearray(self.archive)
with pyzipper.AESZipFile(BytesIO(self.archive)) as archive:
first = archive.infolist()[0]
payload_start = (
first.header_offset
+ 30
+ len(first.filename.encode())
+ len(first.extra)
)
corrupted[payload_start + first.compress_size - 1] ^= 0x40
self.archive = bytes(corrupted)
with self.assertRaises(FileStorageError):
self.read()
def test_independent_crc_check_rejects_mismatched_metadata(self):
with pyzipper.AESZipFile(BytesIO(self.archive)) as archive:
infos = {
_safe_member_path(info.filename): info for info in archive.infolist()
}
infos["first.dat"].CRC ^= 1
with self.assertRaisesRegex(FileStorageError, "CRC verification"):
list(
read_native_zip_members(
native_zip_library(),
self.archive,
infos_by_path=infos,
selected_files={"first.dat"},
normalize_name=_safe_member_path,
password="fixture-only",
max_file_bytes=100_000,
max_total_bytes=100_000,
)
)
def test_legacy_encoded_non_utf8_names_choose_python_before_decoding(self):
encoded_name = "ä.txt".encode("cp437")
placeholder = "x" * len(encoded_name)
self.archive = legacy_archive({placeholder: b"legacy"}).replace(
placeholder.encode(), encoded_name
)
with patch(
"govoplan_files.backend.storage.archives.native_zip_library",
side_effect=AssertionError("Legacy encoding must use original reader"),
):
result = list(
_read_selected_zip_members(
self.archive,
selected_files={"ä.txt"},
password="fixture-only",
max_file_bytes=100,
max_total_bytes=100,
)
)
self.assertEqual([("ä.txt", b"legacy")], result)
def test_suspended_native_reader_is_freed_after_destination_failure(self):
library = native_zip_library()
class LibraryProxy:
def __getattr__(self, name):
return getattr(library, name)
proxy = LibraryProxy()
with (
patch.object(
proxy, "archive_read_free", wraps=library.archive_read_free
) as close,
patch(
"govoplan_files.backend.storage.archives.native_zip_library",
return_value=proxy,
),
patch(
"govoplan_files.backend.storage.archives.create_file_asset",
side_effect=FileStorageError("Destination failure"),
),
):
with self.assertRaisesRegex(FileStorageError, "Destination failure"):
extract_archive_upload(
object(),
tenant_id="tenant",
owner_type="user",
owner_id="user",
user_id="user",
archive_data=self.archive,
filename="archive.zip",
folder="",
campaign_id=None,
password="fixture-only",
)
close.assert_called_once()
def test_native_read_enforces_actual_member_and_total_limits(self):
for member_limit, total_limit in ((10, 100_000), (100_000, 10)):
with self.subTest(member_limit=member_limit, total_limit=total_limit):
with self.assertRaises(FileStorageError):
list(
_read_selected_zip_members(
self.archive,
selected_files={"first.dat"},
password="fixture-only",
max_file_bytes=member_limit,
max_total_bytes=total_limit,
)
)
def test_native_emits_progress_inside_a_large_member(self):
self.archive = legacy_archive(
{"first.dat": random.Random(234).randbytes(3 * 1024 * 1024)}
)
progress = []
result = list(
_read_selected_zip_members(
self.archive,
selected_files={"first.dat"},
password="fixture-only",
max_file_bytes=4 * 1024 * 1024,
max_total_bytes=4 * 1024 * 1024,
total_bytes=3 * 1024 * 1024,
progress=lambda *args: progress.append(args),
)
)
self.assertEqual(3 * 1024 * 1024, len(result[0][1]))
self.assertTrue(
any(item[1] == 0 and 0 < item[3] < 3 * 1024 * 1024 for item in progress)
)
self.assertEqual(
("extracting", 1, 1, 3 * 1024 * 1024, 3 * 1024 * 1024), progress[-1]
)
class ArchiveStreamingTests(unittest.TestCase):
def test_plain_zip_and_tar_store_one_member_before_reading_the_next(self):
entries = {"one.txt": b"one", "two.txt": b"two"}
for filename, archive in (
("files.zip", _zip_bytes(entries)),
("files.tar.gz", _tar_bytes(entries)),
):
with self.subTest(filename=filename):
progress = []
with patch(
"govoplan_files.backend.storage.archives.create_file_asset",
return_value=SimpleNamespace(asset=object()),
) as create:
result = extract_archive_upload(
object(),
tenant_id="tenant",
owner_type="user",
owner_id="user",
user_id="user",
archive_data=archive,
filename=filename,
folder="",
campaign_id=None,
progress=lambda *args: progress.append(args),
)
self.assertEqual(2, len(result))
self.assertEqual(2, create.call_count)
stored_first = progress.index(("storing", 1, 2, 3, 6))
extracted_second = progress.index(("extracting", 2, 2, 6, 6))
self.assertLess(stored_first, extracted_second)
def test_aes_archive_keeps_existing_decoder(self):
with patch(
"govoplan_files.backend.storage.archives.native_zip_library",
side_effect=AssertionError("AES must not use native path"),
):
result = list(
_read_selected_zip_members(
_encrypted_zip_bytes("correct"),
selected_files={"secure/report.txt"},
password="correct",
max_file_bytes=100,
max_total_bytes=100,
)
)
self.assertEqual([("secure/report.txt", b"classified")], result)
def test_late_unsafe_header_prevents_every_write(self):
with patch(
"govoplan_files.backend.storage.archives.create_file_asset"
) as create:
with self.assertRaisesRegex(FileStorageError, "Unsafe archive member"):
extract_archive_upload(
object(),
tenant_id="tenant",
owner_type="user",
owner_id="user",
user_id="user",
archive_data=_zip_bytes({"good.txt": b"good", "../unsafe": b"bad"}),
filename="unsafe.zip",
folder="",
campaign_id=None,
)
create.assert_not_called()
class ArchivePersistencePerformanceTests(unittest.TestCase):
def test_backend_scope_is_lazy_nested_and_reset_after_exception(self):
first_backend = object()
second_backend = object()
with patch(
"govoplan_files.backend.storage.files.get_storage_backend",
side_effect=[first_backend, second_backend],
) as factory:
with self.assertRaisesRegex(RuntimeError, "fixture failure"):
with archive_storage_backend_scope():
factory.assert_not_called()
self.assertIs(first_backend, _archive_write_backend())
with archive_storage_backend_scope():
self.assertIs(first_backend, _archive_write_backend())
factory.assert_called_once()
raise RuntimeError("fixture failure")
with archive_storage_backend_scope():
self.assertIs(second_backend, _archive_write_backend())
self.assertEqual(2, factory.call_count)
def test_backend_reuse_does_not_cache_tenant_identity_or_blob_dedup(self):
fixture = recovery_fixture.StorageRecoveryTests()
fixture.setUp()
self.addCleanup(fixture.doCleanups)
results = []
with patch(
"govoplan_files.backend.storage.files.get_storage_backend",
return_value=fixture.backend,
) as factory:
for tenant in ("tenant-one", "tenant-two"):
results.append(
extract_archive_upload(
fixture.session,
tenant_id=tenant,
owner_type="user",
owner_id="user-1",
user_id="user-1",
archive_data=_zip_bytes({"one.txt": b"one", "two.txt": b"two"}),
filename="files.zip",
folder="",
campaign_id=None,
)
)
fixture.session.commit()
self.assertEqual(2, factory.call_count)
self.assertEqual(
["tenant-one", "tenant-two"], [items[0].blob.tenant_id for items in results]
)
self.assertNotEqual(results[0][0].blob.id, results[1][0].blob.id)
def test_asset_and_version_are_published_once_without_redundant_update(self):
fixture = recovery_fixture.StorageRecoveryTests()
fixture.setUp()
self.addCleanup(fixture.doCleanups)
register_files_change_tracking()
queries = []
event.listen(
fixture.engine,
"before_cursor_execute",
lambda conn, cursor, statement, parameters, context, executemany: (
queries.append(statement)
),
)
with patch(
"govoplan_files.backend.storage.files.get_storage_backend",
return_value=fixture.backend,
):
stored = extract_archive_upload(
fixture.session,
tenant_id="tenant-1",
owner_type="user",
owner_id="user-1",
user_id="user-1",
archive_data=_zip_bytes({"one.txt": b"one", "two.txt": b"two"}),
filename="files.zip",
folder="",
campaign_id=None,
)
fixture.session.commit()
self.assertFalse(
any(statement.startswith("UPDATE file_assets") for statement in queries)
)
events = (
fixture.session.query(ChangeSequenceEntry)
.filter(ChangeSequenceEntry.module_id == "files")
.all()
)
self.assertEqual(["created", "created"], [entry.operation for entry in events])
self.assertTrue(
all(item.asset.current_version_id == item.version.id for item in stored)
)
def test_streamed_late_content_limit_failure_rolls_back_previous_blobs(self):
fixture = recovery_fixture.StorageRecoveryTests()
fixture.setUp()
self.addCleanup(fixture.doCleanups)
with patch(
"govoplan_files.backend.storage.files.get_storage_backend",
return_value=fixture.backend,
):
with self.assertRaisesRegex(FileStorageError, "per-file limit"):
extract_archive_upload(
fixture.session,
tenant_id="tenant-1",
owner_type="user",
owner_id="user-1",
user_id="user-1",
archive_data=_zip_bytes(
{"one.txt": b"one", "too-large.txt": b"x" * 200}
),
filename="files.zip",
folder="",
campaign_id=None,
max_file_bytes=100,
)
fixture.session.rollback()
self.assertEqual([], list(fixture.backend.root.rglob("*.blob")))
if __name__ == "__main__":
unittest.main()
tests/test_archive_staging.py
+290
View File
@@ -0,0 +1,290 @@
from __future__ import annotations
import json
import os
from datetime import datetime
from pathlib import Path
import tempfile
import time
import unittest
from unittest.mock import patch
from uuid import uuid4
from fastapi import FastAPI
from fastapi.testclient import TestClient
from govoplan_core.auth import get_api_principal
from govoplan_core.db.session import get_session
from govoplan_core.security.secrets import open_transient_payload, seal_transient_payload
from govoplan_files.backend.archive_work import ArchiveProgress, use_staged_upload
from govoplan_files.backend import archive_work
from govoplan_files.backend.db.models import FileAsset
from govoplan_files.backend.routes import uploads
from govoplan_files.backend.storage.common import FileStorageError
import test_managed_archives as managed_archive_tests
class ArchiveStagingTests(unittest.TestCase):
store_blob = managed_archive_tests.ManagedArchiveTests.store_blob
seed_source = managed_archive_tests.ManagedArchiveTests.seed_source
assert_no_extraction = managed_archive_tests.ManagedArchiveTests.assert_no_extraction
def setUp(self):
managed_archive_tests.ManagedArchiveTests.setUp(self)
self.directory = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="archive-staging-tests-")))
self.settings.file_archive_work_root = str(self.directory / "work")
self.settings.file_archive_staged_per_actor = 4
self.settings.file_archive_staged_max_bytes = 20 * 1024 * 1024
app = FastAPI()
app.include_router(uploads.router, prefix="/api/v1")
app.dependency_overrides[get_api_principal] = lambda: self.principal
app.dependency_overrides[get_session] = lambda: self.session
self.client = self.enterContext(TestClient(app, raise_server_exceptions=False))
def preview(self, *, archive=None, **changes):
return self.client.post(
"/api/v1/files/archive-preview",
data={"path": "extracted", "retain_upload": "true", **changes},
files={"file": ("private-source.zip", archive or managed_archive_tests.archive_bytes(), "application/zip")},
)
def staged_preview(self, preview, **changes):
return self.client.post(
"/api/v1/files/archive-preview",
data={"path": "extracted", "preview_token": preview["preview_token"], "staged_upload_id": preview["staged_upload_id"], **changes},
)
def confirm(self, preview, **changes):
return self.client.post(
"/api/v1/files/archive-confirm",
data={"path": "extracted", "preview_token": preview["preview_token"], "staged_upload_id": preview["staged_upload_id"], "selected_paths_json": json.dumps(["folder/one.txt"]), **changes},
)
def staged_path(self, preview):
with use_staged_upload(self.settings, preview["staged_upload_id"], tenant_id="tenant-1", user_id="user-1") as filename:
return Path(filename)
def progress(self, operation_id):
return self.client.get(f"/api/v1/files/archive-progress/{operation_id}")
def test_preview_stages_once_password_repreview_and_confirm_reuse_upload(self):
password = "staged secret password"
preview = self.preview(archive=managed_archive_tests.archive_bytes(password=password))
self.assertEqual(preview.status_code, 200, preview.text)
self.assertTrue(preview.json()["requires_password"])
self.assertFalse(preview.json()["password_verified"])
stage = self.staged_path(preview.json())
self.assert_no_extraction()
operation_id = str(uuid4())
with patch.object(uploads, "_spool_limited_upload_to_temp") as spool:
wrong = self.staged_preview(preview.json(), password="incorrect")
self.assertEqual(wrong.status_code, 400, wrong.text)
self.assertTrue(stage.exists())
verified = self.staged_preview(preview.json(), password=password)
self.assertEqual(verified.status_code, 200, verified.text)
self.assertTrue(verified.json()["password_verified"])
self.assertEqual(verified.json()["staged_upload_id"], preview.json()["staged_upload_id"])
confirmed = self.confirm(verified.json(), password=password, operation_id=operation_id)
spool.assert_not_called()
self.assertEqual(confirmed.status_code, 200, confirmed.text)
self.assertEqual([item["display_path"] for item in confirmed.json()["files"]], ["extracted/folder/one.txt"])
self.assertEqual(self.session.query(FileAsset).count(), 2)
self.assertFalse(stage.exists())
progress = self.progress(operation_id)
self.assertEqual(progress.status_code, 200, progress.text)
self.assertEqual(progress.json()["status"], "complete")
for value in (verified.text, confirmed.text, progress.text, str(self.audit.call_args_list)):
self.assertNotIn(password, value)
self.assertNotIn("private-source.zip", progress.text)
self.assertNotIn("folder/one.txt", progress.text)
payload = open_transient_payload(verified.json()["preview_token"], ttl_seconds=1800)
self.assertNotIn(password, json.dumps(payload))
def test_staging_rejects_token_actor_tenant_stage_destination_and_digest_mismatch_without_writes(self):
response = self.preview()
self.assertEqual(response.status_code, 200, response.text)
preview = response.json()
for change in ({"preview_token": "tampered"}, {"staged_upload_id": str(uuid4())}, {"owner_id": "other-user"}, {"path": "elsewhere"}):
with self.subTest(change=change):
rejected = self.confirm(preview, **change)
self.assertEqual(rejected.status_code, 400, rejected.text)
self.assert_no_extraction()
for field, original, foreign in (("tenant_id", "tenant-1", "other-tenant"), ("user_id", "user-1", "other-user")):
target = self.principal if field == "tenant_id" else self.principal.user
attribute = "tenant_id" if field == "tenant_id" else "id"
setattr(target, attribute, foreign)
self.assertEqual(self.confirm(preview).status_code, 400)
setattr(target, attribute, original)
path = self.staged_path(preview)
path.write_bytes(b"tampered archive bytes")
for response in (self.staged_preview(preview), self.confirm(preview)):
self.assertEqual(response.status_code, 400, response.text)
self.assertIn("contents changed", response.text)
self.assert_no_extraction()
self.assertTrue(path.exists())
self.assertEqual(list(path.parent.glob("*.progress")), [])
def test_expired_stage_and_explicit_discard_are_bound_to_current_actor(self):
preview = self.preview().json()
path = self.staged_path(preview)
self.principal.user.id = "other-user"
self.assertEqual(self.client.delete(f"/api/v1/files/archive-staging/{preview['staged_upload_id']}").status_code, 204)
self.assertTrue(path.exists())
self.principal.user.id = "user-1"
self.assertEqual(self.client.delete(f"/api/v1/files/archive-staging/{preview['staged_upload_id']}").status_code, 204)
self.assertFalse(path.exists())
self.assertEqual(self.confirm(preview).status_code, 410)
preview = self.preview().json()
path = self.staged_path(preview)
old = time.time() - self.settings.file_archive_preview_ttl_seconds - 1
os.utime(path, (old, old))
self.assertEqual(self.confirm(preview).status_code, 410)
self.assert_no_extraction()
def test_progress_stays_running_through_commit_and_completes_only_after_commit(self):
preview = self.preview().json()
operation_id = str(uuid4())
original_commit = self.session.commit
observed = []
def commit():
before = self.progress(operation_id).json()
self.assertEqual(before["status"], "running")
self.assertEqual(before["phase"], "finalizing")
original_commit()
observed.append(self.progress(operation_id).json()["status"])
with patch.object(self.session, "commit", side_effect=commit):
response = self.confirm(preview, operation_id=operation_id)
self.assertEqual(response.status_code, 200, response.text)
self.assertEqual(observed, ["running"])
progress = self.progress(operation_id).json()
self.assertEqual(progress["status"], "complete")
self.assertEqual(progress["completed_files"], 1)
self.assertEqual(progress["completed_bytes"], 3)
def test_commit_error_marks_progress_failed_and_preserves_stage_for_retry(self):
preview = self.preview().json()
stage = self.staged_path(preview)
operation_id = str(uuid4())
with patch.object(self.session, "commit", side_effect=FileStorageError("Commit refused")):
response = self.confirm(preview, operation_id=operation_id)
self.assertEqual(response.status_code, 400, response.text)
self.assertEqual(self.progress(operation_id).json()["status"], "failed")
self.assertEqual(self.session.query(FileAsset).count(), 1)
self.assertTrue(stage.exists())
retry = self.confirm(preview, operation_id=str(uuid4()))
self.assertEqual(retry.status_code, 200, retry.text)
def test_post_commit_stage_cleanup_failure_does_not_report_a_failed_import(self):
preview = self.preview().json()
operation_id = str(uuid4())
with patch.object(uploads, "discard_staged_upload", side_effect=OSError("Temporary cleanup failure")):
response = self.confirm(preview, operation_id=operation_id)
self.assertEqual(response.status_code, 200, response.text)
self.assertEqual(self.session.query(FileAsset).count(), 2)
self.assertEqual(self.progress(operation_id).json()["status"], "complete")
def test_duplicate_operation_cannot_start_another_write_or_overwrite_progress(self):
preview = self.preview().json()
operation_id = str(uuid4())
progress = ArchiveProgress(self.settings, operation_id, tenant_id="tenant-1", user_id="user-1")
progress("extracting", 1, 5, 10, 50)
before = self.progress(operation_id).json()
response = self.confirm(preview, operation_id=operation_id)
self.assertEqual(response.status_code, 400, response.text)
self.assertIn("already been used", response.text)
self.assertEqual(self.progress(operation_id).json(), before)
self.assert_no_extraction()
def test_progress_and_discard_require_upload_permission_and_do_not_disclose_foreign_receipts(self):
preview = self.preview().json()
operation_id = str(uuid4())
ArchiveProgress(self.settings, operation_id, tenant_id="tenant-1", user_id="user-1")
self.principal.user.id = "other-user"
self.assertEqual(self.progress(operation_id).status_code, 404)
self.principal.user.id = "user-1"
self.scopes.remove("files:file:upload")
self.assertEqual(self.progress(operation_id).status_code, 403)
self.assertEqual(self.client.delete(f"/api/v1/files/archive-staging/{preview['staged_upload_id']}").status_code, 403)
self.assertTrue(self.staged_path(preview).exists())
def test_malformed_signed_digest_cannot_bypass_stage_binding(self):
preview = self.preview().json()
payload = open_transient_payload(preview["preview_token"], ttl_seconds=1800)
payload["archive_sha256"] = None
invalid = self.confirm(preview, preview_token=seal_transient_payload(payload))
self.assertEqual(invalid.status_code, 400, invalid.text)
self.assert_no_extraction()
def test_progress_persistence_failure_does_not_turn_a_committed_import_into_an_error(self):
preview = self.preview().json()
operation_id = str(uuid4())
with patch.object(archive_work.os, "replace", side_effect=OSError("Progress write unavailable")):
result = self.confirm(preview, operation_id=operation_id)
self.assertEqual(result.status_code, 200, result.text)
self.assertEqual(self.session.query(FileAsset).count(), 2)
# A failed observation write may leave an older running receipt, but
# the confirmation response remains the authoritative outcome.
self.assertEqual(self.progress(operation_id).json()["status"], "running")
def test_lease_cleanup_failure_cannot_mask_a_committed_import(self):
preview = self.preview().json()
operation_id = str(uuid4())
original_unlink = Path.unlink
def unlink(path, *args, **kwargs):
if path.suffix == ".lease":
raise OSError("Lease cleanup temporarily unavailable")
return original_unlink(path, *args, **kwargs)
with patch.object(Path, "unlink", new=unlink):
result = self.confirm(preview, operation_id=operation_id)
self.assertEqual(result.status_code, 200, result.text)
self.assertEqual(self.session.query(FileAsset).count(), 2)
self.assertEqual(self.progress(operation_id).json()["status"], "complete")
def test_invalid_operation_identifier_and_ambiguous_source_are_rejected_before_writes(self):
preview = self.preview().json()
response = self.confirm(preview, operation_id="../../outside")
self.assertEqual(response.status_code, 400, response.text)
ambiguous = self.client.post(
"/api/v1/files/archive-confirm",
data={"path": "extracted", "preview_token": preview["preview_token"], "staged_upload_id": preview["staged_upload_id"], "selected_paths_json": '["folder"]'},
files={"file": ("other.zip", managed_archive_tests.archive_bytes(), "application/zip")},
)
self.assertEqual(ambiguous.status_code, 400, ambiguous.text)
self.assertIn("either", ambiguous.text)
self.assert_no_extraction()
self.assertTrue(self.staged_path(preview).exists())
def test_staged_preview_requires_a_token_without_exposing_a_server_error(self):
preview = self.preview().json()
missing = self.client.post(
"/api/v1/files/archive-preview",
data={"path": "extracted", "staged_upload_id": preview["staged_upload_id"]},
)
self.assertIn(missing.status_code, (400, 422), missing.text)
self.assertTrue(self.staged_path(preview).exists())
self.assert_no_extraction()
def test_repreview_does_not_extend_the_original_stage_expiry(self):
preview = self.preview().json()
path = self.staged_path(preview)
original_mtime = time.time() - 120
os.utime(path, (original_mtime, original_mtime))
original_deadline = path.stat().st_mtime + self.settings.file_archive_preview_ttl_seconds
for _ in range(2):
response = self.staged_preview(preview)
self.assertEqual(response.status_code, 200, response.text)
preview = response.json()
returned_deadline = datetime.fromisoformat(preview["expires_at"]).timestamp()
self.assertLessEqual(returned_deadline, original_deadline + 0.000001)
self.assertAlmostEqual(returned_deadline, original_deadline, places=5)
self.assertEqual(path.stat().st_mtime, original_mtime)
self.assert_no_extraction()
if __name__ == "__main__":
unittest.main()
tests/test_archive_work.py
+191
View File
@@ -0,0 +1,191 @@
from __future__ import annotations
import json
import os
from pathlib import Path
import tempfile
import time
from types import SimpleNamespace
import unittest
from unittest.mock import patch
from uuid import uuid4
from govoplan_files.backend import archive_work
from govoplan_files.backend.archive_work import (
ArchiveProgress,
ArchiveWorkExpired,
discard_staged_upload,
read_progress,
stage_upload,
use_staged_upload,
)
from govoplan_files.backend.storage.common import FileStorageError
class ArchiveWorkTests(unittest.TestCase):
def setUp(self):
self.directory = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="archive-work-tests-")))
self.settings = SimpleNamespace(
file_archive_work_root=str(self.directory / "work"),
file_archive_preview_ttl_seconds=60,
file_archive_staged_per_actor=2,
file_archive_staged_max_bytes=100,
)
self.source = self.directory / "source.zip"
self.source.write_bytes(b"archive bytes")
self.actor = {"tenant_id": "tenant-1", "user_id": "user-1"}
def stage(self, **actor):
return stage_upload(self.settings, str(self.source), **(self.actor | actor))
def use(self, stage_id, **actor):
return use_staged_upload(self.settings, stage_id, **(self.actor | actor))
def progress(self, operation_id=None, **actor):
return ArchiveProgress(self.settings, operation_id or str(uuid4()), **(self.actor | actor))
def test_staging_is_private_and_actor_and_tenant_bound(self):
stage_id = self.stage()
with self.use(stage_id) as filename:
staged = Path(filename)
self.assertEqual(staged.read_bytes(), self.source.read_bytes())
self.assertEqual(staged.stat().st_mode & 0o777, 0o600)
self.assertEqual(staged.parent.stat().st_mode & 0o777, 0o700)
for actor in ({"user_id": "other-user"}, {"tenant_id": "other-tenant"}):
with self.subTest(actor=actor):
with self.assertRaises(ArchiveWorkExpired), self.use(stage_id, **actor):
self.fail("Foreign actor obtained a staged archive")
discard_staged_upload(self.settings, stage_id, **(self.actor | actor))
with self.use(stage_id) as filename:
self.assertTrue(Path(filename).is_file())
def test_noncanonical_identifiers_cannot_escape_the_private_root(self):
stage_id = self.stage()
for invalid in ("../source.zip", "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", stage_id.replace("-", ""), "", "not-a-uuid"):
with self.subTest(invalid=invalid):
with self.assertRaises(FileStorageError), self.use(invalid):
self.fail("Invalid identifier accepted")
with self.assertRaises(FileStorageError):
discard_staged_upload(self.settings, invalid, **self.actor)
with self.assertRaises(FileStorageError):
read_progress(self.settings, invalid, **self.actor)
self.assertEqual(self.source.read_bytes(), b"archive bytes")
def test_expired_stage_and_progress_are_not_readable(self):
stage_id = self.stage()
with self.use(stage_id) as filename:
staged = Path(filename)
progress = self.progress()
old = time.time() - self.settings.file_archive_preview_ttl_seconds - 1
os.utime(staged, (old, old))
os.utime(progress.path, (old, old))
with self.assertRaises(ArchiveWorkExpired), self.use(stage_id):
self.fail("Expired stage accepted")
with self.assertRaises(ArchiveWorkExpired):
read_progress(self.settings, progress.path.stem[-36:], **self.actor)
def test_quota_evicts_oldest_unleased_stage_and_explicit_discard_is_idempotent(self):
first = self.stage()
with self.use(first) as filename:
first_path = Path(filename)
old = time.time() - 5
os.utime(first_path, (old, old))
second = self.stage()
third = self.stage()
self.assertFalse(first_path.exists())
for stage_id in (second, third):
with self.use(stage_id):
pass
discard_staged_upload(self.settings, third, **self.actor)
discard_staged_upload(self.settings, third, **self.actor)
with self.assertRaises(ArchiveWorkExpired), self.use(third):
self.fail("Discarded archive retained")
def test_active_lease_blocks_duplicate_processing_discard_and_quota_eviction(self):
self.settings.file_archive_staged_per_actor = 1
first = self.stage()
with self.use(first) as filename:
with self.assertRaisesRegex(FileStorageError, "already being processed"), self.use(first):
self.fail("Duplicate lease accepted")
discard_staged_upload(self.settings, first, **self.actor)
self.assertTrue(Path(filename).exists())
with self.assertRaises(FileStorageError):
self.stage()
with self.use(first):
pass
def test_long_running_lease_remains_protected_after_preview_ttl(self):
self.settings.file_archive_staged_per_actor = 1
first = self.stage()
with self.use(first) as filename:
path = Path(filename)
old = time.time() - self.settings.file_archive_preview_ttl_seconds - 1
os.utime(path, (old, old))
os.utime(path.with_suffix(".lease"), (old, old))
with self.assertRaises(FileStorageError):
self.stage()
self.assertTrue(path.exists(), "A live import must remain quota-accounted even after its preview TTL")
def test_global_byte_quota_does_not_evict_other_actor_archives(self):
first = self.stage()
self.settings.file_archive_staged_max_bytes = len(self.source.read_bytes())
with self.assertRaisesRegex(FileStorageError, "storage is full"):
self.stage(user_id="other-user")
with self.use(first):
pass
def test_progress_is_private_actor_bound_and_duplicate_operation_is_rejected(self):
operation_id = str(uuid4())
progress = self.progress(operation_id)
progress("extracting", 2, 5, 20, 50)
for actor in ({"user_id": "other-user"}, {"tenant_id": "other-tenant"}):
with self.assertRaises(ArchiveWorkExpired):
read_progress(self.settings, operation_id, **(self.actor | actor))
before = read_progress(self.settings, operation_id, **self.actor)
with self.assertRaisesRegex(FileStorageError, "already been used"):
self.progress(operation_id)
self.assertEqual(read_progress(self.settings, operation_id, **self.actor), before)
self.assertEqual(set(before), {"phase", "status", "completed_files", "total_files", "completed_bytes", "total_bytes"})
self.assertEqual(before["status"], "running")
progress.finish(True)
self.assertEqual(read_progress(self.settings, operation_id, **self.actor)["status"], "complete")
with self.assertRaises(FileStorageError):
self.progress(operation_id)
def test_progress_write_failure_never_interrupts_an_import(self):
progress = self.progress()
with patch.object(archive_work.os, "replace", side_effect=OSError("Temporary storage unavailable")):
progress("extracting", 1, 2, 10, 20)
progress.finish(True)
self.assertEqual(progress.value["status"], "complete")
self.assertEqual(list(progress.path.parent.glob(".progress-*")), [])
self.assertEqual(json.loads(progress.path.read_text())["status"], "running")
def test_missing_optional_operation_id_does_not_create_a_work_directory(self):
progress = ArchiveProgress(self.settings, None, **self.actor)
progress("extracting", 1, 1, 10, 10)
progress.finish(True)
self.assertIsNone(progress.path)
self.assertFalse(Path(self.settings.file_archive_work_root).exists())
def test_work_directory_with_public_permissions_fails_closed(self):
root = Path(self.settings.file_archive_work_root) / "v1"
root.mkdir(parents=True, mode=0o755)
root.chmod(0o755)
with self.assertRaisesRegex(FileStorageError, "must be private"):
self.stage()
self.assertEqual(list(root.glob("*.archive")), [])
def test_orphan_lease_does_not_permanently_block_staging(self):
stage_id = self.stage()
with self.use(stage_id) as filename:
staged = Path(filename)
# Simulate the lock file left behind after a worker exits: no live
# process holds its kernel lock, so a new request can reclaim it.
staged.with_suffix(".lease").touch(mode=0o600)
with self.use(stage_id):
pass
if __name__ == "__main__":
unittest.main()
tests/test_managed_archives.py
+450
View File
@@ -0,0 +1,450 @@
"""Managed archive routes: real authorization, preview and extraction pipeline.
Only external blob persistence, directory membership and audit delivery are
isolated; source/version/share checks and destination writes use SQLite.
"""
from __future__ import annotations
import hashlib
import io
import tarfile
import unittest
import zipfile
from types import SimpleNamespace
from unittest.mock import patch
import pyzipper
from fastapi import FastAPI
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy.pool import StaticPool
from govoplan_access.backend.db.models import Account, Group, User
from govoplan_core.auth import get_api_principal
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.db.base import Base
from govoplan_core.db.session import get_session
from govoplan_files.backend.db.models import (
FileAsset,
FileBlob,
FileConnectorPolicy,
FileShare,
FileVersion,
)
from govoplan_files.backend.routes.managed_archives import router
from govoplan_files.backend.storage.common import FileStorageError, utcnow
from govoplan_files.backend.storage.files import create_file_asset
def archive_bytes(entries=None, *, password=None, tar=False):
entries = entries or {"folder/one.txt": b"one", "other/two.txt": b"two"}
output = io.BytesIO()
if tar:
with tarfile.open(fileobj=output, mode="w:gz") as archive:
for name, data in entries.items():
member = tarfile.TarInfo(name)
member.size = len(data)
archive.addfile(member, io.BytesIO(data))
else:
archive_type = pyzipper.AESZipFile if password else zipfile.ZipFile
options = {"encryption": pyzipper.WZ_AES} if password else {}
with archive_type(
output, "w", compression=zipfile.ZIP_DEFLATED, **options
) as archive:
if password:
archive.setpassword(password.encode())
for name, data in entries.items():
archive.writestr(name, data)
return output.getvalue()
class ManagedArchiveTests(unittest.TestCase):
def setUp(self):
self.engine = create_engine(
"sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool
)
Base.metadata.create_all(
self.engine,
tables=[
model.__table__
for model in (
Account,
User,
Group,
ChangeSequenceEntry,
FileBlob,
FileAsset,
FileVersion,
FileShare,
FileConnectorPolicy,
)
],
)
self.session = Session(self.engine, expire_on_commit=False)
self.addCleanup(self.engine.dispose)
self.addCleanup(self.session.close)
self.objects = {}
self.scopes = {"files:file:read", "files:file:download", "files:file:upload"}
self.principal = SimpleNamespace(
tenant_id="tenant-1",
user=SimpleNamespace(id="user-1"),
has=lambda scope: scope in self.scopes,
)
self.settings = SimpleNamespace(
file_upload_zip_max_bytes=10 * 1024 * 1024,
file_archive_max_entries=10_000,
file_archive_max_expanded_bytes=20 * 1024 * 1024,
file_archive_max_expansion_ratio=100,
file_archive_preview_ttl_seconds=1800,
file_upload_max_bytes=1024 * 1024,
)
for module in ("uploads", "managed_archives"):
self.enterContext(
patch(f"govoplan_files.backend.routes.{module}.settings", self.settings)
)
self.enterContext(
patch(
"govoplan_files.backend.storage.files.user_group_ids", return_value=[]
)
)
self.enterContext(
patch(
"govoplan_files.backend.storage.files._get_or_create_blob",
side_effect=self.store_blob,
)
)
self.enterContext(
patch(
"govoplan_files.backend.storage.files.get_storage_backend",
return_value=SimpleNamespace(
name="test", get_bytes=lambda key: self.objects[key]
),
)
)
self.audit = self.enterContext(
patch("govoplan_files.backend.route_support.audit_from_principal")
)
self.enterContext(
patch(
"govoplan_files.backend.route_support.asset_is_audit_relevant",
return_value=False,
)
)
self.enterContext(
patch(
"govoplan_files.backend.route_support._sent_campaign_asset_ids",
return_value=set(),
)
)
self.source = self.seed_source(archive_bytes())
app = FastAPI()
app.include_router(router, prefix="/api/v1")
app.dependency_overrides[get_api_principal] = lambda: self.principal
app.dependency_overrides[get_session] = lambda: self.session
self.client = self.enterContext(TestClient(app))
def store_blob(self, session, *, tenant_id, data, content_type=None, **kwargs):
checksum = hashlib.sha256(data).hexdigest()
blob = FileBlob(
tenant_id=tenant_id,
storage_backend="test",
storage_key=f"objects/{checksum}",
checksum_sha256=checksum,
size_bytes=len(data),
content_type=content_type,
ref_count=1,
)
session.add(blob)
session.flush()
self.objects[blob.storage_key] = data
return blob
def seed_source(self, data, filename="source.zip"):
stored = create_file_asset(
self.session,
tenant_id="tenant-1",
owner_type="user",
owner_id="user-1",
user_id="user-1",
filename=filename,
data=data,
folder="archives",
)
self.session.commit()
return stored
def preview(self, **changes):
return self.client.post(
f"/api/v1/files/{self.source.asset.id}/archive-preview",
json={
"source_version_id": self.source.version.id,
"path": "extracted",
**changes,
},
)
def confirm(self, token, **changes):
return self.client.post(
f"/api/v1/files/{self.source.asset.id}/archive-confirm",
json={
"source_version_id": self.source.version.id,
"path": "extracted",
"preview_token": token,
"selected_paths": ["folder/one.txt"],
**changes,
},
)
def token(self):
response = self.preview()
self.assertEqual(200, response.status_code, response.text)
return response.json()["preview_token"]
def assert_no_extraction(self):
self.assertEqual(1, self.session.query(FileAsset).count())
self.assertIsNone(self.source.asset.deleted_at)
self.assertEqual(self.source.version.id, self.source.asset.current_version_id)
self.audit.assert_not_called()
def test_preview_is_read_only_and_confirmation_extracts_only_selected_members(self):
token = self.token()
self.assert_no_extraction()
original_data = self.objects[self.source.blob.storage_key]
response = self.confirm(token)
self.assertEqual(200, response.status_code, response.text)
files = response.json()["files"]
self.assertEqual(
["extracted/folder/one.txt"], [file["display_path"] for file in files]
)
self.assertEqual(2, self.session.query(FileAsset).count())
self.assertEqual(original_data, self.objects[self.source.blob.storage_key])
self.assertEqual(self.source.version.id, self.source.asset.current_version_id)
self.assertIsNone(self.source.asset.deleted_at)
self.assertEqual(
self.source.asset.id,
files[0]["source_provenance"]["metadata"]["archive_source_file_id"],
)
self.assertEqual(
self.source.version.id,
files[0]["source_provenance"]["metadata"]["archive_source_version_id"],
)
self.audit.assert_called_once()
def test_all_three_permissions_are_required(self):
token = self.token()
for scope in tuple(self.scopes):
with self.subTest(scope=scope):
self.scopes.remove(scope)
self.assertEqual(403, self.preview().status_code)
self.assertEqual(403, self.confirm(token).status_code)
self.scopes.add(scope)
self.assert_no_extraction()
def test_changed_source_version_is_rejected_before_read_or_write(self):
token = self.token()
self.source.asset.current_version_id = "replacement-version"
self.session.commit()
self.assertIn("version changed", self.confirm(token).text)
self.assertEqual(400, self.preview().status_code)
self.assertEqual(1, self.session.query(FileAsset).count())
def test_mismatched_version_or_blob_tenant_is_rejected(self):
self.source.version.file_asset_id = "other-file"
self.session.commit()
self.assertIn("does not belong", self.preview().text)
self.source.version.file_asset_id = self.source.asset.id
self.source.blob.tenant_id = "other-tenant"
self.session.commit()
self.assertIn("does not belong", self.preview().text)
self.assert_no_extraction()
def test_expired_preview_cannot_extract(self):
with patch("cryptography.fernet.time.time", return_value=1):
token = self.token()
response = self.confirm(token)
self.assertEqual(400, response.status_code, response.text)
self.assertIn("expired", response.text)
self.assert_no_extraction()
def test_source_refresh_during_verified_read_cannot_change_preview_or_extract(self):
token = self.token()
replacement_version = SimpleNamespace(id="refreshed-version")
with patch(
"govoplan_files.backend.routes.managed_archives.read_asset_bytes",
return_value=(
self.objects[self.source.blob.storage_key],
replacement_version,
self.source.blob,
),
):
preview = self.preview()
confirmed = self.confirm(token)
for response in (preview, confirmed):
self.assertEqual(400, response.status_code, response.text)
self.assertIn("version changed during the verified read", response.text)
self.assert_no_extraction()
def test_revoked_shared_access_cannot_reuse_preview(self):
self.source.asset.owner_user_id = "other-user"
share = FileShare(
tenant_id="tenant-1",
file_asset_id=self.source.asset.id,
target_type="user",
target_id="user-1",
permission="read",
)
self.session.add(share)
self.session.commit()
token = self.token()
share.revoked_at = utcnow()
self.session.commit()
response = self.confirm(token)
self.assertEqual(400, response.status_code, response.text)
self.assertIn("No access", response.text)
self.assert_no_extraction()
def test_deleted_or_cross_tenant_sources_are_rejected(self):
token = self.token()
self.source.asset.deleted_at = utcnow()
self.session.commit()
self.assertIn("File not found", self.confirm(token).text)
self.source.asset.deleted_at = None
self.source.asset.tenant_id = "other-tenant"
self.session.commit()
self.assertIn("File not found", self.confirm(token).text)
self.assert_no_extraction()
def test_confirmation_is_bound_to_destination_actor_version_and_file(self):
token = self.token()
for change in (
{"path": "elsewhere"},
{"owner_id": "other-user"},
{"source_version_id": "wrong"},
):
with self.subTest(change=change):
response = self.confirm(token, **change)
self.assertEqual(400, response.status_code, response.text)
self.assertIn("does not match", response.text)
self.principal.user.id = "other-user"
self.assertIn("does not match", self.confirm(token).text)
self.principal.user.id = "user-1"
self.assertEqual(400, self.confirm("tampered-token").status_code)
response = self.client.post(
"/api/v1/files/other-file/archive-confirm",
json={
"source_version_id": self.source.version.id,
"path": "extracted",
"preview_token": token,
"selected_paths": ["folder"],
},
)
self.assertIn("does not match", response.text)
self.assert_no_extraction()
def test_destination_ownership_is_checked_during_preview(self):
response = self.preview(owner_id="other-user")
self.assertEqual(400, response.status_code, response.text)
self.assertIn("No access to this user file space", response.text)
self.assert_no_extraction()
def test_connector_policy_is_rechecked_at_confirmation(self):
self.source.asset.metadata_ = {
"source_provenance": {
"source_type": "webdav",
"provider": "webdav",
"external_id": "archive",
}
}
self.session.commit()
token = self.token()
self.session.add(
FileConnectorPolicy(
tenant_id="tenant-1",
scope_type="tenant",
scope_id="tenant-1",
policy={"deny": {"providers": ["webdav"]}},
)
)
self.session.commit()
response = self.confirm(token)
self.assertEqual(403, response.status_code, response.text)
self.assertEqual(403, self.preview().status_code)
self.assert_no_extraction()
def test_source_collision_rolls_back_all_destination_writes(self):
self.source = self.seed_source(
archive_bytes(
{"a-first.txt": b"first", "collision.zip": b"cannot replace source"}
),
"collision.zip",
)
response = self.preview(path="archives")
self.assertEqual(200, response.status_code, response.text)
result = self.confirm(
response.json()["preview_token"],
path="archives",
selected_paths=["a-first.txt", "collision.zip"],
)
self.assertEqual(400, result.status_code, result.text)
self.assertIn("already exists", result.text)
self.assertEqual(2, self.session.query(FileAsset).count())
self.assertEqual(self.source.version.id, self.source.asset.current_version_id)
self.assertIsNone(self.source.asset.deleted_at)
def test_encrypted_zip_uses_the_same_password_and_selection_flow(self):
self.source = self.seed_source(
archive_bytes(password="correct horse"), "encrypted.zip"
)
preview = self.preview()
self.assertTrue(preview.json()["requires_password"])
self.assertFalse(preview.json()["password_verified"])
token = preview.json()["preview_token"]
self.assertEqual(400, self.confirm(token).status_code)
self.assertEqual(400, self.preview(password="wrong").status_code)
verified = self.preview(password="correct horse")
self.assertTrue(verified.json()["password_verified"])
response = self.confirm(
verified.json()["preview_token"], password="correct horse"
)
self.assertEqual(200, response.status_code, response.text)
self.assertNotIn("correct horse", response.text)
self.assertNotIn("correct horse", str(self.audit.call_args_list))
def test_tar_reuses_the_same_confirmation_flow(self):
self.source = self.seed_source(archive_bytes(tar=True), "source.tar.gz")
preview = self.preview()
self.assertEqual("tar.gz", preview.json()["archive_format"])
self.assertEqual(200, self.confirm(preview.json()["preview_token"]).status_code)
def test_unsafe_members_and_size_limits_fail_before_extraction(self):
self.settings.file_upload_zip_max_bytes = 1
self.assertIn("size limit", self.preview().text)
self.settings.file_upload_zip_max_bytes = 10 * 1024 * 1024
self.source = self.seed_source(
archive_bytes({"../escape": b"no"}), "unsafe.zip"
)
self.assertIn("Unsafe archive member", self.preview().text)
self.audit.assert_not_called()
def test_quarantined_or_protected_sources_use_verified_read_path(self):
self.source.blob.quarantined_at = utcnow()
self.session.commit()
self.assertIn("quarantin", self.preview().text.lower())
self.source.blob.quarantined_at = None
self.source.blob.encryption_envelope_id = "protected-envelope"
self.session.commit()
with patch(
"govoplan_files.backend.storage.content_protection.unprotect_blob_content",
side_effect=FileStorageError("Decryption is unavailable"),
) as decrypt:
response = self.preview()
self.assertEqual(400, response.status_code, response.text)
self.assertIn("Decryption is unavailable", response.text)
decrypt.assert_called_once()
self.assert_no_extraction()
if __name__ == "__main__":
unittest.main()
tests/test_manifest_documentation.py
+12
View File
@@ -8,6 +8,7 @@ STATIC_TOPIC_IDS = {
"files.quick-access-and-product-area",
"files.search.managed-content",
"files.workflow.organize-managed-files",
"files.workflow.unpack-managed-archive",
"files.workflow.find-and-download-files",
"files.workflow.share-managed-files",
"files.workflow.delete-managed-files",
@@ -44,6 +45,17 @@ class FilesManifestDocumentationTests(unittest.TestCase):
def topic(self, topic_id: str):
return self.topics[topic_id]
def test_workspace_action_locations_and_read_only_reload_are_documented_in_both_languages(self) -> None:
topic = self.topic("files.quick-access-and-product-area")
self.assertEqual({"user", "admin"}, set(topic.documentation_types))
for label in ("Reload", "Create folder", "Upload", "Connections and imports", "Manage selection"):
self.assertIn(label, topic.body)
self.assertIn("never imports or synchronizes", topic.body)
german = topic.translations["de"]["body"]
for label in ("Neu laden", "Ordner erstellen", "Hochladen", "Verbindungen und Importe", "Auswahl verwalten"):
self.assertIn(label, german)
self.assertIn("niemals", german)
def test_static_topics_have_role_scope_module_and_link_contracts(self) -> None:
self.assertEqual(STATIC_TOPIC_IDS, set(self.topics))
self.assertEqual(
tests/test_router_contract.py
+7 -1
View File
@@ -22,6 +22,7 @@ from govoplan_files.backend.routes.shares import router as shares_router
from govoplan_files.backend.routes.spaces import router as spaces_router
from govoplan_files.backend.routes.transfers import router as transfers_router
from govoplan_files.backend.routes.uploads import router as uploads_router
from govoplan_files.backend.routes.managed_archives import router as managed_archives_router
class FilesRouterContractTests(unittest.TestCase):
@@ -42,6 +43,7 @@ class FilesRouterContractTests(unittest.TestCase):
lifecycle_router,
listing_router,
uploads_router,
managed_archives_router,
connector_settings_router,
connector_io_router,
connector_profiles_router,
@@ -57,7 +59,7 @@ class FilesRouterContractTests(unittest.TestCase):
actual = self._operation_keys(router)
self.assertEqual(expected, actual)
self.assertEqual(63, len(actual))
self.assertEqual(67, len(actual))
self.assertFalse(
[operation for operation, count in Counter(actual).items() if count > 1]
)
@@ -69,6 +71,10 @@ class FilesRouterContractTests(unittest.TestCase):
self.assertIn((("POST",), "/files/archive-preview"), routes)
self.assertIn((("POST",), "/files/archive-confirm"), routes)
self.assertIn((("GET",), "/files/archive-progress/{operation_id}"), routes)
self.assertIn((("DELETE",), "/files/archive-staging/{stage_id}"), routes)
self.assertIn((("POST",), "/files/{file_id}/archive-preview"), routes)
self.assertIn((("POST",), "/files/{file_id}/archive-confirm"), routes)
def test_connector_routes_keep_existing_api_paths(self) -> None:
routes = {
tests/test_upload_response_batching.py
+274
View File
@@ -0,0 +1,274 @@
"""Response-only parity/query regression with isolated synthetic SQLite data.
This does not benchmark imports, mutate storage, or replace route authorization.
The isolated schema uses string columns for external module identities instead
of importing optional modules. All response, share-state and Campaign-use
queries execute through the real Files ORM.
"""
from __future__ import annotations
from datetime import UTC, datetime, timedelta
import unittest
from sqlalchemy import Column, MetaData, String, Table, create_engine, event, inspect
from sqlalchemy.orm import Session
from sqlalchemy.types import NullType
from govoplan_files.backend.db.models import (
CampaignAttachmentUse,
FileAsset,
FileBlob,
FileShare,
FileVersion,
)
from govoplan_files.backend.route_support import _asset_list_response, _asset_response
class UploadResponseBatchingTests(unittest.TestCase):
def setUp(self):
self.engine = create_engine("sqlite://")
self.addCleanup(self.engine.dispose)
metadata = MetaData()
for model in (
FileBlob,
FileAsset,
FileVersion,
FileShare,
CampaignAttachmentUse,
):
Table(
model.__tablename__,
metadata,
*(
Column(
column.name,
String(36)
if isinstance(column.type, NullType)
else column.type,
primary_key=column.primary_key,
nullable=column.nullable,
default=column.default.arg
if column.default is not None
else None,
server_default=column.server_default.arg
if column.server_default is not None
else None,
)
for column in model.__table__.columns
),
)
metadata.create_all(self.engine)
now = datetime.now(UTC)
with self.engine.begin() as connection:
def seed(row):
# Core inserts into separate fixture metadata avoid resolving
# foreign identities or changing global ORM declarations.
values = {
attribute.columns[0].name: getattr(row, attribute.key)
for attribute in inspect(type(row)).column_attrs
if attribute.key in row.__dict__
}
connection.execute(metadata.tables[row.__tablename__].insert(), values)
for index in range(64):
asset_id = f"asset-{index:03}"
version_id = f"version-{index:03}"
blob_id = f"blob-{index:03}"
checksum = f"{index:064x}"
owner_type = "group" if index % 2 else "user"
seed(
FileBlob(
id=blob_id,
tenant_id="tenant-fixture",
storage_backend="local",
storage_key=f"synthetic/{blob_id}",
checksum_sha256=checksum,
size_bytes=index + 10,
content_type="text/plain",
ref_count=1,
)
)
seed(
FileAsset(
id=asset_id,
tenant_id="tenant-fixture",
owner_type=owner_type,
owner_user_id="user-fixture" if owner_type == "user" else None,
owner_group_id="group-fixture"
if owner_type == "group"
else None,
current_version_id=version_id,
display_path=f"imported/{index:03}.txt",
filename=f"{index:03}.txt",
description=f"Synthetic response {index}",
retained_until=now + timedelta(days=365) if index % 2 else None,
legal_hold=bool(index % 3),
lifecycle_revision=index + 2,
lifecycle_reason=f"Retention decision {index}",
deleted_at=now - timedelta(days=1) if index % 7 == 0 else None,
metadata_={
"fixture": index,
"source_provenance": {
"source_type": "archive",
"revision": f"source-{index}",
"metadata": {
"archive_source_file_id": "synthetic-archive"
},
},
"source_revision": f"source-{index}",
},
)
)
seed(
FileVersion(
id=version_id,
tenant_id="tenant-fixture",
file_asset_id=asset_id,
blob_id=blob_id,
version_number=1,
filename_at_upload=f"{index:03}.txt",
display_path_at_upload=f"imported/{index:03}.txt",
size_bytes=index + 10,
checksum_sha256=checksum,
content_type="text/plain",
)
)
for rank, state in enumerate(("active", "expired", "revoked")):
seed(
FileShare(
id=f"share-{index:03}-{state}",
tenant_id="tenant-fixture",
file_asset_id=asset_id,
target_type="group" if rank == 0 else "user",
target_id=f"target-{state}",
permission="write" if rank == 0 else "read",
created_by_user_id="user-fixture",
created_at=now - timedelta(hours=rank + 1),
expires_at=now - timedelta(days=1)
if state == "expired"
else None,
revoked_at=now - timedelta(days=1)
if state == "revoked"
else None,
revoked_by_user_id="user-fixture"
if state == "revoked"
else None,
)
)
# Duplicate sent evidence must still produce one response;
# built-only usage and missing usage are not audit-relevant.
stages = (
("sent", "sent")
if index % 3 == 0
else ("built",)
if index % 3 == 1
else ()
)
for use_index, stage in enumerate(stages):
seed(
CampaignAttachmentUse(
id=f"use-{index:03}-{use_index}",
tenant_id="tenant-fixture",
campaign_id="campaign-fixture",
campaign_version_id="campaign-version-fixture",
campaign_job_id=f"job-{use_index}",
file_asset_id=asset_id,
file_version_id=version_id,
file_blob_id=blob_id,
filename_used=f"{index:03}.txt",
checksum_sha256=checksum,
size_bytes=index + 10,
content_type="text/plain",
use_stage=stage,
)
)
def render(self, *, batched: bool, include_shares: bool):
statements = []
def record_select(
connection, cursor, statement, parameters, context, executemany
):
if statement.lstrip().upper().startswith("SELECT"):
statements.append(statement)
# A fresh session ensures neither path receives preloaded versions or
# blobs. Loading the already-authorized asset list is deliberately
# outside the measurement; this measures response construction only.
with Session(self.engine) as session:
assets = session.query(FileAsset).order_by(FileAsset.id).all()
event.listen(self.engine, "before_cursor_execute", record_select)
try:
responses = (
_asset_list_response(session, assets, include_shares=include_shares)
if batched
else [
_asset_response(session, asset, include_shares=include_shares)
for asset in assets
]
)
result = [response.model_dump(mode="json") for response in responses]
finally:
event.remove(self.engine, "before_cursor_execute", record_select)
return result, statements
def test_batched_response_matches_all_metadata_lifecycle_shares_and_audit_flags(
self,
):
for include_shares in (False, True):
with self.subTest(include_shares=include_shares):
batched, _ = self.render(batched=True, include_shares=include_shares)
individual, _ = self.render(
batched=False, include_shares=include_shares
)
self.assertEqual(individual, batched)
self.assertEqual(64, len(batched))
for index, item in enumerate(batched):
self.assertEqual(f"asset-{index:03}", item["id"])
self.assertEqual(index + 2, item["lifecycle_revision"])
self.assertEqual(
f"Retention decision {index}", item["lifecycle_reason"]
)
self.assertEqual(
bool(index % 2), item["retained_until"] is not None
)
self.assertEqual(bool(index % 3), item["legal_hold"])
self.assertEqual(index % 3 == 0, item["audit_relevant"])
self.assertEqual(index % 7 == 0, item["deleted_at"] is not None)
self.assertEqual(f"source-{index}", item["source_revision"])
if include_shares:
self.assertEqual(
[
f"share-{index:03}-{state}"
for state in ("active", "expired", "revoked")
],
[share["id"] for share in item["shares"]],
)
self.assertEqual(
[True, False, False],
[share["active"] for share in item["shares"]],
)
else:
self.assertEqual([], item["shares"])
def test_64_asset_response_batches_real_selects_instead_of_querying_per_item(self):
for include_shares, per_item_queries, batch_queries in (
(False, 192, 2),
(True, 256, 3),
):
with self.subTest(include_shares=include_shares):
individual, individual_queries = self.render(
batched=False, include_shares=include_shares
)
batched, batched_queries = self.render(
batched=True, include_shares=include_shares
)
self.assertEqual(individual, batched)
self.assertEqual(per_item_queries, len(individual_queries))
self.assertEqual(batch_queries, len(batched_queries))
if __name__ == "__main__":
unittest.main()