192 lines
8.7 KiB
Python
192 lines
8.7 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import time
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
from uuid import uuid4
|
|
|
|
from govoplan_files.backend import archive_work
|
|
from govoplan_files.backend.archive_work import (
|
|
ArchiveProgress,
|
|
ArchiveWorkExpired,
|
|
discard_staged_upload,
|
|
read_progress,
|
|
stage_upload,
|
|
use_staged_upload,
|
|
)
|
|
from govoplan_files.backend.storage.common import FileStorageError
|
|
|
|
|
|
class ArchiveWorkTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.directory = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="archive-work-tests-")))
|
|
self.settings = SimpleNamespace(
|
|
file_archive_work_root=str(self.directory / "work"),
|
|
file_archive_preview_ttl_seconds=60,
|
|
file_archive_staged_per_actor=2,
|
|
file_archive_staged_max_bytes=100,
|
|
)
|
|
self.source = self.directory / "source.zip"
|
|
self.source.write_bytes(b"archive bytes")
|
|
self.actor = {"tenant_id": "tenant-1", "user_id": "user-1"}
|
|
|
|
def stage(self, **actor):
|
|
return stage_upload(self.settings, str(self.source), **(self.actor | actor))
|
|
|
|
def use(self, stage_id, **actor):
|
|
return use_staged_upload(self.settings, stage_id, **(self.actor | actor))
|
|
|
|
def progress(self, operation_id=None, **actor):
|
|
return ArchiveProgress(self.settings, operation_id or str(uuid4()), **(self.actor | actor))
|
|
|
|
def test_staging_is_private_and_actor_and_tenant_bound(self):
|
|
stage_id = self.stage()
|
|
with self.use(stage_id) as filename:
|
|
staged = Path(filename)
|
|
self.assertEqual(staged.read_bytes(), self.source.read_bytes())
|
|
self.assertEqual(staged.stat().st_mode & 0o777, 0o600)
|
|
self.assertEqual(staged.parent.stat().st_mode & 0o777, 0o700)
|
|
for actor in ({"user_id": "other-user"}, {"tenant_id": "other-tenant"}):
|
|
with self.subTest(actor=actor):
|
|
with self.assertRaises(ArchiveWorkExpired), self.use(stage_id, **actor):
|
|
self.fail("Foreign actor obtained a staged archive")
|
|
discard_staged_upload(self.settings, stage_id, **(self.actor | actor))
|
|
with self.use(stage_id) as filename:
|
|
self.assertTrue(Path(filename).is_file())
|
|
|
|
def test_noncanonical_identifiers_cannot_escape_the_private_root(self):
|
|
stage_id = self.stage()
|
|
for invalid in ("../source.zip", "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", stage_id.replace("-", ""), "", "not-a-uuid"):
|
|
with self.subTest(invalid=invalid):
|
|
with self.assertRaises(FileStorageError), self.use(invalid):
|
|
self.fail("Invalid identifier accepted")
|
|
with self.assertRaises(FileStorageError):
|
|
discard_staged_upload(self.settings, invalid, **self.actor)
|
|
with self.assertRaises(FileStorageError):
|
|
read_progress(self.settings, invalid, **self.actor)
|
|
self.assertEqual(self.source.read_bytes(), b"archive bytes")
|
|
|
|
def test_expired_stage_and_progress_are_not_readable(self):
|
|
stage_id = self.stage()
|
|
with self.use(stage_id) as filename:
|
|
staged = Path(filename)
|
|
progress = self.progress()
|
|
old = time.time() - self.settings.file_archive_preview_ttl_seconds - 1
|
|
os.utime(staged, (old, old))
|
|
os.utime(progress.path, (old, old))
|
|
with self.assertRaises(ArchiveWorkExpired), self.use(stage_id):
|
|
self.fail("Expired stage accepted")
|
|
with self.assertRaises(ArchiveWorkExpired):
|
|
read_progress(self.settings, progress.path.stem[-36:], **self.actor)
|
|
|
|
def test_quota_evicts_oldest_unleased_stage_and_explicit_discard_is_idempotent(self):
|
|
first = self.stage()
|
|
with self.use(first) as filename:
|
|
first_path = Path(filename)
|
|
old = time.time() - 5
|
|
os.utime(first_path, (old, old))
|
|
second = self.stage()
|
|
third = self.stage()
|
|
self.assertFalse(first_path.exists())
|
|
for stage_id in (second, third):
|
|
with self.use(stage_id):
|
|
pass
|
|
discard_staged_upload(self.settings, third, **self.actor)
|
|
discard_staged_upload(self.settings, third, **self.actor)
|
|
with self.assertRaises(ArchiveWorkExpired), self.use(third):
|
|
self.fail("Discarded archive retained")
|
|
|
|
def test_active_lease_blocks_duplicate_processing_discard_and_quota_eviction(self):
|
|
self.settings.file_archive_staged_per_actor = 1
|
|
first = self.stage()
|
|
with self.use(first) as filename:
|
|
with self.assertRaisesRegex(FileStorageError, "already being processed"), self.use(first):
|
|
self.fail("Duplicate lease accepted")
|
|
discard_staged_upload(self.settings, first, **self.actor)
|
|
self.assertTrue(Path(filename).exists())
|
|
with self.assertRaises(FileStorageError):
|
|
self.stage()
|
|
with self.use(first):
|
|
pass
|
|
|
|
def test_long_running_lease_remains_protected_after_preview_ttl(self):
|
|
self.settings.file_archive_staged_per_actor = 1
|
|
first = self.stage()
|
|
with self.use(first) as filename:
|
|
path = Path(filename)
|
|
old = time.time() - self.settings.file_archive_preview_ttl_seconds - 1
|
|
os.utime(path, (old, old))
|
|
os.utime(path.with_suffix(".lease"), (old, old))
|
|
with self.assertRaises(FileStorageError):
|
|
self.stage()
|
|
self.assertTrue(path.exists(), "A live import must remain quota-accounted even after its preview TTL")
|
|
|
|
def test_global_byte_quota_does_not_evict_other_actor_archives(self):
|
|
first = self.stage()
|
|
self.settings.file_archive_staged_max_bytes = len(self.source.read_bytes())
|
|
with self.assertRaisesRegex(FileStorageError, "storage is full"):
|
|
self.stage(user_id="other-user")
|
|
with self.use(first):
|
|
pass
|
|
|
|
def test_progress_is_private_actor_bound_and_duplicate_operation_is_rejected(self):
|
|
operation_id = str(uuid4())
|
|
progress = self.progress(operation_id)
|
|
progress("extracting", 2, 5, 20, 50)
|
|
for actor in ({"user_id": "other-user"}, {"tenant_id": "other-tenant"}):
|
|
with self.assertRaises(ArchiveWorkExpired):
|
|
read_progress(self.settings, operation_id, **(self.actor | actor))
|
|
before = read_progress(self.settings, operation_id, **self.actor)
|
|
with self.assertRaisesRegex(FileStorageError, "already been used"):
|
|
self.progress(operation_id)
|
|
self.assertEqual(read_progress(self.settings, operation_id, **self.actor), before)
|
|
self.assertEqual(set(before), {"phase", "status", "completed_files", "total_files", "completed_bytes", "total_bytes"})
|
|
self.assertEqual(before["status"], "running")
|
|
progress.finish(True)
|
|
self.assertEqual(read_progress(self.settings, operation_id, **self.actor)["status"], "complete")
|
|
with self.assertRaises(FileStorageError):
|
|
self.progress(operation_id)
|
|
|
|
def test_progress_write_failure_never_interrupts_an_import(self):
|
|
progress = self.progress()
|
|
with patch.object(archive_work.os, "replace", side_effect=OSError("Temporary storage unavailable")):
|
|
progress("extracting", 1, 2, 10, 20)
|
|
progress.finish(True)
|
|
self.assertEqual(progress.value["status"], "complete")
|
|
self.assertEqual(list(progress.path.parent.glob(".progress-*")), [])
|
|
self.assertEqual(json.loads(progress.path.read_text())["status"], "running")
|
|
|
|
def test_missing_optional_operation_id_does_not_create_a_work_directory(self):
|
|
progress = ArchiveProgress(self.settings, None, **self.actor)
|
|
progress("extracting", 1, 1, 10, 10)
|
|
progress.finish(True)
|
|
self.assertIsNone(progress.path)
|
|
self.assertFalse(Path(self.settings.file_archive_work_root).exists())
|
|
|
|
def test_work_directory_with_public_permissions_fails_closed(self):
|
|
root = Path(self.settings.file_archive_work_root) / "v1"
|
|
root.mkdir(parents=True, mode=0o755)
|
|
root.chmod(0o755)
|
|
with self.assertRaisesRegex(FileStorageError, "must be private"):
|
|
self.stage()
|
|
self.assertEqual(list(root.glob("*.archive")), [])
|
|
|
|
def test_orphan_lease_does_not_permanently_block_staging(self):
|
|
stage_id = self.stage()
|
|
with self.use(stage_id) as filename:
|
|
staged = Path(filename)
|
|
# Simulate the lock file left behind after a worker exits: no live
|
|
# process holds its kernel lock, so a new request can reclaim it.
|
|
staged.with_suffix(".lease").touch(mode=0o600)
|
|
with self.use(stage_id):
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|