Release govoplan-files v0.1.26: speed archive workflows and unify file tools

tests/test_archive_work.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+import tempfile
+import time
+from types import SimpleNamespace
+import unittest
+from unittest.mock import patch
+from uuid import uuid4
+
+from govoplan_files.backend import archive_work
+from govoplan_files.backend.archive_work import (
+    ArchiveProgress,
+    ArchiveWorkExpired,
+    discard_staged_upload,
+    read_progress,
+    stage_upload,
+    use_staged_upload,
+)
+from govoplan_files.backend.storage.common import FileStorageError
+
+
+class ArchiveWorkTests(unittest.TestCase):
+    def setUp(self):
+        self.directory = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="archive-work-tests-")))
+        self.settings = SimpleNamespace(
+            file_archive_work_root=str(self.directory / "work"),
+            file_archive_preview_ttl_seconds=60,
+            file_archive_staged_per_actor=2,
+            file_archive_staged_max_bytes=100,
+        )
+        self.source = self.directory / "source.zip"
+        self.source.write_bytes(b"archive bytes")
+        self.actor = {"tenant_id": "tenant-1", "user_id": "user-1"}
+
+    def stage(self, **actor):
+        return stage_upload(self.settings, str(self.source), **(self.actor | actor))
+
+    def use(self, stage_id, **actor):
+        return use_staged_upload(self.settings, stage_id, **(self.actor | actor))
+
+    def progress(self, operation_id=None, **actor):
+        return ArchiveProgress(self.settings, operation_id or str(uuid4()), **(self.actor | actor))
+
+    def test_staging_is_private_and_actor_and_tenant_bound(self):
+        stage_id = self.stage()
+        with self.use(stage_id) as filename:
+            staged = Path(filename)
+            self.assertEqual(staged.read_bytes(), self.source.read_bytes())
+            self.assertEqual(staged.stat().st_mode & 0o777, 0o600)
+            self.assertEqual(staged.parent.stat().st_mode & 0o777, 0o700)
+        for actor in ({"user_id": "other-user"}, {"tenant_id": "other-tenant"}):
+            with self.subTest(actor=actor):
+                with self.assertRaises(ArchiveWorkExpired), self.use(stage_id, **actor):
+                    self.fail("Foreign actor obtained a staged archive")
+                discard_staged_upload(self.settings, stage_id, **(self.actor | actor))
+        with self.use(stage_id) as filename:
+            self.assertTrue(Path(filename).is_file())
+
+    def test_noncanonical_identifiers_cannot_escape_the_private_root(self):
+        stage_id = self.stage()
+        for invalid in ("../source.zip", "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA", stage_id.replace("-", ""), "", "not-a-uuid"):
+            with self.subTest(invalid=invalid):
+                with self.assertRaises(FileStorageError), self.use(invalid):
+                    self.fail("Invalid identifier accepted")
+                with self.assertRaises(FileStorageError):
+                    discard_staged_upload(self.settings, invalid, **self.actor)
+                with self.assertRaises(FileStorageError):
+                    read_progress(self.settings, invalid, **self.actor)
+        self.assertEqual(self.source.read_bytes(), b"archive bytes")
+
+    def test_expired_stage_and_progress_are_not_readable(self):
+        stage_id = self.stage()
+        with self.use(stage_id) as filename:
+            staged = Path(filename)
+        progress = self.progress()
+        old = time.time() - self.settings.file_archive_preview_ttl_seconds - 1
+        os.utime(staged, (old, old))
+        os.utime(progress.path, (old, old))
+        with self.assertRaises(ArchiveWorkExpired), self.use(stage_id):
+            self.fail("Expired stage accepted")
+        with self.assertRaises(ArchiveWorkExpired):
+            read_progress(self.settings, progress.path.stem[-36:], **self.actor)
+
+    def test_quota_evicts_oldest_unleased_stage_and_explicit_discard_is_idempotent(self):
+        first = self.stage()
+        with self.use(first) as filename:
+            first_path = Path(filename)
+        old = time.time() - 5
+        os.utime(first_path, (old, old))
+        second = self.stage()
+        third = self.stage()
+        self.assertFalse(first_path.exists())
+        for stage_id in (second, third):
+            with self.use(stage_id):
+                pass
+        discard_staged_upload(self.settings, third, **self.actor)
+        discard_staged_upload(self.settings, third, **self.actor)
+        with self.assertRaises(ArchiveWorkExpired), self.use(third):
+            self.fail("Discarded archive retained")
+
+    def test_active_lease_blocks_duplicate_processing_discard_and_quota_eviction(self):
+        self.settings.file_archive_staged_per_actor = 1
+        first = self.stage()
+        with self.use(first) as filename:
+            with self.assertRaisesRegex(FileStorageError, "already being processed"), self.use(first):
+                self.fail("Duplicate lease accepted")
+            discard_staged_upload(self.settings, first, **self.actor)
+            self.assertTrue(Path(filename).exists())
+            with self.assertRaises(FileStorageError):
+                self.stage()
+        with self.use(first):
+            pass
+
+    def test_long_running_lease_remains_protected_after_preview_ttl(self):
+        self.settings.file_archive_staged_per_actor = 1
+        first = self.stage()
+        with self.use(first) as filename:
+            path = Path(filename)
+            old = time.time() - self.settings.file_archive_preview_ttl_seconds - 1
+            os.utime(path, (old, old))
+            os.utime(path.with_suffix(".lease"), (old, old))
+            with self.assertRaises(FileStorageError):
+                self.stage()
+            self.assertTrue(path.exists(), "A live import must remain quota-accounted even after its preview TTL")
+
+    def test_global_byte_quota_does_not_evict_other_actor_archives(self):
+        first = self.stage()
+        self.settings.file_archive_staged_max_bytes = len(self.source.read_bytes())
+        with self.assertRaisesRegex(FileStorageError, "storage is full"):
+            self.stage(user_id="other-user")
+        with self.use(first):
+            pass
+
+    def test_progress_is_private_actor_bound_and_duplicate_operation_is_rejected(self):
+        operation_id = str(uuid4())
+        progress = self.progress(operation_id)
+        progress("extracting", 2, 5, 20, 50)
+        for actor in ({"user_id": "other-user"}, {"tenant_id": "other-tenant"}):
+            with self.assertRaises(ArchiveWorkExpired):
+                read_progress(self.settings, operation_id, **(self.actor | actor))
+        before = read_progress(self.settings, operation_id, **self.actor)
+        with self.assertRaisesRegex(FileStorageError, "already been used"):
+            self.progress(operation_id)
+        self.assertEqual(read_progress(self.settings, operation_id, **self.actor), before)
+        self.assertEqual(set(before), {"phase", "status", "completed_files", "total_files", "completed_bytes", "total_bytes"})
+        self.assertEqual(before["status"], "running")
+        progress.finish(True)
+        self.assertEqual(read_progress(self.settings, operation_id, **self.actor)["status"], "complete")
+        with self.assertRaises(FileStorageError):
+            self.progress(operation_id)
+
+    def test_progress_write_failure_never_interrupts_an_import(self):
+        progress = self.progress()
+        with patch.object(archive_work.os, "replace", side_effect=OSError("Temporary storage unavailable")):
+            progress("extracting", 1, 2, 10, 20)
+            progress.finish(True)
+        self.assertEqual(progress.value["status"], "complete")
+        self.assertEqual(list(progress.path.parent.glob(".progress-*")), [])
+        self.assertEqual(json.loads(progress.path.read_text())["status"], "running")
+
+    def test_missing_optional_operation_id_does_not_create_a_work_directory(self):
+        progress = ArchiveProgress(self.settings, None, **self.actor)
+        progress("extracting", 1, 1, 10, 10)
+        progress.finish(True)
+        self.assertIsNone(progress.path)
+        self.assertFalse(Path(self.settings.file_archive_work_root).exists())
+
+    def test_work_directory_with_public_permissions_fails_closed(self):
+        root = Path(self.settings.file_archive_work_root) / "v1"
+        root.mkdir(parents=True, mode=0o755)
+        root.chmod(0o755)
+        with self.assertRaisesRegex(FileStorageError, "must be private"):
+            self.stage()
+        self.assertEqual(list(root.glob("*.archive")), [])
+
+    def test_orphan_lease_does_not_permanently_block_staging(self):
+        stage_id = self.stage()
+        with self.use(stage_id) as filename:
+            staged = Path(filename)
+        # Simulate the lock file left behind after a worker exits: no live
+        # process holds its kernel lock, so a new request can reclaim it.
+        staged.with_suffix(".lease").touch(mode=0o600)
+        with self.use(stage_id):
+            pass
+
+
+if __name__ == "__main__":
+    unittest.main()