from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path from typing import Iterable, Protocol from govoplan_core.security.outbound_http import ( OutboundHttpError, response_limit, validate_unpinned_sdk_http_url, ) from govoplan_files.backend.runtime import settings class StorageBackendError(RuntimeError): pass class StorageBackend(Protocol): name: str def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: ... def get_bytes(self, key: str) -> bytes: ... def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: ... def delete(self, key: str) -> None: ... def exists(self, key: str) -> bool: ... @dataclass(slots=True) class LocalFilesystemStorageBackend: root: Path fallback_roots: tuple[Path, ...] = field(default_factory=tuple) name: str = "local" def __post_init__(self) -> None: self.root = self.root.expanduser().resolve() self.fallback_roots = tuple(root.expanduser().resolve() for root in self.fallback_roots if root) self.root.mkdir(parents=True, exist_ok=True) def _path_for_root(self, root: Path, key: str) -> Path: path = (root / key).resolve() if not path.is_relative_to(root): raise StorageBackendError("Storage key escapes local storage root") return path def _path(self, key: str) -> Path: return self._path_for_root(self.root, key) def _readable_path(self, key: str) -> Path: primary = self._path(key) if primary.exists() and primary.is_file(): return primary for root in self.fallback_roots: candidate = self._path_for_root(root, key) if candidate.exists() and candidate.is_file(): return candidate raise StorageBackendError("Stored object does not exist") def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: path = self._path(key) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) def get_bytes(self, key: str) -> bytes: return self._readable_path(key).read_bytes() def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: path = self._readable_path(key) with path.open("rb") as handle: while True: chunk = handle.read(chunk_size) if not chunk: break yield chunk def delete(self, key: str) -> None: path = self._path(key) if path.exists() and path.is_file(): path.unlink() def exists(self, key: str) -> bool: try: self._readable_path(key) except StorageBackendError: return False return True @dataclass(slots=True) class S3StorageBackend: bucket: str endpoint_url: str region_name: str access_key_id: str secret_access_key: str name: str = "s3" @property def client(self): try: import boto3 except ModuleNotFoundError as exc: raise StorageBackendError("boto3 is required for the S3 storage backend") from exc try: endpoint_url = validate_unpinned_sdk_http_url( self.endpoint_url, label="File storage S3 endpoint", ) return boto3.client( "s3", endpoint_url=endpoint_url, region_name=self.region_name, aws_access_key_id=self.access_key_id, aws_secret_access_key=self.secret_access_key, ) except OutboundHttpError as exc: raise StorageBackendError(str(exc)) from exc def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: max_bytes = response_limit("file") if len(data) > max_bytes: raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes") kwargs = {"Bucket": self.bucket, "Key": key, "Body": data} if content_type: kwargs["ContentType"] = content_type self.client.put_object(**kwargs) def get_bytes(self, key: str) -> bytes: try: obj = self.client.get_object(Bucket=self.bucket, Key=key) max_bytes = response_limit("file") body = obj["Body"] try: _reject_declared_object_size(obj, max_bytes=max_bytes) data = body.read(max_bytes + 1) if len(data) > max_bytes: raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes") return data finally: if hasattr(body, "close"): body.close() except Exception as exc: # pragma: no cover - depends on S3 backend raise StorageBackendError(str(exc)) from exc def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: try: obj = self.client.get_object(Bucket=self.bucket, Key=key) max_bytes = response_limit("file") body = obj["Body"] try: _reject_declared_object_size(obj, max_bytes=max_bytes) total = 0 while True: chunk = body.read(chunk_size) if not chunk: break total += len(chunk) if total > max_bytes: raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes") yield chunk finally: if hasattr(body, "close"): body.close() except Exception as exc: # pragma: no cover - depends on S3 backend raise StorageBackendError(str(exc)) from exc def delete(self, key: str) -> None: self.client.delete_object(Bucket=self.bucket, Key=key) def exists(self, key: str) -> bool: try: self.client.head_object(Bucket=self.bucket, Key=key) return True except Exception: return False def _reject_declared_object_size(obj: object, *, max_bytes: int) -> None: if not isinstance(obj, dict): return try: declared_size = int(obj.get("ContentLength")) except (TypeError, ValueError): return if declared_size > max_bytes: raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes") def _fallback_roots() -> tuple[Path, ...]: raw = getattr(settings, "file_storage_local_fallback_roots", "") or "" return tuple(Path(item.strip()) for item in str(raw).split(",") if item.strip()) def get_storage_backend() -> StorageBackend: backend = settings.file_storage_backend.lower().strip() if backend in {"local", "filesystem", "fs"}: return LocalFilesystemStorageBackend(Path(settings.file_storage_local_root), fallback_roots=_fallback_roots()) if backend in {"s3", "garage"}: return S3StorageBackend( bucket=settings.file_storage_s3_bucket or settings.s3_bucket, endpoint_url=settings.file_storage_s3_endpoint_url or settings.s3_endpoint_url, region_name=settings.file_storage_s3_region or settings.s3_region, access_key_id=settings.file_storage_s3_access_key_id or settings.s3_access_key_id, secret_access_key=settings.file_storage_s3_secret_access_key or settings.s3_secret_access_key, ) raise StorageBackendError(f"Unsupported file storage backend: {settings.file_storage_backend}")