feat: declare governed external provider state
This commit is contained in:
@@ -24,11 +24,22 @@ from govoplan_core.core.modules import (
|
||||
RoleTemplate,
|
||||
)
|
||||
from govoplan_core.core.operations import OperationalCheckProviderRegistration
|
||||
from govoplan_core.core.provider_governance import (
|
||||
ExternalProviderDeclaration,
|
||||
ExternalProviderStateProviderRegistration,
|
||||
ProviderBehaviorDeclaration,
|
||||
ProviderObjectDeclaration,
|
||||
declared_module_architecture,
|
||||
)
|
||||
from govoplan_core.core.views import ViewSurface
|
||||
from govoplan_core.db.base import Base
|
||||
from govoplan_files.backend.change_tracking import register_files_change_tracking
|
||||
from govoplan_files.backend.db import models as file_models # noqa: F401 - populate Files ORM metadata
|
||||
from govoplan_files.backend.documentation import documentation_topics
|
||||
from govoplan_files.backend.provider_state import (
|
||||
REMOTE_STORAGE_PROVIDER_ID,
|
||||
remote_storage_provider_states,
|
||||
)
|
||||
|
||||
register_files_change_tracking()
|
||||
|
||||
@@ -207,6 +218,58 @@ def _files_router(context: ModuleContext):
|
||||
return router
|
||||
|
||||
|
||||
REMOTE_STORAGE_PROVIDER = ExternalProviderDeclaration(
|
||||
id=REMOTE_STORAGE_PROVIDER_ID,
|
||||
module_id="files",
|
||||
label="Remote file storage mirror",
|
||||
maturity="synchronize",
|
||||
operations=("discover", "search", "read", "synchronize", "preview"),
|
||||
objects=(
|
||||
ProviderObjectDeclaration(
|
||||
object_type="remote_folder",
|
||||
field_groups=("identity", "hierarchy", "display", "source_metadata"),
|
||||
authority_modes=("external_authoritative", "external_mirror"),
|
||||
default_authority_mode="external_mirror",
|
||||
),
|
||||
ProviderObjectDeclaration(
|
||||
object_type="remote_file",
|
||||
field_groups=("identity", "content", "version", "source_metadata"),
|
||||
authority_modes=("external_authoritative", "external_mirror"),
|
||||
default_authority_mode="external_mirror",
|
||||
),
|
||||
),
|
||||
behavior=ProviderBehaviorDeclaration(
|
||||
revision_tokens="Remote path, provider revision, size, and content digest are retained on managed imports.",
|
||||
concurrency="A sync compares the frozen source reference and revision before creating a new managed version.",
|
||||
freshness="Provider state distinguishes software availability from an unobserved live remote binding.",
|
||||
health="Unsupported providers or missing optional transports fail closed; live health remains unknown without a probe.",
|
||||
max_read_items=5000,
|
||||
idempotency="Source profile, remote object reference, and revision/digest suppress duplicate managed versions.",
|
||||
retry="Operators repeat bounded browse/import after a classified transport failure; effects are not blindly retried.",
|
||||
timeout_seconds=30,
|
||||
conflicts="Managed-file conflict policy is explicit; the current provider never mutates the remote source.",
|
||||
outcome_unknown="An interrupted download is discarded unless its complete digest and managed version commit are confirmed.",
|
||||
outcome_unknown_supported=True,
|
||||
evidence="Managed versions retain connector profile, remote object identity, source revision, digest, and acquisition time.",
|
||||
audit_event_types=(
|
||||
"files.connector.accessed",
|
||||
"files.connector.imported",
|
||||
"files.connector.synced",
|
||||
),
|
||||
correction="A later acquisition creates a new managed version and preserves prior provenance.",
|
||||
rollback="Remote reads require no remote rollback; incomplete local objects are reconciled as orphans.",
|
||||
compensation="A wrongly imported managed version can be retired under Files policy without deleting the source.",
|
||||
reconciliation="Re-read source metadata and digest, then compare the committed managed version and object-store inventory.",
|
||||
outage="Previously imported managed versions remain available while remote spaces report unknown or stale state.",
|
||||
classifications=("internal", "confidential", "personal"),
|
||||
purposes=("governed file acquisition", "managed evidence snapshot"),
|
||||
retention="Files retention applies to managed versions; external retention remains provider-owned.",
|
||||
secret_handling="Credentials remain encrypted or deployment-owned and never appear in provider state or provenance.",
|
||||
),
|
||||
documentation_topic_ids=("files.governed-connectors-and-provenance",),
|
||||
)
|
||||
|
||||
|
||||
manifest = ModuleManifest(
|
||||
id="files",
|
||||
name="Files",
|
||||
@@ -682,6 +745,46 @@ manifest = ModuleManifest(
|
||||
],
|
||||
},
|
||||
),
|
||||
DocumentationTopic(
|
||||
id="files.reference.shared-storage-profile",
|
||||
title="Operate Files with shared object storage",
|
||||
summary="Choose local, host-shared, or S3-backed storage consistently with the runtime topology.",
|
||||
body="Core supplies the common local/S3 object-storage backend while Files owns file metadata and object-key semantics. Local storage is valid for one runtime process; a shared host volume supports same-host replicas; independent hosts require an explicitly trusted HTTPS S3-compatible endpoint. Restore PostgreSQL, objects, and the master key to one coordinated recovery point.",
|
||||
layer="configured",
|
||||
documentation_types=("admin",),
|
||||
audience=("file_admin", "operator", "system_admin"),
|
||||
order=54,
|
||||
conditions=(
|
||||
DocumentationCondition(
|
||||
required_modules=("files",),
|
||||
any_scopes=("files:file:admin", "system:settings:read"),
|
||||
),
|
||||
),
|
||||
links=(
|
||||
DocumentationLink(label="Files", href="/files", kind="runtime"),
|
||||
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
|
||||
),
|
||||
related_modules=("ops", "campaigns"),
|
||||
configuration_keys=(
|
||||
"GOVOPLAN_STATE_PROFILE",
|
||||
"GOVOPLAN_INSTALLATION_ID",
|
||||
"FILE_STORAGE_BACKEND",
|
||||
"FILE_STORAGE_LOCAL_ROOT",
|
||||
"FILE_STORAGE_S3_ENDPOINT_URL",
|
||||
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
|
||||
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
|
||||
),
|
||||
metadata={
|
||||
"kind": "reference",
|
||||
"route": "/ops",
|
||||
"screen": "Storage and runtime posture",
|
||||
"limitations": [
|
||||
"Installer-managed Garage is single-node unless an external multi-node cluster is operated separately.",
|
||||
"The application does not create or verify production PostgreSQL/object/key backups.",
|
||||
],
|
||||
"verification": "Run the Files storage round-trip check and a coordinated restore drill against the exact deployment topology.",
|
||||
},
|
||||
),
|
||||
),
|
||||
documentation_providers=(documentation_topics,),
|
||||
migration_spec=MigrationSpec(
|
||||
@@ -722,6 +825,33 @@ manifest = ModuleManifest(
|
||||
cache_seconds=60,
|
||||
),
|
||||
),
|
||||
external_providers=(REMOTE_STORAGE_PROVIDER,),
|
||||
external_provider_state_providers=(
|
||||
ExternalProviderStateProviderRegistration(
|
||||
module_id="files",
|
||||
provider_id=REMOTE_STORAGE_PROVIDER_ID,
|
||||
provider=remote_storage_provider_states,
|
||||
),
|
||||
),
|
||||
architecture=declared_module_architecture(
|
||||
layer="content_records_evidence",
|
||||
kind="domain",
|
||||
maturity="vertical_slice",
|
||||
documentation_ref="docs/FILES_HANDBOOK.md",
|
||||
test_ref="tests/test_storage_backends.py",
|
||||
known_limits=("Multi-node object-storage recovery evidence and every remote connector profile are not reference-ready.",),
|
||||
supported_authority_modes=(
|
||||
"native_authoritative",
|
||||
"external_authoritative",
|
||||
"external_mirror",
|
||||
),
|
||||
owned_concepts=("file asset", "file version", "folder", "share", "connector space"),
|
||||
non_owned_concepts=("record disposition", "campaign attachment rule", "external storage object"),
|
||||
target_tested_providers=(REMOTE_STORAGE_PROVIDER_ID,),
|
||||
recovery_docs=("docs/FILES_HANDBOOK.md",),
|
||||
security_docs=("docs/CONNECTOR_BOUNDARY.md",),
|
||||
operations_docs=("docs/FILES_HANDBOOK.md",),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.provider_governance import (
|
||||
ExternalProviderRuntimeState,
|
||||
ExternalProviderStateContext,
|
||||
)
|
||||
from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace
|
||||
from govoplan_files.backend.storage.connector_providers import (
|
||||
ConnectorProviderDescriptor,
|
||||
connector_provider_descriptors,
|
||||
)
|
||||
|
||||
|
||||
REMOTE_STORAGE_PROVIDER_ID = "files.remote_storage"
|
||||
|
||||
|
||||
def remote_storage_provider_states(
|
||||
context: ExternalProviderStateContext,
|
||||
) -> tuple[ExternalProviderRuntimeState, ...]:
|
||||
if not isinstance(context.session, Session):
|
||||
raise RuntimeError("Files provider state requires a database session.")
|
||||
statement = select(FileConnectorProfile)
|
||||
if context.tenant_id is not None:
|
||||
statement = statement.where(
|
||||
or_(
|
||||
FileConnectorProfile.tenant_id.is_(None),
|
||||
FileConnectorProfile.tenant_id == context.tenant_id,
|
||||
)
|
||||
)
|
||||
profiles = tuple(
|
||||
context.session.scalars(
|
||||
statement.order_by(
|
||||
FileConnectorProfile.tenant_id,
|
||||
FileConnectorProfile.id,
|
||||
).limit(context.max_items + 1)
|
||||
)
|
||||
)
|
||||
if not profiles:
|
||||
return ()
|
||||
|
||||
profile_ids = tuple(item.id for item in profiles)
|
||||
space_statement = select(FileConnectorSpace).where(
|
||||
FileConnectorSpace.connector_profile_id.in_(profile_ids),
|
||||
FileConnectorSpace.deleted_at.is_(None),
|
||||
)
|
||||
if context.tenant_id is not None:
|
||||
space_statement = space_statement.where(
|
||||
FileConnectorSpace.tenant_id == context.tenant_id
|
||||
)
|
||||
spaces_by_profile: dict[str, list[FileConnectorSpace]] = defaultdict(list)
|
||||
for space in context.session.scalars(space_statement):
|
||||
spaces_by_profile[space.connector_profile_id].append(space)
|
||||
|
||||
descriptors = {
|
||||
item.provider: item for item in connector_provider_descriptors()
|
||||
}
|
||||
observed_at = datetime.now(UTC)
|
||||
return tuple(
|
||||
_profile_state(
|
||||
profile,
|
||||
spaces=spaces_by_profile.get(profile.id, []),
|
||||
descriptor=descriptors.get(profile.provider),
|
||||
observed_at=observed_at,
|
||||
)
|
||||
for profile in profiles
|
||||
)
|
||||
|
||||
|
||||
def _profile_state(
|
||||
profile: FileConnectorProfile,
|
||||
*,
|
||||
spaces: list[FileConnectorSpace],
|
||||
descriptor: ConnectorProviderDescriptor | None,
|
||||
observed_at: datetime,
|
||||
) -> ExternalProviderRuntimeState:
|
||||
active_spaces = tuple(item for item in spaces if item.is_active)
|
||||
active = bool(profile.enabled)
|
||||
implementation_ready = bool(
|
||||
descriptor is not None and descriptor.implemented and descriptor.installed
|
||||
)
|
||||
health = (
|
||||
"inactive"
|
||||
if not active
|
||||
else "error"
|
||||
if not implementation_ready
|
||||
else "unknown"
|
||||
)
|
||||
recovery = (
|
||||
"not_applicable"
|
||||
if not active
|
||||
else "unsupported"
|
||||
if not implementation_ready
|
||||
else "attention"
|
||||
)
|
||||
detail = (
|
||||
"Remote-storage connector profile is disabled."
|
||||
if not active
|
||||
else "The configured provider is not available in this runtime."
|
||||
if not implementation_ready
|
||||
else "Software support is available; no live remote health observation is retained."
|
||||
)
|
||||
return ExternalProviderRuntimeState(
|
||||
provider_id=REMOTE_STORAGE_PROVIDER_ID,
|
||||
binding_ref=f"files:connector-profile:{profile.id}",
|
||||
authority_mode="external_mirror",
|
||||
observed_at=observed_at,
|
||||
configured=True,
|
||||
active=active,
|
||||
health=health,
|
||||
freshness="unknown" if active else "not_applicable",
|
||||
conflict="not_applicable",
|
||||
recovery=recovery,
|
||||
detail=detail,
|
||||
metrics={
|
||||
"provider": profile.provider,
|
||||
"configured_spaces": len(spaces),
|
||||
"active_spaces": len(active_spaces),
|
||||
"write_requested_spaces": sum(
|
||||
1 for item in active_spaces if not item.read_only
|
||||
),
|
||||
"software_implemented": bool(descriptor and descriptor.implemented),
|
||||
"optional_dependency_available": bool(descriptor and descriptor.installed),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["REMOTE_STORAGE_PROVIDER_ID", "remote_storage_provider_states"]
|
||||
@@ -1,346 +1,32 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Iterable, Protocol
|
||||
|
||||
from govoplan_core.security.outbound_http import (
|
||||
OutboundHttpError,
|
||||
response_limit,
|
||||
validate_unpinned_sdk_http_url,
|
||||
from govoplan_core.core.object_storage import (
|
||||
LocalFilesystemStorageBackend,
|
||||
S3StorageBackend,
|
||||
StorageBackend,
|
||||
StorageBackendError,
|
||||
StorageObjectInfo,
|
||||
StorageObjectMissing,
|
||||
StorageObjectPage,
|
||||
configured_storage_backend,
|
||||
)
|
||||
|
||||
from govoplan_files.backend.runtime import settings
|
||||
|
||||
|
||||
class StorageBackendError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class StorageObjectMissing(StorageBackendError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StorageObjectInfo:
|
||||
key: str
|
||||
size_bytes: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StorageObjectPage:
|
||||
objects: tuple[StorageObjectInfo, ...]
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class StorageBackend(Protocol):
|
||||
name: str
|
||||
|
||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: ...
|
||||
def get_bytes(self, key: str) -> bytes: ...
|
||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: ...
|
||||
def delete(self, key: str) -> None: ...
|
||||
def exists(self, key: str) -> bool: ...
|
||||
def stat(self, key: str) -> StorageObjectInfo: ...
|
||||
def list_objects(
|
||||
self,
|
||||
*,
|
||||
prefix: str,
|
||||
after: str | None = None,
|
||||
limit: int = 500,
|
||||
) -> StorageObjectPage: ...
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class LocalFilesystemStorageBackend:
|
||||
root: Path
|
||||
fallback_roots: tuple[Path, ...] = field(default_factory=tuple)
|
||||
name: str = "local"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self.root = self.root.expanduser().resolve()
|
||||
self.fallback_roots = tuple(root.expanduser().resolve() for root in self.fallback_roots if root)
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _path_for_root(self, root: Path, key: str) -> Path:
|
||||
path = (root / key).resolve()
|
||||
if not path.is_relative_to(root):
|
||||
raise StorageBackendError("Storage key escapes local storage root")
|
||||
return path
|
||||
|
||||
def _path(self, key: str) -> Path:
|
||||
return self._path_for_root(self.root, key)
|
||||
|
||||
def _readable_path(self, key: str) -> Path:
|
||||
primary = self._path(key)
|
||||
if primary.exists() and primary.is_file():
|
||||
return primary
|
||||
for root in self.fallback_roots:
|
||||
candidate = self._path_for_root(root, key)
|
||||
if candidate.exists() and candidate.is_file():
|
||||
return candidate
|
||||
raise StorageObjectMissing("Stored object does not exist")
|
||||
|
||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
|
||||
path = self._path(key)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
|
||||
def get_bytes(self, key: str) -> bytes:
|
||||
return self._readable_path(key).read_bytes()
|
||||
|
||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]:
|
||||
path = self._readable_path(key)
|
||||
with path.open("rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
yield chunk
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
path = self._path(key)
|
||||
if path.exists() and path.is_file():
|
||||
path.unlink()
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
try:
|
||||
self._readable_path(key)
|
||||
except StorageBackendError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def stat(self, key: str) -> StorageObjectInfo:
|
||||
path = self._readable_path(key)
|
||||
return StorageObjectInfo(key=key, size_bytes=path.stat().st_size)
|
||||
|
||||
def list_objects(
|
||||
self,
|
||||
*,
|
||||
prefix: str,
|
||||
after: str | None = None,
|
||||
limit: int = 500,
|
||||
) -> StorageObjectPage:
|
||||
normalized_limit = max(1, min(int(limit), 5000))
|
||||
candidates: list[StorageObjectInfo] = []
|
||||
for path in _iter_local_files(self.root):
|
||||
key = path.relative_to(self.root).as_posix()
|
||||
if not key.startswith(prefix) or (after is not None and key <= after):
|
||||
continue
|
||||
candidates.append(StorageObjectInfo(key=key, size_bytes=path.stat().st_size))
|
||||
if len(candidates) > normalized_limit:
|
||||
break
|
||||
has_more = len(candidates) > normalized_limit
|
||||
page = tuple(candidates[:normalized_limit])
|
||||
return StorageObjectPage(
|
||||
objects=page,
|
||||
next_cursor=page[-1].key if has_more and page else None,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class S3StorageBackend:
|
||||
bucket: str
|
||||
endpoint_url: str
|
||||
region_name: str
|
||||
access_key_id: str
|
||||
secret_access_key: str
|
||||
deployment_managed: bool = False
|
||||
name: str = "s3"
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
if self.deployment_managed:
|
||||
endpoint_url = _deployment_managed_garage_endpoint(self.endpoint_url)
|
||||
else:
|
||||
try:
|
||||
endpoint_url = validate_unpinned_sdk_http_url(
|
||||
self.endpoint_url,
|
||||
label="File storage S3 endpoint",
|
||||
)
|
||||
except OutboundHttpError as exc:
|
||||
raise StorageBackendError(str(exc)) from exc
|
||||
try:
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
except ModuleNotFoundError as exc:
|
||||
raise StorageBackendError("boto3 is required for the S3 storage backend") from exc
|
||||
options: dict[str, object] = {
|
||||
"endpoint_url": endpoint_url,
|
||||
"region_name": self.region_name,
|
||||
"aws_access_key_id": self.access_key_id,
|
||||
"aws_secret_access_key": self.secret_access_key,
|
||||
}
|
||||
if self.deployment_managed:
|
||||
options["config"] = Config(s3={"addressing_style": "path"})
|
||||
return boto3.client("s3", **options)
|
||||
|
||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
|
||||
max_bytes = response_limit("file")
|
||||
if len(data) > max_bytes:
|
||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
||||
kwargs = {"Bucket": self.bucket, "Key": key, "Body": data}
|
||||
if content_type:
|
||||
kwargs["ContentType"] = content_type
|
||||
self.client.put_object(**kwargs)
|
||||
|
||||
def get_bytes(self, key: str) -> bytes:
|
||||
try:
|
||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
max_bytes = response_limit("file")
|
||||
body = obj["Body"]
|
||||
try:
|
||||
_reject_declared_object_size(obj, max_bytes=max_bytes)
|
||||
data = body.read(max_bytes + 1)
|
||||
if len(data) > max_bytes:
|
||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
||||
return data
|
||||
finally:
|
||||
if hasattr(body, "close"):
|
||||
body.close()
|
||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
||||
raise StorageBackendError(str(exc)) from exc
|
||||
|
||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]:
|
||||
try:
|
||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
max_bytes = response_limit("file")
|
||||
body = obj["Body"]
|
||||
try:
|
||||
_reject_declared_object_size(obj, max_bytes=max_bytes)
|
||||
total = 0
|
||||
while True:
|
||||
chunk = body.read(chunk_size)
|
||||
if not chunk:
|
||||
break
|
||||
total += len(chunk)
|
||||
if total > max_bytes:
|
||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
||||
yield chunk
|
||||
finally:
|
||||
if hasattr(body, "close"):
|
||||
body.close()
|
||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
||||
raise StorageBackendError(str(exc)) from exc
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
try:
|
||||
self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def stat(self, key: str) -> StorageObjectInfo:
|
||||
try:
|
||||
response = self.client.head_object(Bucket=self.bucket, Key=key)
|
||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
||||
if _s3_missing_error(exc):
|
||||
raise StorageObjectMissing("Stored object does not exist") from exc
|
||||
raise StorageBackendError(str(exc)) from exc
|
||||
try:
|
||||
size = int(response.get("ContentLength"))
|
||||
except (AttributeError, TypeError, ValueError) as exc:
|
||||
raise StorageBackendError("S3 object metadata did not include a valid size") from exc
|
||||
return StorageObjectInfo(key=key, size_bytes=size)
|
||||
|
||||
def list_objects(
|
||||
self,
|
||||
*,
|
||||
prefix: str,
|
||||
after: str | None = None,
|
||||
limit: int = 500,
|
||||
) -> StorageObjectPage:
|
||||
normalized_limit = max(1, min(int(limit), 1000))
|
||||
kwargs: dict[str, object] = {
|
||||
"Bucket": self.bucket,
|
||||
"Prefix": prefix,
|
||||
"MaxKeys": normalized_limit,
|
||||
}
|
||||
if after:
|
||||
kwargs["StartAfter"] = after
|
||||
try:
|
||||
response = self.client.list_objects_v2(**kwargs)
|
||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
||||
raise StorageBackendError(str(exc)) from exc
|
||||
objects = tuple(
|
||||
StorageObjectInfo(
|
||||
key=str(item["Key"]),
|
||||
size_bytes=int(item.get("Size") or 0),
|
||||
)
|
||||
for item in response.get("Contents", ())
|
||||
if isinstance(item, dict) and item.get("Key")
|
||||
)
|
||||
has_more = bool(response.get("IsTruncated"))
|
||||
return StorageObjectPage(
|
||||
objects=objects,
|
||||
next_cursor=objects[-1].key if has_more and objects else None,
|
||||
)
|
||||
|
||||
|
||||
def _reject_declared_object_size(obj: object, *, max_bytes: int) -> None:
|
||||
if not isinstance(obj, dict):
|
||||
return
|
||||
try:
|
||||
declared_size = int(obj.get("ContentLength"))
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
if declared_size > max_bytes:
|
||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
||||
|
||||
|
||||
def _iter_local_files(root: Path):
|
||||
for entry in sorted(root.iterdir(), key=lambda item: item.name):
|
||||
if entry.is_dir():
|
||||
yield from _iter_local_files(entry)
|
||||
elif entry.is_file():
|
||||
yield entry
|
||||
|
||||
|
||||
def _s3_missing_error(exc: Exception) -> bool:
|
||||
response = getattr(exc, "response", None)
|
||||
if not isinstance(response, dict):
|
||||
return False
|
||||
error = response.get("Error")
|
||||
metadata = response.get("ResponseMetadata")
|
||||
code = str(error.get("Code") if isinstance(error, dict) else "")
|
||||
status_code = metadata.get("HTTPStatusCode") if isinstance(metadata, dict) else None
|
||||
return code in {"404", "NoSuchKey", "NotFound"} or status_code == 404
|
||||
|
||||
|
||||
def _deployment_managed_garage_endpoint(value: str) -> str:
|
||||
endpoint = str(value or "").strip()
|
||||
if endpoint != "http://garage:3900":
|
||||
raise StorageBackendError(
|
||||
"Deployment-managed S3 trust is restricted to http://garage:3900"
|
||||
)
|
||||
return endpoint
|
||||
|
||||
|
||||
def _fallback_roots() -> tuple[Path, ...]:
|
||||
raw = getattr(settings, "file_storage_local_fallback_roots", "") or ""
|
||||
return tuple(Path(item.strip()) for item in str(raw).split(",") if item.strip())
|
||||
|
||||
|
||||
def get_storage_backend() -> StorageBackend:
|
||||
backend = settings.file_storage_backend.lower().strip()
|
||||
if backend in {"local", "filesystem", "fs"}:
|
||||
return LocalFilesystemStorageBackend(Path(settings.file_storage_local_root), fallback_roots=_fallback_roots())
|
||||
if backend in {"s3", "garage"}:
|
||||
return S3StorageBackend(
|
||||
bucket=settings.file_storage_s3_bucket or settings.s3_bucket,
|
||||
endpoint_url=settings.file_storage_s3_endpoint_url or settings.s3_endpoint_url,
|
||||
region_name=settings.file_storage_s3_region or settings.s3_region,
|
||||
access_key_id=settings.file_storage_s3_access_key_id or settings.s3_access_key_id,
|
||||
secret_access_key=settings.file_storage_s3_secret_access_key or settings.s3_secret_access_key,
|
||||
deployment_managed=bool(
|
||||
getattr(
|
||||
settings,
|
||||
"file_storage_s3_deployment_managed",
|
||||
False,
|
||||
)
|
||||
),
|
||||
)
|
||||
raise StorageBackendError(f"Unsupported file storage backend: {settings.file_storage_backend}")
|
||||
"""Return the deployment-owned backend for Files-managed objects."""
|
||||
|
||||
return configured_storage_backend(settings)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"LocalFilesystemStorageBackend",
|
||||
"S3StorageBackend",
|
||||
"StorageBackend",
|
||||
"StorageBackendError",
|
||||
"StorageObjectInfo",
|
||||
"StorageObjectMissing",
|
||||
"StorageObjectPage",
|
||||
"get_storage_backend",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user