From 233ce409838b6f42ae04a83591ebf8262026ff65 Mon Sep 17 00:00:00 2001 From: Albrecht Degering Date: Sat, 1 Aug 2026 17:48:32 +0200 Subject: [PATCH] feat: declare governed external provider state --- AGENTS.md | 6 + README.md | 22 +- docs/FILES_HANDBOOK.md | 44 ++- pyproject.toml | 2 +- src/govoplan_files/backend/manifest.py | 130 +++++++ src/govoplan_files/backend/provider_state.py | 133 +++++++ .../backend/storage/backends.py | 362 ++---------------- tests/test_provider_state.py | 97 +++++ 8 files changed, 430 insertions(+), 366 deletions(-) create mode 100644 src/govoplan_files/backend/provider_state.py create mode 100644 tests/test_provider_state.py diff --git a/AGENTS.md b/AGENTS.md index c885548..aa9da7c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,11 @@ # GovOPlaN Files Codex Guide +## Documentation Contract + +- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior. +- Keep feature content here; `govoplan-docs` projects it without importing Files internals. +- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes. + ## Scope This repository owns the `files` module: managed file storage APIs, file metadata, shares, uploads/downloads, folder and pattern helpers, backend module manifest, and `@govoplan/files-webui`. diff --git a/README.md b/README.md index ff471e4..fae344c 100644 --- a/README.md +++ b/README.md @@ -119,14 +119,20 @@ access fails closed in both public-only and private-network deployments. It will remain disabled until every initial connection and referral target can be policy-validated and pinned. -S3 browse/import and arbitrary external S3 managed-storage endpoints currently -fail closed before creating a `boto3` client. Botocore does not yet use the -GovOPlaN pinned HTTP transport and may manage redirects itself. The only -exception is the deployment-owned Garage service installed at the exact -`http://garage:3900` endpoint with -`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`; this flag cannot authorize another -host. External endpoints remain disabled until peer pinning and redirect -revalidation are enforced. +S3 connector browse/import remains fail-closed because a user-governed connector +endpoint must use the GovOPlaN pinned HTTP trust model. Durable platform storage +has a separate deployment boundary: installer-owned Garage is accepted only at +its exact generated endpoint, while an operator-selected external backend +requires a clean HTTPS origin and +`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. That flag is deployment configuration, +cannot be supplied through a Files connector profile, and does not replace +operator responsibility for DNS, certificates, egress, bucket policy, +versioning, and recovery. + +The actual local/S3 backend implementation is owned by Core so Files, Campaign, +and workers resolve the same object namespace. Files owns file metadata and key +layout. Node-local storage is supported only for `local` or one-host +`host-shared` profiles; multi-host `shared` deployments require S3. Destructive Files-module retirement applies the same credential lifecycle before dropping tables. Every remaining Files-owned encrypted connector secret is diff --git a/docs/FILES_HANDBOOK.md b/docs/FILES_HANDBOOK.md index 052dea1..305afca 100644 --- a/docs/FILES_HANDBOOK.md +++ b/docs/FILES_HANDBOOK.md @@ -408,6 +408,8 @@ the table above says live access is disabled. | `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Comma-separated older read-only roots checked after the primary root | | `FILE_STORAGE_S3_ENDPOINT_URL` and related `FILE_STORAGE_S3_*` values | deployment-specific | S3-compatible endpoint, region, credentials, and bucket | | `FILE_STORAGE_S3_DEPLOYMENT_MANAGED` | `false` | Installer-only trust marker for the exact `http://garage:3900` service; never use it for another endpoint | +| `FILE_STORAGE_S3_ENDPOINT_TRUSTED` | `false` | Deployment-owner acknowledgement for one clean HTTPS external S3 origin; never expose this through connector configuration | +| `GOVOPLAN_STATE_PROFILE` | `local` | Selects `local`, one-host `host-shared`, or multi-host `shared` state validation | | `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and extracted archive-member maximum | | `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | Compressed archive request maximum (legacy name retained for compatibility) | | `FILE_ARCHIVE_MAX_ENTRIES` | 10,000 | Maximum declared archive entries | @@ -421,20 +423,19 @@ under the configured root and rejects escape attempts. Fallback roots support a controlled storage-root migration: new writes go to the primary root while reads can still find older objects. -The S3 managed-storage adapter fails closed before creating a boto3 client for -arbitrary external endpoints because the SDK cannot yet guarantee -connection-time DNS/IP pinning and redirect revalidation. The supported -installer may provision a deployment-owned Garage service at the exact -`http://garage:3900` endpoint and set -`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`. Files accepts only that exact -service-discovery endpoint and forces path-style S3 addressing. The marker is -deployment authority, not a general private-network bypass. +The supported installer may provision a deployment-owned Garage service at the +exact `http://garage:3900` endpoint and set +`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`. An operator-selected external S3 +backend instead requires a clean HTTPS origin and +`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. Both are deployment authority, not a +general connector or private-network bypass. Core owns the shared backend +implementation; Files owns metadata and the Files key namespace. Multiple API replicas require the same durable blob namespace. Separate local -container filesystems will produce incomplete reads. Until a pinned shared -object-storage transport is available, use one durable shared mount or constrain -Files traffic to a deployment topology that preserves one consistent local -root. +container filesystems will produce incomplete reads. Use `host-shared` with one +durable shared mount only for same-host replicas. Independent hosts require the +`shared` profile with external S3, PostgreSQL, Redis, a stable installation id, +and one immutable module composition. ### Connector egress @@ -464,10 +465,12 @@ Override the connector response limits with `GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES`. The smaller applicable limit wins when an import is also subject to `FILE_UPLOAD_MAX_BYTES`. -Never work around a pinning failure by adding a raw IP, disabling TLS, or -enabling private networks. SMB may redirect through DFS, and S3 SDKs may perform -their own redirects or endpoint discovery; both remain disabled even for an IP -literal until every connection peer can be governed. +Never work around a connector pinning failure by adding a raw IP, disabling TLS, +or enabling private networks. SMB may redirect through DFS, and user-configured +S3 connectors may perform their own redirects or endpoint discovery; those +connector transports remain disabled until every connection peer can be +governed. The separately configured platform S3 backend is trusted only by the +deployment owner and is not selectable by a user or connector profile. ### Backup and restore @@ -477,6 +480,8 @@ include: - Files database rows, including asset/version/blob relationships, shares, connector settings, and campaign attachment-use evidence; - every object below `FILE_STORAGE_LOCAL_ROOT` and any still-used fallback root; + or the complete S3 bucket/prefix and version/lifecycle evidence for an S3 + backend; - the exact `MASTER_KEY_B64` needed to decrypt retained connector credentials; - deployment-owned connector profile files, referenced CA bundles, and secret environment configuration where those definitions are in use. @@ -779,7 +784,7 @@ returning different content or credentials. | Area | Implemented now | Planned or explicitly outside the current boundary | | --- | --- | --- | -| Managed storage | Local durable-root backend, fallback read roots, tenant blob deduplication, checksums, bounded resumable integrity scans, quarantine, and dry-run-first orphan cleanup | Operational S3 after pinned SDK transport ([#34](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/34)); scheduled scan execution | +| Managed storage | Core local/S3 backend, exact managed-Garage or explicitly trusted HTTPS external S3, state-profile validation, fallback local read roots, tenant blob deduplication, checksums, bounded resumable integrity scans, quarantine, and dry-run-first orphan cleanup | Scheduled scan execution and deployment-specific S3 HA/backup automation | | Upload | Bounded direct upload, drag-and-drop UI, archive preview/selective extraction, password-protected ZIP support, explicit conflicts | Malware scanning, quotas, type policy, resumable/chunked upload | | Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution | General file-history UI and user-driven append-version/restore | | Sharing | User/group/tenant/campaign grants, expiry, idempotent revocation, searchable share-management UI, and campaign linkage display | Richer policy-driven share lifecycles | @@ -808,8 +813,9 @@ Before releasing Files: 5. Exercise an allowed and denied owner/share path. 6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion. 7. Exercise connector policy explanation and one pinned HTTP provider where - configured; verify managed Garage if selected, and verify arbitrary external - S3 and SMB still fail closed. + configured; verify the deployment-managed or trusted external S3 backend if + selected, and verify user-configured S3 and SMB connector peers still fail + closed. 8. Verify credential deletion scrubs dependents and produces audit evidence. 9. Verify a campaign attachment snapshot still identifies its exact version and checksum after the current file changes. diff --git a/pyproject.toml b/pyproject.toml index 8c4b741..1faba13 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ requires-python = ">=3.12" license = { file = "LICENSE" } authors = [{ name = "GovOPlaN" }] dependencies = [ - "govoplan-core>=0.1.9", + "govoplan-core>=0.1.14", "defusedxml>=0.7,<1", "pyzipper>=0.3.6,<1", "python-multipart>=0.0.31,<1", diff --git a/src/govoplan_files/backend/manifest.py b/src/govoplan_files/backend/manifest.py index 9185a37..e5b58b2 100644 --- a/src/govoplan_files/backend/manifest.py +++ b/src/govoplan_files/backend/manifest.py @@ -24,11 +24,22 @@ from govoplan_core.core.modules import ( RoleTemplate, ) from govoplan_core.core.operations import OperationalCheckProviderRegistration +from govoplan_core.core.provider_governance import ( + ExternalProviderDeclaration, + ExternalProviderStateProviderRegistration, + ProviderBehaviorDeclaration, + ProviderObjectDeclaration, + declared_module_architecture, +) from govoplan_core.core.views import ViewSurface from govoplan_core.db.base import Base from govoplan_files.backend.change_tracking import register_files_change_tracking from govoplan_files.backend.db import models as file_models # noqa: F401 - populate Files ORM metadata from govoplan_files.backend.documentation import documentation_topics +from govoplan_files.backend.provider_state import ( + REMOTE_STORAGE_PROVIDER_ID, + remote_storage_provider_states, +) register_files_change_tracking() @@ -207,6 +218,58 @@ def _files_router(context: ModuleContext): return router +REMOTE_STORAGE_PROVIDER = ExternalProviderDeclaration( + id=REMOTE_STORAGE_PROVIDER_ID, + module_id="files", + label="Remote file storage mirror", + maturity="synchronize", + operations=("discover", "search", "read", "synchronize", "preview"), + objects=( + ProviderObjectDeclaration( + object_type="remote_folder", + field_groups=("identity", "hierarchy", "display", "source_metadata"), + authority_modes=("external_authoritative", "external_mirror"), + default_authority_mode="external_mirror", + ), + ProviderObjectDeclaration( + object_type="remote_file", + field_groups=("identity", "content", "version", "source_metadata"), + authority_modes=("external_authoritative", "external_mirror"), + default_authority_mode="external_mirror", + ), + ), + behavior=ProviderBehaviorDeclaration( + revision_tokens="Remote path, provider revision, size, and content digest are retained on managed imports.", + concurrency="A sync compares the frozen source reference and revision before creating a new managed version.", + freshness="Provider state distinguishes software availability from an unobserved live remote binding.", + health="Unsupported providers or missing optional transports fail closed; live health remains unknown without a probe.", + max_read_items=5000, + idempotency="Source profile, remote object reference, and revision/digest suppress duplicate managed versions.", + retry="Operators repeat bounded browse/import after a classified transport failure; effects are not blindly retried.", + timeout_seconds=30, + conflicts="Managed-file conflict policy is explicit; the current provider never mutates the remote source.", + outcome_unknown="An interrupted download is discarded unless its complete digest and managed version commit are confirmed.", + outcome_unknown_supported=True, + evidence="Managed versions retain connector profile, remote object identity, source revision, digest, and acquisition time.", + audit_event_types=( + "files.connector.accessed", + "files.connector.imported", + "files.connector.synced", + ), + correction="A later acquisition creates a new managed version and preserves prior provenance.", + rollback="Remote reads require no remote rollback; incomplete local objects are reconciled as orphans.", + compensation="A wrongly imported managed version can be retired under Files policy without deleting the source.", + reconciliation="Re-read source metadata and digest, then compare the committed managed version and object-store inventory.", + outage="Previously imported managed versions remain available while remote spaces report unknown or stale state.", + classifications=("internal", "confidential", "personal"), + purposes=("governed file acquisition", "managed evidence snapshot"), + retention="Files retention applies to managed versions; external retention remains provider-owned.", + secret_handling="Credentials remain encrypted or deployment-owned and never appear in provider state or provenance.", + ), + documentation_topic_ids=("files.governed-connectors-and-provenance",), +) + + manifest = ModuleManifest( id="files", name="Files", @@ -682,6 +745,46 @@ manifest = ModuleManifest( ], }, ), + DocumentationTopic( + id="files.reference.shared-storage-profile", + title="Operate Files with shared object storage", + summary="Choose local, host-shared, or S3-backed storage consistently with the runtime topology.", + body="Core supplies the common local/S3 object-storage backend while Files owns file metadata and object-key semantics. Local storage is valid for one runtime process; a shared host volume supports same-host replicas; independent hosts require an explicitly trusted HTTPS S3-compatible endpoint. Restore PostgreSQL, objects, and the master key to one coordinated recovery point.", + layer="configured", + documentation_types=("admin",), + audience=("file_admin", "operator", "system_admin"), + order=54, + conditions=( + DocumentationCondition( + required_modules=("files",), + any_scopes=("files:file:admin", "system:settings:read"), + ), + ), + links=( + DocumentationLink(label="Files", href="/files", kind="runtime"), + DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"), + ), + related_modules=("ops", "campaigns"), + configuration_keys=( + "GOVOPLAN_STATE_PROFILE", + "GOVOPLAN_INSTALLATION_ID", + "FILE_STORAGE_BACKEND", + "FILE_STORAGE_LOCAL_ROOT", + "FILE_STORAGE_S3_ENDPOINT_URL", + "FILE_STORAGE_S3_ENDPOINT_TRUSTED", + "FILE_STORAGE_S3_DEPLOYMENT_MANAGED", + ), + metadata={ + "kind": "reference", + "route": "/ops", + "screen": "Storage and runtime posture", + "limitations": [ + "Installer-managed Garage is single-node unless an external multi-node cluster is operated separately.", + "The application does not create or verify production PostgreSQL/object/key backups.", + ], + "verification": "Run the Files storage round-trip check and a coordinated restore drill against the exact deployment topology.", + }, + ), ), documentation_providers=(documentation_topics,), migration_spec=MigrationSpec( @@ -722,6 +825,33 @@ manifest = ModuleManifest( cache_seconds=60, ), ), + external_providers=(REMOTE_STORAGE_PROVIDER,), + external_provider_state_providers=( + ExternalProviderStateProviderRegistration( + module_id="files", + provider_id=REMOTE_STORAGE_PROVIDER_ID, + provider=remote_storage_provider_states, + ), + ), + architecture=declared_module_architecture( + layer="content_records_evidence", + kind="domain", + maturity="vertical_slice", + documentation_ref="docs/FILES_HANDBOOK.md", + test_ref="tests/test_storage_backends.py", + known_limits=("Multi-node object-storage recovery evidence and every remote connector profile are not reference-ready.",), + supported_authority_modes=( + "native_authoritative", + "external_authoritative", + "external_mirror", + ), + owned_concepts=("file asset", "file version", "folder", "share", "connector space"), + non_owned_concepts=("record disposition", "campaign attachment rule", "external storage object"), + target_tested_providers=(REMOTE_STORAGE_PROVIDER_ID,), + recovery_docs=("docs/FILES_HANDBOOK.md",), + security_docs=("docs/CONNECTOR_BOUNDARY.md",), + operations_docs=("docs/FILES_HANDBOOK.md",), + ), ) diff --git a/src/govoplan_files/backend/provider_state.py b/src/govoplan_files/backend/provider_state.py new file mode 100644 index 0000000..95b07a9 --- /dev/null +++ b/src/govoplan_files/backend/provider_state.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +from collections import defaultdict +from datetime import UTC, datetime + +from sqlalchemy import or_, select +from sqlalchemy.orm import Session + +from govoplan_core.core.provider_governance import ( + ExternalProviderRuntimeState, + ExternalProviderStateContext, +) +from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace +from govoplan_files.backend.storage.connector_providers import ( + ConnectorProviderDescriptor, + connector_provider_descriptors, +) + + +REMOTE_STORAGE_PROVIDER_ID = "files.remote_storage" + + +def remote_storage_provider_states( + context: ExternalProviderStateContext, +) -> tuple[ExternalProviderRuntimeState, ...]: + if not isinstance(context.session, Session): + raise RuntimeError("Files provider state requires a database session.") + statement = select(FileConnectorProfile) + if context.tenant_id is not None: + statement = statement.where( + or_( + FileConnectorProfile.tenant_id.is_(None), + FileConnectorProfile.tenant_id == context.tenant_id, + ) + ) + profiles = tuple( + context.session.scalars( + statement.order_by( + FileConnectorProfile.tenant_id, + FileConnectorProfile.id, + ).limit(context.max_items + 1) + ) + ) + if not profiles: + return () + + profile_ids = tuple(item.id for item in profiles) + space_statement = select(FileConnectorSpace).where( + FileConnectorSpace.connector_profile_id.in_(profile_ids), + FileConnectorSpace.deleted_at.is_(None), + ) + if context.tenant_id is not None: + space_statement = space_statement.where( + FileConnectorSpace.tenant_id == context.tenant_id + ) + spaces_by_profile: dict[str, list[FileConnectorSpace]] = defaultdict(list) + for space in context.session.scalars(space_statement): + spaces_by_profile[space.connector_profile_id].append(space) + + descriptors = { + item.provider: item for item in connector_provider_descriptors() + } + observed_at = datetime.now(UTC) + return tuple( + _profile_state( + profile, + spaces=spaces_by_profile.get(profile.id, []), + descriptor=descriptors.get(profile.provider), + observed_at=observed_at, + ) + for profile in profiles + ) + + +def _profile_state( + profile: FileConnectorProfile, + *, + spaces: list[FileConnectorSpace], + descriptor: ConnectorProviderDescriptor | None, + observed_at: datetime, +) -> ExternalProviderRuntimeState: + active_spaces = tuple(item for item in spaces if item.is_active) + active = bool(profile.enabled) + implementation_ready = bool( + descriptor is not None and descriptor.implemented and descriptor.installed + ) + health = ( + "inactive" + if not active + else "error" + if not implementation_ready + else "unknown" + ) + recovery = ( + "not_applicable" + if not active + else "unsupported" + if not implementation_ready + else "attention" + ) + detail = ( + "Remote-storage connector profile is disabled." + if not active + else "The configured provider is not available in this runtime." + if not implementation_ready + else "Software support is available; no live remote health observation is retained." + ) + return ExternalProviderRuntimeState( + provider_id=REMOTE_STORAGE_PROVIDER_ID, + binding_ref=f"files:connector-profile:{profile.id}", + authority_mode="external_mirror", + observed_at=observed_at, + configured=True, + active=active, + health=health, + freshness="unknown" if active else "not_applicable", + conflict="not_applicable", + recovery=recovery, + detail=detail, + metrics={ + "provider": profile.provider, + "configured_spaces": len(spaces), + "active_spaces": len(active_spaces), + "write_requested_spaces": sum( + 1 for item in active_spaces if not item.read_only + ), + "software_implemented": bool(descriptor and descriptor.implemented), + "optional_dependency_available": bool(descriptor and descriptor.installed), + }, + ) + + +__all__ = ["REMOTE_STORAGE_PROVIDER_ID", "remote_storage_provider_states"] diff --git a/src/govoplan_files/backend/storage/backends.py b/src/govoplan_files/backend/storage/backends.py index cf599ea..7ea7c28 100644 --- a/src/govoplan_files/backend/storage/backends.py +++ b/src/govoplan_files/backend/storage/backends.py @@ -1,346 +1,32 @@ from __future__ import annotations -from dataclasses import dataclass, field -from pathlib import Path -from typing import Iterable, Protocol - -from govoplan_core.security.outbound_http import ( - OutboundHttpError, - response_limit, - validate_unpinned_sdk_http_url, +from govoplan_core.core.object_storage import ( + LocalFilesystemStorageBackend, + S3StorageBackend, + StorageBackend, + StorageBackendError, + StorageObjectInfo, + StorageObjectMissing, + StorageObjectPage, + configured_storage_backend, ) from govoplan_files.backend.runtime import settings -class StorageBackendError(RuntimeError): - pass - - -class StorageObjectMissing(StorageBackendError): - pass - - -@dataclass(frozen=True, slots=True) -class StorageObjectInfo: - key: str - size_bytes: int - - -@dataclass(frozen=True, slots=True) -class StorageObjectPage: - objects: tuple[StorageObjectInfo, ...] - next_cursor: str | None = None - - -class StorageBackend(Protocol): - name: str - - def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: ... - def get_bytes(self, key: str) -> bytes: ... - def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: ... - def delete(self, key: str) -> None: ... - def exists(self, key: str) -> bool: ... - def stat(self, key: str) -> StorageObjectInfo: ... - def list_objects( - self, - *, - prefix: str, - after: str | None = None, - limit: int = 500, - ) -> StorageObjectPage: ... - - -@dataclass(slots=True) -class LocalFilesystemStorageBackend: - root: Path - fallback_roots: tuple[Path, ...] = field(default_factory=tuple) - name: str = "local" - - def __post_init__(self) -> None: - self.root = self.root.expanduser().resolve() - self.fallback_roots = tuple(root.expanduser().resolve() for root in self.fallback_roots if root) - self.root.mkdir(parents=True, exist_ok=True) - - def _path_for_root(self, root: Path, key: str) -> Path: - path = (root / key).resolve() - if not path.is_relative_to(root): - raise StorageBackendError("Storage key escapes local storage root") - return path - - def _path(self, key: str) -> Path: - return self._path_for_root(self.root, key) - - def _readable_path(self, key: str) -> Path: - primary = self._path(key) - if primary.exists() and primary.is_file(): - return primary - for root in self.fallback_roots: - candidate = self._path_for_root(root, key) - if candidate.exists() and candidate.is_file(): - return candidate - raise StorageObjectMissing("Stored object does not exist") - - def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: - path = self._path(key) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(data) - - def get_bytes(self, key: str) -> bytes: - return self._readable_path(key).read_bytes() - - def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: - path = self._readable_path(key) - with path.open("rb") as handle: - while True: - chunk = handle.read(chunk_size) - if not chunk: - break - yield chunk - - def delete(self, key: str) -> None: - path = self._path(key) - if path.exists() and path.is_file(): - path.unlink() - - def exists(self, key: str) -> bool: - try: - self._readable_path(key) - except StorageBackendError: - return False - return True - - def stat(self, key: str) -> StorageObjectInfo: - path = self._readable_path(key) - return StorageObjectInfo(key=key, size_bytes=path.stat().st_size) - - def list_objects( - self, - *, - prefix: str, - after: str | None = None, - limit: int = 500, - ) -> StorageObjectPage: - normalized_limit = max(1, min(int(limit), 5000)) - candidates: list[StorageObjectInfo] = [] - for path in _iter_local_files(self.root): - key = path.relative_to(self.root).as_posix() - if not key.startswith(prefix) or (after is not None and key <= after): - continue - candidates.append(StorageObjectInfo(key=key, size_bytes=path.stat().st_size)) - if len(candidates) > normalized_limit: - break - has_more = len(candidates) > normalized_limit - page = tuple(candidates[:normalized_limit]) - return StorageObjectPage( - objects=page, - next_cursor=page[-1].key if has_more and page else None, - ) - - -@dataclass(slots=True) -class S3StorageBackend: - bucket: str - endpoint_url: str - region_name: str - access_key_id: str - secret_access_key: str - deployment_managed: bool = False - name: str = "s3" - - @property - def client(self): - if self.deployment_managed: - endpoint_url = _deployment_managed_garage_endpoint(self.endpoint_url) - else: - try: - endpoint_url = validate_unpinned_sdk_http_url( - self.endpoint_url, - label="File storage S3 endpoint", - ) - except OutboundHttpError as exc: - raise StorageBackendError(str(exc)) from exc - try: - import boto3 - from botocore.config import Config - except ModuleNotFoundError as exc: - raise StorageBackendError("boto3 is required for the S3 storage backend") from exc - options: dict[str, object] = { - "endpoint_url": endpoint_url, - "region_name": self.region_name, - "aws_access_key_id": self.access_key_id, - "aws_secret_access_key": self.secret_access_key, - } - if self.deployment_managed: - options["config"] = Config(s3={"addressing_style": "path"}) - return boto3.client("s3", **options) - - def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: - max_bytes = response_limit("file") - if len(data) > max_bytes: - raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes") - kwargs = {"Bucket": self.bucket, "Key": key, "Body": data} - if content_type: - kwargs["ContentType"] = content_type - self.client.put_object(**kwargs) - - def get_bytes(self, key: str) -> bytes: - try: - obj = self.client.get_object(Bucket=self.bucket, Key=key) - max_bytes = response_limit("file") - body = obj["Body"] - try: - _reject_declared_object_size(obj, max_bytes=max_bytes) - data = body.read(max_bytes + 1) - if len(data) > max_bytes: - raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes") - return data - finally: - if hasattr(body, "close"): - body.close() - except Exception as exc: # pragma: no cover - depends on S3 backend - raise StorageBackendError(str(exc)) from exc - - def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: - try: - obj = self.client.get_object(Bucket=self.bucket, Key=key) - max_bytes = response_limit("file") - body = obj["Body"] - try: - _reject_declared_object_size(obj, max_bytes=max_bytes) - total = 0 - while True: - chunk = body.read(chunk_size) - if not chunk: - break - total += len(chunk) - if total > max_bytes: - raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes") - yield chunk - finally: - if hasattr(body, "close"): - body.close() - except Exception as exc: # pragma: no cover - depends on S3 backend - raise StorageBackendError(str(exc)) from exc - - def delete(self, key: str) -> None: - self.client.delete_object(Bucket=self.bucket, Key=key) - - def exists(self, key: str) -> bool: - try: - self.client.head_object(Bucket=self.bucket, Key=key) - return True - except Exception: - return False - - def stat(self, key: str) -> StorageObjectInfo: - try: - response = self.client.head_object(Bucket=self.bucket, Key=key) - except Exception as exc: # pragma: no cover - depends on S3 backend - if _s3_missing_error(exc): - raise StorageObjectMissing("Stored object does not exist") from exc - raise StorageBackendError(str(exc)) from exc - try: - size = int(response.get("ContentLength")) - except (AttributeError, TypeError, ValueError) as exc: - raise StorageBackendError("S3 object metadata did not include a valid size") from exc - return StorageObjectInfo(key=key, size_bytes=size) - - def list_objects( - self, - *, - prefix: str, - after: str | None = None, - limit: int = 500, - ) -> StorageObjectPage: - normalized_limit = max(1, min(int(limit), 1000)) - kwargs: dict[str, object] = { - "Bucket": self.bucket, - "Prefix": prefix, - "MaxKeys": normalized_limit, - } - if after: - kwargs["StartAfter"] = after - try: - response = self.client.list_objects_v2(**kwargs) - except Exception as exc: # pragma: no cover - depends on S3 backend - raise StorageBackendError(str(exc)) from exc - objects = tuple( - StorageObjectInfo( - key=str(item["Key"]), - size_bytes=int(item.get("Size") or 0), - ) - for item in response.get("Contents", ()) - if isinstance(item, dict) and item.get("Key") - ) - has_more = bool(response.get("IsTruncated")) - return StorageObjectPage( - objects=objects, - next_cursor=objects[-1].key if has_more and objects else None, - ) - - -def _reject_declared_object_size(obj: object, *, max_bytes: int) -> None: - if not isinstance(obj, dict): - return - try: - declared_size = int(obj.get("ContentLength")) - except (TypeError, ValueError): - return - if declared_size > max_bytes: - raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes") - - -def _iter_local_files(root: Path): - for entry in sorted(root.iterdir(), key=lambda item: item.name): - if entry.is_dir(): - yield from _iter_local_files(entry) - elif entry.is_file(): - yield entry - - -def _s3_missing_error(exc: Exception) -> bool: - response = getattr(exc, "response", None) - if not isinstance(response, dict): - return False - error = response.get("Error") - metadata = response.get("ResponseMetadata") - code = str(error.get("Code") if isinstance(error, dict) else "") - status_code = metadata.get("HTTPStatusCode") if isinstance(metadata, dict) else None - return code in {"404", "NoSuchKey", "NotFound"} or status_code == 404 - - -def _deployment_managed_garage_endpoint(value: str) -> str: - endpoint = str(value or "").strip() - if endpoint != "http://garage:3900": - raise StorageBackendError( - "Deployment-managed S3 trust is restricted to http://garage:3900" - ) - return endpoint - - -def _fallback_roots() -> tuple[Path, ...]: - raw = getattr(settings, "file_storage_local_fallback_roots", "") or "" - return tuple(Path(item.strip()) for item in str(raw).split(",") if item.strip()) - - def get_storage_backend() -> StorageBackend: - backend = settings.file_storage_backend.lower().strip() - if backend in {"local", "filesystem", "fs"}: - return LocalFilesystemStorageBackend(Path(settings.file_storage_local_root), fallback_roots=_fallback_roots()) - if backend in {"s3", "garage"}: - return S3StorageBackend( - bucket=settings.file_storage_s3_bucket or settings.s3_bucket, - endpoint_url=settings.file_storage_s3_endpoint_url or settings.s3_endpoint_url, - region_name=settings.file_storage_s3_region or settings.s3_region, - access_key_id=settings.file_storage_s3_access_key_id or settings.s3_access_key_id, - secret_access_key=settings.file_storage_s3_secret_access_key or settings.s3_secret_access_key, - deployment_managed=bool( - getattr( - settings, - "file_storage_s3_deployment_managed", - False, - ) - ), - ) - raise StorageBackendError(f"Unsupported file storage backend: {settings.file_storage_backend}") + """Return the deployment-owned backend for Files-managed objects.""" + + return configured_storage_backend(settings) + + +__all__ = [ + "LocalFilesystemStorageBackend", + "S3StorageBackend", + "StorageBackend", + "StorageBackendError", + "StorageObjectInfo", + "StorageObjectMissing", + "StorageObjectPage", + "get_storage_backend", +] diff --git a/tests/test_provider_state.py b/tests/test_provider_state.py new file mode 100644 index 0000000..4f2eaa2 --- /dev/null +++ b/tests/test_provider_state.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import unittest + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from govoplan_access.backend.db import models as access_models # noqa: F401 +from govoplan_core.core.provider_governance import ExternalProviderStateContext +from govoplan_core.db.base import Base +from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace +from govoplan_files.backend.manifest import manifest +from govoplan_files.backend.provider_state import ( + REMOTE_STORAGE_PROVIDER_ID, + remote_storage_provider_states, +) + + +class FilesProviderStateTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True) + Base.metadata.create_all( + self.engine, + tables=(FileConnectorProfile.__table__, FileConnectorSpace.__table__), + ) + self.session = sessionmaker(bind=self.engine, expire_on_commit=False)() + self.profile = FileConnectorProfile( + id="profile-1", + tenant_id="tenant-1", + scope_type="tenant", + scope_id="tenant-1", + label="Remote WebDAV", + provider="webdav", + endpoint_url="https://files.example.test/dav/", + enabled=True, + ) + self.session.add_all( + ( + self.profile, + FileConnectorSpace( + id="space-1", + tenant_id="tenant-1", + owner_type="tenant", + label="Documents", + connector_profile_id=self.profile.id, + provider="webdav", + remote_path="/documents", + read_only=True, + is_active=True, + ), + ) + ) + self.session.commit() + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def test_state_does_not_overstate_live_health_or_expose_endpoint(self) -> None: + state = remote_storage_provider_states( + ExternalProviderStateContext(session=self.session, tenant_id="tenant-1") + )[0] + + self.assertEqual("external_mirror", state.authority_mode) + self.assertEqual("unknown", state.health) + self.assertEqual("attention", state.recovery) + self.assertEqual(1, state.metrics["active_spaces"]) + self.assertNotIn("files.example.test", str(state.to_dict())) + self.assertEqual( + (), + remote_storage_provider_states( + ExternalProviderStateContext( + session=self.session, + tenant_id="tenant-2", + ) + ), + ) + + def test_unimplemented_profile_is_fail_closed_and_manifest_registered(self) -> None: + self.profile.provider = "sharepoint" + self.session.flush() + + state = remote_storage_provider_states( + ExternalProviderStateContext(session=self.session, tenant_id="tenant-1") + )[0] + + self.assertEqual("error", state.health) + self.assertEqual("unsupported", state.recovery) + self.assertEqual(REMOTE_STORAGE_PROVIDER_ID, manifest.external_providers[0].id) + self.assertEqual( + REMOTE_STORAGE_PROVIDER_ID, + manifest.external_provider_state_providers[0].provider_id, + ) + + +if __name__ == "__main__": + unittest.main()