feat: declare governed external provider state
This commit is contained in:
@@ -1,5 +1,11 @@
|
|||||||
# GovOPlaN Files Codex Guide
|
# GovOPlaN Files Codex Guide
|
||||||
|
|
||||||
|
## Documentation Contract
|
||||||
|
|
||||||
|
- Treat documentation as part of every behavior change. Update this module's manifest-driven `DocumentationTopic` contributions for affected user and administrator behavior.
|
||||||
|
- Keep feature content here; `govoplan-docs` projects it without importing Files internals.
|
||||||
|
- Maintain a static user/admin baseline and run `/mnt/DATA/git/govoplan/tools/checks/check-manifest-shapes.py` after behavior or manifest changes.
|
||||||
|
|
||||||
## Scope
|
## Scope
|
||||||
|
|
||||||
This repository owns the `files` module: managed file storage APIs, file metadata, shares, uploads/downloads, folder and pattern helpers, backend module manifest, and `@govoplan/files-webui`.
|
This repository owns the `files` module: managed file storage APIs, file metadata, shares, uploads/downloads, folder and pattern helpers, backend module manifest, and `@govoplan/files-webui`.
|
||||||
|
|||||||
@@ -119,14 +119,20 @@ access fails closed in both public-only and private-network deployments. It will
|
|||||||
remain disabled until every initial connection and referral target can be
|
remain disabled until every initial connection and referral target can be
|
||||||
policy-validated and pinned.
|
policy-validated and pinned.
|
||||||
|
|
||||||
S3 browse/import and arbitrary external S3 managed-storage endpoints currently
|
S3 connector browse/import remains fail-closed because a user-governed connector
|
||||||
fail closed before creating a `boto3` client. Botocore does not yet use the
|
endpoint must use the GovOPlaN pinned HTTP trust model. Durable platform storage
|
||||||
GovOPlaN pinned HTTP transport and may manage redirects itself. The only
|
has a separate deployment boundary: installer-owned Garage is accepted only at
|
||||||
exception is the deployment-owned Garage service installed at the exact
|
its exact generated endpoint, while an operator-selected external backend
|
||||||
`http://garage:3900` endpoint with
|
requires a clean HTTPS origin and
|
||||||
`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`; this flag cannot authorize another
|
`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. That flag is deployment configuration,
|
||||||
host. External endpoints remain disabled until peer pinning and redirect
|
cannot be supplied through a Files connector profile, and does not replace
|
||||||
revalidation are enforced.
|
operator responsibility for DNS, certificates, egress, bucket policy,
|
||||||
|
versioning, and recovery.
|
||||||
|
|
||||||
|
The actual local/S3 backend implementation is owned by Core so Files, Campaign,
|
||||||
|
and workers resolve the same object namespace. Files owns file metadata and key
|
||||||
|
layout. Node-local storage is supported only for `local` or one-host
|
||||||
|
`host-shared` profiles; multi-host `shared` deployments require S3.
|
||||||
|
|
||||||
Destructive Files-module retirement applies the same credential lifecycle before
|
Destructive Files-module retirement applies the same credential lifecycle before
|
||||||
dropping tables. Every remaining Files-owned encrypted connector secret is
|
dropping tables. Every remaining Files-owned encrypted connector secret is
|
||||||
|
|||||||
+25
-19
@@ -408,6 +408,8 @@ the table above says live access is disabled.
|
|||||||
| `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Comma-separated older read-only roots checked after the primary root |
|
| `FILE_STORAGE_LOCAL_FALLBACK_ROOTS` | empty | Comma-separated older read-only roots checked after the primary root |
|
||||||
| `FILE_STORAGE_S3_ENDPOINT_URL` and related `FILE_STORAGE_S3_*` values | deployment-specific | S3-compatible endpoint, region, credentials, and bucket |
|
| `FILE_STORAGE_S3_ENDPOINT_URL` and related `FILE_STORAGE_S3_*` values | deployment-specific | S3-compatible endpoint, region, credentials, and bucket |
|
||||||
| `FILE_STORAGE_S3_DEPLOYMENT_MANAGED` | `false` | Installer-only trust marker for the exact `http://garage:3900` service; never use it for another endpoint |
|
| `FILE_STORAGE_S3_DEPLOYMENT_MANAGED` | `false` | Installer-only trust marker for the exact `http://garage:3900` service; never use it for another endpoint |
|
||||||
|
| `FILE_STORAGE_S3_ENDPOINT_TRUSTED` | `false` | Deployment-owner acknowledgement for one clean HTTPS external S3 origin; never expose this through connector configuration |
|
||||||
|
| `GOVOPLAN_STATE_PROFILE` | `local` | Selects `local`, one-host `host-shared`, or multi-host `shared` state validation |
|
||||||
| `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and extracted archive-member maximum |
|
| `FILE_UPLOAD_MAX_BYTES` | 50 MiB | Direct-upload and extracted archive-member maximum |
|
||||||
| `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | Compressed archive request maximum (legacy name retained for compatibility) |
|
| `FILE_UPLOAD_ZIP_MAX_BYTES` | 250 MiB | Compressed archive request maximum (legacy name retained for compatibility) |
|
||||||
| `FILE_ARCHIVE_MAX_ENTRIES` | 10,000 | Maximum declared archive entries |
|
| `FILE_ARCHIVE_MAX_ENTRIES` | 10,000 | Maximum declared archive entries |
|
||||||
@@ -421,20 +423,19 @@ under the configured root and rejects escape attempts. Fallback roots support a
|
|||||||
controlled storage-root migration: new writes go to the primary root while
|
controlled storage-root migration: new writes go to the primary root while
|
||||||
reads can still find older objects.
|
reads can still find older objects.
|
||||||
|
|
||||||
The S3 managed-storage adapter fails closed before creating a boto3 client for
|
The supported installer may provision a deployment-owned Garage service at the
|
||||||
arbitrary external endpoints because the SDK cannot yet guarantee
|
exact `http://garage:3900` endpoint and set
|
||||||
connection-time DNS/IP pinning and redirect revalidation. The supported
|
`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`. An operator-selected external S3
|
||||||
installer may provision a deployment-owned Garage service at the exact
|
backend instead requires a clean HTTPS origin and
|
||||||
`http://garage:3900` endpoint and set
|
`FILE_STORAGE_S3_ENDPOINT_TRUSTED=true`. Both are deployment authority, not a
|
||||||
`FILE_STORAGE_S3_DEPLOYMENT_MANAGED=true`. Files accepts only that exact
|
general connector or private-network bypass. Core owns the shared backend
|
||||||
service-discovery endpoint and forces path-style S3 addressing. The marker is
|
implementation; Files owns metadata and the Files key namespace.
|
||||||
deployment authority, not a general private-network bypass.
|
|
||||||
|
|
||||||
Multiple API replicas require the same durable blob namespace. Separate local
|
Multiple API replicas require the same durable blob namespace. Separate local
|
||||||
container filesystems will produce incomplete reads. Until a pinned shared
|
container filesystems will produce incomplete reads. Use `host-shared` with one
|
||||||
object-storage transport is available, use one durable shared mount or constrain
|
durable shared mount only for same-host replicas. Independent hosts require the
|
||||||
Files traffic to a deployment topology that preserves one consistent local
|
`shared` profile with external S3, PostgreSQL, Redis, a stable installation id,
|
||||||
root.
|
and one immutable module composition.
|
||||||
|
|
||||||
### Connector egress
|
### Connector egress
|
||||||
|
|
||||||
@@ -464,10 +465,12 @@ Override the connector response limits with
|
|||||||
`GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES`. The smaller applicable limit wins
|
`GOVOPLAN_CONNECTOR_MAX_FILE_TRANSFER_BYTES`. The smaller applicable limit wins
|
||||||
when an import is also subject to `FILE_UPLOAD_MAX_BYTES`.
|
when an import is also subject to `FILE_UPLOAD_MAX_BYTES`.
|
||||||
|
|
||||||
Never work around a pinning failure by adding a raw IP, disabling TLS, or
|
Never work around a connector pinning failure by adding a raw IP, disabling TLS,
|
||||||
enabling private networks. SMB may redirect through DFS, and S3 SDKs may perform
|
or enabling private networks. SMB may redirect through DFS, and user-configured
|
||||||
their own redirects or endpoint discovery; both remain disabled even for an IP
|
S3 connectors may perform their own redirects or endpoint discovery; those
|
||||||
literal until every connection peer can be governed.
|
connector transports remain disabled until every connection peer can be
|
||||||
|
governed. The separately configured platform S3 backend is trusted only by the
|
||||||
|
deployment owner and is not selectable by a user or connector profile.
|
||||||
|
|
||||||
### Backup and restore
|
### Backup and restore
|
||||||
|
|
||||||
@@ -477,6 +480,8 @@ include:
|
|||||||
- Files database rows, including asset/version/blob relationships, shares,
|
- Files database rows, including asset/version/blob relationships, shares,
|
||||||
connector settings, and campaign attachment-use evidence;
|
connector settings, and campaign attachment-use evidence;
|
||||||
- every object below `FILE_STORAGE_LOCAL_ROOT` and any still-used fallback root;
|
- every object below `FILE_STORAGE_LOCAL_ROOT` and any still-used fallback root;
|
||||||
|
or the complete S3 bucket/prefix and version/lifecycle evidence for an S3
|
||||||
|
backend;
|
||||||
- the exact `MASTER_KEY_B64` needed to decrypt retained connector credentials;
|
- the exact `MASTER_KEY_B64` needed to decrypt retained connector credentials;
|
||||||
- deployment-owned connector profile files, referenced CA bundles, and secret
|
- deployment-owned connector profile files, referenced CA bundles, and secret
|
||||||
environment configuration where those definitions are in use.
|
environment configuration where those definitions are in use.
|
||||||
@@ -779,7 +784,7 @@ returning different content or credentials.
|
|||||||
|
|
||||||
| Area | Implemented now | Planned or explicitly outside the current boundary |
|
| Area | Implemented now | Planned or explicitly outside the current boundary |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Managed storage | Local durable-root backend, fallback read roots, tenant blob deduplication, checksums, bounded resumable integrity scans, quarantine, and dry-run-first orphan cleanup | Operational S3 after pinned SDK transport ([#34](https://git.add-ideas.de/GovOPlaN/govoplan-files/issues/34)); scheduled scan execution |
|
| Managed storage | Core local/S3 backend, exact managed-Garage or explicitly trusted HTTPS external S3, state-profile validation, fallback local read roots, tenant blob deduplication, checksums, bounded resumable integrity scans, quarantine, and dry-run-first orphan cleanup | Scheduled scan execution and deployment-specific S3 HA/backup automation |
|
||||||
| Upload | Bounded direct upload, drag-and-drop UI, archive preview/selective extraction, password-protected ZIP support, explicit conflicts | Malware scanning, quotas, type policy, resumable/chunked upload |
|
| Upload | Bounded direct upload, drag-and-drop UI, archive preview/selective extraction, password-protected ZIP support, explicit conflicts | Malware scanning, quotas, type policy, resumable/chunked upload |
|
||||||
| Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution | General file-history UI and user-driven append-version/restore |
|
| Organization | Folders, bulk rename preview/apply, move/copy, drag-and-drop, ZIP download, pattern resolution | General file-history UI and user-driven append-version/restore |
|
||||||
| Sharing | User/group/tenant/campaign grants, expiry, idempotent revocation, searchable share-management UI, and campaign linkage display | Richer policy-driven share lifecycles |
|
| Sharing | User/group/tenant/campaign grants, expiry, idempotent revocation, searchable share-management UI, and campaign linkage display | Richer policy-driven share lifecycles |
|
||||||
@@ -808,8 +813,9 @@ Before releasing Files:
|
|||||||
5. Exercise an allowed and denied owner/share path.
|
5. Exercise an allowed and denied owner/share path.
|
||||||
6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion.
|
6. Exercise upload, ZIP bounds, conflict handling, download, and soft deletion.
|
||||||
7. Exercise connector policy explanation and one pinned HTTP provider where
|
7. Exercise connector policy explanation and one pinned HTTP provider where
|
||||||
configured; verify managed Garage if selected, and verify arbitrary external
|
configured; verify the deployment-managed or trusted external S3 backend if
|
||||||
S3 and SMB still fail closed.
|
selected, and verify user-configured S3 and SMB connector peers still fail
|
||||||
|
closed.
|
||||||
8. Verify credential deletion scrubs dependents and produces audit evidence.
|
8. Verify credential deletion scrubs dependents and produces audit evidence.
|
||||||
9. Verify a campaign attachment snapshot still identifies its exact version and
|
9. Verify a campaign attachment snapshot still identifies its exact version and
|
||||||
checksum after the current file changes.
|
checksum after the current file changes.
|
||||||
|
|||||||
+1
-1
@@ -11,7 +11,7 @@ requires-python = ">=3.12"
|
|||||||
license = { file = "LICENSE" }
|
license = { file = "LICENSE" }
|
||||||
authors = [{ name = "GovOPlaN" }]
|
authors = [{ name = "GovOPlaN" }]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"govoplan-core>=0.1.9",
|
"govoplan-core>=0.1.14",
|
||||||
"defusedxml>=0.7,<1",
|
"defusedxml>=0.7,<1",
|
||||||
"pyzipper>=0.3.6,<1",
|
"pyzipper>=0.3.6,<1",
|
||||||
"python-multipart>=0.0.31,<1",
|
"python-multipart>=0.0.31,<1",
|
||||||
|
|||||||
@@ -24,11 +24,22 @@ from govoplan_core.core.modules import (
|
|||||||
RoleTemplate,
|
RoleTemplate,
|
||||||
)
|
)
|
||||||
from govoplan_core.core.operations import OperationalCheckProviderRegistration
|
from govoplan_core.core.operations import OperationalCheckProviderRegistration
|
||||||
|
from govoplan_core.core.provider_governance import (
|
||||||
|
ExternalProviderDeclaration,
|
||||||
|
ExternalProviderStateProviderRegistration,
|
||||||
|
ProviderBehaviorDeclaration,
|
||||||
|
ProviderObjectDeclaration,
|
||||||
|
declared_module_architecture,
|
||||||
|
)
|
||||||
from govoplan_core.core.views import ViewSurface
|
from govoplan_core.core.views import ViewSurface
|
||||||
from govoplan_core.db.base import Base
|
from govoplan_core.db.base import Base
|
||||||
from govoplan_files.backend.change_tracking import register_files_change_tracking
|
from govoplan_files.backend.change_tracking import register_files_change_tracking
|
||||||
from govoplan_files.backend.db import models as file_models # noqa: F401 - populate Files ORM metadata
|
from govoplan_files.backend.db import models as file_models # noqa: F401 - populate Files ORM metadata
|
||||||
from govoplan_files.backend.documentation import documentation_topics
|
from govoplan_files.backend.documentation import documentation_topics
|
||||||
|
from govoplan_files.backend.provider_state import (
|
||||||
|
REMOTE_STORAGE_PROVIDER_ID,
|
||||||
|
remote_storage_provider_states,
|
||||||
|
)
|
||||||
|
|
||||||
register_files_change_tracking()
|
register_files_change_tracking()
|
||||||
|
|
||||||
@@ -207,6 +218,58 @@ def _files_router(context: ModuleContext):
|
|||||||
return router
|
return router
|
||||||
|
|
||||||
|
|
||||||
|
REMOTE_STORAGE_PROVIDER = ExternalProviderDeclaration(
|
||||||
|
id=REMOTE_STORAGE_PROVIDER_ID,
|
||||||
|
module_id="files",
|
||||||
|
label="Remote file storage mirror",
|
||||||
|
maturity="synchronize",
|
||||||
|
operations=("discover", "search", "read", "synchronize", "preview"),
|
||||||
|
objects=(
|
||||||
|
ProviderObjectDeclaration(
|
||||||
|
object_type="remote_folder",
|
||||||
|
field_groups=("identity", "hierarchy", "display", "source_metadata"),
|
||||||
|
authority_modes=("external_authoritative", "external_mirror"),
|
||||||
|
default_authority_mode="external_mirror",
|
||||||
|
),
|
||||||
|
ProviderObjectDeclaration(
|
||||||
|
object_type="remote_file",
|
||||||
|
field_groups=("identity", "content", "version", "source_metadata"),
|
||||||
|
authority_modes=("external_authoritative", "external_mirror"),
|
||||||
|
default_authority_mode="external_mirror",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
behavior=ProviderBehaviorDeclaration(
|
||||||
|
revision_tokens="Remote path, provider revision, size, and content digest are retained on managed imports.",
|
||||||
|
concurrency="A sync compares the frozen source reference and revision before creating a new managed version.",
|
||||||
|
freshness="Provider state distinguishes software availability from an unobserved live remote binding.",
|
||||||
|
health="Unsupported providers or missing optional transports fail closed; live health remains unknown without a probe.",
|
||||||
|
max_read_items=5000,
|
||||||
|
idempotency="Source profile, remote object reference, and revision/digest suppress duplicate managed versions.",
|
||||||
|
retry="Operators repeat bounded browse/import after a classified transport failure; effects are not blindly retried.",
|
||||||
|
timeout_seconds=30,
|
||||||
|
conflicts="Managed-file conflict policy is explicit; the current provider never mutates the remote source.",
|
||||||
|
outcome_unknown="An interrupted download is discarded unless its complete digest and managed version commit are confirmed.",
|
||||||
|
outcome_unknown_supported=True,
|
||||||
|
evidence="Managed versions retain connector profile, remote object identity, source revision, digest, and acquisition time.",
|
||||||
|
audit_event_types=(
|
||||||
|
"files.connector.accessed",
|
||||||
|
"files.connector.imported",
|
||||||
|
"files.connector.synced",
|
||||||
|
),
|
||||||
|
correction="A later acquisition creates a new managed version and preserves prior provenance.",
|
||||||
|
rollback="Remote reads require no remote rollback; incomplete local objects are reconciled as orphans.",
|
||||||
|
compensation="A wrongly imported managed version can be retired under Files policy without deleting the source.",
|
||||||
|
reconciliation="Re-read source metadata and digest, then compare the committed managed version and object-store inventory.",
|
||||||
|
outage="Previously imported managed versions remain available while remote spaces report unknown or stale state.",
|
||||||
|
classifications=("internal", "confidential", "personal"),
|
||||||
|
purposes=("governed file acquisition", "managed evidence snapshot"),
|
||||||
|
retention="Files retention applies to managed versions; external retention remains provider-owned.",
|
||||||
|
secret_handling="Credentials remain encrypted or deployment-owned and never appear in provider state or provenance.",
|
||||||
|
),
|
||||||
|
documentation_topic_ids=("files.governed-connectors-and-provenance",),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
manifest = ModuleManifest(
|
manifest = ModuleManifest(
|
||||||
id="files",
|
id="files",
|
||||||
name="Files",
|
name="Files",
|
||||||
@@ -682,6 +745,46 @@ manifest = ModuleManifest(
|
|||||||
],
|
],
|
||||||
},
|
},
|
||||||
),
|
),
|
||||||
|
DocumentationTopic(
|
||||||
|
id="files.reference.shared-storage-profile",
|
||||||
|
title="Operate Files with shared object storage",
|
||||||
|
summary="Choose local, host-shared, or S3-backed storage consistently with the runtime topology.",
|
||||||
|
body="Core supplies the common local/S3 object-storage backend while Files owns file metadata and object-key semantics. Local storage is valid for one runtime process; a shared host volume supports same-host replicas; independent hosts require an explicitly trusted HTTPS S3-compatible endpoint. Restore PostgreSQL, objects, and the master key to one coordinated recovery point.",
|
||||||
|
layer="configured",
|
||||||
|
documentation_types=("admin",),
|
||||||
|
audience=("file_admin", "operator", "system_admin"),
|
||||||
|
order=54,
|
||||||
|
conditions=(
|
||||||
|
DocumentationCondition(
|
||||||
|
required_modules=("files",),
|
||||||
|
any_scopes=("files:file:admin", "system:settings:read"),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
links=(
|
||||||
|
DocumentationLink(label="Files", href="/files", kind="runtime"),
|
||||||
|
DocumentationLink(label="Files handbook", href="govoplan-files/docs/FILES_HANDBOOK.md", kind="repository"),
|
||||||
|
),
|
||||||
|
related_modules=("ops", "campaigns"),
|
||||||
|
configuration_keys=(
|
||||||
|
"GOVOPLAN_STATE_PROFILE",
|
||||||
|
"GOVOPLAN_INSTALLATION_ID",
|
||||||
|
"FILE_STORAGE_BACKEND",
|
||||||
|
"FILE_STORAGE_LOCAL_ROOT",
|
||||||
|
"FILE_STORAGE_S3_ENDPOINT_URL",
|
||||||
|
"FILE_STORAGE_S3_ENDPOINT_TRUSTED",
|
||||||
|
"FILE_STORAGE_S3_DEPLOYMENT_MANAGED",
|
||||||
|
),
|
||||||
|
metadata={
|
||||||
|
"kind": "reference",
|
||||||
|
"route": "/ops",
|
||||||
|
"screen": "Storage and runtime posture",
|
||||||
|
"limitations": [
|
||||||
|
"Installer-managed Garage is single-node unless an external multi-node cluster is operated separately.",
|
||||||
|
"The application does not create or verify production PostgreSQL/object/key backups.",
|
||||||
|
],
|
||||||
|
"verification": "Run the Files storage round-trip check and a coordinated restore drill against the exact deployment topology.",
|
||||||
|
},
|
||||||
|
),
|
||||||
),
|
),
|
||||||
documentation_providers=(documentation_topics,),
|
documentation_providers=(documentation_topics,),
|
||||||
migration_spec=MigrationSpec(
|
migration_spec=MigrationSpec(
|
||||||
@@ -722,6 +825,33 @@ manifest = ModuleManifest(
|
|||||||
cache_seconds=60,
|
cache_seconds=60,
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
external_providers=(REMOTE_STORAGE_PROVIDER,),
|
||||||
|
external_provider_state_providers=(
|
||||||
|
ExternalProviderStateProviderRegistration(
|
||||||
|
module_id="files",
|
||||||
|
provider_id=REMOTE_STORAGE_PROVIDER_ID,
|
||||||
|
provider=remote_storage_provider_states,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
architecture=declared_module_architecture(
|
||||||
|
layer="content_records_evidence",
|
||||||
|
kind="domain",
|
||||||
|
maturity="vertical_slice",
|
||||||
|
documentation_ref="docs/FILES_HANDBOOK.md",
|
||||||
|
test_ref="tests/test_storage_backends.py",
|
||||||
|
known_limits=("Multi-node object-storage recovery evidence and every remote connector profile are not reference-ready.",),
|
||||||
|
supported_authority_modes=(
|
||||||
|
"native_authoritative",
|
||||||
|
"external_authoritative",
|
||||||
|
"external_mirror",
|
||||||
|
),
|
||||||
|
owned_concepts=("file asset", "file version", "folder", "share", "connector space"),
|
||||||
|
non_owned_concepts=("record disposition", "campaign attachment rule", "external storage object"),
|
||||||
|
target_tested_providers=(REMOTE_STORAGE_PROVIDER_ID,),
|
||||||
|
recovery_docs=("docs/FILES_HANDBOOK.md",),
|
||||||
|
security_docs=("docs/CONNECTOR_BOUNDARY.md",),
|
||||||
|
operations_docs=("docs/FILES_HANDBOOK.md",),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections import defaultdict
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
|
||||||
|
from sqlalchemy import or_, select
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from govoplan_core.core.provider_governance import (
|
||||||
|
ExternalProviderRuntimeState,
|
||||||
|
ExternalProviderStateContext,
|
||||||
|
)
|
||||||
|
from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace
|
||||||
|
from govoplan_files.backend.storage.connector_providers import (
|
||||||
|
ConnectorProviderDescriptor,
|
||||||
|
connector_provider_descriptors,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
REMOTE_STORAGE_PROVIDER_ID = "files.remote_storage"
|
||||||
|
|
||||||
|
|
||||||
|
def remote_storage_provider_states(
|
||||||
|
context: ExternalProviderStateContext,
|
||||||
|
) -> tuple[ExternalProviderRuntimeState, ...]:
|
||||||
|
if not isinstance(context.session, Session):
|
||||||
|
raise RuntimeError("Files provider state requires a database session.")
|
||||||
|
statement = select(FileConnectorProfile)
|
||||||
|
if context.tenant_id is not None:
|
||||||
|
statement = statement.where(
|
||||||
|
or_(
|
||||||
|
FileConnectorProfile.tenant_id.is_(None),
|
||||||
|
FileConnectorProfile.tenant_id == context.tenant_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
profiles = tuple(
|
||||||
|
context.session.scalars(
|
||||||
|
statement.order_by(
|
||||||
|
FileConnectorProfile.tenant_id,
|
||||||
|
FileConnectorProfile.id,
|
||||||
|
).limit(context.max_items + 1)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if not profiles:
|
||||||
|
return ()
|
||||||
|
|
||||||
|
profile_ids = tuple(item.id for item in profiles)
|
||||||
|
space_statement = select(FileConnectorSpace).where(
|
||||||
|
FileConnectorSpace.connector_profile_id.in_(profile_ids),
|
||||||
|
FileConnectorSpace.deleted_at.is_(None),
|
||||||
|
)
|
||||||
|
if context.tenant_id is not None:
|
||||||
|
space_statement = space_statement.where(
|
||||||
|
FileConnectorSpace.tenant_id == context.tenant_id
|
||||||
|
)
|
||||||
|
spaces_by_profile: dict[str, list[FileConnectorSpace]] = defaultdict(list)
|
||||||
|
for space in context.session.scalars(space_statement):
|
||||||
|
spaces_by_profile[space.connector_profile_id].append(space)
|
||||||
|
|
||||||
|
descriptors = {
|
||||||
|
item.provider: item for item in connector_provider_descriptors()
|
||||||
|
}
|
||||||
|
observed_at = datetime.now(UTC)
|
||||||
|
return tuple(
|
||||||
|
_profile_state(
|
||||||
|
profile,
|
||||||
|
spaces=spaces_by_profile.get(profile.id, []),
|
||||||
|
descriptor=descriptors.get(profile.provider),
|
||||||
|
observed_at=observed_at,
|
||||||
|
)
|
||||||
|
for profile in profiles
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _profile_state(
|
||||||
|
profile: FileConnectorProfile,
|
||||||
|
*,
|
||||||
|
spaces: list[FileConnectorSpace],
|
||||||
|
descriptor: ConnectorProviderDescriptor | None,
|
||||||
|
observed_at: datetime,
|
||||||
|
) -> ExternalProviderRuntimeState:
|
||||||
|
active_spaces = tuple(item for item in spaces if item.is_active)
|
||||||
|
active = bool(profile.enabled)
|
||||||
|
implementation_ready = bool(
|
||||||
|
descriptor is not None and descriptor.implemented and descriptor.installed
|
||||||
|
)
|
||||||
|
health = (
|
||||||
|
"inactive"
|
||||||
|
if not active
|
||||||
|
else "error"
|
||||||
|
if not implementation_ready
|
||||||
|
else "unknown"
|
||||||
|
)
|
||||||
|
recovery = (
|
||||||
|
"not_applicable"
|
||||||
|
if not active
|
||||||
|
else "unsupported"
|
||||||
|
if not implementation_ready
|
||||||
|
else "attention"
|
||||||
|
)
|
||||||
|
detail = (
|
||||||
|
"Remote-storage connector profile is disabled."
|
||||||
|
if not active
|
||||||
|
else "The configured provider is not available in this runtime."
|
||||||
|
if not implementation_ready
|
||||||
|
else "Software support is available; no live remote health observation is retained."
|
||||||
|
)
|
||||||
|
return ExternalProviderRuntimeState(
|
||||||
|
provider_id=REMOTE_STORAGE_PROVIDER_ID,
|
||||||
|
binding_ref=f"files:connector-profile:{profile.id}",
|
||||||
|
authority_mode="external_mirror",
|
||||||
|
observed_at=observed_at,
|
||||||
|
configured=True,
|
||||||
|
active=active,
|
||||||
|
health=health,
|
||||||
|
freshness="unknown" if active else "not_applicable",
|
||||||
|
conflict="not_applicable",
|
||||||
|
recovery=recovery,
|
||||||
|
detail=detail,
|
||||||
|
metrics={
|
||||||
|
"provider": profile.provider,
|
||||||
|
"configured_spaces": len(spaces),
|
||||||
|
"active_spaces": len(active_spaces),
|
||||||
|
"write_requested_spaces": sum(
|
||||||
|
1 for item in active_spaces if not item.read_only
|
||||||
|
),
|
||||||
|
"software_implemented": bool(descriptor and descriptor.implemented),
|
||||||
|
"optional_dependency_available": bool(descriptor and descriptor.installed),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
__all__ = ["REMOTE_STORAGE_PROVIDER_ID", "remote_storage_provider_states"]
|
||||||
@@ -1,346 +1,32 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
from govoplan_core.core.object_storage import (
|
||||||
from pathlib import Path
|
LocalFilesystemStorageBackend,
|
||||||
from typing import Iterable, Protocol
|
S3StorageBackend,
|
||||||
|
StorageBackend,
|
||||||
from govoplan_core.security.outbound_http import (
|
StorageBackendError,
|
||||||
OutboundHttpError,
|
StorageObjectInfo,
|
||||||
response_limit,
|
StorageObjectMissing,
|
||||||
validate_unpinned_sdk_http_url,
|
StorageObjectPage,
|
||||||
|
configured_storage_backend,
|
||||||
)
|
)
|
||||||
|
|
||||||
from govoplan_files.backend.runtime import settings
|
from govoplan_files.backend.runtime import settings
|
||||||
|
|
||||||
|
|
||||||
class StorageBackendError(RuntimeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class StorageObjectMissing(StorageBackendError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class StorageObjectInfo:
|
|
||||||
key: str
|
|
||||||
size_bytes: int
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True, slots=True)
|
|
||||||
class StorageObjectPage:
|
|
||||||
objects: tuple[StorageObjectInfo, ...]
|
|
||||||
next_cursor: str | None = None
|
|
||||||
|
|
||||||
|
|
||||||
class StorageBackend(Protocol):
|
|
||||||
name: str
|
|
||||||
|
|
||||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None: ...
|
|
||||||
def get_bytes(self, key: str) -> bytes: ...
|
|
||||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]: ...
|
|
||||||
def delete(self, key: str) -> None: ...
|
|
||||||
def exists(self, key: str) -> bool: ...
|
|
||||||
def stat(self, key: str) -> StorageObjectInfo: ...
|
|
||||||
def list_objects(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
prefix: str,
|
|
||||||
after: str | None = None,
|
|
||||||
limit: int = 500,
|
|
||||||
) -> StorageObjectPage: ...
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
|
||||||
class LocalFilesystemStorageBackend:
|
|
||||||
root: Path
|
|
||||||
fallback_roots: tuple[Path, ...] = field(default_factory=tuple)
|
|
||||||
name: str = "local"
|
|
||||||
|
|
||||||
def __post_init__(self) -> None:
|
|
||||||
self.root = self.root.expanduser().resolve()
|
|
||||||
self.fallback_roots = tuple(root.expanduser().resolve() for root in self.fallback_roots if root)
|
|
||||||
self.root.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
def _path_for_root(self, root: Path, key: str) -> Path:
|
|
||||||
path = (root / key).resolve()
|
|
||||||
if not path.is_relative_to(root):
|
|
||||||
raise StorageBackendError("Storage key escapes local storage root")
|
|
||||||
return path
|
|
||||||
|
|
||||||
def _path(self, key: str) -> Path:
|
|
||||||
return self._path_for_root(self.root, key)
|
|
||||||
|
|
||||||
def _readable_path(self, key: str) -> Path:
|
|
||||||
primary = self._path(key)
|
|
||||||
if primary.exists() and primary.is_file():
|
|
||||||
return primary
|
|
||||||
for root in self.fallback_roots:
|
|
||||||
candidate = self._path_for_root(root, key)
|
|
||||||
if candidate.exists() and candidate.is_file():
|
|
||||||
return candidate
|
|
||||||
raise StorageObjectMissing("Stored object does not exist")
|
|
||||||
|
|
||||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
|
|
||||||
path = self._path(key)
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
path.write_bytes(data)
|
|
||||||
|
|
||||||
def get_bytes(self, key: str) -> bytes:
|
|
||||||
return self._readable_path(key).read_bytes()
|
|
||||||
|
|
||||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]:
|
|
||||||
path = self._readable_path(key)
|
|
||||||
with path.open("rb") as handle:
|
|
||||||
while True:
|
|
||||||
chunk = handle.read(chunk_size)
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
yield chunk
|
|
||||||
|
|
||||||
def delete(self, key: str) -> None:
|
|
||||||
path = self._path(key)
|
|
||||||
if path.exists() and path.is_file():
|
|
||||||
path.unlink()
|
|
||||||
|
|
||||||
def exists(self, key: str) -> bool:
|
|
||||||
try:
|
|
||||||
self._readable_path(key)
|
|
||||||
except StorageBackendError:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
def stat(self, key: str) -> StorageObjectInfo:
|
|
||||||
path = self._readable_path(key)
|
|
||||||
return StorageObjectInfo(key=key, size_bytes=path.stat().st_size)
|
|
||||||
|
|
||||||
def list_objects(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
prefix: str,
|
|
||||||
after: str | None = None,
|
|
||||||
limit: int = 500,
|
|
||||||
) -> StorageObjectPage:
|
|
||||||
normalized_limit = max(1, min(int(limit), 5000))
|
|
||||||
candidates: list[StorageObjectInfo] = []
|
|
||||||
for path in _iter_local_files(self.root):
|
|
||||||
key = path.relative_to(self.root).as_posix()
|
|
||||||
if not key.startswith(prefix) or (after is not None and key <= after):
|
|
||||||
continue
|
|
||||||
candidates.append(StorageObjectInfo(key=key, size_bytes=path.stat().st_size))
|
|
||||||
if len(candidates) > normalized_limit:
|
|
||||||
break
|
|
||||||
has_more = len(candidates) > normalized_limit
|
|
||||||
page = tuple(candidates[:normalized_limit])
|
|
||||||
return StorageObjectPage(
|
|
||||||
objects=page,
|
|
||||||
next_cursor=page[-1].key if has_more and page else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(slots=True)
|
|
||||||
class S3StorageBackend:
|
|
||||||
bucket: str
|
|
||||||
endpoint_url: str
|
|
||||||
region_name: str
|
|
||||||
access_key_id: str
|
|
||||||
secret_access_key: str
|
|
||||||
deployment_managed: bool = False
|
|
||||||
name: str = "s3"
|
|
||||||
|
|
||||||
@property
|
|
||||||
def client(self):
|
|
||||||
if self.deployment_managed:
|
|
||||||
endpoint_url = _deployment_managed_garage_endpoint(self.endpoint_url)
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
endpoint_url = validate_unpinned_sdk_http_url(
|
|
||||||
self.endpoint_url,
|
|
||||||
label="File storage S3 endpoint",
|
|
||||||
)
|
|
||||||
except OutboundHttpError as exc:
|
|
||||||
raise StorageBackendError(str(exc)) from exc
|
|
||||||
try:
|
|
||||||
import boto3
|
|
||||||
from botocore.config import Config
|
|
||||||
except ModuleNotFoundError as exc:
|
|
||||||
raise StorageBackendError("boto3 is required for the S3 storage backend") from exc
|
|
||||||
options: dict[str, object] = {
|
|
||||||
"endpoint_url": endpoint_url,
|
|
||||||
"region_name": self.region_name,
|
|
||||||
"aws_access_key_id": self.access_key_id,
|
|
||||||
"aws_secret_access_key": self.secret_access_key,
|
|
||||||
}
|
|
||||||
if self.deployment_managed:
|
|
||||||
options["config"] = Config(s3={"addressing_style": "path"})
|
|
||||||
return boto3.client("s3", **options)
|
|
||||||
|
|
||||||
def put_bytes(self, key: str, data: bytes, *, content_type: str | None = None) -> None:
|
|
||||||
max_bytes = response_limit("file")
|
|
||||||
if len(data) > max_bytes:
|
|
||||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
|
||||||
kwargs = {"Bucket": self.bucket, "Key": key, "Body": data}
|
|
||||||
if content_type:
|
|
||||||
kwargs["ContentType"] = content_type
|
|
||||||
self.client.put_object(**kwargs)
|
|
||||||
|
|
||||||
def get_bytes(self, key: str) -> bytes:
|
|
||||||
try:
|
|
||||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
|
||||||
max_bytes = response_limit("file")
|
|
||||||
body = obj["Body"]
|
|
||||||
try:
|
|
||||||
_reject_declared_object_size(obj, max_bytes=max_bytes)
|
|
||||||
data = body.read(max_bytes + 1)
|
|
||||||
if len(data) > max_bytes:
|
|
||||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
|
||||||
return data
|
|
||||||
finally:
|
|
||||||
if hasattr(body, "close"):
|
|
||||||
body.close()
|
|
||||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
|
||||||
raise StorageBackendError(str(exc)) from exc
|
|
||||||
|
|
||||||
def iter_bytes(self, key: str, *, chunk_size: int = 1024 * 1024) -> Iterable[bytes]:
|
|
||||||
try:
|
|
||||||
obj = self.client.get_object(Bucket=self.bucket, Key=key)
|
|
||||||
max_bytes = response_limit("file")
|
|
||||||
body = obj["Body"]
|
|
||||||
try:
|
|
||||||
_reject_declared_object_size(obj, max_bytes=max_bytes)
|
|
||||||
total = 0
|
|
||||||
while True:
|
|
||||||
chunk = body.read(chunk_size)
|
|
||||||
if not chunk:
|
|
||||||
break
|
|
||||||
total += len(chunk)
|
|
||||||
if total > max_bytes:
|
|
||||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
|
||||||
yield chunk
|
|
||||||
finally:
|
|
||||||
if hasattr(body, "close"):
|
|
||||||
body.close()
|
|
||||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
|
||||||
raise StorageBackendError(str(exc)) from exc
|
|
||||||
|
|
||||||
def delete(self, key: str) -> None:
|
|
||||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
|
||||||
|
|
||||||
def exists(self, key: str) -> bool:
|
|
||||||
try:
|
|
||||||
self.client.head_object(Bucket=self.bucket, Key=key)
|
|
||||||
return True
|
|
||||||
except Exception:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def stat(self, key: str) -> StorageObjectInfo:
|
|
||||||
try:
|
|
||||||
response = self.client.head_object(Bucket=self.bucket, Key=key)
|
|
||||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
|
||||||
if _s3_missing_error(exc):
|
|
||||||
raise StorageObjectMissing("Stored object does not exist") from exc
|
|
||||||
raise StorageBackendError(str(exc)) from exc
|
|
||||||
try:
|
|
||||||
size = int(response.get("ContentLength"))
|
|
||||||
except (AttributeError, TypeError, ValueError) as exc:
|
|
||||||
raise StorageBackendError("S3 object metadata did not include a valid size") from exc
|
|
||||||
return StorageObjectInfo(key=key, size_bytes=size)
|
|
||||||
|
|
||||||
def list_objects(
|
|
||||||
self,
|
|
||||||
*,
|
|
||||||
prefix: str,
|
|
||||||
after: str | None = None,
|
|
||||||
limit: int = 500,
|
|
||||||
) -> StorageObjectPage:
|
|
||||||
normalized_limit = max(1, min(int(limit), 1000))
|
|
||||||
kwargs: dict[str, object] = {
|
|
||||||
"Bucket": self.bucket,
|
|
||||||
"Prefix": prefix,
|
|
||||||
"MaxKeys": normalized_limit,
|
|
||||||
}
|
|
||||||
if after:
|
|
||||||
kwargs["StartAfter"] = after
|
|
||||||
try:
|
|
||||||
response = self.client.list_objects_v2(**kwargs)
|
|
||||||
except Exception as exc: # pragma: no cover - depends on S3 backend
|
|
||||||
raise StorageBackendError(str(exc)) from exc
|
|
||||||
objects = tuple(
|
|
||||||
StorageObjectInfo(
|
|
||||||
key=str(item["Key"]),
|
|
||||||
size_bytes=int(item.get("Size") or 0),
|
|
||||||
)
|
|
||||||
for item in response.get("Contents", ())
|
|
||||||
if isinstance(item, dict) and item.get("Key")
|
|
||||||
)
|
|
||||||
has_more = bool(response.get("IsTruncated"))
|
|
||||||
return StorageObjectPage(
|
|
||||||
objects=objects,
|
|
||||||
next_cursor=objects[-1].key if has_more and objects else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _reject_declared_object_size(obj: object, *, max_bytes: int) -> None:
|
|
||||||
if not isinstance(obj, dict):
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
declared_size = int(obj.get("ContentLength"))
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
return
|
|
||||||
if declared_size > max_bytes:
|
|
||||||
raise StorageBackendError(f"Stored object exceeds the deployment limit of {max_bytes} bytes")
|
|
||||||
|
|
||||||
|
|
||||||
def _iter_local_files(root: Path):
|
|
||||||
for entry in sorted(root.iterdir(), key=lambda item: item.name):
|
|
||||||
if entry.is_dir():
|
|
||||||
yield from _iter_local_files(entry)
|
|
||||||
elif entry.is_file():
|
|
||||||
yield entry
|
|
||||||
|
|
||||||
|
|
||||||
def _s3_missing_error(exc: Exception) -> bool:
|
|
||||||
response = getattr(exc, "response", None)
|
|
||||||
if not isinstance(response, dict):
|
|
||||||
return False
|
|
||||||
error = response.get("Error")
|
|
||||||
metadata = response.get("ResponseMetadata")
|
|
||||||
code = str(error.get("Code") if isinstance(error, dict) else "")
|
|
||||||
status_code = metadata.get("HTTPStatusCode") if isinstance(metadata, dict) else None
|
|
||||||
return code in {"404", "NoSuchKey", "NotFound"} or status_code == 404
|
|
||||||
|
|
||||||
|
|
||||||
def _deployment_managed_garage_endpoint(value: str) -> str:
|
|
||||||
endpoint = str(value or "").strip()
|
|
||||||
if endpoint != "http://garage:3900":
|
|
||||||
raise StorageBackendError(
|
|
||||||
"Deployment-managed S3 trust is restricted to http://garage:3900"
|
|
||||||
)
|
|
||||||
return endpoint
|
|
||||||
|
|
||||||
|
|
||||||
def _fallback_roots() -> tuple[Path, ...]:
|
|
||||||
raw = getattr(settings, "file_storage_local_fallback_roots", "") or ""
|
|
||||||
return tuple(Path(item.strip()) for item in str(raw).split(",") if item.strip())
|
|
||||||
|
|
||||||
|
|
||||||
def get_storage_backend() -> StorageBackend:
|
def get_storage_backend() -> StorageBackend:
|
||||||
backend = settings.file_storage_backend.lower().strip()
|
"""Return the deployment-owned backend for Files-managed objects."""
|
||||||
if backend in {"local", "filesystem", "fs"}:
|
|
||||||
return LocalFilesystemStorageBackend(Path(settings.file_storage_local_root), fallback_roots=_fallback_roots())
|
return configured_storage_backend(settings)
|
||||||
if backend in {"s3", "garage"}:
|
|
||||||
return S3StorageBackend(
|
|
||||||
bucket=settings.file_storage_s3_bucket or settings.s3_bucket,
|
__all__ = [
|
||||||
endpoint_url=settings.file_storage_s3_endpoint_url or settings.s3_endpoint_url,
|
"LocalFilesystemStorageBackend",
|
||||||
region_name=settings.file_storage_s3_region or settings.s3_region,
|
"S3StorageBackend",
|
||||||
access_key_id=settings.file_storage_s3_access_key_id or settings.s3_access_key_id,
|
"StorageBackend",
|
||||||
secret_access_key=settings.file_storage_s3_secret_access_key or settings.s3_secret_access_key,
|
"StorageBackendError",
|
||||||
deployment_managed=bool(
|
"StorageObjectInfo",
|
||||||
getattr(
|
"StorageObjectMissing",
|
||||||
settings,
|
"StorageObjectPage",
|
||||||
"file_storage_s3_deployment_managed",
|
"get_storage_backend",
|
||||||
False,
|
]
|
||||||
)
|
|
||||||
),
|
|
||||||
)
|
|
||||||
raise StorageBackendError(f"Unsupported file storage backend: {settings.file_storage_backend}")
|
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
|
from govoplan_access.backend.db import models as access_models # noqa: F401
|
||||||
|
from govoplan_core.core.provider_governance import ExternalProviderStateContext
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_files.backend.db.models import FileConnectorProfile, FileConnectorSpace
|
||||||
|
from govoplan_files.backend.manifest import manifest
|
||||||
|
from govoplan_files.backend.provider_state import (
|
||||||
|
REMOTE_STORAGE_PROVIDER_ID,
|
||||||
|
remote_storage_provider_states,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FilesProviderStateTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
|
||||||
|
Base.metadata.create_all(
|
||||||
|
self.engine,
|
||||||
|
tables=(FileConnectorProfile.__table__, FileConnectorSpace.__table__),
|
||||||
|
)
|
||||||
|
self.session = sessionmaker(bind=self.engine, expire_on_commit=False)()
|
||||||
|
self.profile = FileConnectorProfile(
|
||||||
|
id="profile-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
scope_type="tenant",
|
||||||
|
scope_id="tenant-1",
|
||||||
|
label="Remote WebDAV",
|
||||||
|
provider="webdav",
|
||||||
|
endpoint_url="https://files.example.test/dav/",
|
||||||
|
enabled=True,
|
||||||
|
)
|
||||||
|
self.session.add_all(
|
||||||
|
(
|
||||||
|
self.profile,
|
||||||
|
FileConnectorSpace(
|
||||||
|
id="space-1",
|
||||||
|
tenant_id="tenant-1",
|
||||||
|
owner_type="tenant",
|
||||||
|
label="Documents",
|
||||||
|
connector_profile_id=self.profile.id,
|
||||||
|
provider="webdav",
|
||||||
|
remote_path="/documents",
|
||||||
|
read_only=True,
|
||||||
|
is_active=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.session.commit()
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.session.close()
|
||||||
|
self.engine.dispose()
|
||||||
|
|
||||||
|
def test_state_does_not_overstate_live_health_or_expose_endpoint(self) -> None:
|
||||||
|
state = remote_storage_provider_states(
|
||||||
|
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("external_mirror", state.authority_mode)
|
||||||
|
self.assertEqual("unknown", state.health)
|
||||||
|
self.assertEqual("attention", state.recovery)
|
||||||
|
self.assertEqual(1, state.metrics["active_spaces"])
|
||||||
|
self.assertNotIn("files.example.test", str(state.to_dict()))
|
||||||
|
self.assertEqual(
|
||||||
|
(),
|
||||||
|
remote_storage_provider_states(
|
||||||
|
ExternalProviderStateContext(
|
||||||
|
session=self.session,
|
||||||
|
tenant_id="tenant-2",
|
||||||
|
)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_unimplemented_profile_is_fail_closed_and_manifest_registered(self) -> None:
|
||||||
|
self.profile.provider = "sharepoint"
|
||||||
|
self.session.flush()
|
||||||
|
|
||||||
|
state = remote_storage_provider_states(
|
||||||
|
ExternalProviderStateContext(session=self.session, tenant_id="tenant-1")
|
||||||
|
)[0]
|
||||||
|
|
||||||
|
self.assertEqual("error", state.health)
|
||||||
|
self.assertEqual("unsupported", state.recovery)
|
||||||
|
self.assertEqual(REMOTE_STORAGE_PROVIDER_ID, manifest.external_providers[0].id)
|
||||||
|
self.assertEqual(
|
||||||
|
REMOTE_STORAGE_PROVIDER_ID,
|
||||||
|
manifest.external_provider_state_providers[0].provider_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user