feat: govern file lifecycle and connector writes
This commit is contained in:
@@ -32,6 +32,7 @@ class ConnectorProviderDescriptor:
|
||||
"installed": self.installed,
|
||||
"browse_supported": self.browse_supported,
|
||||
"import_supported": self.import_supported,
|
||||
"write_supported": self.provider == "s3",
|
||||
"optional_dependency": self.optional_dependency,
|
||||
"permission_model": self.permission_model,
|
||||
"sync_strategy": self.sync_strategy,
|
||||
|
||||
@@ -14,6 +14,7 @@ from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||
|
||||
|
||||
SYNC_MODES = {"manual"}
|
||||
WRITABLE_PROVIDERS = {"s3"}
|
||||
|
||||
|
||||
def connector_space_owner_id(space: FileConnectorSpace) -> str:
|
||||
@@ -40,6 +41,7 @@ def create_connector_space(
|
||||
ensure_owner_access(session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id, user_id=user_id, is_admin=is_admin)
|
||||
label = _normalize_label(label)
|
||||
sync_mode = _normalize_sync_mode(sync_mode)
|
||||
validate_connector_space_write_mode(profile, read_only=read_only)
|
||||
remote_path = normalize_connector_browse_path(remote_path)
|
||||
library_id = _clean_optional(library_id)
|
||||
|
||||
@@ -145,6 +147,7 @@ def update_connector_space(
|
||||
library_id: str | None = None,
|
||||
remote_path: str | None = None,
|
||||
sync_mode: str | None = None,
|
||||
read_only: bool | None = None,
|
||||
is_active: bool | None = None,
|
||||
metadata: Mapping[str, Any] | None = None,
|
||||
is_admin: bool = False,
|
||||
@@ -178,6 +181,8 @@ def update_connector_space(
|
||||
space.remote_path = normalize_connector_browse_path(remote_path)
|
||||
if sync_mode is not None:
|
||||
space.sync_mode = _normalize_sync_mode(sync_mode)
|
||||
if read_only is not None:
|
||||
space.read_only = bool(read_only)
|
||||
if is_active is not None:
|
||||
space.is_active = bool(is_active)
|
||||
if metadata is not None:
|
||||
@@ -187,6 +192,17 @@ def update_connector_space(
|
||||
return space
|
||||
|
||||
|
||||
def validate_connector_space_write_mode(
|
||||
profile: ConnectorProfile, *, read_only: bool
|
||||
) -> None:
|
||||
if read_only:
|
||||
return
|
||||
if profile.provider not in WRITABLE_PROVIDERS or "write" not in profile.capabilities:
|
||||
raise FileStorageError(
|
||||
"Two-way mode requires an S3 connection with the write capability enabled"
|
||||
)
|
||||
|
||||
|
||||
def soft_delete_connector_space(
|
||||
session: Session,
|
||||
space: FileConnectorSpace,
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
from typing import Any, Mapping
|
||||
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryGuaranteeError,
|
||||
RecoveryMode,
|
||||
RecoveryPlan,
|
||||
RecoveryStatus,
|
||||
)
|
||||
from govoplan_core.core.recovery_runtime import (
|
||||
RecoveryOperationBusy,
|
||||
RecoveryOperationStateConflict,
|
||||
begin_durable_recovery_operation,
|
||||
)
|
||||
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||
from govoplan_core.db.session import get_database
|
||||
from govoplan_files.backend.storage.common import FileStorageError
|
||||
from govoplan_files.backend.storage.connector_browse import (
|
||||
ConnectorBrowseError,
|
||||
ConnectorBrowseUnsupported,
|
||||
_clean,
|
||||
_s3_bucket,
|
||||
_s3_client,
|
||||
_s3_object_key,
|
||||
normalize_connector_browse_path,
|
||||
)
|
||||
from govoplan_files.backend.storage.connector_profiles import ConnectorProfile
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConnectorWriteResult:
|
||||
recovery_operation_id: str
|
||||
status: str
|
||||
replayed: bool
|
||||
provider: str
|
||||
remote_path: str
|
||||
revision: str | None
|
||||
checksum_sha256: str
|
||||
size_bytes: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _HeadProbe:
|
||||
observed: Mapping[str, Any] | None
|
||||
verified: bool
|
||||
exception_type: str | None = None
|
||||
|
||||
|
||||
def write_connector_file(
|
||||
profile: ConnectorProfile,
|
||||
*,
|
||||
tenant_id: str,
|
||||
library_id: str | None,
|
||||
remote_path: str,
|
||||
data: bytes,
|
||||
content_type: str | None,
|
||||
expected_revision: str | None,
|
||||
idempotency_key: str,
|
||||
) -> ConnectorWriteResult:
|
||||
if profile.provider != "s3" or "write" not in profile.capabilities:
|
||||
raise FileStorageError(
|
||||
"Remote writes require an S3 connection with the write capability enabled"
|
||||
)
|
||||
try:
|
||||
normalized_path = normalize_connector_browse_path(remote_path)
|
||||
except ConnectorBrowseError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
if not normalized_path:
|
||||
raise FileStorageError("Remote write requires an object path")
|
||||
try:
|
||||
bucket = _s3_bucket(profile, library_id)
|
||||
key = _s3_object_key(profile, normalized_path)
|
||||
except ConnectorBrowseError as exc:
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
if not bucket or not key:
|
||||
raise FileStorageError("S3 remote write requires a bucket and object path")
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
target_digest = hashlib.sha256(f"{bucket}:{key}".encode("utf-8")).hexdigest()
|
||||
request = {
|
||||
"tenant_id": tenant_id,
|
||||
"connector_profile_id": profile.id,
|
||||
"provider": profile.provider,
|
||||
"remote_target_sha256": target_digest,
|
||||
"content_sha256": checksum,
|
||||
"size_bytes": len(data),
|
||||
"expected_revision": expected_revision,
|
||||
}
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
get_database().SessionLocal,
|
||||
identity=process_runtime_identity(),
|
||||
module_id="files",
|
||||
operation_type="connector-s3-write",
|
||||
idempotency_key=f"files-connector-write:{idempotency_key}",
|
||||
request=request,
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||
preconditions=(
|
||||
"the connector space is explicitly configured for two-way writes",
|
||||
"the connection and inherited policy allow the exact remote path",
|
||||
"a conditional create or expected remote revision prevents blind overwrite",
|
||||
),
|
||||
forward_recovery_steps=(
|
||||
"inspect provider metadata for the request and content digests",
|
||||
"resolve success only for an exact digest match or retry after verified absence",
|
||||
),
|
||||
verification_steps=(
|
||||
"read S3 object metadata after the write",
|
||||
"match the recorded request and content digests without downloading content",
|
||||
),
|
||||
),
|
||||
precondition_evidence={
|
||||
"remote_target_sha256": target_digest,
|
||||
"content_sha256": checksum,
|
||||
"size_bytes": len(data),
|
||||
"conditional_write": True,
|
||||
},
|
||||
lease_resource_key=f"files:connector:{tenant_id}:{profile.id}:{target_digest[:40]}",
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type="file_connector_object",
|
||||
resource_id=target_digest,
|
||||
metadata={
|
||||
"resources": ["postgresql", "s3-connector"],
|
||||
"provider": "s3",
|
||||
"connector_profile_id": profile.id,
|
||||
},
|
||||
block_unresolved_resource=True,
|
||||
)
|
||||
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||
raise FileStorageError(
|
||||
"This remote object is owned by another write or unresolved recovery operation"
|
||||
) from exc
|
||||
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||
raise FileStorageError(
|
||||
"The Files recovery ledger is unavailable; the remote object was not changed"
|
||||
) from exc
|
||||
if started.replayed or started.operation is None:
|
||||
return ConnectorWriteResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=started.status,
|
||||
replayed=True,
|
||||
provider="s3",
|
||||
remote_path=normalized_path,
|
||||
revision=None,
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=len(data),
|
||||
)
|
||||
operation = started.operation
|
||||
try:
|
||||
client = _s3_client(profile)
|
||||
except (ConnectorBrowseError, ConnectorBrowseUnsupported) as exc:
|
||||
operation.reject(
|
||||
summary="The S3 client was unavailable before the remote effect",
|
||||
evidence=_verified(
|
||||
{"external_effect_started": False}, exception_type=type(exc).__name__
|
||||
),
|
||||
)
|
||||
raise FileStorageError(str(exc)) from exc
|
||||
try:
|
||||
try:
|
||||
before = _head_object(client, bucket=bucket, key=key)
|
||||
except Exception as exc:
|
||||
operation.reject(
|
||||
summary="Remote preconditions could not be inspected before the effect",
|
||||
evidence=_verified(
|
||||
{"external_effect_started": False},
|
||||
exception_type=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
raise FileStorageError("S3 remote revision lookup failed") from exc
|
||||
expected = _normalize_revision(expected_revision)
|
||||
if before is not None and expected is None:
|
||||
operation.reject(
|
||||
summary="A blind remote overwrite was rejected",
|
||||
evidence=_verified(
|
||||
{"remote_object_exists": True, "expected_revision_supplied": False}
|
||||
),
|
||||
)
|
||||
raise FileStorageError(
|
||||
"The remote object already exists; reload it and supply its expected revision"
|
||||
)
|
||||
if expected is not None and (
|
||||
before is None or expected not in _observed_revisions(before)
|
||||
):
|
||||
operation.reject(
|
||||
summary="The expected remote revision did not match",
|
||||
evidence=_verified(
|
||||
{
|
||||
"remote_object_exists": before is not None,
|
||||
"expected_revision_matches": False,
|
||||
}
|
||||
),
|
||||
)
|
||||
raise FileStorageError("The remote object changed; reload before writing")
|
||||
params: dict[str, object] = {
|
||||
"Bucket": bucket,
|
||||
"Key": key,
|
||||
"Body": data,
|
||||
"Metadata": {
|
||||
"govoplan-sha256": checksum,
|
||||
"govoplan-request-id": started.operation_id,
|
||||
},
|
||||
}
|
||||
if content_type:
|
||||
params["ContentType"] = content_type
|
||||
if before is None:
|
||||
params["IfNoneMatch"] = "*"
|
||||
else:
|
||||
params["IfMatch"] = str(before.get("ETag") or expected_revision or "")
|
||||
try:
|
||||
client.put_object(**params)
|
||||
except Exception as exc:
|
||||
probe = _probe_head_object(client, bucket=bucket, key=key)
|
||||
if probe.verified and _matches_effect(
|
||||
probe.observed,
|
||||
checksum=checksum,
|
||||
operation_id=started.operation_id,
|
||||
):
|
||||
operation.succeed(
|
||||
evidence=_success_evidence(
|
||||
probe.observed,
|
||||
checksum=checksum,
|
||||
operation_id=started.operation_id,
|
||||
)
|
||||
)
|
||||
return _result(
|
||||
started.operation_id,
|
||||
normalized_path,
|
||||
checksum,
|
||||
len(data),
|
||||
probe.observed,
|
||||
)
|
||||
operation.unresolved(
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
summary="The S3 write outcome requires reconciliation",
|
||||
evidence={
|
||||
"remote_target_sha256": target_digest,
|
||||
"observed": _public_observation(probe),
|
||||
"exception_type": type(exc).__name__,
|
||||
},
|
||||
failure_summary="Inspect the remote object metadata before retrying this path",
|
||||
)
|
||||
return ConnectorWriteResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN.value,
|
||||
replayed=False,
|
||||
provider="s3",
|
||||
remote_path=normalized_path,
|
||||
revision=_revision(probe.observed),
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=len(data),
|
||||
)
|
||||
probe = _probe_head_object(client, bucket=bucket, key=key)
|
||||
if not probe.verified:
|
||||
operation.unresolved(
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
summary="The S3 write returned but provider evidence could not be queried",
|
||||
evidence={
|
||||
"remote_target_sha256": target_digest,
|
||||
"observed": _public_observation(probe),
|
||||
},
|
||||
failure_summary="Inspect the remote object metadata before retrying this path",
|
||||
)
|
||||
return ConnectorWriteResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN.value,
|
||||
replayed=False,
|
||||
provider="s3",
|
||||
remote_path=normalized_path,
|
||||
revision=None,
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=len(data),
|
||||
)
|
||||
observed = probe.observed
|
||||
if not _matches_effect(
|
||||
observed, checksum=checksum, operation_id=started.operation_id
|
||||
):
|
||||
operation.unresolved(
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
summary="The S3 write returned but exact provider evidence did not match",
|
||||
evidence={
|
||||
"remote_target_sha256": target_digest,
|
||||
"observed": _public_observation(probe),
|
||||
},
|
||||
failure_summary="Reconcile the remote object before another write",
|
||||
)
|
||||
return ConnectorWriteResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED.value,
|
||||
replayed=False,
|
||||
provider="s3",
|
||||
remote_path=normalized_path,
|
||||
revision=_revision(observed),
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=len(data),
|
||||
)
|
||||
operation.succeed(
|
||||
evidence=_success_evidence(
|
||||
observed, checksum=checksum, operation_id=started.operation_id
|
||||
)
|
||||
)
|
||||
return _result(
|
||||
started.operation_id,
|
||||
normalized_path,
|
||||
checksum,
|
||||
len(data),
|
||||
observed,
|
||||
)
|
||||
finally:
|
||||
close = getattr(client, "close", None)
|
||||
if callable(close):
|
||||
close()
|
||||
|
||||
|
||||
def _head_object(client: Any, *, bucket: str, key: str) -> Mapping[str, Any] | None:
|
||||
try:
|
||||
response = client.head_object(Bucket=bucket, Key=key)
|
||||
except Exception as exc:
|
||||
if _is_not_found(exc):
|
||||
return None
|
||||
raise
|
||||
if not isinstance(response, Mapping):
|
||||
raise FileStorageError("S3 connector returned invalid object metadata")
|
||||
return response
|
||||
|
||||
|
||||
def _probe_head_object(client: Any, *, bucket: str, key: str) -> _HeadProbe:
|
||||
try:
|
||||
return _HeadProbe(
|
||||
observed=_head_object(client, bucket=bucket, key=key), verified=True
|
||||
)
|
||||
except Exception as exc:
|
||||
return _HeadProbe(
|
||||
observed=None,
|
||||
verified=False,
|
||||
exception_type=type(exc).__name__,
|
||||
)
|
||||
|
||||
|
||||
def _is_not_found(exc: Exception) -> bool:
|
||||
response = getattr(exc, "response", None)
|
||||
if not isinstance(response, Mapping):
|
||||
return False
|
||||
error = response.get("Error")
|
||||
code = error.get("Code") if isinstance(error, Mapping) else None
|
||||
status = response.get("ResponseMetadata")
|
||||
http_status = status.get("HTTPStatusCode") if isinstance(status, Mapping) else None
|
||||
return str(code).casefold() in {"404", "nosuchkey", "notfound"} or http_status == 404
|
||||
|
||||
|
||||
def _metadata(observed: Mapping[str, Any] | None) -> Mapping[str, Any]:
|
||||
value = observed.get("Metadata") if observed else None
|
||||
return value if isinstance(value, Mapping) else {}
|
||||
|
||||
|
||||
def _matches_effect(
|
||||
observed: Mapping[str, Any] | None, *, checksum: str, operation_id: str
|
||||
) -> bool:
|
||||
metadata = _metadata(observed)
|
||||
return bool(
|
||||
observed is not None
|
||||
and _clean(metadata.get("govoplan-sha256")) == checksum
|
||||
and _clean(metadata.get("govoplan-request-id")) == operation_id
|
||||
)
|
||||
|
||||
|
||||
def _revision(observed: Mapping[str, Any] | None) -> str | None:
|
||||
if observed is None:
|
||||
return None
|
||||
return _clean(observed.get("VersionId") or observed.get("ETag"))
|
||||
|
||||
|
||||
def _normalize_revision(value: str | None) -> str | None:
|
||||
cleaned = _clean(value)
|
||||
return cleaned.strip('"') if cleaned else None
|
||||
|
||||
|
||||
def _observed_revisions(observed: Mapping[str, Any]) -> set[str]:
|
||||
return {
|
||||
normalized
|
||||
for value in (observed.get("VersionId"), observed.get("ETag"))
|
||||
if (normalized := _normalize_revision(_clean(value))) is not None
|
||||
}
|
||||
|
||||
|
||||
def _success_evidence(
|
||||
observed: Mapping[str, Any] | None, *, checksum: str, operation_id: str
|
||||
) -> dict[str, object]:
|
||||
return _verified(
|
||||
{
|
||||
"remote_object_present": observed is not None,
|
||||
"content_digest_matches": _clean(_metadata(observed).get("govoplan-sha256")) == checksum,
|
||||
"request_marker_matches": _clean(_metadata(observed).get("govoplan-request-id")) == operation_id,
|
||||
},
|
||||
revision=_revision(observed),
|
||||
content_sha256=checksum,
|
||||
)
|
||||
|
||||
|
||||
def _public_observation(probe: _HeadProbe) -> dict[str, object]:
|
||||
observed = probe.observed
|
||||
return {
|
||||
"probe_verified": probe.verified,
|
||||
"probe_exception_type": probe.exception_type,
|
||||
"present": observed is not None,
|
||||
"revision": _revision(observed),
|
||||
"has_content_digest": bool(_clean(_metadata(observed).get("govoplan-sha256"))),
|
||||
"has_request_marker": bool(_clean(_metadata(observed).get("govoplan-request-id"))),
|
||||
}
|
||||
|
||||
|
||||
def _verified(checks: dict[str, object], **details: object) -> dict[str, object]:
|
||||
return {"verified": True, "checks": checks, **details}
|
||||
|
||||
|
||||
def _result(
|
||||
operation_id: str,
|
||||
remote_path: str,
|
||||
checksum: str,
|
||||
size_bytes: int,
|
||||
observed: Mapping[str, Any] | None,
|
||||
) -> ConnectorWriteResult:
|
||||
return ConnectorWriteResult(
|
||||
recovery_operation_id=operation_id,
|
||||
status=RecoveryStatus.SUCCEEDED.value,
|
||||
replayed=False,
|
||||
provider="s3",
|
||||
remote_path=remote_path,
|
||||
revision=_revision(observed),
|
||||
checksum_sha256=checksum,
|
||||
size_bytes=size_bytes,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["ConnectorWriteResult", "write_connector_file"]
|
||||
@@ -84,6 +84,7 @@ def _get_or_create_blob(
|
||||
blob = (
|
||||
session.query(FileBlob)
|
||||
.filter(FileBlob.tenant_id == tenant_id, FileBlob.checksum_sha256 == checksum, FileBlob.size_bytes == size, FileBlob.protection_discriminator == protection_discriminator)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
if blob:
|
||||
|
||||
@@ -0,0 +1,796 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Callable, Iterable
|
||||
|
||||
from sqlalchemy import func, inspect, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from govoplan_core.core.recovery import (
|
||||
RecoveryGuaranteeError,
|
||||
RecoveryMode,
|
||||
RecoveryPlan,
|
||||
RecoveryStatus,
|
||||
)
|
||||
from govoplan_core.core.recovery_runtime import (
|
||||
RecoveryOperationBusy,
|
||||
RecoveryOperationStateConflict,
|
||||
begin_durable_recovery_operation,
|
||||
)
|
||||
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
||||
from govoplan_core.db.session import get_database
|
||||
from govoplan_files.backend.db.models import (
|
||||
CampaignAttachmentUse,
|
||||
FileAsset,
|
||||
FileBlob,
|
||||
FileConnectorSpace,
|
||||
FileFolder,
|
||||
FileFormEvidenceGrant,
|
||||
FileShare,
|
||||
FileVersion,
|
||||
)
|
||||
from govoplan_files.backend.storage.access import ensure_owner_access
|
||||
from govoplan_files.backend.storage.backends import (
|
||||
StorageBackendError,
|
||||
get_storage_backend,
|
||||
)
|
||||
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
||||
from govoplan_files.backend.storage.connector_spaces import connector_space_owner_id
|
||||
from govoplan_files.backend.storage.paths import normalize_folder
|
||||
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PurgePreviewItem:
|
||||
file_id: str
|
||||
filename: str
|
||||
lifecycle_revision: int
|
||||
deleted_at: datetime | None
|
||||
retained_until: datetime | None
|
||||
legal_hold: bool
|
||||
blockers: tuple[str, ...]
|
||||
blob_ids: tuple[str, ...]
|
||||
|
||||
def digest_payload(self) -> dict[str, object]:
|
||||
return {
|
||||
"file_id": self.file_id,
|
||||
"filename": self.filename,
|
||||
"lifecycle_revision": self.lifecycle_revision,
|
||||
"deleted_at": _iso(self.deleted_at),
|
||||
"retained_until": _iso(self.retained_until),
|
||||
"legal_hold": self.legal_hold,
|
||||
"blockers": list(self.blockers),
|
||||
"blob_ids": list(self.blob_ids),
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PurgePreview:
|
||||
preview_sha256: str
|
||||
items: tuple[PurgePreviewItem, ...]
|
||||
|
||||
@property
|
||||
def eligible(self) -> bool:
|
||||
return all(not item.blockers for item in self.items)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PurgeResult:
|
||||
recovery_operation_id: str
|
||||
status: str
|
||||
replayed: bool
|
||||
purged_files: int
|
||||
released_blobs: int
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BlobGcResult:
|
||||
inspected_blobs: int
|
||||
deleted_blobs: int
|
||||
unresolved_operation_ids: tuple[str, ...]
|
||||
|
||||
|
||||
def set_asset_lifecycle(
|
||||
session: Session,
|
||||
asset: FileAsset,
|
||||
*,
|
||||
retained_until: datetime | None,
|
||||
legal_hold: bool,
|
||||
reason: str,
|
||||
expected_revision: int,
|
||||
) -> FileAsset:
|
||||
if asset.lifecycle_revision != expected_revision:
|
||||
raise FileStorageError("File lifecycle settings changed; reload before saving")
|
||||
asset.retained_until = retained_until
|
||||
asset.legal_hold = bool(legal_hold)
|
||||
asset.lifecycle_reason = reason.strip()
|
||||
asset.lifecycle_revision += 1
|
||||
session.add(asset)
|
||||
session.flush()
|
||||
return asset
|
||||
|
||||
|
||||
def get_asset_for_lifecycle(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
asset_id: str,
|
||||
is_admin: bool,
|
||||
) -> FileAsset:
|
||||
asset = session.get(FileAsset, asset_id)
|
||||
if asset is None or asset.tenant_id != tenant_id:
|
||||
raise FileStorageError("File not found")
|
||||
ensure_owner_access(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=asset.owner_type,
|
||||
owner_id=_asset_owner_id(asset),
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
return asset
|
||||
|
||||
|
||||
def restore_asset(session: Session, asset: FileAsset) -> bool:
|
||||
if asset.deleted_at is None:
|
||||
return False
|
||||
collision = _asset_owner_query(session, asset).filter(
|
||||
FileAsset.id != asset.id,
|
||||
FileAsset.display_path == asset.display_path,
|
||||
FileAsset.deleted_at.is_(None),
|
||||
).first()
|
||||
if collision is not None:
|
||||
raise FileStorageError(
|
||||
"The file cannot be restored because its path is already in use"
|
||||
)
|
||||
asset.deleted_at = None
|
||||
asset.lifecycle_revision += 1
|
||||
session.add(asset)
|
||||
session.flush()
|
||||
return True
|
||||
|
||||
|
||||
def restore_folder(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
owner_type: str,
|
||||
owner_id: str,
|
||||
user_id: str,
|
||||
path: str,
|
||||
recursive: bool,
|
||||
is_admin: bool,
|
||||
) -> tuple[int, int]:
|
||||
ensure_owner_access(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
normalized = normalize_folder(path)
|
||||
if not normalized:
|
||||
raise FileStorageError("Folder path is required")
|
||||
prefix = f"{normalized}/"
|
||||
folders = _folder_owner_query(
|
||||
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
|
||||
).filter(FileFolder.deleted_at.is_not(None))
|
||||
assets = _asset_owner_query_by_values(
|
||||
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
|
||||
).filter(FileAsset.deleted_at.is_not(None))
|
||||
if recursive:
|
||||
folders = folders.filter(
|
||||
(FileFolder.path == normalized) | FileFolder.path.like(f"{prefix}%")
|
||||
)
|
||||
assets = assets.filter(FileAsset.display_path.like(f"{prefix}%"))
|
||||
else:
|
||||
folders = folders.filter(FileFolder.path == normalized)
|
||||
assets = assets.filter(False)
|
||||
folder_rows = folders.order_by(FileFolder.path.asc()).all()
|
||||
asset_rows = assets.order_by(FileAsset.display_path.asc()).all()
|
||||
if not folder_rows:
|
||||
raise FileStorageError("Deleted folder not found")
|
||||
for folder in folder_rows:
|
||||
collision = _folder_owner_query(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=owner_type,
|
||||
owner_id=owner_id,
|
||||
).filter(
|
||||
FileFolder.id != folder.id,
|
||||
FileFolder.path == folder.path,
|
||||
FileFolder.deleted_at.is_(None),
|
||||
).first()
|
||||
if collision is not None:
|
||||
raise FileStorageError(
|
||||
f"The folder cannot be restored because its path is in use: {folder.path}"
|
||||
)
|
||||
for asset in asset_rows:
|
||||
collision = _asset_owner_query(session, asset).filter(
|
||||
FileAsset.id != asset.id,
|
||||
FileAsset.display_path == asset.display_path,
|
||||
FileAsset.deleted_at.is_(None),
|
||||
).first()
|
||||
if collision is not None:
|
||||
raise FileStorageError(
|
||||
f"A file cannot be restored because its path is in use: {asset.display_path}"
|
||||
)
|
||||
for folder in folder_rows:
|
||||
folder.deleted_at = None
|
||||
session.add(folder)
|
||||
for asset in asset_rows:
|
||||
asset.deleted_at = None
|
||||
asset.lifecycle_revision += 1
|
||||
session.add(asset)
|
||||
session.flush()
|
||||
return len(folder_rows), len(asset_rows)
|
||||
|
||||
|
||||
def restore_connector_space(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
user_id: str,
|
||||
space_id: str,
|
||||
is_admin: bool,
|
||||
) -> FileConnectorSpace:
|
||||
space = session.get(FileConnectorSpace, space_id)
|
||||
if space is None or space.tenant_id != tenant_id or space.deleted_at is None:
|
||||
raise FileStorageError("Deleted connector space not found")
|
||||
ensure_owner_access(
|
||||
session,
|
||||
tenant_id=tenant_id,
|
||||
owner_type=space.owner_type,
|
||||
owner_id=connector_space_owner_id(space),
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
)
|
||||
collision = _connector_space_owner_query(session, space).filter(
|
||||
FileConnectorSpace.id != space.id,
|
||||
FileConnectorSpace.label == space.label,
|
||||
FileConnectorSpace.deleted_at.is_(None),
|
||||
).first()
|
||||
if collision is not None:
|
||||
raise FileStorageError(
|
||||
"The connector space cannot be restored because its label is in use"
|
||||
)
|
||||
space.deleted_at = None
|
||||
space.is_active = True
|
||||
session.add(space)
|
||||
session.flush()
|
||||
return space
|
||||
|
||||
|
||||
def preview_asset_purge(
|
||||
session: Session, *, tenant_id: str, file_ids: Iterable[str]
|
||||
) -> PurgePreview:
|
||||
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
|
||||
if not normalized_ids or len(normalized_ids) > 100:
|
||||
raise FileStorageError("Purge preview must contain between 1 and 100 files")
|
||||
assets = (
|
||||
session.query(FileAsset)
|
||||
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
|
||||
.order_by(FileAsset.id.asc())
|
||||
.all()
|
||||
)
|
||||
by_id = {asset.id: asset for asset in assets}
|
||||
items: list[PurgePreviewItem] = []
|
||||
for file_id in normalized_ids:
|
||||
asset = by_id.get(file_id)
|
||||
if asset is None:
|
||||
items.append(
|
||||
PurgePreviewItem(
|
||||
file_id=file_id,
|
||||
filename="",
|
||||
lifecycle_revision=0,
|
||||
deleted_at=None,
|
||||
retained_until=None,
|
||||
legal_hold=False,
|
||||
blockers=("not_found",),
|
||||
blob_ids=(),
|
||||
)
|
||||
)
|
||||
continue
|
||||
blockers = _asset_purge_blockers(session, asset)
|
||||
blob_ids = tuple(
|
||||
sorted(
|
||||
row[0]
|
||||
for row in session.query(FileVersion.blob_id)
|
||||
.filter(FileVersion.file_asset_id == asset.id)
|
||||
.distinct()
|
||||
.all()
|
||||
)
|
||||
)
|
||||
items.append(
|
||||
PurgePreviewItem(
|
||||
file_id=asset.id,
|
||||
filename=asset.filename,
|
||||
lifecycle_revision=asset.lifecycle_revision,
|
||||
deleted_at=asset.deleted_at,
|
||||
retained_until=asset.retained_until,
|
||||
legal_hold=asset.legal_hold,
|
||||
blockers=tuple(blockers),
|
||||
blob_ids=blob_ids,
|
||||
)
|
||||
)
|
||||
payload = [item.digest_payload() for item in items]
|
||||
digest = hashlib.sha256(
|
||||
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
).hexdigest()
|
||||
return PurgePreview(preview_sha256=digest, items=tuple(items))
|
||||
|
||||
|
||||
def execute_asset_purge(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
file_ids: Iterable[str],
|
||||
preview_sha256: str,
|
||||
idempotency_key: str,
|
||||
approval_reference: str,
|
||||
before_commit: Callable[[list[FileAsset]], None] | None = None,
|
||||
) -> PurgeResult:
|
||||
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
|
||||
if not normalized_ids or len(normalized_ids) > 100:
|
||||
raise FileStorageError("Purge must contain between 1 and 100 files")
|
||||
request = {
|
||||
"tenant_id": tenant_id,
|
||||
"file_ids": normalized_ids,
|
||||
"preview_sha256": preview_sha256,
|
||||
"approval_reference": approval_reference,
|
||||
}
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
get_database().SessionLocal,
|
||||
identity=process_runtime_identity(),
|
||||
module_id="files",
|
||||
operation_type="asset-hard-purge",
|
||||
idempotency_key=f"files-asset-purge:{idempotency_key}",
|
||||
request=request,
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.IRREVERSIBLE,
|
||||
preconditions=(
|
||||
"the actor holds the dedicated Files purge permission",
|
||||
"the signed-off preview still matches current lifecycle state",
|
||||
"every target is soft-deleted and free of retention or legal-hold blockers",
|
||||
),
|
||||
verification_steps=(
|
||||
"verify every target FileAsset and FileVersion row is absent",
|
||||
"recalculate retained FileBlob reference counts before later garbage collection",
|
||||
),
|
||||
approval_reference=approval_reference,
|
||||
),
|
||||
precondition_evidence={
|
||||
"preview_sha256": preview_sha256,
|
||||
"target_count": len(normalized_ids),
|
||||
},
|
||||
lease_resource_key=f"files:purge:{tenant_id}",
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type="file_asset_batch",
|
||||
resource_id=preview_sha256,
|
||||
metadata={"resources": ["postgresql"], "bounded_target_count": len(normalized_ids)},
|
||||
block_unresolved_resource=True,
|
||||
)
|
||||
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
||||
raise FileStorageError(
|
||||
"Another purge or unresolved retry owns the tenant purge fence"
|
||||
) from exc
|
||||
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||
raise FileStorageError("The Files recovery ledger is unavailable; nothing was purged") from exc
|
||||
if started.replayed or started.operation is None:
|
||||
return PurgeResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=started.status,
|
||||
replayed=True,
|
||||
purged_files=0,
|
||||
released_blobs=0,
|
||||
)
|
||||
operation = started.operation
|
||||
try:
|
||||
preview = preview_asset_purge(session, tenant_id=tenant_id, file_ids=normalized_ids)
|
||||
if preview.preview_sha256 != preview_sha256:
|
||||
operation.reject(
|
||||
summary="The purge preview became stale before execution",
|
||||
evidence=_verified_evidence(
|
||||
{"preview_matches": False},
|
||||
expected_preview_sha256=preview_sha256,
|
||||
observed_preview_sha256=preview.preview_sha256,
|
||||
),
|
||||
)
|
||||
raise FileStorageError("The purge preview is stale; preview again")
|
||||
if not preview.eligible:
|
||||
operation.reject(
|
||||
summary="The purge was blocked by current lifecycle policy",
|
||||
evidence=_verified_evidence(
|
||||
{"lifecycle_policy_allows": False},
|
||||
blocked_files=[
|
||||
{"file_id": item.file_id, "blockers": list(item.blockers)}
|
||||
for item in preview.items
|
||||
if item.blockers
|
||||
]
|
||||
),
|
||||
)
|
||||
raise FileStorageError("One or more files are not eligible for purge")
|
||||
assets = (
|
||||
session.query(FileAsset)
|
||||
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
|
||||
.with_for_update()
|
||||
.all()
|
||||
)
|
||||
blob_ids = sorted({blob_id for item in preview.items for blob_id in item.blob_ids})
|
||||
if before_commit is not None:
|
||||
before_commit(assets)
|
||||
session.query(FileShare).filter(
|
||||
FileShare.file_asset_id.in_(normalized_ids)
|
||||
).delete(synchronize_session=False)
|
||||
session.query(FileVersion).filter(
|
||||
FileVersion.file_asset_id.in_(normalized_ids)
|
||||
).delete(synchronize_session=False)
|
||||
for asset in assets:
|
||||
session.delete(asset)
|
||||
session.flush()
|
||||
released_blobs = 0
|
||||
for blob_id in blob_ids:
|
||||
references = int(
|
||||
session.query(func.count(FileVersion.id))
|
||||
.filter(FileVersion.blob_id == blob_id)
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
blob = session.get(FileBlob, blob_id)
|
||||
if blob is not None:
|
||||
blob.ref_count = references
|
||||
session.add(blob)
|
||||
if references == 0:
|
||||
released_blobs += 1
|
||||
operation.commit_verified_success(
|
||||
session,
|
||||
evidence=_verified_evidence(
|
||||
{
|
||||
"assets_absent": True,
|
||||
"versions_absent": True,
|
||||
"blob_reference_counts_recalculated": True,
|
||||
},
|
||||
purged_file_ids=normalized_ids,
|
||||
purged_files=len(assets),
|
||||
released_blobs=released_blobs,
|
||||
preview_sha256=preview_sha256,
|
||||
),
|
||||
)
|
||||
return PurgeResult(
|
||||
recovery_operation_id=started.operation_id,
|
||||
status=RecoveryStatus.SUCCEEDED.value,
|
||||
replayed=False,
|
||||
purged_files=len(assets),
|
||||
released_blobs=released_blobs,
|
||||
)
|
||||
except FileStorageError:
|
||||
session.rollback()
|
||||
if not operation.closed:
|
||||
operation.release_unresolved()
|
||||
raise
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
if not operation.closed:
|
||||
operation.reject(
|
||||
summary="The purge failed before its database transaction committed",
|
||||
evidence=_verified_evidence(
|
||||
{"database_transaction_committed": False},
|
||||
exception_type=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
def garbage_collect_unreferenced_blobs(
|
||||
session: Session,
|
||||
*,
|
||||
tenant_id: str,
|
||||
limit: int,
|
||||
approval_reference: str,
|
||||
) -> BlobGcResult:
|
||||
candidate_rows = (
|
||||
session.query(FileBlob)
|
||||
.filter(FileBlob.tenant_id == tenant_id)
|
||||
.filter(
|
||||
or_(FileBlob.retained_until.is_(None), FileBlob.retained_until <= utcnow())
|
||||
)
|
||||
.filter(
|
||||
~session.query(FileVersion.id)
|
||||
.filter(FileVersion.blob_id == FileBlob.id)
|
||||
.exists()
|
||||
)
|
||||
.order_by(FileBlob.created_at.asc(), FileBlob.id.asc())
|
||||
.limit(limit)
|
||||
.all()
|
||||
)
|
||||
candidates = tuple(
|
||||
(row.id, row.storage_key) for row in candidate_rows
|
||||
)
|
||||
# Close the read snapshot before the independent recovery transaction takes
|
||||
# its durable SQLite/PostgreSQL write lock.
|
||||
session.commit()
|
||||
deleted = 0
|
||||
unresolved: list[str] = []
|
||||
backend = get_storage_backend()
|
||||
for candidate_id, storage_key in candidates:
|
||||
key_digest = hashlib.sha256(storage_key.encode("utf-8")).hexdigest()
|
||||
request = {
|
||||
"tenant_id": tenant_id,
|
||||
"blob_id": candidate_id,
|
||||
"storage_key_sha256": key_digest,
|
||||
"approval_reference": approval_reference,
|
||||
}
|
||||
try:
|
||||
started = begin_durable_recovery_operation(
|
||||
get_database().SessionLocal,
|
||||
identity=process_runtime_identity(),
|
||||
module_id="files",
|
||||
operation_type="blob-garbage-collection",
|
||||
idempotency_key=(
|
||||
f"files-blob-gc:{candidate_id}:{key_digest[:12]}:"
|
||||
f"{hashlib.sha256(approval_reference.encode('utf-8')).hexdigest()[:12]}"
|
||||
),
|
||||
request=request,
|
||||
recovery_plan=RecoveryPlan(
|
||||
mode=RecoveryMode.FORWARD_RECOVERY,
|
||||
preconditions=(
|
||||
"the caller holds dedicated Files purge authority",
|
||||
"a fresh database check shows no FileVersion reference",
|
||||
),
|
||||
forward_recovery_steps=(
|
||||
"verify the object is absent",
|
||||
"delete the unreferenced FileBlob row after absence is proven",
|
||||
),
|
||||
verification_steps=(
|
||||
"recheck FileVersion references while holding the blob fence",
|
||||
"probe the exact managed storage key after deletion",
|
||||
),
|
||||
approval_reference=approval_reference,
|
||||
),
|
||||
precondition_evidence={
|
||||
"blob_id": candidate_id,
|
||||
"storage_key_sha256": key_digest,
|
||||
"observed_reference_count": 0,
|
||||
},
|
||||
lease_resource_key=f"files:blob:{tenant_id}:{candidate_id}",
|
||||
lease_ttl_seconds=15 * 60,
|
||||
resource_type="file_blob",
|
||||
resource_id=candidate_id,
|
||||
metadata={"resources": ["postgresql", "object-storage"], "storage_backend": backend.name},
|
||||
block_unresolved_resource=True,
|
||||
)
|
||||
except (RecoveryOperationBusy, RecoveryOperationStateConflict):
|
||||
continue
|
||||
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
||||
raise FileStorageError(
|
||||
"The Files recovery ledger is unavailable; no blob bytes were deleted"
|
||||
) from exc
|
||||
if started.replayed or started.operation is None:
|
||||
continue
|
||||
operation = started.operation
|
||||
try:
|
||||
blob = (
|
||||
session.query(FileBlob)
|
||||
.filter(FileBlob.id == candidate_id, FileBlob.tenant_id == tenant_id)
|
||||
.with_for_update()
|
||||
.one_or_none()
|
||||
)
|
||||
references = int(
|
||||
session.query(func.count(FileVersion.id))
|
||||
.filter(FileVersion.blob_id == candidate_id)
|
||||
.scalar()
|
||||
or 0
|
||||
)
|
||||
blob_retained = bool(
|
||||
blob is not None
|
||||
and blob.retained_until is not None
|
||||
and _as_utc(blob.retained_until) > utcnow()
|
||||
)
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
if not operation.closed:
|
||||
operation.reject(
|
||||
summary="Blob eligibility could not be rechecked before deletion",
|
||||
evidence=_verified_evidence(
|
||||
{"external_effect_started": False},
|
||||
exception_type=type(exc).__name__,
|
||||
),
|
||||
)
|
||||
raise FileStorageError(
|
||||
"Blob eligibility could not be rechecked; no bytes were deleted"
|
||||
) from exc
|
||||
if blob is None or references or blob_retained:
|
||||
operation.reject(
|
||||
summary="Blob garbage collection was no longer eligible",
|
||||
evidence=_verified_evidence(
|
||||
{
|
||||
"blob_present": blob is not None,
|
||||
"reference_count_zero": references == 0,
|
||||
"blob_retention_expired": not blob_retained,
|
||||
},
|
||||
reference_count=references,
|
||||
),
|
||||
)
|
||||
session.rollback()
|
||||
continue
|
||||
try:
|
||||
backend.delete(blob.storage_key)
|
||||
object_present = backend.exists(blob.storage_key)
|
||||
except (StorageBackendError, OSError) as exc:
|
||||
try:
|
||||
object_present = backend.exists(blob.storage_key)
|
||||
except (StorageBackendError, OSError):
|
||||
object_present = None
|
||||
session.rollback()
|
||||
if object_present is True:
|
||||
operation.fail(
|
||||
summary="The unreferenced blob object was retained",
|
||||
evidence={"object_present": True, "exception_type": type(exc).__name__},
|
||||
)
|
||||
else:
|
||||
operation.unresolved(
|
||||
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
||||
summary="The blob deletion outcome could not be verified",
|
||||
evidence={"object_present": object_present, "exception_type": type(exc).__name__},
|
||||
failure_summary="Reconcile the exact blob key before retrying garbage collection",
|
||||
)
|
||||
unresolved.append(started.operation_id)
|
||||
continue
|
||||
if object_present:
|
||||
session.rollback()
|
||||
operation.fail(
|
||||
summary="The object store did not delete the unreferenced blob",
|
||||
evidence={"object_present": True},
|
||||
)
|
||||
continue
|
||||
try:
|
||||
session.delete(blob)
|
||||
session.flush()
|
||||
operation.commit_verified_success(
|
||||
session,
|
||||
evidence=_verified_evidence(
|
||||
{
|
||||
"object_absent": True,
|
||||
"database_blob_absent": True,
|
||||
"reference_count_zero": True,
|
||||
},
|
||||
blob_id=candidate_id,
|
||||
storage_key_sha256=key_digest,
|
||||
object_present=False,
|
||||
database_blob_present=False,
|
||||
reference_count=0,
|
||||
),
|
||||
)
|
||||
deleted += 1
|
||||
except Exception as exc:
|
||||
session.rollback()
|
||||
if not operation.closed:
|
||||
operation.unresolved(
|
||||
status=RecoveryStatus.RECOVERY_REQUIRED,
|
||||
summary="The object is absent but blob metadata still requires forward recovery",
|
||||
evidence={"object_present": False, "exception_type": type(exc).__name__},
|
||||
failure_summary="Delete the FileBlob row only after rechecking all FileVersion references",
|
||||
)
|
||||
unresolved.append(started.operation_id)
|
||||
return BlobGcResult(
|
||||
inspected_blobs=len(candidates),
|
||||
deleted_blobs=deleted,
|
||||
unresolved_operation_ids=tuple(unresolved),
|
||||
)
|
||||
|
||||
|
||||
def _asset_purge_blockers(session: Session, asset: FileAsset) -> list[str]:
|
||||
blockers: list[str] = []
|
||||
if asset.deleted_at is None:
|
||||
blockers.append("not_soft_deleted")
|
||||
if asset.legal_hold:
|
||||
blockers.append("legal_hold")
|
||||
if asset.retained_until is not None and _as_utc(asset.retained_until) > utcnow():
|
||||
blockers.append("retention_active")
|
||||
if _table_exists(session, CampaignAttachmentUse.__tablename__) and session.query(
|
||||
CampaignAttachmentUse.id
|
||||
).filter(CampaignAttachmentUse.file_asset_id == asset.id).first() is not None:
|
||||
blockers.append("campaign_evidence")
|
||||
if _table_exists(session, FileFormEvidenceGrant.__tablename__) and session.query(
|
||||
FileFormEvidenceGrant.id
|
||||
).filter(FileFormEvidenceGrant.file_asset_id == asset.id).first() is not None:
|
||||
blockers.append("form_evidence")
|
||||
if session.query(FileShare.id).filter(
|
||||
FileShare.file_asset_id == asset.id,
|
||||
effective_file_share_clause(),
|
||||
).first() is not None:
|
||||
blockers.append("active_share")
|
||||
return blockers
|
||||
|
||||
|
||||
def _asset_owner_id(asset: FileAsset) -> str:
|
||||
owner_id = asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id
|
||||
if not owner_id:
|
||||
raise FileStorageError("File has no valid owner")
|
||||
return owner_id
|
||||
|
||||
|
||||
def _asset_owner_query(session: Session, asset: FileAsset):
|
||||
return _asset_owner_query_by_values(
|
||||
session,
|
||||
tenant_id=asset.tenant_id,
|
||||
owner_type=asset.owner_type,
|
||||
owner_id=_asset_owner_id(asset),
|
||||
)
|
||||
|
||||
|
||||
def _asset_owner_query_by_values(
|
||||
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
|
||||
):
|
||||
query = session.query(FileAsset).filter(
|
||||
FileAsset.tenant_id == tenant_id, FileAsset.owner_type == owner_type
|
||||
)
|
||||
if owner_type == "user":
|
||||
return query.filter(FileAsset.owner_user_id == owner_id)
|
||||
if owner_type == "group":
|
||||
return query.filter(FileAsset.owner_group_id == owner_id)
|
||||
raise FileStorageError("Files must be owned by a user or group")
|
||||
|
||||
|
||||
def _folder_owner_query(
|
||||
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
|
||||
):
|
||||
query = session.query(FileFolder).filter(
|
||||
FileFolder.tenant_id == tenant_id, FileFolder.owner_type == owner_type
|
||||
)
|
||||
if owner_type == "user":
|
||||
return query.filter(FileFolder.owner_user_id == owner_id)
|
||||
if owner_type == "group":
|
||||
return query.filter(FileFolder.owner_group_id == owner_id)
|
||||
raise FileStorageError("Folders must be owned by a user or group")
|
||||
|
||||
|
||||
def _connector_space_owner_query(session: Session, space: FileConnectorSpace):
|
||||
query = session.query(FileConnectorSpace).filter(
|
||||
FileConnectorSpace.tenant_id == space.tenant_id,
|
||||
FileConnectorSpace.owner_type == space.owner_type,
|
||||
)
|
||||
owner_id = connector_space_owner_id(space)
|
||||
if space.owner_type == "user":
|
||||
return query.filter(FileConnectorSpace.owner_user_id == owner_id)
|
||||
return query.filter(FileConnectorSpace.owner_group_id == owner_id)
|
||||
|
||||
|
||||
def _as_utc(value: datetime) -> datetime:
|
||||
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return _as_utc(value).isoformat() if value is not None else None
|
||||
|
||||
|
||||
def _table_exists(session: Session, table_name: str) -> bool:
|
||||
return bool(session.bind is not None and inspect(session.bind).has_table(table_name))
|
||||
|
||||
|
||||
def _verified_evidence(
|
||||
checks: dict[str, object], **details: object
|
||||
) -> dict[str, object]:
|
||||
return {"verified": True, "checks": checks, **details}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BlobGcResult",
|
||||
"PurgePreview",
|
||||
"PurgePreviewItem",
|
||||
"PurgeResult",
|
||||
"execute_asset_purge",
|
||||
"garbage_collect_unreferenced_blobs",
|
||||
"get_asset_for_lifecycle",
|
||||
"preview_asset_purge",
|
||||
"restore_asset",
|
||||
"restore_connector_space",
|
||||
"restore_folder",
|
||||
"set_asset_lifecycle",
|
||||
]
|
||||
Reference in New Issue
Block a user