797 lines
28 KiB
Python
797 lines
28 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
import hashlib
|
|
import json
|
|
from typing import Callable, Iterable
|
|
|
|
from sqlalchemy import func, inspect, or_
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_core.core.recovery import (
|
|
RecoveryGuaranteeError,
|
|
RecoveryMode,
|
|
RecoveryPlan,
|
|
RecoveryStatus,
|
|
)
|
|
from govoplan_core.core.recovery_runtime import (
|
|
RecoveryOperationBusy,
|
|
RecoveryOperationStateConflict,
|
|
begin_durable_recovery_operation,
|
|
)
|
|
from govoplan_core.core.runtime_coordination import process_runtime_identity
|
|
from govoplan_core.db.session import get_database
|
|
from govoplan_files.backend.db.models import (
|
|
CampaignAttachmentUse,
|
|
FileAsset,
|
|
FileBlob,
|
|
FileConnectorSpace,
|
|
FileFolder,
|
|
FileFormEvidenceGrant,
|
|
FileShare,
|
|
FileVersion,
|
|
)
|
|
from govoplan_files.backend.storage.access import ensure_owner_access
|
|
from govoplan_files.backend.storage.backends import (
|
|
StorageBackendError,
|
|
get_storage_backend,
|
|
)
|
|
from govoplan_files.backend.storage.common import FileStorageError, utcnow
|
|
from govoplan_files.backend.storage.connector_spaces import connector_space_owner_id
|
|
from govoplan_files.backend.storage.paths import normalize_folder
|
|
from govoplan_files.backend.storage.share_state import effective_file_share_clause
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PurgePreviewItem:
|
|
file_id: str
|
|
filename: str
|
|
lifecycle_revision: int
|
|
deleted_at: datetime | None
|
|
retained_until: datetime | None
|
|
legal_hold: bool
|
|
blockers: tuple[str, ...]
|
|
blob_ids: tuple[str, ...]
|
|
|
|
def digest_payload(self) -> dict[str, object]:
|
|
return {
|
|
"file_id": self.file_id,
|
|
"filename": self.filename,
|
|
"lifecycle_revision": self.lifecycle_revision,
|
|
"deleted_at": _iso(self.deleted_at),
|
|
"retained_until": _iso(self.retained_until),
|
|
"legal_hold": self.legal_hold,
|
|
"blockers": list(self.blockers),
|
|
"blob_ids": list(self.blob_ids),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PurgePreview:
|
|
preview_sha256: str
|
|
items: tuple[PurgePreviewItem, ...]
|
|
|
|
@property
|
|
def eligible(self) -> bool:
|
|
return all(not item.blockers for item in self.items)
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class PurgeResult:
|
|
recovery_operation_id: str
|
|
status: str
|
|
replayed: bool
|
|
purged_files: int
|
|
released_blobs: int
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class BlobGcResult:
|
|
inspected_blobs: int
|
|
deleted_blobs: int
|
|
unresolved_operation_ids: tuple[str, ...]
|
|
|
|
|
|
def set_asset_lifecycle(
|
|
session: Session,
|
|
asset: FileAsset,
|
|
*,
|
|
retained_until: datetime | None,
|
|
legal_hold: bool,
|
|
reason: str,
|
|
expected_revision: int,
|
|
) -> FileAsset:
|
|
if asset.lifecycle_revision != expected_revision:
|
|
raise FileStorageError("File lifecycle settings changed; reload before saving")
|
|
asset.retained_until = retained_until
|
|
asset.legal_hold = bool(legal_hold)
|
|
asset.lifecycle_reason = reason.strip()
|
|
asset.lifecycle_revision += 1
|
|
session.add(asset)
|
|
session.flush()
|
|
return asset
|
|
|
|
|
|
def get_asset_for_lifecycle(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
user_id: str,
|
|
asset_id: str,
|
|
is_admin: bool,
|
|
) -> FileAsset:
|
|
asset = session.get(FileAsset, asset_id)
|
|
if asset is None or asset.tenant_id != tenant_id:
|
|
raise FileStorageError("File not found")
|
|
ensure_owner_access(
|
|
session,
|
|
tenant_id=tenant_id,
|
|
owner_type=asset.owner_type,
|
|
owner_id=_asset_owner_id(asset),
|
|
user_id=user_id,
|
|
is_admin=is_admin,
|
|
)
|
|
return asset
|
|
|
|
|
|
def restore_asset(session: Session, asset: FileAsset) -> bool:
|
|
if asset.deleted_at is None:
|
|
return False
|
|
collision = _asset_owner_query(session, asset).filter(
|
|
FileAsset.id != asset.id,
|
|
FileAsset.display_path == asset.display_path,
|
|
FileAsset.deleted_at.is_(None),
|
|
).first()
|
|
if collision is not None:
|
|
raise FileStorageError(
|
|
"The file cannot be restored because its path is already in use"
|
|
)
|
|
asset.deleted_at = None
|
|
asset.lifecycle_revision += 1
|
|
session.add(asset)
|
|
session.flush()
|
|
return True
|
|
|
|
|
|
def restore_folder(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
owner_type: str,
|
|
owner_id: str,
|
|
user_id: str,
|
|
path: str,
|
|
recursive: bool,
|
|
is_admin: bool,
|
|
) -> tuple[int, int]:
|
|
ensure_owner_access(
|
|
session,
|
|
tenant_id=tenant_id,
|
|
owner_type=owner_type,
|
|
owner_id=owner_id,
|
|
user_id=user_id,
|
|
is_admin=is_admin,
|
|
)
|
|
normalized = normalize_folder(path)
|
|
if not normalized:
|
|
raise FileStorageError("Folder path is required")
|
|
prefix = f"{normalized}/"
|
|
folders = _folder_owner_query(
|
|
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
|
|
).filter(FileFolder.deleted_at.is_not(None))
|
|
assets = _asset_owner_query_by_values(
|
|
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
|
|
).filter(FileAsset.deleted_at.is_not(None))
|
|
if recursive:
|
|
folders = folders.filter(
|
|
(FileFolder.path == normalized) | FileFolder.path.like(f"{prefix}%")
|
|
)
|
|
assets = assets.filter(FileAsset.display_path.like(f"{prefix}%"))
|
|
else:
|
|
folders = folders.filter(FileFolder.path == normalized)
|
|
assets = assets.filter(False)
|
|
folder_rows = folders.order_by(FileFolder.path.asc()).all()
|
|
asset_rows = assets.order_by(FileAsset.display_path.asc()).all()
|
|
if not folder_rows:
|
|
raise FileStorageError("Deleted folder not found")
|
|
for folder in folder_rows:
|
|
collision = _folder_owner_query(
|
|
session,
|
|
tenant_id=tenant_id,
|
|
owner_type=owner_type,
|
|
owner_id=owner_id,
|
|
).filter(
|
|
FileFolder.id != folder.id,
|
|
FileFolder.path == folder.path,
|
|
FileFolder.deleted_at.is_(None),
|
|
).first()
|
|
if collision is not None:
|
|
raise FileStorageError(
|
|
f"The folder cannot be restored because its path is in use: {folder.path}"
|
|
)
|
|
for asset in asset_rows:
|
|
collision = _asset_owner_query(session, asset).filter(
|
|
FileAsset.id != asset.id,
|
|
FileAsset.display_path == asset.display_path,
|
|
FileAsset.deleted_at.is_(None),
|
|
).first()
|
|
if collision is not None:
|
|
raise FileStorageError(
|
|
f"A file cannot be restored because its path is in use: {asset.display_path}"
|
|
)
|
|
for folder in folder_rows:
|
|
folder.deleted_at = None
|
|
session.add(folder)
|
|
for asset in asset_rows:
|
|
asset.deleted_at = None
|
|
asset.lifecycle_revision += 1
|
|
session.add(asset)
|
|
session.flush()
|
|
return len(folder_rows), len(asset_rows)
|
|
|
|
|
|
def restore_connector_space(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
user_id: str,
|
|
space_id: str,
|
|
is_admin: bool,
|
|
) -> FileConnectorSpace:
|
|
space = session.get(FileConnectorSpace, space_id)
|
|
if space is None or space.tenant_id != tenant_id or space.deleted_at is None:
|
|
raise FileStorageError("Deleted connector space not found")
|
|
ensure_owner_access(
|
|
session,
|
|
tenant_id=tenant_id,
|
|
owner_type=space.owner_type,
|
|
owner_id=connector_space_owner_id(space),
|
|
user_id=user_id,
|
|
is_admin=is_admin,
|
|
)
|
|
collision = _connector_space_owner_query(session, space).filter(
|
|
FileConnectorSpace.id != space.id,
|
|
FileConnectorSpace.label == space.label,
|
|
FileConnectorSpace.deleted_at.is_(None),
|
|
).first()
|
|
if collision is not None:
|
|
raise FileStorageError(
|
|
"The connector space cannot be restored because its label is in use"
|
|
)
|
|
space.deleted_at = None
|
|
space.is_active = True
|
|
session.add(space)
|
|
session.flush()
|
|
return space
|
|
|
|
|
|
def preview_asset_purge(
|
|
session: Session, *, tenant_id: str, file_ids: Iterable[str]
|
|
) -> PurgePreview:
|
|
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
|
|
if not normalized_ids or len(normalized_ids) > 100:
|
|
raise FileStorageError("Purge preview must contain between 1 and 100 files")
|
|
assets = (
|
|
session.query(FileAsset)
|
|
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
|
|
.order_by(FileAsset.id.asc())
|
|
.all()
|
|
)
|
|
by_id = {asset.id: asset for asset in assets}
|
|
items: list[PurgePreviewItem] = []
|
|
for file_id in normalized_ids:
|
|
asset = by_id.get(file_id)
|
|
if asset is None:
|
|
items.append(
|
|
PurgePreviewItem(
|
|
file_id=file_id,
|
|
filename="",
|
|
lifecycle_revision=0,
|
|
deleted_at=None,
|
|
retained_until=None,
|
|
legal_hold=False,
|
|
blockers=("not_found",),
|
|
blob_ids=(),
|
|
)
|
|
)
|
|
continue
|
|
blockers = _asset_purge_blockers(session, asset)
|
|
blob_ids = tuple(
|
|
sorted(
|
|
row[0]
|
|
for row in session.query(FileVersion.blob_id)
|
|
.filter(FileVersion.file_asset_id == asset.id)
|
|
.distinct()
|
|
.all()
|
|
)
|
|
)
|
|
items.append(
|
|
PurgePreviewItem(
|
|
file_id=asset.id,
|
|
filename=asset.filename,
|
|
lifecycle_revision=asset.lifecycle_revision,
|
|
deleted_at=asset.deleted_at,
|
|
retained_until=asset.retained_until,
|
|
legal_hold=asset.legal_hold,
|
|
blockers=tuple(blockers),
|
|
blob_ids=blob_ids,
|
|
)
|
|
)
|
|
payload = [item.digest_payload() for item in items]
|
|
digest = hashlib.sha256(
|
|
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
|
).hexdigest()
|
|
return PurgePreview(preview_sha256=digest, items=tuple(items))
|
|
|
|
|
|
def execute_asset_purge(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
file_ids: Iterable[str],
|
|
preview_sha256: str,
|
|
idempotency_key: str,
|
|
approval_reference: str,
|
|
before_commit: Callable[[list[FileAsset]], None] | None = None,
|
|
) -> PurgeResult:
|
|
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
|
|
if not normalized_ids or len(normalized_ids) > 100:
|
|
raise FileStorageError("Purge must contain between 1 and 100 files")
|
|
request = {
|
|
"tenant_id": tenant_id,
|
|
"file_ids": normalized_ids,
|
|
"preview_sha256": preview_sha256,
|
|
"approval_reference": approval_reference,
|
|
}
|
|
try:
|
|
started = begin_durable_recovery_operation(
|
|
get_database().SessionLocal,
|
|
identity=process_runtime_identity(),
|
|
module_id="files",
|
|
operation_type="asset-hard-purge",
|
|
idempotency_key=f"files-asset-purge:{idempotency_key}",
|
|
request=request,
|
|
recovery_plan=RecoveryPlan(
|
|
mode=RecoveryMode.IRREVERSIBLE,
|
|
preconditions=(
|
|
"the actor holds the dedicated Files purge permission",
|
|
"the signed-off preview still matches current lifecycle state",
|
|
"every target is soft-deleted and free of retention or legal-hold blockers",
|
|
),
|
|
verification_steps=(
|
|
"verify every target FileAsset and FileVersion row is absent",
|
|
"recalculate retained FileBlob reference counts before later garbage collection",
|
|
),
|
|
approval_reference=approval_reference,
|
|
),
|
|
precondition_evidence={
|
|
"preview_sha256": preview_sha256,
|
|
"target_count": len(normalized_ids),
|
|
},
|
|
lease_resource_key=f"files:purge:{tenant_id}",
|
|
lease_ttl_seconds=15 * 60,
|
|
resource_type="file_asset_batch",
|
|
resource_id=preview_sha256,
|
|
metadata={"resources": ["postgresql"], "bounded_target_count": len(normalized_ids)},
|
|
block_unresolved_resource=True,
|
|
)
|
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
|
raise FileStorageError(
|
|
"Another purge or unresolved retry owns the tenant purge fence"
|
|
) from exc
|
|
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
|
raise FileStorageError("The Files recovery ledger is unavailable; nothing was purged") from exc
|
|
if started.replayed or started.operation is None:
|
|
return PurgeResult(
|
|
recovery_operation_id=started.operation_id,
|
|
status=started.status,
|
|
replayed=True,
|
|
purged_files=0,
|
|
released_blobs=0,
|
|
)
|
|
operation = started.operation
|
|
try:
|
|
preview = preview_asset_purge(session, tenant_id=tenant_id, file_ids=normalized_ids)
|
|
if preview.preview_sha256 != preview_sha256:
|
|
operation.reject(
|
|
summary="The purge preview became stale before execution",
|
|
evidence=_verified_evidence(
|
|
{"preview_matches": False},
|
|
expected_preview_sha256=preview_sha256,
|
|
observed_preview_sha256=preview.preview_sha256,
|
|
),
|
|
)
|
|
raise FileStorageError("The purge preview is stale; preview again")
|
|
if not preview.eligible:
|
|
operation.reject(
|
|
summary="The purge was blocked by current lifecycle policy",
|
|
evidence=_verified_evidence(
|
|
{"lifecycle_policy_allows": False},
|
|
blocked_files=[
|
|
{"file_id": item.file_id, "blockers": list(item.blockers)}
|
|
for item in preview.items
|
|
if item.blockers
|
|
]
|
|
),
|
|
)
|
|
raise FileStorageError("One or more files are not eligible for purge")
|
|
assets = (
|
|
session.query(FileAsset)
|
|
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
|
|
.with_for_update()
|
|
.all()
|
|
)
|
|
blob_ids = sorted({blob_id for item in preview.items for blob_id in item.blob_ids})
|
|
if before_commit is not None:
|
|
before_commit(assets)
|
|
session.query(FileShare).filter(
|
|
FileShare.file_asset_id.in_(normalized_ids)
|
|
).delete(synchronize_session=False)
|
|
session.query(FileVersion).filter(
|
|
FileVersion.file_asset_id.in_(normalized_ids)
|
|
).delete(synchronize_session=False)
|
|
for asset in assets:
|
|
session.delete(asset)
|
|
session.flush()
|
|
released_blobs = 0
|
|
for blob_id in blob_ids:
|
|
references = int(
|
|
session.query(func.count(FileVersion.id))
|
|
.filter(FileVersion.blob_id == blob_id)
|
|
.scalar()
|
|
or 0
|
|
)
|
|
blob = session.get(FileBlob, blob_id)
|
|
if blob is not None:
|
|
blob.ref_count = references
|
|
session.add(blob)
|
|
if references == 0:
|
|
released_blobs += 1
|
|
operation.commit_verified_success(
|
|
session,
|
|
evidence=_verified_evidence(
|
|
{
|
|
"assets_absent": True,
|
|
"versions_absent": True,
|
|
"blob_reference_counts_recalculated": True,
|
|
},
|
|
purged_file_ids=normalized_ids,
|
|
purged_files=len(assets),
|
|
released_blobs=released_blobs,
|
|
preview_sha256=preview_sha256,
|
|
),
|
|
)
|
|
return PurgeResult(
|
|
recovery_operation_id=started.operation_id,
|
|
status=RecoveryStatus.SUCCEEDED.value,
|
|
replayed=False,
|
|
purged_files=len(assets),
|
|
released_blobs=released_blobs,
|
|
)
|
|
except FileStorageError:
|
|
session.rollback()
|
|
if not operation.closed:
|
|
operation.release_unresolved()
|
|
raise
|
|
except Exception as exc:
|
|
session.rollback()
|
|
if not operation.closed:
|
|
operation.reject(
|
|
summary="The purge failed before its database transaction committed",
|
|
evidence=_verified_evidence(
|
|
{"database_transaction_committed": False},
|
|
exception_type=type(exc).__name__,
|
|
),
|
|
)
|
|
raise
|
|
|
|
|
|
def garbage_collect_unreferenced_blobs(
|
|
session: Session,
|
|
*,
|
|
tenant_id: str,
|
|
limit: int,
|
|
approval_reference: str,
|
|
) -> BlobGcResult:
|
|
candidate_rows = (
|
|
session.query(FileBlob)
|
|
.filter(FileBlob.tenant_id == tenant_id)
|
|
.filter(
|
|
or_(FileBlob.retained_until.is_(None), FileBlob.retained_until <= utcnow())
|
|
)
|
|
.filter(
|
|
~session.query(FileVersion.id)
|
|
.filter(FileVersion.blob_id == FileBlob.id)
|
|
.exists()
|
|
)
|
|
.order_by(FileBlob.created_at.asc(), FileBlob.id.asc())
|
|
.limit(limit)
|
|
.all()
|
|
)
|
|
candidates = tuple(
|
|
(row.id, row.storage_key) for row in candidate_rows
|
|
)
|
|
# Close the read snapshot before the independent recovery transaction takes
|
|
# its durable SQLite/PostgreSQL write lock.
|
|
session.commit()
|
|
deleted = 0
|
|
unresolved: list[str] = []
|
|
backend = get_storage_backend()
|
|
for candidate_id, storage_key in candidates:
|
|
key_digest = hashlib.sha256(storage_key.encode("utf-8")).hexdigest()
|
|
request = {
|
|
"tenant_id": tenant_id,
|
|
"blob_id": candidate_id,
|
|
"storage_key_sha256": key_digest,
|
|
"approval_reference": approval_reference,
|
|
}
|
|
try:
|
|
started = begin_durable_recovery_operation(
|
|
get_database().SessionLocal,
|
|
identity=process_runtime_identity(),
|
|
module_id="files",
|
|
operation_type="blob-garbage-collection",
|
|
idempotency_key=(
|
|
f"files-blob-gc:{candidate_id}:{key_digest[:12]}:"
|
|
f"{hashlib.sha256(approval_reference.encode('utf-8')).hexdigest()[:12]}"
|
|
),
|
|
request=request,
|
|
recovery_plan=RecoveryPlan(
|
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
|
preconditions=(
|
|
"the caller holds dedicated Files purge authority",
|
|
"a fresh database check shows no FileVersion reference",
|
|
),
|
|
forward_recovery_steps=(
|
|
"verify the object is absent",
|
|
"delete the unreferenced FileBlob row after absence is proven",
|
|
),
|
|
verification_steps=(
|
|
"recheck FileVersion references while holding the blob fence",
|
|
"probe the exact managed storage key after deletion",
|
|
),
|
|
approval_reference=approval_reference,
|
|
),
|
|
precondition_evidence={
|
|
"blob_id": candidate_id,
|
|
"storage_key_sha256": key_digest,
|
|
"observed_reference_count": 0,
|
|
},
|
|
lease_resource_key=f"files:blob:{tenant_id}:{candidate_id}",
|
|
lease_ttl_seconds=15 * 60,
|
|
resource_type="file_blob",
|
|
resource_id=candidate_id,
|
|
metadata={"resources": ["postgresql", "object-storage"], "storage_backend": backend.name},
|
|
block_unresolved_resource=True,
|
|
)
|
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict):
|
|
continue
|
|
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
|
|
raise FileStorageError(
|
|
"The Files recovery ledger is unavailable; no blob bytes were deleted"
|
|
) from exc
|
|
if started.replayed or started.operation is None:
|
|
continue
|
|
operation = started.operation
|
|
try:
|
|
blob = (
|
|
session.query(FileBlob)
|
|
.filter(FileBlob.id == candidate_id, FileBlob.tenant_id == tenant_id)
|
|
.with_for_update()
|
|
.one_or_none()
|
|
)
|
|
references = int(
|
|
session.query(func.count(FileVersion.id))
|
|
.filter(FileVersion.blob_id == candidate_id)
|
|
.scalar()
|
|
or 0
|
|
)
|
|
blob_retained = bool(
|
|
blob is not None
|
|
and blob.retained_until is not None
|
|
and _as_utc(blob.retained_until) > utcnow()
|
|
)
|
|
except Exception as exc:
|
|
session.rollback()
|
|
if not operation.closed:
|
|
operation.reject(
|
|
summary="Blob eligibility could not be rechecked before deletion",
|
|
evidence=_verified_evidence(
|
|
{"external_effect_started": False},
|
|
exception_type=type(exc).__name__,
|
|
),
|
|
)
|
|
raise FileStorageError(
|
|
"Blob eligibility could not be rechecked; no bytes were deleted"
|
|
) from exc
|
|
if blob is None or references or blob_retained:
|
|
operation.reject(
|
|
summary="Blob garbage collection was no longer eligible",
|
|
evidence=_verified_evidence(
|
|
{
|
|
"blob_present": blob is not None,
|
|
"reference_count_zero": references == 0,
|
|
"blob_retention_expired": not blob_retained,
|
|
},
|
|
reference_count=references,
|
|
),
|
|
)
|
|
session.rollback()
|
|
continue
|
|
try:
|
|
backend.delete(blob.storage_key)
|
|
object_present = backend.exists(blob.storage_key)
|
|
except (StorageBackendError, OSError) as exc:
|
|
try:
|
|
object_present = backend.exists(blob.storage_key)
|
|
except (StorageBackendError, OSError):
|
|
object_present = None
|
|
session.rollback()
|
|
if object_present is True:
|
|
operation.fail(
|
|
summary="The unreferenced blob object was retained",
|
|
evidence={"object_present": True, "exception_type": type(exc).__name__},
|
|
)
|
|
else:
|
|
operation.unresolved(
|
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
|
summary="The blob deletion outcome could not be verified",
|
|
evidence={"object_present": object_present, "exception_type": type(exc).__name__},
|
|
failure_summary="Reconcile the exact blob key before retrying garbage collection",
|
|
)
|
|
unresolved.append(started.operation_id)
|
|
continue
|
|
if object_present:
|
|
session.rollback()
|
|
operation.fail(
|
|
summary="The object store did not delete the unreferenced blob",
|
|
evidence={"object_present": True},
|
|
)
|
|
continue
|
|
try:
|
|
session.delete(blob)
|
|
session.flush()
|
|
operation.commit_verified_success(
|
|
session,
|
|
evidence=_verified_evidence(
|
|
{
|
|
"object_absent": True,
|
|
"database_blob_absent": True,
|
|
"reference_count_zero": True,
|
|
},
|
|
blob_id=candidate_id,
|
|
storage_key_sha256=key_digest,
|
|
object_present=False,
|
|
database_blob_present=False,
|
|
reference_count=0,
|
|
),
|
|
)
|
|
deleted += 1
|
|
except Exception as exc:
|
|
session.rollback()
|
|
if not operation.closed:
|
|
operation.unresolved(
|
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
|
summary="The object is absent but blob metadata still requires forward recovery",
|
|
evidence={"object_present": False, "exception_type": type(exc).__name__},
|
|
failure_summary="Delete the FileBlob row only after rechecking all FileVersion references",
|
|
)
|
|
unresolved.append(started.operation_id)
|
|
return BlobGcResult(
|
|
inspected_blobs=len(candidates),
|
|
deleted_blobs=deleted,
|
|
unresolved_operation_ids=tuple(unresolved),
|
|
)
|
|
|
|
|
|
def _asset_purge_blockers(session: Session, asset: FileAsset) -> list[str]:
|
|
blockers: list[str] = []
|
|
if asset.deleted_at is None:
|
|
blockers.append("not_soft_deleted")
|
|
if asset.legal_hold:
|
|
blockers.append("legal_hold")
|
|
if asset.retained_until is not None and _as_utc(asset.retained_until) > utcnow():
|
|
blockers.append("retention_active")
|
|
if _table_exists(session, CampaignAttachmentUse.__tablename__) and session.query(
|
|
CampaignAttachmentUse.id
|
|
).filter(CampaignAttachmentUse.file_asset_id == asset.id).first() is not None:
|
|
blockers.append("campaign_evidence")
|
|
if _table_exists(session, FileFormEvidenceGrant.__tablename__) and session.query(
|
|
FileFormEvidenceGrant.id
|
|
).filter(FileFormEvidenceGrant.file_asset_id == asset.id).first() is not None:
|
|
blockers.append("form_evidence")
|
|
if session.query(FileShare.id).filter(
|
|
FileShare.file_asset_id == asset.id,
|
|
effective_file_share_clause(),
|
|
).first() is not None:
|
|
blockers.append("active_share")
|
|
return blockers
|
|
|
|
|
|
def _asset_owner_id(asset: FileAsset) -> str:
|
|
owner_id = asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id
|
|
if not owner_id:
|
|
raise FileStorageError("File has no valid owner")
|
|
return owner_id
|
|
|
|
|
|
def _asset_owner_query(session: Session, asset: FileAsset):
|
|
return _asset_owner_query_by_values(
|
|
session,
|
|
tenant_id=asset.tenant_id,
|
|
owner_type=asset.owner_type,
|
|
owner_id=_asset_owner_id(asset),
|
|
)
|
|
|
|
|
|
def _asset_owner_query_by_values(
|
|
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
|
|
):
|
|
query = session.query(FileAsset).filter(
|
|
FileAsset.tenant_id == tenant_id, FileAsset.owner_type == owner_type
|
|
)
|
|
if owner_type == "user":
|
|
return query.filter(FileAsset.owner_user_id == owner_id)
|
|
if owner_type == "group":
|
|
return query.filter(FileAsset.owner_group_id == owner_id)
|
|
raise FileStorageError("Files must be owned by a user or group")
|
|
|
|
|
|
def _folder_owner_query(
|
|
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
|
|
):
|
|
query = session.query(FileFolder).filter(
|
|
FileFolder.tenant_id == tenant_id, FileFolder.owner_type == owner_type
|
|
)
|
|
if owner_type == "user":
|
|
return query.filter(FileFolder.owner_user_id == owner_id)
|
|
if owner_type == "group":
|
|
return query.filter(FileFolder.owner_group_id == owner_id)
|
|
raise FileStorageError("Folders must be owned by a user or group")
|
|
|
|
|
|
def _connector_space_owner_query(session: Session, space: FileConnectorSpace):
|
|
query = session.query(FileConnectorSpace).filter(
|
|
FileConnectorSpace.tenant_id == space.tenant_id,
|
|
FileConnectorSpace.owner_type == space.owner_type,
|
|
)
|
|
owner_id = connector_space_owner_id(space)
|
|
if space.owner_type == "user":
|
|
return query.filter(FileConnectorSpace.owner_user_id == owner_id)
|
|
return query.filter(FileConnectorSpace.owner_group_id == owner_id)
|
|
|
|
|
|
def _as_utc(value: datetime) -> datetime:
|
|
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
|
|
|
|
|
|
def _iso(value: datetime | None) -> str | None:
|
|
return _as_utc(value).isoformat() if value is not None else None
|
|
|
|
|
|
def _table_exists(session: Session, table_name: str) -> bool:
|
|
return bool(session.bind is not None and inspect(session.bind).has_table(table_name))
|
|
|
|
|
|
def _verified_evidence(
|
|
checks: dict[str, object], **details: object
|
|
) -> dict[str, object]:
|
|
return {"verified": True, "checks": checks, **details}
|
|
|
|
|
|
__all__ = [
|
|
"BlobGcResult",
|
|
"PurgePreview",
|
|
"PurgePreviewItem",
|
|
"PurgeResult",
|
|
"execute_asset_purge",
|
|
"garbage_collect_unreferenced_blobs",
|
|
"get_asset_for_lifecycle",
|
|
"preview_asset_purge",
|
|
"restore_asset",
|
|
"restore_connector_space",
|
|
"restore_folder",
|
|
"set_asset_lifecycle",
|
|
]
|