Files
govoplan-files/src/govoplan_files/backend/storage/lifecycle.py
T

797 lines
28 KiB
Python

from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
import hashlib
import json
from typing import Callable, Iterable
from sqlalchemy import func, inspect, or_
from sqlalchemy.orm import Session
from govoplan_core.core.recovery import (
RecoveryGuaranteeError,
RecoveryMode,
RecoveryPlan,
RecoveryStatus,
)
from govoplan_core.core.recovery_runtime import (
RecoveryOperationBusy,
RecoveryOperationStateConflict,
begin_durable_recovery_operation,
)
from govoplan_core.core.runtime_coordination import process_runtime_identity
from govoplan_core.db.session import get_database
from govoplan_files.backend.db.models import (
CampaignAttachmentUse,
FileAsset,
FileBlob,
FileConnectorSpace,
FileFolder,
FileFormEvidenceGrant,
FileShare,
FileVersion,
)
from govoplan_files.backend.storage.access import ensure_owner_access
from govoplan_files.backend.storage.backends import (
StorageBackendError,
get_storage_backend,
)
from govoplan_files.backend.storage.common import FileStorageError, utcnow
from govoplan_files.backend.storage.connector_spaces import connector_space_owner_id
from govoplan_files.backend.storage.paths import normalize_folder
from govoplan_files.backend.storage.share_state import effective_file_share_clause
@dataclass(frozen=True, slots=True)
class PurgePreviewItem:
file_id: str
filename: str
lifecycle_revision: int
deleted_at: datetime | None
retained_until: datetime | None
legal_hold: bool
blockers: tuple[str, ...]
blob_ids: tuple[str, ...]
def digest_payload(self) -> dict[str, object]:
return {
"file_id": self.file_id,
"filename": self.filename,
"lifecycle_revision": self.lifecycle_revision,
"deleted_at": _iso(self.deleted_at),
"retained_until": _iso(self.retained_until),
"legal_hold": self.legal_hold,
"blockers": list(self.blockers),
"blob_ids": list(self.blob_ids),
}
@dataclass(frozen=True, slots=True)
class PurgePreview:
preview_sha256: str
items: tuple[PurgePreviewItem, ...]
@property
def eligible(self) -> bool:
return all(not item.blockers for item in self.items)
@dataclass(frozen=True, slots=True)
class PurgeResult:
recovery_operation_id: str
status: str
replayed: bool
purged_files: int
released_blobs: int
@dataclass(frozen=True, slots=True)
class BlobGcResult:
inspected_blobs: int
deleted_blobs: int
unresolved_operation_ids: tuple[str, ...]
def set_asset_lifecycle(
session: Session,
asset: FileAsset,
*,
retained_until: datetime | None,
legal_hold: bool,
reason: str,
expected_revision: int,
) -> FileAsset:
if asset.lifecycle_revision != expected_revision:
raise FileStorageError("File lifecycle settings changed; reload before saving")
asset.retained_until = retained_until
asset.legal_hold = bool(legal_hold)
asset.lifecycle_reason = reason.strip()
asset.lifecycle_revision += 1
session.add(asset)
session.flush()
return asset
def get_asset_for_lifecycle(
session: Session,
*,
tenant_id: str,
user_id: str,
asset_id: str,
is_admin: bool,
) -> FileAsset:
asset = session.get(FileAsset, asset_id)
if asset is None or asset.tenant_id != tenant_id:
raise FileStorageError("File not found")
ensure_owner_access(
session,
tenant_id=tenant_id,
owner_type=asset.owner_type,
owner_id=_asset_owner_id(asset),
user_id=user_id,
is_admin=is_admin,
)
return asset
def restore_asset(session: Session, asset: FileAsset) -> bool:
if asset.deleted_at is None:
return False
collision = _asset_owner_query(session, asset).filter(
FileAsset.id != asset.id,
FileAsset.display_path == asset.display_path,
FileAsset.deleted_at.is_(None),
).first()
if collision is not None:
raise FileStorageError(
"The file cannot be restored because its path is already in use"
)
asset.deleted_at = None
asset.lifecycle_revision += 1
session.add(asset)
session.flush()
return True
def restore_folder(
session: Session,
*,
tenant_id: str,
owner_type: str,
owner_id: str,
user_id: str,
path: str,
recursive: bool,
is_admin: bool,
) -> tuple[int, int]:
ensure_owner_access(
session,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
user_id=user_id,
is_admin=is_admin,
)
normalized = normalize_folder(path)
if not normalized:
raise FileStorageError("Folder path is required")
prefix = f"{normalized}/"
folders = _folder_owner_query(
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
).filter(FileFolder.deleted_at.is_not(None))
assets = _asset_owner_query_by_values(
session, tenant_id=tenant_id, owner_type=owner_type, owner_id=owner_id
).filter(FileAsset.deleted_at.is_not(None))
if recursive:
folders = folders.filter(
(FileFolder.path == normalized) | FileFolder.path.like(f"{prefix}%")
)
assets = assets.filter(FileAsset.display_path.like(f"{prefix}%"))
else:
folders = folders.filter(FileFolder.path == normalized)
assets = assets.filter(False)
folder_rows = folders.order_by(FileFolder.path.asc()).all()
asset_rows = assets.order_by(FileAsset.display_path.asc()).all()
if not folder_rows:
raise FileStorageError("Deleted folder not found")
for folder in folder_rows:
collision = _folder_owner_query(
session,
tenant_id=tenant_id,
owner_type=owner_type,
owner_id=owner_id,
).filter(
FileFolder.id != folder.id,
FileFolder.path == folder.path,
FileFolder.deleted_at.is_(None),
).first()
if collision is not None:
raise FileStorageError(
f"The folder cannot be restored because its path is in use: {folder.path}"
)
for asset in asset_rows:
collision = _asset_owner_query(session, asset).filter(
FileAsset.id != asset.id,
FileAsset.display_path == asset.display_path,
FileAsset.deleted_at.is_(None),
).first()
if collision is not None:
raise FileStorageError(
f"A file cannot be restored because its path is in use: {asset.display_path}"
)
for folder in folder_rows:
folder.deleted_at = None
session.add(folder)
for asset in asset_rows:
asset.deleted_at = None
asset.lifecycle_revision += 1
session.add(asset)
session.flush()
return len(folder_rows), len(asset_rows)
def restore_connector_space(
session: Session,
*,
tenant_id: str,
user_id: str,
space_id: str,
is_admin: bool,
) -> FileConnectorSpace:
space = session.get(FileConnectorSpace, space_id)
if space is None or space.tenant_id != tenant_id or space.deleted_at is None:
raise FileStorageError("Deleted connector space not found")
ensure_owner_access(
session,
tenant_id=tenant_id,
owner_type=space.owner_type,
owner_id=connector_space_owner_id(space),
user_id=user_id,
is_admin=is_admin,
)
collision = _connector_space_owner_query(session, space).filter(
FileConnectorSpace.id != space.id,
FileConnectorSpace.label == space.label,
FileConnectorSpace.deleted_at.is_(None),
).first()
if collision is not None:
raise FileStorageError(
"The connector space cannot be restored because its label is in use"
)
space.deleted_at = None
space.is_active = True
session.add(space)
session.flush()
return space
def preview_asset_purge(
session: Session, *, tenant_id: str, file_ids: Iterable[str]
) -> PurgePreview:
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
if not normalized_ids or len(normalized_ids) > 100:
raise FileStorageError("Purge preview must contain between 1 and 100 files")
assets = (
session.query(FileAsset)
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
.order_by(FileAsset.id.asc())
.all()
)
by_id = {asset.id: asset for asset in assets}
items: list[PurgePreviewItem] = []
for file_id in normalized_ids:
asset = by_id.get(file_id)
if asset is None:
items.append(
PurgePreviewItem(
file_id=file_id,
filename="",
lifecycle_revision=0,
deleted_at=None,
retained_until=None,
legal_hold=False,
blockers=("not_found",),
blob_ids=(),
)
)
continue
blockers = _asset_purge_blockers(session, asset)
blob_ids = tuple(
sorted(
row[0]
for row in session.query(FileVersion.blob_id)
.filter(FileVersion.file_asset_id == asset.id)
.distinct()
.all()
)
)
items.append(
PurgePreviewItem(
file_id=asset.id,
filename=asset.filename,
lifecycle_revision=asset.lifecycle_revision,
deleted_at=asset.deleted_at,
retained_until=asset.retained_until,
legal_hold=asset.legal_hold,
blockers=tuple(blockers),
blob_ids=blob_ids,
)
)
payload = [item.digest_payload() for item in items]
digest = hashlib.sha256(
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
).hexdigest()
return PurgePreview(preview_sha256=digest, items=tuple(items))
def execute_asset_purge(
session: Session,
*,
tenant_id: str,
file_ids: Iterable[str],
preview_sha256: str,
idempotency_key: str,
approval_reference: str,
before_commit: Callable[[list[FileAsset]], None] | None = None,
) -> PurgeResult:
normalized_ids = sorted(set(str(value).strip() for value in file_ids if str(value).strip()))
if not normalized_ids or len(normalized_ids) > 100:
raise FileStorageError("Purge must contain between 1 and 100 files")
request = {
"tenant_id": tenant_id,
"file_ids": normalized_ids,
"preview_sha256": preview_sha256,
"approval_reference": approval_reference,
}
try:
started = begin_durable_recovery_operation(
get_database().SessionLocal,
identity=process_runtime_identity(),
module_id="files",
operation_type="asset-hard-purge",
idempotency_key=f"files-asset-purge:{idempotency_key}",
request=request,
recovery_plan=RecoveryPlan(
mode=RecoveryMode.IRREVERSIBLE,
preconditions=(
"the actor holds the dedicated Files purge permission",
"the signed-off preview still matches current lifecycle state",
"every target is soft-deleted and free of retention or legal-hold blockers",
),
verification_steps=(
"verify every target FileAsset and FileVersion row is absent",
"recalculate retained FileBlob reference counts before later garbage collection",
),
approval_reference=approval_reference,
),
precondition_evidence={
"preview_sha256": preview_sha256,
"target_count": len(normalized_ids),
},
lease_resource_key=f"files:purge:{tenant_id}",
lease_ttl_seconds=15 * 60,
resource_type="file_asset_batch",
resource_id=preview_sha256,
metadata={"resources": ["postgresql"], "bounded_target_count": len(normalized_ids)},
block_unresolved_resource=True,
)
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
raise FileStorageError(
"Another purge or unresolved retry owns the tenant purge fence"
) from exc
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
raise FileStorageError("The Files recovery ledger is unavailable; nothing was purged") from exc
if started.replayed or started.operation is None:
return PurgeResult(
recovery_operation_id=started.operation_id,
status=started.status,
replayed=True,
purged_files=0,
released_blobs=0,
)
operation = started.operation
try:
preview = preview_asset_purge(session, tenant_id=tenant_id, file_ids=normalized_ids)
if preview.preview_sha256 != preview_sha256:
operation.reject(
summary="The purge preview became stale before execution",
evidence=_verified_evidence(
{"preview_matches": False},
expected_preview_sha256=preview_sha256,
observed_preview_sha256=preview.preview_sha256,
),
)
raise FileStorageError("The purge preview is stale; preview again")
if not preview.eligible:
operation.reject(
summary="The purge was blocked by current lifecycle policy",
evidence=_verified_evidence(
{"lifecycle_policy_allows": False},
blocked_files=[
{"file_id": item.file_id, "blockers": list(item.blockers)}
for item in preview.items
if item.blockers
]
),
)
raise FileStorageError("One or more files are not eligible for purge")
assets = (
session.query(FileAsset)
.filter(FileAsset.tenant_id == tenant_id, FileAsset.id.in_(normalized_ids))
.with_for_update()
.all()
)
blob_ids = sorted({blob_id for item in preview.items for blob_id in item.blob_ids})
if before_commit is not None:
before_commit(assets)
session.query(FileShare).filter(
FileShare.file_asset_id.in_(normalized_ids)
).delete(synchronize_session=False)
session.query(FileVersion).filter(
FileVersion.file_asset_id.in_(normalized_ids)
).delete(synchronize_session=False)
for asset in assets:
session.delete(asset)
session.flush()
released_blobs = 0
for blob_id in blob_ids:
references = int(
session.query(func.count(FileVersion.id))
.filter(FileVersion.blob_id == blob_id)
.scalar()
or 0
)
blob = session.get(FileBlob, blob_id)
if blob is not None:
blob.ref_count = references
session.add(blob)
if references == 0:
released_blobs += 1
operation.commit_verified_success(
session,
evidence=_verified_evidence(
{
"assets_absent": True,
"versions_absent": True,
"blob_reference_counts_recalculated": True,
},
purged_file_ids=normalized_ids,
purged_files=len(assets),
released_blobs=released_blobs,
preview_sha256=preview_sha256,
),
)
return PurgeResult(
recovery_operation_id=started.operation_id,
status=RecoveryStatus.SUCCEEDED.value,
replayed=False,
purged_files=len(assets),
released_blobs=released_blobs,
)
except FileStorageError:
session.rollback()
if not operation.closed:
operation.release_unresolved()
raise
except Exception as exc:
session.rollback()
if not operation.closed:
operation.reject(
summary="The purge failed before its database transaction committed",
evidence=_verified_evidence(
{"database_transaction_committed": False},
exception_type=type(exc).__name__,
),
)
raise
def garbage_collect_unreferenced_blobs(
session: Session,
*,
tenant_id: str,
limit: int,
approval_reference: str,
) -> BlobGcResult:
candidate_rows = (
session.query(FileBlob)
.filter(FileBlob.tenant_id == tenant_id)
.filter(
or_(FileBlob.retained_until.is_(None), FileBlob.retained_until <= utcnow())
)
.filter(
~session.query(FileVersion.id)
.filter(FileVersion.blob_id == FileBlob.id)
.exists()
)
.order_by(FileBlob.created_at.asc(), FileBlob.id.asc())
.limit(limit)
.all()
)
candidates = tuple(
(row.id, row.storage_key) for row in candidate_rows
)
# Close the read snapshot before the independent recovery transaction takes
# its durable SQLite/PostgreSQL write lock.
session.commit()
deleted = 0
unresolved: list[str] = []
backend = get_storage_backend()
for candidate_id, storage_key in candidates:
key_digest = hashlib.sha256(storage_key.encode("utf-8")).hexdigest()
request = {
"tenant_id": tenant_id,
"blob_id": candidate_id,
"storage_key_sha256": key_digest,
"approval_reference": approval_reference,
}
try:
started = begin_durable_recovery_operation(
get_database().SessionLocal,
identity=process_runtime_identity(),
module_id="files",
operation_type="blob-garbage-collection",
idempotency_key=(
f"files-blob-gc:{candidate_id}:{key_digest[:12]}:"
f"{hashlib.sha256(approval_reference.encode('utf-8')).hexdigest()[:12]}"
),
request=request,
recovery_plan=RecoveryPlan(
mode=RecoveryMode.FORWARD_RECOVERY,
preconditions=(
"the caller holds dedicated Files purge authority",
"a fresh database check shows no FileVersion reference",
),
forward_recovery_steps=(
"verify the object is absent",
"delete the unreferenced FileBlob row after absence is proven",
),
verification_steps=(
"recheck FileVersion references while holding the blob fence",
"probe the exact managed storage key after deletion",
),
approval_reference=approval_reference,
),
precondition_evidence={
"blob_id": candidate_id,
"storage_key_sha256": key_digest,
"observed_reference_count": 0,
},
lease_resource_key=f"files:blob:{tenant_id}:{candidate_id}",
lease_ttl_seconds=15 * 60,
resource_type="file_blob",
resource_id=candidate_id,
metadata={"resources": ["postgresql", "object-storage"], "storage_backend": backend.name},
block_unresolved_resource=True,
)
except (RecoveryOperationBusy, RecoveryOperationStateConflict):
continue
except (RecoveryGuaranteeError, RuntimeError, ValueError) as exc:
raise FileStorageError(
"The Files recovery ledger is unavailable; no blob bytes were deleted"
) from exc
if started.replayed or started.operation is None:
continue
operation = started.operation
try:
blob = (
session.query(FileBlob)
.filter(FileBlob.id == candidate_id, FileBlob.tenant_id == tenant_id)
.with_for_update()
.one_or_none()
)
references = int(
session.query(func.count(FileVersion.id))
.filter(FileVersion.blob_id == candidate_id)
.scalar()
or 0
)
blob_retained = bool(
blob is not None
and blob.retained_until is not None
and _as_utc(blob.retained_until) > utcnow()
)
except Exception as exc:
session.rollback()
if not operation.closed:
operation.reject(
summary="Blob eligibility could not be rechecked before deletion",
evidence=_verified_evidence(
{"external_effect_started": False},
exception_type=type(exc).__name__,
),
)
raise FileStorageError(
"Blob eligibility could not be rechecked; no bytes were deleted"
) from exc
if blob is None or references or blob_retained:
operation.reject(
summary="Blob garbage collection was no longer eligible",
evidence=_verified_evidence(
{
"blob_present": blob is not None,
"reference_count_zero": references == 0,
"blob_retention_expired": not blob_retained,
},
reference_count=references,
),
)
session.rollback()
continue
try:
backend.delete(blob.storage_key)
object_present = backend.exists(blob.storage_key)
except (StorageBackendError, OSError) as exc:
try:
object_present = backend.exists(blob.storage_key)
except (StorageBackendError, OSError):
object_present = None
session.rollback()
if object_present is True:
operation.fail(
summary="The unreferenced blob object was retained",
evidence={"object_present": True, "exception_type": type(exc).__name__},
)
else:
operation.unresolved(
status=RecoveryStatus.OUTCOME_UNKNOWN,
summary="The blob deletion outcome could not be verified",
evidence={"object_present": object_present, "exception_type": type(exc).__name__},
failure_summary="Reconcile the exact blob key before retrying garbage collection",
)
unresolved.append(started.operation_id)
continue
if object_present:
session.rollback()
operation.fail(
summary="The object store did not delete the unreferenced blob",
evidence={"object_present": True},
)
continue
try:
session.delete(blob)
session.flush()
operation.commit_verified_success(
session,
evidence=_verified_evidence(
{
"object_absent": True,
"database_blob_absent": True,
"reference_count_zero": True,
},
blob_id=candidate_id,
storage_key_sha256=key_digest,
object_present=False,
database_blob_present=False,
reference_count=0,
),
)
deleted += 1
except Exception as exc:
session.rollback()
if not operation.closed:
operation.unresolved(
status=RecoveryStatus.RECOVERY_REQUIRED,
summary="The object is absent but blob metadata still requires forward recovery",
evidence={"object_present": False, "exception_type": type(exc).__name__},
failure_summary="Delete the FileBlob row only after rechecking all FileVersion references",
)
unresolved.append(started.operation_id)
return BlobGcResult(
inspected_blobs=len(candidates),
deleted_blobs=deleted,
unresolved_operation_ids=tuple(unresolved),
)
def _asset_purge_blockers(session: Session, asset: FileAsset) -> list[str]:
blockers: list[str] = []
if asset.deleted_at is None:
blockers.append("not_soft_deleted")
if asset.legal_hold:
blockers.append("legal_hold")
if asset.retained_until is not None and _as_utc(asset.retained_until) > utcnow():
blockers.append("retention_active")
if _table_exists(session, CampaignAttachmentUse.__tablename__) and session.query(
CampaignAttachmentUse.id
).filter(CampaignAttachmentUse.file_asset_id == asset.id).first() is not None:
blockers.append("campaign_evidence")
if _table_exists(session, FileFormEvidenceGrant.__tablename__) and session.query(
FileFormEvidenceGrant.id
).filter(FileFormEvidenceGrant.file_asset_id == asset.id).first() is not None:
blockers.append("form_evidence")
if session.query(FileShare.id).filter(
FileShare.file_asset_id == asset.id,
effective_file_share_clause(),
).first() is not None:
blockers.append("active_share")
return blockers
def _asset_owner_id(asset: FileAsset) -> str:
owner_id = asset.owner_user_id if asset.owner_type == "user" else asset.owner_group_id
if not owner_id:
raise FileStorageError("File has no valid owner")
return owner_id
def _asset_owner_query(session: Session, asset: FileAsset):
return _asset_owner_query_by_values(
session,
tenant_id=asset.tenant_id,
owner_type=asset.owner_type,
owner_id=_asset_owner_id(asset),
)
def _asset_owner_query_by_values(
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
):
query = session.query(FileAsset).filter(
FileAsset.tenant_id == tenant_id, FileAsset.owner_type == owner_type
)
if owner_type == "user":
return query.filter(FileAsset.owner_user_id == owner_id)
if owner_type == "group":
return query.filter(FileAsset.owner_group_id == owner_id)
raise FileStorageError("Files must be owned by a user or group")
def _folder_owner_query(
session: Session, *, tenant_id: str, owner_type: str, owner_id: str
):
query = session.query(FileFolder).filter(
FileFolder.tenant_id == tenant_id, FileFolder.owner_type == owner_type
)
if owner_type == "user":
return query.filter(FileFolder.owner_user_id == owner_id)
if owner_type == "group":
return query.filter(FileFolder.owner_group_id == owner_id)
raise FileStorageError("Folders must be owned by a user or group")
def _connector_space_owner_query(session: Session, space: FileConnectorSpace):
query = session.query(FileConnectorSpace).filter(
FileConnectorSpace.tenant_id == space.tenant_id,
FileConnectorSpace.owner_type == space.owner_type,
)
owner_id = connector_space_owner_id(space)
if space.owner_type == "user":
return query.filter(FileConnectorSpace.owner_user_id == owner_id)
return query.filter(FileConnectorSpace.owner_group_id == owner_id)
def _as_utc(value: datetime) -> datetime:
return value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
def _iso(value: datetime | None) -> str | None:
return _as_utc(value).isoformat() if value is not None else None
def _table_exists(session: Session, table_name: str) -> bool:
return bool(session.bind is not None and inspect(session.bind).has_table(table_name))
def _verified_evidence(
checks: dict[str, object], **details: object
) -> dict[str, object]:
return {"verified": True, "checks": checks, **details}
__all__ = [
"BlobGcResult",
"PurgePreview",
"PurgePreviewItem",
"PurgeResult",
"execute_asset_purge",
"garbage_collect_unreferenced_blobs",
"get_asset_for_lifecycle",
"preview_asset_purge",
"restore_asset",
"restore_connector_space",
"restore_folder",
"set_asset_lifecycle",
]