932 lines
34 KiB
Python
932 lines
34 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
import hashlib
|
|
from threading import Lock
|
|
from typing import Protocol
|
|
|
|
from sqlalchemy import event, select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from govoplan_core.core.recovery import (
|
|
RecoveryGuaranteeError,
|
|
RecoveryMode,
|
|
RecoveryOperation,
|
|
RecoveryPlan,
|
|
RecoveryStatus,
|
|
plan_recovery_operation,
|
|
prepare_recovery_operation,
|
|
start_recovery_operation,
|
|
verify_recovery_evidence_chain,
|
|
)
|
|
from govoplan_core.core.recovery_runtime import (
|
|
DurableRecoveryOperation,
|
|
DurableRecoveryStart,
|
|
RecoveryOperationBusy,
|
|
RecoveryOperationStateConflict,
|
|
begin_durable_recovery_operation,
|
|
)
|
|
from govoplan_core.core.runtime_coordination import (
|
|
RuntimeIdentity,
|
|
acquire_lease,
|
|
process_runtime_identity,
|
|
release_lease,
|
|
)
|
|
from govoplan_core.db.session import get_database
|
|
from govoplan_files.backend.db.models import (
|
|
FileBlob,
|
|
FileIntegrityFinding,
|
|
FileIntegrityScan,
|
|
)
|
|
from govoplan_files.backend.storage.backends import (
|
|
StorageBackend,
|
|
StorageBackendError,
|
|
StorageObjectMissing,
|
|
)
|
|
from govoplan_files.backend.storage.common import FileStorageError
|
|
|
|
|
|
_PENDING_EFFECTS_KEY = "govoplan_files_pending_recovery_effects"
|
|
_HOOKS_INSTALLED_KEY = "govoplan_files_recovery_hooks_installed"
|
|
_ROLLBACK_ERRORS_KEY = "govoplan_files_recovery_rollback_errors"
|
|
_SQLITE_FENCES: set[str] = set()
|
|
_SQLITE_FENCES_LOCK = Lock()
|
|
|
|
|
|
class _PendingEffect(Protocol):
|
|
def settle(self, *, committed: bool) -> None: ...
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class PendingBlobWrite:
|
|
operation: DurableRecoveryOperation
|
|
backend: StorageBackend
|
|
tenant_id: str
|
|
blob_id: str
|
|
storage_key: str
|
|
semantic_checksum_sha256: str
|
|
semantic_size_bytes: int
|
|
protection_discriminator: str
|
|
created_new: bool
|
|
expected_storage_checksum_sha256: str | None = None
|
|
expected_storage_size_bytes: int | None = None
|
|
expected_envelope_id: str | None = None
|
|
sqlite_fence_key: str | None = None
|
|
restart_after_rollback: Callable[[], DurableRecoveryOperation] | None = None
|
|
|
|
def prepare_stored_bytes(
|
|
self,
|
|
data: bytes,
|
|
*,
|
|
envelope_id: str | None,
|
|
) -> None:
|
|
"""Retain process-local verification evidence without recording content."""
|
|
|
|
self.expected_storage_checksum_sha256 = hashlib.sha256(data).hexdigest()
|
|
self.expected_storage_size_bytes = len(data)
|
|
self.expected_envelope_id = envelope_id
|
|
|
|
def settle(self, *, committed: bool) -> None:
|
|
if not committed and self.restart_after_rollback is not None:
|
|
try:
|
|
self.operation = self.restart_after_rollback()
|
|
except Exception:
|
|
# SQLite cannot make the pre-effect ledger row independent of
|
|
# the caller transaction. If reconstructing the evidence row
|
|
# also fails, still avoid leaving a known new orphan behind.
|
|
if self.created_new:
|
|
try:
|
|
self.backend.delete(self.storage_key)
|
|
except (StorageBackendError, OSError):
|
|
pass
|
|
raise
|
|
evidence = _blob_write_evidence(self)
|
|
if _blob_write_complete(evidence):
|
|
self.operation.succeed(evidence=evidence)
|
|
return
|
|
|
|
if self.created_new and evidence.get("database_blob_present") is False:
|
|
self._settle_unreferenced_new_object(evidence)
|
|
return
|
|
|
|
if not self.created_new and _object_matches(evidence):
|
|
if _forward_complete_blob_repair(self):
|
|
completed = _blob_write_evidence(self)
|
|
if _blob_write_complete(completed):
|
|
self.operation.succeed(evidence=completed)
|
|
return
|
|
|
|
if evidence.get("database_blob_present") is True and not _object_matches(
|
|
evidence
|
|
):
|
|
_quarantine_blob_after_failed_verification(self, evidence)
|
|
evidence = _blob_write_evidence(self)
|
|
|
|
status = (
|
|
RecoveryStatus.OUTCOME_UNKNOWN
|
|
if not evidence.get("verified")
|
|
else RecoveryStatus.RECOVERY_REQUIRED
|
|
)
|
|
transaction = "committed" if committed else "rolled back"
|
|
self.operation.unresolved(
|
|
status=status,
|
|
summary=f"Managed Files blob write remained unresolved after the database transaction {transaction}",
|
|
evidence=evidence,
|
|
failure_summary=(
|
|
"The managed object and Files blob metadata require reconciliation"
|
|
),
|
|
)
|
|
|
|
def _settle_unreferenced_new_object(self, evidence: dict[str, object]) -> None:
|
|
object_present = evidence.get("object_present")
|
|
if object_present is False:
|
|
self.operation.reject(
|
|
summary="The Files blob write left no durable object or database row",
|
|
evidence=evidence,
|
|
)
|
|
return
|
|
if object_present is not True:
|
|
self.operation.unresolved(
|
|
status=RecoveryStatus.OUTCOME_UNKNOWN,
|
|
summary="The unreferenced Files object could not be probed",
|
|
evidence=evidence,
|
|
failure_summary="Object storage availability prevented upload compensation",
|
|
)
|
|
return
|
|
try:
|
|
self.backend.delete(self.storage_key)
|
|
except (StorageBackendError, OSError):
|
|
self.operation.unresolved(
|
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
|
summary="An unreferenced Files object could not be compensated",
|
|
evidence=evidence,
|
|
failure_summary="Delete the unreferenced managed object after verifying that no FileBlob references it",
|
|
)
|
|
return
|
|
recovered = _blob_write_evidence(self)
|
|
if (
|
|
recovered.get("verified") is True
|
|
and recovered.get("database_blob_present") is False
|
|
and recovered.get("object_present") is False
|
|
):
|
|
self.operation.compensate(
|
|
failure_summary="The Files database transaction did not retain the new blob",
|
|
failure_evidence=evidence,
|
|
recovery_evidence=recovered,
|
|
)
|
|
return
|
|
self.operation.unresolved(
|
|
status=RecoveryStatus.RECOVERY_REQUIRED,
|
|
summary="Files upload compensation could not be verified",
|
|
evidence=recovered,
|
|
failure_summary="The unreferenced managed object requires operator reconciliation",
|
|
)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class PendingOrphanCleanup:
|
|
operation: DurableRecoveryOperation
|
|
backend: StorageBackend
|
|
finding_id: str
|
|
tenant_id: str
|
|
storage_key: str
|
|
resolved_by_user_id: str
|
|
|
|
def settle(self, *, committed: bool) -> None:
|
|
del committed
|
|
evidence = _orphan_cleanup_evidence(self)
|
|
if _orphan_cleanup_complete(evidence):
|
|
self.operation.succeed(evidence=evidence)
|
|
return
|
|
if (
|
|
evidence.get("verified") is True
|
|
and evidence.get("object_present") is True
|
|
and evidence.get("finding_deleted") is False
|
|
):
|
|
self.operation.reject(
|
|
summary="The orphan object was retained and the cleanup finding stayed open",
|
|
evidence=evidence,
|
|
)
|
|
return
|
|
if evidence.get("object_present") is False:
|
|
if _forward_complete_orphan_finding(self):
|
|
completed = _orphan_cleanup_evidence(self)
|
|
if _orphan_cleanup_complete(completed):
|
|
self.operation.succeed(evidence=completed)
|
|
return
|
|
status = (
|
|
RecoveryStatus.OUTCOME_UNKNOWN
|
|
if not evidence.get("verified")
|
|
else RecoveryStatus.RECOVERY_REQUIRED
|
|
)
|
|
self.operation.unresolved(
|
|
status=status,
|
|
summary="Files orphan cleanup requires reconciliation",
|
|
evidence=evidence,
|
|
failure_summary="Recheck the object and integrity-finding state before another cleanup attempt",
|
|
)
|
|
|
|
|
|
def begin_blob_write_recovery(
|
|
session: Session,
|
|
*,
|
|
backend: StorageBackend,
|
|
tenant_id: str,
|
|
blob_id: str,
|
|
storage_key: str,
|
|
semantic_checksum_sha256: str,
|
|
semantic_size_bytes: int,
|
|
protection_discriminator: str,
|
|
created_new: bool,
|
|
repair_token: str | None = None,
|
|
) -> PendingBlobWrite:
|
|
disposition = "create" if created_new else "repair"
|
|
state_token = repair_token or blob_id
|
|
key_digest = hashlib.sha256(storage_key.encode("utf-8")).hexdigest()
|
|
idempotency_key = (
|
|
f"files-blob-{disposition}:{blob_id}:{state_token[:48]}"
|
|
)
|
|
try:
|
|
identity = process_runtime_identity()
|
|
except RuntimeError as exc:
|
|
raise FileStorageError(
|
|
"The Files recovery ledger is unavailable; no object was written"
|
|
) from exc
|
|
lease_resource_key = f"files:blob:{tenant_id}:{blob_id}"
|
|
request = {
|
|
"tenant_id": tenant_id,
|
|
"blob_id": blob_id,
|
|
"storage_key": storage_key if created_new else None,
|
|
"storage_key_sha256": key_digest,
|
|
"semantic_checksum_sha256": semantic_checksum_sha256,
|
|
"semantic_size_bytes": semantic_size_bytes,
|
|
"protection_discriminator": protection_discriminator,
|
|
"disposition": disposition,
|
|
}
|
|
recovery_plan = RecoveryPlan(
|
|
mode=(
|
|
RecoveryMode.COMPENSATION
|
|
if created_new
|
|
else RecoveryMode.FORWARD_RECOVERY
|
|
),
|
|
preconditions=(
|
|
"the caller has Files write authority for the target owner",
|
|
"the storage key belongs to the tenant Files namespace",
|
|
"the request records digests rather than file contents",
|
|
),
|
|
compensation_steps=(
|
|
"verify that no FileBlob references the newly reserved key",
|
|
"delete only that unreferenced key and verify absence",
|
|
)
|
|
if created_new
|
|
else (),
|
|
forward_recovery_steps=(
|
|
"verify the expected bytes at the existing blob key",
|
|
"forward-complete matching integrity metadata or quarantine the blob",
|
|
)
|
|
if not created_new
|
|
else (),
|
|
verification_steps=(
|
|
"reload FileBlob metadata through an independent session",
|
|
"stream and hash the managed object independently",
|
|
),
|
|
)
|
|
precondition_evidence = {
|
|
"blob_id": blob_id,
|
|
"storage_key_sha256": key_digest,
|
|
"semantic_checksum_sha256": semantic_checksum_sha256,
|
|
"semantic_size_bytes": semantic_size_bytes,
|
|
"created_new": created_new,
|
|
}
|
|
metadata = {
|
|
"resources": ["postgresql", "object-storage"],
|
|
"storage_backend": backend.name,
|
|
"durability_mode": "independent_transaction",
|
|
}
|
|
session_factory = get_database().SessionLocal
|
|
sqlite_mode = session.get_bind().dialect.name == "sqlite"
|
|
sqlite_fence_key: str | None = None
|
|
|
|
def start_independent(*, reconstructed_after_rollback: bool = False) -> DurableRecoveryStart:
|
|
return begin_durable_recovery_operation(
|
|
session_factory,
|
|
identity=identity,
|
|
module_id="files",
|
|
operation_type=f"blob-{disposition}",
|
|
idempotency_key=idempotency_key,
|
|
request=request,
|
|
recovery_plan=recovery_plan,
|
|
precondition_evidence={
|
|
**precondition_evidence,
|
|
**(
|
|
{
|
|
"sqlite_caller_transaction_rolled_back": True,
|
|
"effect_may_have_preceded_durable_intent": True,
|
|
}
|
|
if reconstructed_after_rollback
|
|
else {}
|
|
),
|
|
},
|
|
lease_resource_key=lease_resource_key,
|
|
lease_ttl_seconds=15 * 60,
|
|
resource_type="file_blob",
|
|
resource_id=blob_id,
|
|
metadata={
|
|
**metadata,
|
|
"durability_mode": (
|
|
"sqlite_post_rollback_reconstruction"
|
|
if reconstructed_after_rollback
|
|
else "independent_transaction"
|
|
),
|
|
},
|
|
)
|
|
|
|
try:
|
|
if sqlite_mode:
|
|
_reserve_sqlite_fence(lease_resource_key)
|
|
sqlite_fence_key = lease_resource_key
|
|
started = _begin_caller_transaction_recovery_operation(
|
|
session,
|
|
session_factory=session_factory,
|
|
identity=identity,
|
|
module_id="files",
|
|
operation_type=f"blob-{disposition}",
|
|
idempotency_key=idempotency_key,
|
|
request=request,
|
|
recovery_plan=recovery_plan,
|
|
precondition_evidence={
|
|
**precondition_evidence,
|
|
"sqlite_caller_transaction": True,
|
|
"reduced_crash_durability": True,
|
|
},
|
|
lease_resource_key=lease_resource_key,
|
|
lease_ttl_seconds=15 * 60,
|
|
resource_type="file_blob",
|
|
resource_id=blob_id,
|
|
metadata={
|
|
**metadata,
|
|
"resources": ["sqlite", "object-storage"],
|
|
"durability_mode": "sqlite_caller_transaction",
|
|
},
|
|
)
|
|
else:
|
|
started = start_independent()
|
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
|
if sqlite_fence_key is not None:
|
|
_release_sqlite_fence(sqlite_fence_key)
|
|
raise FileStorageError(
|
|
"This managed blob is already owned by another recovery operation"
|
|
) from exc
|
|
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
|
if sqlite_fence_key is not None:
|
|
_release_sqlite_fence(sqlite_fence_key)
|
|
raise FileStorageError(
|
|
"The Files recovery ledger is unavailable; no object was written"
|
|
) from exc
|
|
if started.replayed or started.operation is None:
|
|
if sqlite_fence_key is not None:
|
|
_release_sqlite_fence(sqlite_fence_key)
|
|
raise FileStorageError(
|
|
"The matching Files blob operation was already completed; reload before retrying"
|
|
)
|
|
pending = PendingBlobWrite(
|
|
operation=started.operation,
|
|
backend=backend,
|
|
tenant_id=tenant_id,
|
|
blob_id=blob_id,
|
|
storage_key=storage_key,
|
|
semantic_checksum_sha256=semantic_checksum_sha256,
|
|
semantic_size_bytes=semantic_size_bytes,
|
|
protection_discriminator=protection_discriminator,
|
|
created_new=created_new,
|
|
sqlite_fence_key=sqlite_fence_key,
|
|
restart_after_rollback=(
|
|
lambda: _require_started_operation(
|
|
start_independent(reconstructed_after_rollback=True)
|
|
)
|
|
if sqlite_mode
|
|
else None
|
|
),
|
|
)
|
|
_register_pending_effect(session, pending)
|
|
return pending
|
|
|
|
|
|
def begin_orphan_cleanup_recovery(
|
|
session: Session,
|
|
finding: FileIntegrityFinding,
|
|
*,
|
|
backend: StorageBackend,
|
|
user_id: str,
|
|
) -> PendingOrphanCleanup:
|
|
key_digest = hashlib.sha256(finding.storage_key.encode("utf-8")).hexdigest()
|
|
try:
|
|
started = begin_durable_recovery_operation(
|
|
get_database().SessionLocal,
|
|
identity=process_runtime_identity(),
|
|
module_id="files",
|
|
operation_type="integrity-orphan-cleanup",
|
|
idempotency_key=f"files-orphan-cleanup:{finding.id}",
|
|
request={
|
|
"tenant_id": finding.tenant_id,
|
|
"finding_id": finding.id,
|
|
"storage_key_sha256": key_digest,
|
|
},
|
|
recovery_plan=RecoveryPlan(
|
|
mode=RecoveryMode.FORWARD_RECOVERY,
|
|
preconditions=(
|
|
"the integrity finding identifies an unreferenced object",
|
|
"the object key remains inside the completed scan scope",
|
|
"a fresh database reference check found no FileBlob",
|
|
),
|
|
forward_recovery_steps=(
|
|
"verify object absence independently",
|
|
"mark the durable finding deleted only after absence is proven",
|
|
),
|
|
verification_steps=(
|
|
"reload the finding and its scan through an independent session",
|
|
"probe the original storage key through the configured backend",
|
|
),
|
|
),
|
|
precondition_evidence={
|
|
"finding_id": finding.id,
|
|
"scan_id": finding.scan_id,
|
|
"storage_key_sha256": key_digest,
|
|
"finding_state": finding.state,
|
|
},
|
|
lease_resource_key=(
|
|
f"files:orphan-cleanup:{finding.tenant_id}:{key_digest[:40]}"
|
|
),
|
|
lease_ttl_seconds=15 * 60,
|
|
resource_type="file_integrity_finding",
|
|
resource_id=finding.id,
|
|
metadata={
|
|
"resources": ["postgresql", "object-storage"],
|
|
"storage_backend": backend.name,
|
|
},
|
|
)
|
|
except (RecoveryOperationBusy, RecoveryOperationStateConflict) as exc:
|
|
raise FileStorageError(
|
|
"This orphan object is already owned by another recovery operation"
|
|
) from exc
|
|
except (RecoveryGuaranteeError, RuntimeError) as exc:
|
|
raise FileStorageError(
|
|
"The Files recovery ledger is unavailable; no object was deleted"
|
|
) from exc
|
|
if started.replayed or started.operation is None:
|
|
raise FileStorageError(
|
|
"This orphan cleanup was already completed; reload the finding"
|
|
)
|
|
pending = PendingOrphanCleanup(
|
|
operation=started.operation,
|
|
backend=backend,
|
|
finding_id=finding.id,
|
|
tenant_id=finding.tenant_id,
|
|
storage_key=finding.storage_key,
|
|
resolved_by_user_id=user_id,
|
|
)
|
|
_register_pending_effect(session, pending)
|
|
return pending
|
|
|
|
|
|
def _register_pending_effect(session: Session, effect: _PendingEffect) -> None:
|
|
if not session.in_transaction():
|
|
session.begin()
|
|
pending = session.info.setdefault(_PENDING_EFFECTS_KEY, [])
|
|
pending.append(effect)
|
|
if session.info.get(_HOOKS_INSTALLED_KEY):
|
|
return
|
|
event.listen(session, "after_commit", _after_session_commit)
|
|
event.listen(session, "after_rollback", _after_session_rollback)
|
|
session.info[_HOOKS_INSTALLED_KEY] = True
|
|
|
|
|
|
def _after_session_commit(session: Session) -> None:
|
|
# SQLAlchemy also emits after_commit for a released SAVEPOINT. A Files
|
|
# batch may open one while creating a later recovery row; settle only when
|
|
# the outer business transaction has actually become visible.
|
|
if session.in_nested_transaction():
|
|
return
|
|
_settle_pending_effects(session, committed=True)
|
|
|
|
|
|
def _after_session_rollback(session: Session) -> None:
|
|
if session.in_nested_transaction():
|
|
return
|
|
try:
|
|
_settle_pending_effects(session, committed=False)
|
|
except RecoveryGuaranteeError as exc:
|
|
session.info.setdefault(_ROLLBACK_ERRORS_KEY, []).append(str(exc))
|
|
|
|
|
|
def _settle_pending_effects(session: Session, *, committed: bool) -> None:
|
|
pending = list(session.info.pop(_PENDING_EFFECTS_KEY, []))
|
|
failures: list[Exception] = []
|
|
for effect in pending:
|
|
try:
|
|
effect.settle(committed=committed)
|
|
except Exception as exc: # preserve every effect's chance to settle
|
|
failures.append(exc)
|
|
operation = getattr(effect, "operation", None)
|
|
if operation is not None and not operation.closed:
|
|
try:
|
|
operation.release_unresolved()
|
|
except Exception:
|
|
pass
|
|
finally:
|
|
sqlite_fence_key = getattr(effect, "sqlite_fence_key", None)
|
|
if sqlite_fence_key:
|
|
_release_sqlite_fence(sqlite_fence_key)
|
|
if failures:
|
|
raise RecoveryGuaranteeError(
|
|
f"{len(failures)} Files recovery operation(s) could not be finalized"
|
|
) from failures[0]
|
|
|
|
|
|
def _reserve_sqlite_fence(resource_key: str) -> None:
|
|
with _SQLITE_FENCES_LOCK:
|
|
if resource_key in _SQLITE_FENCES:
|
|
raise RecoveryOperationBusy(
|
|
f"Another local SQLite transaction owns {resource_key}"
|
|
)
|
|
_SQLITE_FENCES.add(resource_key)
|
|
|
|
|
|
def _release_sqlite_fence(resource_key: str) -> None:
|
|
with _SQLITE_FENCES_LOCK:
|
|
_SQLITE_FENCES.discard(resource_key)
|
|
|
|
|
|
def _require_started_operation(started: DurableRecoveryStart) -> DurableRecoveryOperation:
|
|
if started.replayed or started.operation is None:
|
|
raise RecoveryOperationStateConflict(started.operation_id, started.status)
|
|
return started.operation
|
|
|
|
|
|
def _begin_caller_transaction_recovery_operation(
|
|
session: Session,
|
|
*,
|
|
session_factory: Callable[[], Session],
|
|
identity: RuntimeIdentity,
|
|
module_id: str,
|
|
operation_type: str,
|
|
idempotency_key: str,
|
|
request: dict[str, object],
|
|
recovery_plan: RecoveryPlan,
|
|
precondition_evidence: dict[str, object],
|
|
lease_resource_key: str,
|
|
lease_ttl_seconds: int,
|
|
resource_type: str,
|
|
resource_id: str,
|
|
metadata: dict[str, object],
|
|
) -> DurableRecoveryStart:
|
|
"""Start SQLite recovery evidence inside the caller transaction.
|
|
|
|
SQLite permits only one writer, so opening the normal independent ledger
|
|
transaction after an earlier member of a batch has written will deadlock
|
|
until the busy timeout. The caller-transaction mode preserves fencing,
|
|
request hashing, checkpoints, commit verification, and rollback
|
|
compensation, but cannot make the intent survive a hard process loss
|
|
before the caller commits. PostgreSQL never uses this reduced mode.
|
|
"""
|
|
|
|
claim = acquire_lease(
|
|
session,
|
|
installation_id=identity.installation_id,
|
|
resource_key=lease_resource_key,
|
|
holder_node_id=identity.node_id,
|
|
holder_incarnation=identity.incarnation,
|
|
ttl_seconds=lease_ttl_seconds,
|
|
metadata={
|
|
"module_id": module_id,
|
|
"operation_type": operation_type,
|
|
"durability_mode": "sqlite_caller_transaction",
|
|
},
|
|
)
|
|
if claim is None:
|
|
raise RecoveryOperationBusy(
|
|
f"Another runtime owns the recovery fence for {lease_resource_key}"
|
|
)
|
|
existing = session.execute(
|
|
select(RecoveryOperation).where(
|
|
RecoveryOperation.installation_id == identity.installation_id,
|
|
RecoveryOperation.module_id == module_id,
|
|
RecoveryOperation.idempotency_key == idempotency_key,
|
|
)
|
|
).scalar_one_or_none()
|
|
operation = plan_recovery_operation(
|
|
session,
|
|
installation_id=identity.installation_id,
|
|
module_id=module_id,
|
|
operation_type=operation_type,
|
|
idempotency_key=idempotency_key,
|
|
request=request,
|
|
recovery_plan=recovery_plan,
|
|
resource_type=resource_type,
|
|
resource_id=resource_id,
|
|
lease_claim=claim,
|
|
metadata=metadata,
|
|
)
|
|
if existing is not None:
|
|
if operation.status == RecoveryStatus.SUCCEEDED.value:
|
|
release_lease(session, claim)
|
|
return DurableRecoveryStart(
|
|
operation_id=operation.id,
|
|
status=operation.status,
|
|
replayed=True,
|
|
operation=None,
|
|
)
|
|
raise RecoveryOperationStateConflict(operation.id, operation.status)
|
|
prepare_recovery_operation(
|
|
session,
|
|
operation,
|
|
evidence=precondition_evidence,
|
|
lease_claim=claim,
|
|
)
|
|
start_recovery_operation(
|
|
session,
|
|
operation,
|
|
evidence={"lease_resource_key": lease_resource_key},
|
|
lease_claim=claim,
|
|
)
|
|
if not verify_recovery_evidence_chain(session, operation.id):
|
|
raise RecoveryGuaranteeError(
|
|
"Recovery checkpoint chain verification failed before side effects"
|
|
)
|
|
return DurableRecoveryStart(
|
|
operation_id=operation.id,
|
|
status=operation.status,
|
|
replayed=False,
|
|
operation=DurableRecoveryOperation(
|
|
session_factory=session_factory,
|
|
operation_id=operation.id,
|
|
lease_claim=claim,
|
|
lease_ttl_seconds=lease_ttl_seconds,
|
|
),
|
|
)
|
|
|
|
|
|
def _blob_write_evidence(effect: PendingBlobWrite) -> dict[str, object]:
|
|
database_blob_present: bool | None
|
|
database_matches: bool | None
|
|
database_integrity_verified: bool | None
|
|
try:
|
|
with effect.operation.session_factory() as session:
|
|
blob = session.get(FileBlob, effect.blob_id)
|
|
database_blob_present = blob is not None
|
|
database_matches = bool(
|
|
blob is not None
|
|
and blob.tenant_id == effect.tenant_id
|
|
and blob.storage_key == effect.storage_key
|
|
and blob.checksum_sha256 == effect.semantic_checksum_sha256
|
|
and blob.size_bytes == effect.semantic_size_bytes
|
|
and blob.protection_discriminator
|
|
== effect.protection_discriminator
|
|
and blob.encryption_envelope_id == effect.expected_envelope_id
|
|
and (
|
|
effect.expected_storage_checksum_sha256
|
|
== (
|
|
blob.storage_checksum_sha256
|
|
or blob.checksum_sha256
|
|
)
|
|
)
|
|
) if blob is not None else False
|
|
database_integrity_verified = bool(
|
|
blob is not None
|
|
and blob.integrity_status == "verified"
|
|
and blob.quarantined_at is None
|
|
) if blob is not None else False
|
|
except Exception:
|
|
database_blob_present = None
|
|
database_matches = None
|
|
database_integrity_verified = None
|
|
|
|
object_present: bool | None
|
|
observed_size: int | None = None
|
|
observed_checksum: str | None = None
|
|
try:
|
|
digest = hashlib.sha256()
|
|
observed_size = 0
|
|
for chunk in effect.backend.iter_bytes(effect.storage_key):
|
|
observed_size += len(chunk)
|
|
digest.update(chunk)
|
|
observed_checksum = digest.hexdigest()
|
|
object_present = True
|
|
except StorageObjectMissing:
|
|
object_present = False
|
|
except (StorageBackendError, OSError):
|
|
object_present = None
|
|
|
|
object_size_matches = (
|
|
observed_size == effect.expected_storage_size_bytes
|
|
if object_present is True and effect.expected_storage_size_bytes is not None
|
|
else False if object_present is False else None
|
|
)
|
|
object_checksum_matches = (
|
|
observed_checksum == effect.expected_storage_checksum_sha256
|
|
if object_present is True
|
|
and effect.expected_storage_checksum_sha256 is not None
|
|
else False if object_present is False else None
|
|
)
|
|
verified = database_blob_present is not None and object_present is not None
|
|
return {
|
|
"verified": verified,
|
|
"checks": {
|
|
"database_reloaded": database_blob_present is not None,
|
|
"object_probed": object_present is not None,
|
|
"database_matches_request": database_matches,
|
|
"database_integrity_verified": database_integrity_verified,
|
|
"object_size_matches": object_size_matches,
|
|
"object_checksum_matches": object_checksum_matches,
|
|
},
|
|
"blob_id": effect.blob_id,
|
|
"database_blob_present": database_blob_present,
|
|
"database_matches_request": database_matches,
|
|
"database_integrity_verified": database_integrity_verified,
|
|
"object_present": object_present,
|
|
"observed_size_bytes": observed_size,
|
|
"observed_checksum_sha256": observed_checksum,
|
|
"expected_storage_size_bytes": effect.expected_storage_size_bytes,
|
|
"expected_storage_checksum_sha256": effect.expected_storage_checksum_sha256,
|
|
}
|
|
|
|
|
|
def _blob_write_complete(evidence: dict[str, object]) -> bool:
|
|
return bool(
|
|
evidence.get("verified") is True
|
|
and evidence.get("database_matches_request") is True
|
|
and evidence.get("database_integrity_verified") is True
|
|
and _object_matches(evidence)
|
|
)
|
|
|
|
|
|
def _object_matches(evidence: dict[str, object]) -> bool:
|
|
checks = evidence.get("checks")
|
|
return bool(
|
|
isinstance(checks, dict)
|
|
and evidence.get("object_present") is True
|
|
and checks.get("object_size_matches") is True
|
|
and checks.get("object_checksum_matches") is True
|
|
)
|
|
|
|
|
|
def _forward_complete_blob_repair(effect: PendingBlobWrite) -> bool:
|
|
try:
|
|
with effect.operation.session_factory() as session:
|
|
blob = session.get(FileBlob, effect.blob_id)
|
|
if (
|
|
blob is None
|
|
or blob.tenant_id != effect.tenant_id
|
|
or blob.storage_key != effect.storage_key
|
|
or blob.checksum_sha256 != effect.semantic_checksum_sha256
|
|
or blob.size_bytes != effect.semantic_size_bytes
|
|
or blob.protection_discriminator
|
|
!= effect.protection_discriminator
|
|
or blob.encryption_envelope_id != effect.expected_envelope_id
|
|
):
|
|
return False
|
|
blob.storage_checksum_sha256 = (
|
|
effect.expected_storage_checksum_sha256
|
|
if effect.expected_envelope_id
|
|
else None
|
|
)
|
|
blob.storage_size_bytes = (
|
|
effect.expected_storage_size_bytes
|
|
if effect.expected_envelope_id
|
|
else None
|
|
)
|
|
blob.integrity_status = "verified"
|
|
blob.integrity_checked_at = datetime.now(UTC)
|
|
blob.integrity_failure = None
|
|
blob.quarantined_at = None
|
|
session.add(blob)
|
|
session.commit()
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def _quarantine_blob_after_failed_verification(
|
|
effect: PendingBlobWrite,
|
|
evidence: dict[str, object],
|
|
) -> None:
|
|
try:
|
|
with effect.operation.session_factory() as session:
|
|
blob = session.get(FileBlob, effect.blob_id)
|
|
if blob is None or blob.storage_key != effect.storage_key:
|
|
return
|
|
blob.integrity_status = (
|
|
"missing"
|
|
if evidence.get("object_present") is False
|
|
else "checksum_mismatch"
|
|
)
|
|
blob.integrity_checked_at = datetime.now(UTC)
|
|
blob.integrity_failure = "recovery_verification_failed"
|
|
blob.quarantined_at = datetime.now(UTC)
|
|
session.add(blob)
|
|
session.commit()
|
|
except Exception:
|
|
return
|
|
|
|
|
|
def _orphan_cleanup_evidence(effect: PendingOrphanCleanup) -> dict[str, object]:
|
|
try:
|
|
object_present: bool | None = effect.backend.exists(effect.storage_key)
|
|
except (StorageBackendError, OSError):
|
|
object_present = None
|
|
finding_present: bool | None
|
|
finding_deleted: bool | None
|
|
still_unreferenced: bool | None
|
|
try:
|
|
with effect.operation.session_factory() as session:
|
|
finding = session.get(FileIntegrityFinding, effect.finding_id)
|
|
finding_present = finding is not None
|
|
finding_deleted = bool(
|
|
finding is not None and finding.state == "deleted"
|
|
) if finding is not None else False
|
|
referenced = (
|
|
session.query(FileBlob.id)
|
|
.filter(
|
|
FileBlob.tenant_id == effect.tenant_id,
|
|
FileBlob.storage_key == effect.storage_key,
|
|
)
|
|
.first()
|
|
)
|
|
still_unreferenced = referenced is None
|
|
except Exception:
|
|
finding_present = None
|
|
finding_deleted = None
|
|
still_unreferenced = None
|
|
verified = object_present is not None and finding_present is not None
|
|
return {
|
|
"verified": verified,
|
|
"checks": {
|
|
"database_reloaded": finding_present is not None,
|
|
"object_probed": object_present is not None,
|
|
"finding_marked_deleted": finding_deleted,
|
|
"object_absent": (
|
|
not object_present if object_present is not None else None
|
|
),
|
|
"still_unreferenced": still_unreferenced,
|
|
},
|
|
"finding_id": effect.finding_id,
|
|
"finding_present": finding_present,
|
|
"finding_deleted": finding_deleted,
|
|
"object_present": object_present,
|
|
"still_unreferenced": still_unreferenced,
|
|
}
|
|
|
|
|
|
def _orphan_cleanup_complete(evidence: dict[str, object]) -> bool:
|
|
return bool(
|
|
evidence.get("verified") is True
|
|
and evidence.get("finding_deleted") is True
|
|
and evidence.get("object_present") is False
|
|
and evidence.get("still_unreferenced") is True
|
|
)
|
|
|
|
|
|
def _forward_complete_orphan_finding(effect: PendingOrphanCleanup) -> bool:
|
|
try:
|
|
with effect.operation.session_factory() as session:
|
|
finding = session.get(FileIntegrityFinding, effect.finding_id)
|
|
if (
|
|
finding is None
|
|
or finding.tenant_id != effect.tenant_id
|
|
or finding.storage_key != effect.storage_key
|
|
):
|
|
return False
|
|
scan = session.get(FileIntegrityScan, finding.scan_id)
|
|
if scan is None or not finding.storage_key.startswith(
|
|
scan.storage_prefix
|
|
):
|
|
return False
|
|
referenced = (
|
|
session.query(FileBlob.id)
|
|
.filter(
|
|
FileBlob.tenant_id == effect.tenant_id,
|
|
FileBlob.storage_key == effect.storage_key,
|
|
)
|
|
.first()
|
|
)
|
|
if referenced:
|
|
return False
|
|
finding.state = "deleted"
|
|
finding.resolved_at = datetime.now(UTC)
|
|
finding.resolved_by_user_id = effect.resolved_by_user_id
|
|
session.add(finding)
|
|
session.commit()
|
|
return True
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
__all__ = [
|
|
"PendingBlobWrite",
|
|
"PendingOrphanCleanup",
|
|
"begin_blob_write_recovery",
|
|
"begin_orphan_cleanup_recovery",
|
|
]
|