from __future__ import annotations from dataclasses import dataclass from datetime import datetime, timedelta, timezone import hashlib import json from typing import Any, Callable from sqlalchemy import or_ from sqlalchemy.orm import Session from govoplan_campaign.backend.db.models import ( Campaign, CampaignJob, CampaignVersion, ) from govoplan_core.core.object_storage import ( StorageBackend, StorageBackendError, StorageObjectInfo, ) from govoplan_core.core.recovery import ( RecoveryMode, RecoveryOperation, RecoveryPlan, RecoveryStatus, TERMINAL_RECOVERY_STATUSES, ) from govoplan_core.core.recovery_runtime import begin_durable_recovery_operation from govoplan_core.core.runtime_coordination import ( DistributedLease, RuntimeIdentity, ) CAMPAIGN_ARTIFACT_NAMESPACE = "campaign-artifacts" MINIMUM_GRACE_HOURS = 24 MAXIMUM_PAGE_SIZE = 1000 _CHECKPOINT_BATCH_SIZE = 25 SessionFactory = Callable[[], Session] class CampaignArtifactReconciliationError(RuntimeError): pass @dataclass(slots=True) class ArtifactCandidate: key: str size_bytes: int modified_at: datetime age_seconds: int reason: str = "unreferenced_after_grace_period" disposition: str = "candidate" failure_type: str | None = None def as_dict(self) -> dict[str, Any]: return { "key": self.key, "size_bytes": self.size_bytes, "modified_at": self.modified_at.isoformat(), "age_seconds": self.age_seconds, "reason": self.reason, "disposition": self.disposition, "failure_type": self.failure_type, } @dataclass(slots=True) class ArtifactInventory: tenant_prefix: str cursor: str | None next_cursor: str | None scanned_count: int scanned_bytes: int referenced_count: int active_build_count: int young_count: int unknown_age_count: int invalid_shape_count: int candidates: list[ArtifactCandidate] manifest_sha256: str @property def candidate_bytes(self) -> int: return sum(candidate.size_bytes for candidate in self.candidates) def response( self, *, apply: bool, status: str, recovery_operation_id: str | None = None, ) -> dict[str, Any]: deleted = [ candidate for candidate in self.candidates if candidate.disposition == "deleted" ] failures = [ candidate for candidate in self.candidates if candidate.disposition in {"delete_failed", "delete_outcome_unknown"} ] return { "apply": apply, "status": status, "recovery_operation_id": recovery_operation_id, "tenant_prefix": self.tenant_prefix, "cursor": self.cursor, "next_cursor": self.next_cursor, "scanned_count": self.scanned_count, "scanned_bytes": self.scanned_bytes, "referenced_count": self.referenced_count, "active_build_count": self.active_build_count, "young_count": self.young_count, "unknown_age_count": self.unknown_age_count, "invalid_shape_count": self.invalid_shape_count, "candidate_count": len(self.candidates), "candidate_bytes": self.candidate_bytes, "deleted_count": len(deleted), "deleted_bytes": sum(candidate.size_bytes for candidate in deleted), "failure_count": len(failures), "manifest_sha256": self.manifest_sha256, "candidates": [candidate.as_dict() for candidate in self.candidates], } def campaign_artifact_inventory( session: Session, *, storage: StorageBackend, tenant_id: str, grace_period: timedelta, cursor: str | None = None, page_size: int = 250, now: datetime | None = None, ) -> ArtifactInventory: observed_at = _as_utc(now or datetime.now(timezone.utc)) if grace_period < timedelta(hours=MINIMUM_GRACE_HOURS): raise ValueError( f"Campaign artifact grace period must be at least {MINIMUM_GRACE_HOURS} hours" ) bounded_page_size = max(1, min(int(page_size), MAXIMUM_PAGE_SIZE)) prefix = _tenant_artifact_prefix(tenant_id) if cursor is not None and not cursor.startswith(prefix): raise ValueError("Campaign artifact cursor is outside the tenant namespace") page = storage.list_objects( prefix=prefix, after=cursor, limit=bounded_page_size, ) page_keys = {info.key for info in page.objects} referenced_keys = _referenced_artifact_keys( session, tenant_id=tenant_id, prefix=prefix, artifact_keys=page_keys, ) active_build_ids = _active_build_ids( session, tenant_id=tenant_id, now=observed_at, ) candidates: list[ArtifactCandidate] = [] referenced_count = 0 active_build_count = 0 young_count = 0 unknown_age_count = 0 invalid_shape_count = 0 scanned_bytes = 0 manifest_rows: list[dict[str, Any]] = [] for info in page.objects: scanned_bytes += info.size_bytes build_id = _build_id_for_key(info.key, prefix=prefix) modified_at = _object_modified_at(info) manifest_rows.append( { "key_sha256": _sha256(info.key), "size_bytes": info.size_bytes, "modified_at": modified_at.isoformat() if modified_at else None, } ) if build_id is None: invalid_shape_count += 1 continue if info.key in referenced_keys: referenced_count += 1 continue if build_id in active_build_ids: active_build_count += 1 continue if modified_at is None: unknown_age_count += 1 continue age = observed_at - modified_at if age < grace_period: young_count += 1 continue candidates.append( ArtifactCandidate( key=info.key, size_bytes=info.size_bytes, modified_at=modified_at, age_seconds=max(0, int(age.total_seconds())), ) ) return ArtifactInventory( tenant_prefix=prefix, cursor=cursor, next_cursor=page.next_cursor, scanned_count=len(page.objects), scanned_bytes=scanned_bytes, referenced_count=referenced_count, active_build_count=active_build_count, young_count=young_count, unknown_age_count=unknown_age_count, invalid_shape_count=invalid_shape_count, candidates=candidates, manifest_sha256=_canonical_sha256(manifest_rows), ) def reconcile_campaign_artifacts( session_factory: SessionFactory, *, storage: StorageBackend, identity: RuntimeIdentity, tenant_id: str, apply: bool = False, idempotency_key: str | None = None, grace_period_hours: int = MINIMUM_GRACE_HOURS, cursor: str | None = None, page_size: int = 250, now: datetime | None = None, ) -> dict[str, Any]: if apply and not (idempotency_key or "").strip(): raise ValueError("Applied Campaign artifact cleanup requires an idempotency key") if grace_period_hours < MINIMUM_GRACE_HOURS: raise ValueError( f"Campaign artifact grace period must be at least {MINIMUM_GRACE_HOURS} hours" ) observed_at = _as_utc(now or datetime.now(timezone.utc)) grace_period = timedelta(hours=grace_period_hours) if not apply: with session_factory() as session: inventory = campaign_artifact_inventory( session, storage=storage, tenant_id=tenant_id, grace_period=grace_period, cursor=cursor, page_size=page_size, now=observed_at, ) return inventory.response(apply=False, status="dry_run") prefix = _tenant_artifact_prefix(tenant_id) request = { "tenant_id": tenant_id, "prefix": prefix, "cursor_sha256": _sha256(cursor) if cursor else None, "page_size": max(1, min(int(page_size), MAXIMUM_PAGE_SIZE)), "grace_period_hours": grace_period_hours, } recovery_start = begin_durable_recovery_operation( session_factory, identity=identity, module_id="campaigns", operation_type="artifact-orphan-reconciliation", idempotency_key=str(idempotency_key).strip(), request=request, recovery_plan=RecoveryPlan( mode=RecoveryMode.FORWARD_RECOVERY, preconditions=( "inventory is bounded to one tenant Campaign artifact prefix", "objects younger than the conservative grace period are excluded", ), forward_recovery_steps=( "retry only objects still unreferenced by committed Campaign state", ), verification_steps=( "probe every attempted object after deletion", "preserve database references without mutation", ), ), precondition_evidence={ "tenant_prefix_sha256": _sha256(prefix), "grace_period_hours": grace_period_hours, "page_size": request["page_size"], }, lease_resource_key=f"campaign:artifact-reconcile:{tenant_id}", lease_ttl_seconds=15 * 60, resource_type="campaign_artifact_namespace", resource_id=tenant_id, metadata={"tenant_id": tenant_id}, ) if recovery_start.replayed: return _replayed_response( prefix=prefix, cursor=cursor, operation_id=recovery_start.operation_id, ) operation = recovery_start.operation if operation is None: # pragma: no cover - guarded by replay branch raise CampaignArtifactReconciliationError( "Campaign artifact cleanup authority was not created" ) try: try: with session_factory() as session: inventory = campaign_artifact_inventory( session, storage=storage, tenant_id=tenant_id, grace_period=grace_period, cursor=cursor, page_size=page_size, now=observed_at, ) except Exception as exc: operation.fail( summary="Campaign artifact inventory failed before deletion", evidence={ "effect_started": False, "failure_type": type(exc).__name__, }, ) raise operation.checkpoint( kind="artifact-inventory", summary="The bounded Campaign artifact inventory was classified", evidence={ "manifest_sha256": inventory.manifest_sha256, "scanned_count": inventory.scanned_count, "candidate_count": len(inventory.candidates), "candidate_bytes": inventory.candidate_bytes, "next_page": inventory.next_cursor is not None, }, ) if inventory.candidates: _apply_inventory( session_factory, storage=storage, operation=operation, inventory=inventory, tenant_id=tenant_id, now=observed_at, ) failed = [ item for item in inventory.candidates if item.disposition == "delete_failed" ] unknown = [ item for item in inventory.candidates if item.disposition == "delete_outcome_unknown" ] evidence = _cleanup_evidence(inventory) if unknown: operation.unresolved( status=RecoveryStatus.OUTCOME_UNKNOWN, summary="Campaign artifact deletion could not be verified", evidence=evidence, failure_summary=( "One or more Campaign artifact deletion outcomes are unknown" ), ) status = "outcome_unknown" elif failed: operation.unresolved( status=RecoveryStatus.RECOVERY_REQUIRED, summary="Campaign artifact deletion requires a retry", evidence=evidence, failure_summary=( "One or more unreferenced Campaign artifacts remain" ), ) status = "recovery_required" else: operation.succeed(evidence=evidence) status = "applied" return inventory.response( apply=True, status=status, recovery_operation_id=recovery_start.operation_id, ) except Exception: if not operation.closed: try: operation.release_unresolved() except Exception: pass raise def _apply_inventory( session_factory: SessionFactory, *, storage: StorageBackend, operation: Any, inventory: ArtifactInventory, tenant_id: str, now: datetime, ) -> None: for index in range(0, len(inventory.candidates), _CHECKPOINT_BATCH_SIZE): batch = inventory.candidates[index : index + _CHECKPOINT_BATCH_SIZE] with session_factory() as session: referenced = _referenced_artifact_keys( session, tenant_id=tenant_id, prefix=inventory.tenant_prefix, artifact_keys={candidate.key for candidate in batch}, ) active_build_ids = _active_build_ids( session, tenant_id=tenant_id, now=now, ) operation.checkpoint( kind="artifact-delete-batch-authorized", summary="Deletion authority was renewed for a bounded object batch", evidence={ "batch_index": index // _CHECKPOINT_BATCH_SIZE, "batch_count": len(batch), "batch_manifest_sha256": _canonical_sha256( [_sha256(candidate.key) for candidate in batch] ), }, ) for candidate in batch: build_id = _build_id_for_key( candidate.key, prefix=inventory.tenant_prefix, ) if candidate.key in referenced or build_id in active_build_ids: candidate.disposition = "protected_before_delete" candidate.reason = "reference_or_active_build_appeared" continue _delete_and_verify(storage, candidate) def _delete_and_verify( storage: StorageBackend, candidate: ArtifactCandidate, ) -> None: delete_failure: Exception | None = None try: storage.delete(candidate.key) except (OSError, StorageBackendError) as exc: delete_failure = exc try: remains = storage.exists(candidate.key) except (OSError, StorageBackendError) as exc: candidate.disposition = "delete_outcome_unknown" candidate.failure_type = type(exc).__name__ return if not remains: candidate.disposition = "deleted" return candidate.disposition = "delete_failed" candidate.failure_type = ( type(delete_failure).__name__ if delete_failure is not None else None ) def _referenced_artifact_keys( session: Session, *, tenant_id: str, prefix: str, artifact_keys: set[str] | None = None, ) -> set[str]: if artifact_keys == set(): return set() keys: set[str] = set() version_ids = { identity[1] for key in artifact_keys or () if (identity := _artifact_identity(key, prefix=prefix)) is not None } job_query = session.query( CampaignJob.eml_storage_key, CampaignJob.resolved_print_output, ).filter(CampaignJob.tenant_id == tenant_id) if artifact_keys is not None: filters = [CampaignJob.eml_storage_key.in_(artifact_keys)] if version_ids: filters.append(CampaignJob.campaign_version_id.in_(version_ids)) job_query = job_query.filter(or_(*filters)) job_rows = job_query.yield_per(1000) for eml_storage_key, print_output in job_rows: _add_key( keys, eml_storage_key, prefix=prefix, allowed_keys=artifact_keys, ) _add_key( keys, _print_output_storage_key(print_output), prefix=prefix, allowed_keys=artifact_keys, ) version_query = session.query(CampaignVersion.build_summary).join( Campaign, Campaign.id == CampaignVersion.campaign_id, ).filter(Campaign.tenant_id == tenant_id) if artifact_keys is not None: if not version_ids: return keys version_query = version_query.filter(CampaignVersion.id.in_(version_ids)) version_rows = version_query.yield_per(500) for (build_summary,) in version_rows: print_output = ( build_summary.get("print_output") if isinstance(build_summary, dict) else None ) _add_key( keys, _print_output_storage_key(print_output), prefix=prefix, allowed_keys=artifact_keys, ) return keys def _artifact_identity( key: str, *, prefix: str, ) -> tuple[str, str, str] | None: if not key.startswith(prefix): return None parts = key[len(prefix) :].split("/") if len(parts) < 4 or any(not part for part in parts[:4]): return None return parts[0], parts[1], parts[2] def _active_build_ids( session: Session, *, tenant_id: str, now: datetime, ) -> set[str]: operations = ( session.query(RecoveryOperation) .join( CampaignVersion, CampaignVersion.id == RecoveryOperation.resource_id, ) .join(Campaign, Campaign.id == CampaignVersion.campaign_id) .filter( Campaign.tenant_id == tenant_id, RecoveryOperation.module_id == "campaigns", RecoveryOperation.operation_type == "build-artifacts", RecoveryOperation.status.not_in(TERMINAL_RECOVERY_STATUSES), ) .all() ) resource_keys = { operation.lease_resource_key for operation in operations if operation.lease_resource_key } if not resource_keys: return set() installation_ids = {operation.installation_id for operation in operations} leases = ( session.query(DistributedLease) .filter( DistributedLease.installation_id.in_(installation_ids), DistributedLease.resource_key.in_(resource_keys), ) .all() ) leases_by_key = { (lease.installation_id, lease.resource_key): lease for lease in leases } active: set[str] = set() for operation in operations: lease = leases_by_key.get( (operation.installation_id, operation.lease_resource_key or "") ) if ( lease is not None and lease.holder_node_id == operation.holder_node_id and lease.holder_incarnation == operation.holder_incarnation and lease.fencing_token == operation.fencing_token and _as_utc(lease.expires_at) > now ): active.add(operation.id) return active def _cleanup_evidence(inventory: ArtifactInventory) -> dict[str, Any]: dispositions: dict[str, int] = {} for candidate in inventory.candidates: dispositions[candidate.disposition] = ( dispositions.get(candidate.disposition, 0) + 1 ) verified = not any( key in dispositions for key in ("delete_failed", "delete_outcome_unknown") ) return { "verified": verified, "checks": { "candidate_objects": ( "absent-or-newly-protected" if verified else "incomplete" ), "database_references": "unchanged", "lease_fence": "renewed-before-each-batch", }, "inventory_manifest_sha256": inventory.manifest_sha256, "candidate_manifest_sha256": _canonical_sha256( [_sha256(candidate.key) for candidate in inventory.candidates] ), "candidate_count": len(inventory.candidates), "candidate_bytes": inventory.candidate_bytes, "dispositions": dispositions, "database_references_mutated": False, } def _replayed_response( *, prefix: str, cursor: str | None, operation_id: str, ) -> dict[str, Any]: return { "apply": True, "status": "already_completed", "recovery_operation_id": operation_id, "tenant_prefix": prefix, "cursor": cursor, "next_cursor": None, "scanned_count": 0, "scanned_bytes": 0, "referenced_count": 0, "active_build_count": 0, "young_count": 0, "unknown_age_count": 0, "invalid_shape_count": 0, "candidate_count": 0, "candidate_bytes": 0, "deleted_count": 0, "deleted_bytes": 0, "failure_count": 0, "manifest_sha256": None, "candidates": [], } def _tenant_artifact_prefix(tenant_id: str) -> str: normalized = str(tenant_id or "").strip() if not normalized or "/" in normalized or normalized in {".", ".."}: raise ValueError("Campaign artifact inventory requires a valid tenant id") return f"{CAMPAIGN_ARTIFACT_NAMESPACE}/{normalized}/" def _build_id_for_key(key: str, *, prefix: str) -> str | None: identity = _artifact_identity(key, prefix=prefix) return identity[2] if identity is not None else None def _print_output_storage_key(value: object) -> str | None: if not isinstance(value, dict): return None artifact = value.get("artifact") if not isinstance(artifact, dict): return None key = artifact.get("storage_key") return str(key) if key else None def _add_key( keys: set[str], value: object, *, prefix: str, allowed_keys: set[str] | None = None, ) -> None: if value is None: return key = str(value) if key.startswith(prefix) and (allowed_keys is None or key in allowed_keys): keys.add(key) def _object_modified_at(info: StorageObjectInfo) -> datetime | None: return _as_utc(info.modified_at) if info.modified_at is not None else None def _as_utc(value: datetime) -> datetime: if value.tzinfo is None: return value.replace(tzinfo=timezone.utc) return value.astimezone(timezone.utc) def _sha256(value: object) -> str: return hashlib.sha256(str(value).encode("utf-8")).hexdigest() def _canonical_sha256(value: object) -> str: return hashlib.sha256( json.dumps( value, sort_keys=True, separators=(",", ":"), default=str, ).encode("utf-8") ).hexdigest() __all__ = [ "CAMPAIGN_ARTIFACT_NAMESPACE", "CampaignArtifactReconciliationError", "campaign_artifact_inventory", "reconcile_campaign_artifacts", ]