from __future__ import annotations import copy import hashlib import json from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Callable from sqlalchemy.orm import Session from govoplan_campaign.backend.db.models import CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus FINAL_VERSION_STATES = { "completed", "partially_completed", "outcome_unknown", "failed", "cancelled", "archived", } FINAL_EML_SEND_STATUSES = { "smtp_accepted", "sent", "failed_permanent", "cancelled", "outcome_unknown", } def _cutoff(days: int | None, *, now: datetime) -> datetime | None: if days is None: return None return now - timedelta(days=days) def _is_before_cutoff(value: datetime | None, cutoff: datetime | None) -> bool: if value is None or cutoff is None: return False candidate = value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value return candidate < cutoff def _json_sha256(value: Any) -> str: payload = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8") return hashlib.sha256(payload).hexdigest() def _summary_was_redacted(summary: Any) -> bool: return isinstance(summary, dict) and bool(summary.get("_retention", {}).get("report_detail_redacted")) def _redacted_report_summary(summary: Any, *, now: datetime) -> tuple[dict[str, Any] | None, bool]: if not isinstance(summary, dict): return summary, False if _summary_was_redacted(summary): return summary, False detail_keys = {"messages", "issues", "recent_failures", "jobs"} if not any(key in summary for key in detail_keys): return summary, False next_summary = copy.deepcopy(summary) for key in detail_keys: next_summary.pop(key, None) retention = dict(next_summary.get("_retention") or {}) retention.update({"report_detail_redacted": True, "redacted_at": now.isoformat()}) next_summary["_retention"] = retention return next_summary, True def _apply_raw_json_retention( session: Session, *, dry_run: bool, now: datetime, policy_for_campaign_id: Callable[[str | None], object], ) -> dict[str, int]: result = {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0} versions = session.query(CampaignVersion).order_by(CampaignVersion.updated_at.asc()).all() for version in versions: policy = policy_for_campaign_id(version.campaign_id) cutoff = _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now) if not _is_before_cutoff(version.updated_at, cutoff): continue raw_json = version.raw_json if isinstance(version.raw_json, dict) else {} if raw_json.get("_retention", {}).get("raw_json_redacted"): result["already_redacted"] += 1 continue if version.workflow_state not in FINAL_VERSION_STATES: result["skipped_not_final"] += 1 continue result["eligible"] += 1 if dry_run: continue snapshot = version.execution_snapshot if isinstance(version.execution_snapshot, dict) else {} version.raw_json = { "version": version.schema_version or "1.0", "_retention": { "raw_json_redacted": True, "redacted_at": now.isoformat(), "original_sha256": snapshot.get("campaign_json_sha256") or _json_sha256(raw_json), }, } session.add(version) result["redacted"] += 1 return result def _apply_eml_retention( session: Session, *, dry_run: bool, now: datetime, policy_for_campaign_id: Callable[[str | None], object], ) -> dict[str, int]: result = {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0} jobs = ( session.query(CampaignJob) .filter((CampaignJob.eml_local_path.is_not(None)) | (CampaignJob.eml_storage_key.is_not(None))) .order_by(CampaignJob.updated_at.asc()) .all() ) for job in jobs: policy = policy_for_campaign_id(job.campaign_id) cutoff = _cutoff(policy.generated_eml_retention_days, now=now) if not _is_before_cutoff(job.updated_at, cutoff): continue if job.queue_status in {JobQueueStatus.QUEUED.value, JobQueueStatus.SENDING.value} or job.send_status not in FINAL_EML_SEND_STATUSES: result["skipped_not_final"] += 1 continue if job.imap_status == JobImapStatus.PENDING.value: result["skipped_not_final"] += 1 continue result["eligible"] += 1 if dry_run: continue if job.eml_local_path: path = Path(job.eml_local_path) if path.exists(): path.unlink() result["files_deleted"] += 1 else: result["files_missing"] += 1 job.eml_local_path = None job.eml_storage_key = None session.add(job) result["metadata_cleared"] += 1 return result def _apply_report_detail_retention( session: Session, *, dry_run: bool, now: datetime, policy_for_campaign_id: Callable[[str | None], object], ) -> dict[str, int]: result = {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0} versions = session.query(CampaignVersion).order_by(CampaignVersion.updated_at.asc()).all() for version in versions: policy = policy_for_campaign_id(version.campaign_id) cutoff = _cutoff(policy.stored_report_detail_retention_days, now=now) if not _is_before_cutoff(version.updated_at, cutoff): continue next_validation, validation_changed = _redacted_report_summary(version.validation_summary, now=now) next_build, build_changed = _redacted_report_summary(version.build_summary, now=now) if not validation_changed and not build_changed: if _summary_was_redacted(version.validation_summary) or _summary_was_redacted(version.build_summary): result["already_redacted"] += 1 continue result["eligible_versions"] += 1 if dry_run: continue version.validation_summary = next_validation version.build_summary = next_build session.add(version) result["summaries_redacted"] += int(validation_changed) + int(build_changed) return result def apply_campaign_retention( session: Session, *, dry_run: bool, now: datetime, policy_for_campaign_id: Callable[[str | None], object], ) -> dict[str, dict[str, int]]: return { "raw_campaign_json": _apply_raw_json_retention(session, dry_run=dry_run, now=now, policy_for_campaign_id=policy_for_campaign_id), "generated_eml": _apply_eml_retention(session, dry_run=dry_run, now=now, policy_for_campaign_id=policy_for_campaign_id), "stored_report_detail": _apply_report_detail_retention(session, dry_run=dry_run, now=now, policy_for_campaign_id=policy_for_campaign_id), }