Release v0.1.5

src/govoplan_campaign/backend/retention.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+import copy
+import hashlib
+import json
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Callable
+
+from sqlalchemy.orm import Session
+
+from govoplan_campaign.backend.db.models import CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus
+
+FINAL_VERSION_STATES = {
+    "completed",
+    "partially_completed",
+    "outcome_unknown",
+    "failed",
+    "cancelled",
+    "archived",
+}
+FINAL_EML_SEND_STATUSES = {
+    "smtp_accepted",
+    "sent",
+    "failed_permanent",
+    "cancelled",
+    "outcome_unknown",
+}
+
+
+def _cutoff(days: int | None, *, now: datetime) -> datetime | None:
+    if days is None:
+        return None
+    return now - timedelta(days=days)
+
+
+def _is_before_cutoff(value: datetime | None, cutoff: datetime | None) -> bool:
+    if value is None or cutoff is None:
+        return False
+    candidate = value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value
+    return candidate < cutoff
+
+
+def _json_sha256(value: Any) -> str:
+    payload = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8")
+    return hashlib.sha256(payload).hexdigest()
+
+
+def _summary_was_redacted(summary: Any) -> bool:
+    return isinstance(summary, dict) and bool(summary.get("_retention", {}).get("report_detail_redacted"))
+
+
+def _redacted_report_summary(summary: Any, *, now: datetime) -> tuple[dict[str, Any] | None, bool]:
+    if not isinstance(summary, dict):
+        return summary, False
+    if _summary_was_redacted(summary):
+        return summary, False
+    detail_keys = {"messages", "issues", "recent_failures", "jobs"}
+    if not any(key in summary for key in detail_keys):
+        return summary, False
+    next_summary = copy.deepcopy(summary)
+    for key in detail_keys:
+        next_summary.pop(key, None)
+    retention = dict(next_summary.get("_retention") or {})
+    retention.update({"report_detail_redacted": True, "redacted_at": now.isoformat()})
+    next_summary["_retention"] = retention
+    return next_summary, True
+
+
+def _apply_raw_json_retention(
+    session: Session,
+    *,
+    dry_run: bool,
+    now: datetime,
+    policy_for_campaign_id: Callable[[str | None], object],
+) -> dict[str, int]:
+    result = {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0}
+    versions = session.query(CampaignVersion).order_by(CampaignVersion.updated_at.asc()).all()
+    for version in versions:
+        policy = policy_for_campaign_id(version.campaign_id)
+        cutoff = _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now)
+        if not _is_before_cutoff(version.updated_at, cutoff):
+            continue
+        raw_json = version.raw_json if isinstance(version.raw_json, dict) else {}
+        if raw_json.get("_retention", {}).get("raw_json_redacted"):
+            result["already_redacted"] += 1
+            continue
+        if version.workflow_state not in FINAL_VERSION_STATES:
+            result["skipped_not_final"] += 1
+            continue
+        result["eligible"] += 1
+        if dry_run:
+            continue
+        snapshot = version.execution_snapshot if isinstance(version.execution_snapshot, dict) else {}
+        version.raw_json = {
+            "version": version.schema_version or "1.0",
+            "_retention": {
+                "raw_json_redacted": True,
+                "redacted_at": now.isoformat(),
+                "original_sha256": snapshot.get("campaign_json_sha256") or _json_sha256(raw_json),
+            },
+        }
+        session.add(version)
+        result["redacted"] += 1
+    return result
+
+
+def _apply_eml_retention(
+    session: Session,
+    *,
+    dry_run: bool,
+    now: datetime,
+    policy_for_campaign_id: Callable[[str | None], object],
+) -> dict[str, int]:
+    result = {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0}
+    jobs = (
+        session.query(CampaignJob)
+        .filter((CampaignJob.eml_local_path.is_not(None)) | (CampaignJob.eml_storage_key.is_not(None)))
+        .order_by(CampaignJob.updated_at.asc())
+        .all()
+    )
+    for job in jobs:
+        policy = policy_for_campaign_id(job.campaign_id)
+        cutoff = _cutoff(policy.generated_eml_retention_days, now=now)
+        if not _is_before_cutoff(job.updated_at, cutoff):
+            continue
+        if job.queue_status in {JobQueueStatus.QUEUED.value, JobQueueStatus.SENDING.value} or job.send_status not in FINAL_EML_SEND_STATUSES:
+            result["skipped_not_final"] += 1
+            continue
+        if job.imap_status == JobImapStatus.PENDING.value:
+            result["skipped_not_final"] += 1
+            continue
+        result["eligible"] += 1
+        if dry_run:
+            continue
+        if job.eml_local_path:
+            path = Path(job.eml_local_path)
+            if path.exists():
+                path.unlink()
+                result["files_deleted"] += 1
+            else:
+                result["files_missing"] += 1
+        job.eml_local_path = None
+        job.eml_storage_key = None
+        session.add(job)
+        result["metadata_cleared"] += 1
+    return result
+
+
+def _apply_report_detail_retention(
+    session: Session,
+    *,
+    dry_run: bool,
+    now: datetime,
+    policy_for_campaign_id: Callable[[str | None], object],
+) -> dict[str, int]:
+    result = {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0}
+    versions = session.query(CampaignVersion).order_by(CampaignVersion.updated_at.asc()).all()
+    for version in versions:
+        policy = policy_for_campaign_id(version.campaign_id)
+        cutoff = _cutoff(policy.stored_report_detail_retention_days, now=now)
+        if not _is_before_cutoff(version.updated_at, cutoff):
+            continue
+        next_validation, validation_changed = _redacted_report_summary(version.validation_summary, now=now)
+        next_build, build_changed = _redacted_report_summary(version.build_summary, now=now)
+        if not validation_changed and not build_changed:
+            if _summary_was_redacted(version.validation_summary) or _summary_was_redacted(version.build_summary):
+                result["already_redacted"] += 1
+            continue
+        result["eligible_versions"] += 1
+        if dry_run:
+            continue
+        version.validation_summary = next_validation
+        version.build_summary = next_build
+        session.add(version)
+        result["summaries_redacted"] += int(validation_changed) + int(build_changed)
+    return result
+
+
+def apply_campaign_retention(
+    session: Session,
+    *,
+    dry_run: bool,
+    now: datetime,
+    policy_for_campaign_id: Callable[[str | None], object],
+) -> dict[str, dict[str, int]]:
+    return {
+        "raw_campaign_json": _apply_raw_json_retention(session, dry_run=dry_run, now=now, policy_for_campaign_id=policy_for_campaign_id),
+        "generated_eml": _apply_eml_retention(session, dry_run=dry_run, now=now, policy_for_campaign_id=policy_for_campaign_id),
+        "stored_report_detail": _apply_report_detail_retention(session, dry_run=dry_run, now=now, policy_for_campaign_id=policy_for_campaign_id),
+    }