govoplan-campaign/src/govoplan_campaign/backend/retention.py

from __future__ import annotations

import copy
import hashlib
import json
from datetime import datetime, timedelta, timezone
from pathlib import Path
from typing import Any, Callable

from sqlalchemy.orm import Session

from govoplan_campaign.backend.db.models import CampaignJob, CampaignVersion, JobImapStatus, JobQueueStatus

FINAL_VERSION_STATES = {
    "completed",
    "partially_completed",
    "outcome_unknown",
    "failed",
    "cancelled",
    "archived",
}
FINAL_EML_SEND_STATUSES = {
    "smtp_accepted",
    "sent",
    "failed_permanent",
    "cancelled",
    "outcome_unknown",
}


def _cutoff(days: int | None, *, now: datetime) -> datetime | None:
    if days is None:
        return None
    return now - timedelta(days=days)


def _is_before_cutoff(value: datetime | None, cutoff: datetime | None) -> bool:
    if value is None or cutoff is None:
        return False
    candidate = value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value
    return candidate < cutoff


def _json_sha256(value: Any) -> str:
    payload = json.dumps(value, sort_keys=True, ensure_ascii=False, default=str).encode("utf-8")
    return hashlib.sha256(payload).hexdigest()


def _summary_was_redacted(summary: Any) -> bool:
    return isinstance(summary, dict) and bool(summary.get("_retention", {}).get("report_detail_redacted"))


def _redacted_report_summary(summary: Any, *, now: datetime) -> tuple[dict[str, Any] | None, bool]:
    if not isinstance(summary, dict):
        return summary, False
    if _summary_was_redacted(summary):
        return summary, False
    detail_keys = {"messages", "issues", "recent_failures", "jobs"}
    if not any(key in summary for key in detail_keys):
        return summary, False
    next_summary = copy.deepcopy(summary)
    for key in detail_keys:
        next_summary.pop(key, None)
    retention = dict(next_summary.get("_retention") or {})
    retention.update({"report_detail_redacted": True, "redacted_at": now.isoformat()})
    next_summary["_retention"] = retention
    return next_summary, True


def _apply_raw_json_retention(
    session: Session,
    *,
    dry_run: bool,
    now: datetime,
    policy_for_campaign_id: Callable[[str | None], object],
) -> dict[str, int]:
    result = {"eligible": 0, "redacted": 0, "skipped_not_final": 0, "already_redacted": 0}
    versions = session.query(CampaignVersion).order_by(CampaignVersion.updated_at.asc()).all()
    for version in versions:
        policy = policy_for_campaign_id(version.campaign_id)
        cutoff = _cutoff(0 if not policy.store_raw_campaign_json else policy.raw_campaign_json_retention_days, now=now)
        if not _is_before_cutoff(version.updated_at, cutoff):
            continue
        raw_json = version.raw_json if isinstance(version.raw_json, dict) else {}
        if raw_json.get("_retention", {}).get("raw_json_redacted"):
            result["already_redacted"] += 1
            continue
        if version.workflow_state not in FINAL_VERSION_STATES:
            result["skipped_not_final"] += 1
            continue
        result["eligible"] += 1
        if dry_run:
            continue
        snapshot = version.execution_snapshot if isinstance(version.execution_snapshot, dict) else {}
        version.raw_json = {
            "version": version.schema_version or "1.0",
            "_retention": {
                "raw_json_redacted": True,
                "redacted_at": now.isoformat(),
                "original_sha256": snapshot.get("campaign_json_sha256") or _json_sha256(raw_json),
            },
        }
        session.add(version)
        result["redacted"] += 1
    return result


def _apply_eml_retention(
    session: Session,
    *,
    dry_run: bool,
    now: datetime,
    policy_for_campaign_id: Callable[[str | None], object],
) -> dict[str, int]:
    result = {"eligible": 0, "metadata_cleared": 0, "files_deleted": 0, "files_missing": 0, "skipped_not_final": 0}
    jobs = (
        session.query(CampaignJob)
        .filter((CampaignJob.eml_local_path.is_not(None)) | (CampaignJob.eml_storage_key.is_not(None)))
        .order_by(CampaignJob.updated_at.asc())
        .all()
    )
    for job in jobs:
        policy = policy_for_campaign_id(job.campaign_id)
        cutoff = _cutoff(policy.generated_eml_retention_days, now=now)
        if not _is_before_cutoff(job.updated_at, cutoff):
            continue
        if job.queue_status in {JobQueueStatus.QUEUED.value, JobQueueStatus.SENDING.value} or job.send_status not in FINAL_EML_SEND_STATUSES:
            result["skipped_not_final"] += 1
            continue
        if job.imap_status == JobImapStatus.PENDING.value:
            result["skipped_not_final"] += 1
            continue
        result["eligible"] += 1
        if dry_run:
            continue
        if job.eml_local_path:
            path = Path(job.eml_local_path)
            if path.exists():
                path.unlink()
                result["files_deleted"] += 1
            else:
                result["files_missing"] += 1
        job.eml_local_path = None
        job.eml_storage_key = None
        session.add(job)
        result["metadata_cleared"] += 1
    return result


def _apply_report_detail_retention(
    session: Session,
    *,
    dry_run: bool,
    now: datetime,
    policy_for_campaign_id: Callable[[str | None], object],
) -> dict[str, int]:
    result = {"eligible_versions": 0, "summaries_redacted": 0, "already_redacted": 0}
    versions = session.query(CampaignVersion).order_by(CampaignVersion.updated_at.asc()).all()
    for version in versions:
        policy = policy_for_campaign_id(version.campaign_id)
        cutoff = _cutoff(policy.stored_report_detail_retention_days, now=now)
        if not _is_before_cutoff(version.updated_at, cutoff):
            continue
        next_validation, validation_changed = _redacted_report_summary(version.validation_summary, now=now)
        next_build, build_changed = _redacted_report_summary(version.build_summary, now=now)
        if not validation_changed and not build_changed:
            if _summary_was_redacted(version.validation_summary) or _summary_was_redacted(version.build_summary):
                result["already_redacted"] += 1
            continue
        result["eligible_versions"] += 1
        if dry_run:
            continue
        version.validation_summary = next_validation
        version.build_summary = next_build
        session.add(version)
        result["summaries_redacted"] += int(validation_changed) + int(build_changed)
    return result


def apply_campaign_retention(
    session: Session,
    *,
    dry_run: bool,
    now: datetime,
    policy_for_campaign_id: Callable[[str | None], object],
) -> dict[str, dict[str, int]]:
    return {
        "raw_campaign_json": _apply_raw_json_retention(session, dry_run=dry_run, now=now, policy_for_campaign_id=policy_for_campaign_id),
        "generated_eml": _apply_eml_retention(session, dry_run=dry_run, now=now, policy_for_campaign_id=policy_for_campaign_id),
        "stored_report_detail": _apply_report_detail_retention(session, dry_run=dry_run, now=now, policy_for_campaign_id=policy_for_campaign_id),
    }