govoplan-campaign/tests/test_imap_append_idempotency.py

from __future__ import annotations

from datetime import datetime, timedelta, timezone
import importlib
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch

import pytest
from alembic.migration import MigrationContext
from alembic.operations import Operations
from sqlalchemy import create_engine, inspect, text

from govoplan_campaign.backend.routes import delivery as router
from govoplan_campaign.backend.db.models import (
    CampaignJob,
    ImapAppendAttempt,
    JobImapStatus,
    JobSendStatus,
)
from govoplan_campaign.backend.retention import _apply_eml_retention
from govoplan_campaign.backend.schemas import CampaignResolveOutcomeRequest
from govoplan_campaign.backend.sending.jobs import (
    AppendSentResult,
    QueueingError,
    _claim_job_for_imap_append,
    _record_imap_append_success,
    append_sent_for_job,
    reconcile_job_outcome,
)


@pytest.mark.parametrize(
    ("imap_status", "expected_status"),
    [
        (JobImapStatus.APPENDING.value, "append_in_progress"),
        (JobImapStatus.OUTCOME_UNKNOWN.value, JobImapStatus.OUTCOME_UNKNOWN.value),
    ],
)
def test_in_progress_and_unknown_jobs_never_reinvoke_mail_provider(
    imap_status: str,
    expected_status: str,
) -> None:
    job = SimpleNamespace(
        id="job-1",
        send_status=JobSendStatus.SMTP_ACCEPTED.value,
        imap_status=imap_status,
    )
    session = SimpleNamespace(get=lambda _model, _id: job)

    with (
        patch("govoplan_campaign.backend.sending.jobs._imap_attempt_count", return_value=1),
        patch("govoplan_campaign.backend.sending.jobs.mail_integration") as mail,
    ):
        result = append_sent_for_job(
            session,  # type: ignore[arg-type]
            job_id="job-1",
        )

    assert result.status == expected_status
    mail.assert_not_called()


def test_imap_claim_is_a_single_atomic_state_transition() -> None:
    session = MagicMock()
    query = session.query.return_value
    query.filter.return_value.update.return_value = 1
    job = SimpleNamespace(id="job-1")

    claim_token = _claim_job_for_imap_append(
        session,
        job,  # type: ignore[arg-type]
    )

    assert claim_token
    changes = query.filter.return_value.update.call_args.args[0]
    assert changes[CampaignJob.imap_status] == JobImapStatus.APPENDING.value
    assert changes[CampaignJob.imap_claim_token] == claim_token
    session.commit.assert_called_once_with()
    session.expire_all.assert_called_once_with()


def test_late_provider_acknowledgement_cannot_overwrite_a_changed_claim() -> None:
    session = MagicMock()
    session.query.return_value.filter.return_value.update.return_value = 0
    job = SimpleNamespace(id="job-1")
    attempt = SimpleNamespace(id="attempt-1", attempt_number=1)
    expected = AppendSentResult(
        job_id="job-1",
        status=JobImapStatus.OUTCOME_UNKNOWN.value,
        attempt_number=1,
    )

    with (
        patch(
            "govoplan_campaign.backend.sending.jobs._imap_result_after_lost_claim",
            return_value=expected,
        ) as lost_claim,
        patch("govoplan_campaign.backend.sending.jobs.files_integration") as files,
    ):
        result = _record_imap_append_success(
            session,
            job=job,  # type: ignore[arg-type]
            attempt=attempt,  # type: ignore[arg-type]
            claim_token="claim-1",
            folder="Sent",
        )

    assert result is expected
    lost_claim.assert_called_once_with(
        session,
        job_id="job-1",
        attempt=attempt,
        provider_succeeded=True,
    )
    files.assert_not_called()


def test_post_provider_persistence_failure_freezes_imap_retry() -> None:
    job = SimpleNamespace(
        id="job-1",
        tenant_id="tenant-1",
        campaign_id="campaign-1",
        campaign_version_id="version-1",
        send_status=JobSendStatus.SMTP_ACCEPTED.value,
        imap_status=JobImapStatus.PENDING.value,
    )
    version = SimpleNamespace(id="version-1")
    snapshot = SimpleNamespace(
        mail_profile_id="profile-1",
        smtp_transport_revision="smtp-revision",
        imap_transport_revision="imap-revision",
        smtp_server_id=None,
        smtp_credential_id=None,
        imap_server_id=None,
        imap_credential_id=None,
        delivery=SimpleNamespace(
            imap_append_sent=SimpleNamespace(enabled=True, folder="Sent"),
        ),
    )
    attempt = SimpleNamespace(id="attempt-1", attempt_number=1)

    class Session:
        def get(self, model, _object_id):
            return version if model.__name__ == "CampaignVersion" else job

    class Mail:
        def append_campaign_message_to_sent(self, *_args, **_kwargs):
            return SimpleNamespace(folder="Sent")

    expected = AppendSentResult(
        job_id="job-1",
        status=JobImapStatus.OUTCOME_UNKNOWN.value,
        attempt_number=1,
    )
    session = Session()
    with (
        patch("govoplan_campaign.backend.sending.jobs.ensure_execution_snapshot", return_value=snapshot),
        patch("govoplan_campaign.backend.sending.jobs._load_eml_bytes_for_job", return_value=b"message"),
        patch("govoplan_campaign.backend.sending.jobs._claim_job_for_imap_append", return_value="claim-1"),
        patch("govoplan_campaign.backend.sending.jobs._record_imap_attempt_start", return_value=attempt),
        patch("govoplan_campaign.backend.sending.jobs.mail_integration", return_value=Mail()),
        patch(
            "govoplan_campaign.backend.sending.jobs._record_imap_append_success",
            side_effect=OSError("database unavailable"),
        ),
        patch(
            "govoplan_campaign.backend.sending.jobs._mark_imap_append_outcome_unknown_after_effect",
            return_value=expected,
        ) as mark_unknown,
    ):
        result = append_sent_for_job(
            session,  # type: ignore[arg-type]
            job_id="job-1",
        )

    assert result is expected
    assert "Automatic retry is stopped" in mark_unknown.call_args.kwargs["reason"]


def test_attempt_numbers_are_unique_per_job_in_the_model_contract() -> None:
    constraints = {
        constraint.name
        for constraint in ImapAppendAttempt.__table__.constraints
    }
    assert "uq_imap_append_attempts_job_attempt" in constraints


def test_imap_claim_migration_preserves_and_renumbers_duplicate_attempts() -> None:
    migration = importlib.import_module(
        "govoplan_campaign.backend.migrations.versions.3c4d5e6f8192_v019_imap_append_claim"
    )
    engine = create_engine("sqlite+pysqlite:///:memory:")
    with engine.begin() as connection:
        connection.execute(
            text(
                "CREATE TABLE imap_append_attempts ("
                "id VARCHAR(36) PRIMARY KEY, job_id VARCHAR(36) NOT NULL, "
                "attempt_number INTEGER NOT NULL, created_at DATETIME NOT NULL)"
            )
        )
        connection.execute(
            text(
                "INSERT INTO imap_append_attempts "
                "(id, job_id, attempt_number, created_at) VALUES "
                "('a2', 'job-1', 1, '2026-01-02'), "
                "('a1', 'job-1', 1, '2026-01-01'), "
                "('b1', 'job-2', 7, '2026-01-01')"
            )
        )
        with patch.object(migration.op, "get_bind", return_value=connection):
            migration._renumber_attempts()
        rows = connection.execute(
            text(
                "SELECT id, job_id, attempt_number FROM imap_append_attempts "
                "ORDER BY job_id, attempt_number"
            )
        ).all()

    assert rows == [
        ("a1", "job-1", 1),
        ("a2", "job-1", 2),
        ("b1", "job-2", 1),
    ]


def test_imap_attempt_claim_repair_adds_only_the_missing_column() -> None:
    migration = importlib.import_module(
        "govoplan_campaign.backend.migrations.versions.e9f0a1b2c3d4_v0114_repair_imap_attempt_claim"
    )
    engine = create_engine("sqlite+pysqlite:///:memory:")
    with engine.begin() as connection:
        connection.execute(
            text(
                "CREATE TABLE imap_append_attempts ("
                "id VARCHAR(36) PRIMARY KEY, job_id VARCHAR(36) NOT NULL)"
            )
        )
        context = MigrationContext.configure(connection)
        with patch.object(migration, "op", Operations(context)):
            migration.upgrade()
            migration.upgrade()

        columns = {
            column["name"]
            for column in inspect(connection).get_columns("imap_append_attempts")
        }

    assert columns == {"id", "job_id", "claim_token"}


@pytest.mark.parametrize("decision", ["smtp_accepted", "not_sent", "imap_appended", "imap_not_appended"])
def test_reconciliation_requires_an_evidence_note(decision: str) -> None:
    with pytest.raises(ValueError, match="evidence note"):
        CampaignResolveOutcomeRequest(
            decision=decision,  # type: ignore[arg-type]
            note="  ",
        )


@pytest.mark.parametrize(
    ("decision", "expected_status", "attempt_status"),
    [
        (
            "imap_appended",
            JobImapStatus.APPENDED.value,
            "reconciled_imap_appended",
        ),
        (
            "imap_not_appended",
            JobImapStatus.FAILED.value,
            "reconciled_imap_not_appended",
        ),
    ],
)
def test_imap_reconciliation_preserves_attempt_and_only_not_appended_is_retryable(
    decision: str,
    expected_status: str,
    attempt_status: str,
) -> None:
    campaign = SimpleNamespace(id="campaign-1")
    job = SimpleNamespace(
        id="job-1",
        tenant_id="tenant-1",
        campaign_id="campaign-1",
        campaign_version_id="version-1",
        send_status=JobSendStatus.SMTP_ACCEPTED.value,
        imap_status=JobImapStatus.OUTCOME_UNKNOWN.value,
        imap_claimed_at=datetime.now(timezone.utc),
        imap_claim_token="claim-1",
        last_error="unknown",
    )
    attempt = SimpleNamespace(
        id="attempt-1",
        status=JobImapStatus.OUTCOME_UNKNOWN.value,
        error_message="unknown",
    )
    session = MagicMock()
    session.get.return_value = job
    session.query.return_value.filter.return_value.order_by.return_value.first.return_value = attempt

    with (
        patch(
            "govoplan_campaign.backend.sending.jobs._get_campaign_for_tenant",
            return_value=campaign,
        ),
        patch("govoplan_campaign.backend.sending.jobs.files_integration") as files,
    ):
        result = reconcile_job_outcome(
            session,
            tenant_id="tenant-1",
            campaign_id="campaign-1",
            job_id="job-1",
            decision=decision,
            note="Mailbox UID evidence checked by operator 42.",
            commit=False,
        )

    assert result["channel"] == "imap"
    assert job.imap_status == expected_status
    assert job.imap_claimed_at is None
    assert job.imap_claim_token is None
    assert attempt.status == attempt_status
    assert attempt.error_message == "Mailbox UID evidence checked by operator 42."
    assert attempt in session.add.call_args_list[0].args
    session.flush.assert_called_once_with()
    session.commit.assert_not_called()
    if decision == "imap_appended":
        files().mark_job_attachment_uses_sent.assert_called_once_with(session, job)
    else:
        files().mark_job_attachment_uses_sent.assert_not_called()


@pytest.mark.parametrize(
    "imap_status",
    [JobImapStatus.APPENDING.value, JobImapStatus.FAILED.value, JobImapStatus.APPENDED.value],
)
def test_imap_reconciliation_rejects_live_retryable_or_completed_state(imap_status: str) -> None:
    campaign = SimpleNamespace(id="campaign-1")
    job = SimpleNamespace(
        id="job-1",
        tenant_id="tenant-1",
        campaign_id="campaign-1",
        campaign_version_id="version-1",
        send_status=JobSendStatus.SMTP_ACCEPTED.value,
        imap_status=imap_status,
    )
    session = MagicMock()
    session.get.return_value = job
    with patch(
        "govoplan_campaign.backend.sending.jobs._get_campaign_for_tenant",
        return_value=campaign,
    ):
        with pytest.raises(QueueingError, match="does not require reconciliation"):
            reconcile_job_outcome(
                session,
                tenant_id="tenant-1",
                campaign_id="campaign-1",
                job_id="job-1",
                decision="imap_not_appended",
                note="Checked mailbox.",
            )


@pytest.mark.parametrize("send_status", [JobSendStatus.CLAIMED.value, JobSendStatus.SENDING.value])
def test_smtp_reconciliation_rejects_a_live_worker_state(send_status: str) -> None:
    campaign = SimpleNamespace(id="campaign-1")
    job = SimpleNamespace(
        id="job-1",
        tenant_id="tenant-1",
        campaign_id="campaign-1",
        campaign_version_id="version-1",
        send_status=send_status,
    )
    session = MagicMock()
    session.get.return_value = job
    with patch(
        "govoplan_campaign.backend.sending.jobs._get_campaign_for_tenant",
        return_value=campaign,
    ):
        with pytest.raises(QueueingError, match="does not require reconciliation"):
            reconcile_job_outcome(
                session,
                tenant_id="tenant-1",
                campaign_id="campaign-1",
                job_id="job-1",
                decision="not_sent",
                note="Checked provider logs.",
            )


@pytest.mark.parametrize(
    ("decision", "note"),
    [
        ("smtp_accepted", "SMTP provider evidence checked."),
        ("imap_appended", "Mailbox evidence checked."),
    ],
)
def test_reconciliation_rolls_back_state_when_audit_fails(
    decision: str,
    note: str | None,
) -> None:
    payload = CampaignResolveOutcomeRequest(decision=decision, note=note)  # type: ignore[arg-type]
    principal = SimpleNamespace(tenant_id="tenant-1")
    session = MagicMock()

    def mutate_without_commit(*_args, **kwargs):
        assert kwargs["commit"] is False
        session.flush()
        return {"decision": decision, "job_id": "job-1"}

    with (
        patch("govoplan_campaign.backend.routes.delivery._get_campaign_for_principal"),
        patch("govoplan_campaign.backend.routes.delivery._require_permission"),
        patch(
            "govoplan_campaign.backend.routes.delivery.reconcile_job_outcome",
            side_effect=mutate_without_commit,
        ),
        patch(
            "govoplan_campaign.backend.routes.delivery.audit_from_principal",
            side_effect=RuntimeError("audit unavailable"),
        ),
    ):
        with pytest.raises(RuntimeError, match="audit unavailable"):
            router.resolve_campaign_job_outcome(
                "campaign-1",
                "job-1",
                payload,
                session=session,
                principal=principal,  # type: ignore[arg-type]
            )

    session.commit.assert_not_called()
    session.rollback.assert_called_once_with()


def test_retention_preserves_eml_for_unknown_imap_outcome(tmp_path: Path) -> None:
    eml_path = tmp_path / "message.eml"
    eml_path.write_bytes(b"message")
    job = SimpleNamespace(
        campaign_id="campaign-1",
        updated_at=datetime.now(timezone.utc) - timedelta(days=10),
        queue_status="draft",
        send_status=JobSendStatus.SMTP_ACCEPTED.value,
        imap_status=JobImapStatus.OUTCOME_UNKNOWN.value,
        eml_local_path=str(eml_path),
        eml_storage_key=None,
    )
    query = MagicMock()
    query.filter.return_value.order_by.return_value.all.return_value = [job]
    session = MagicMock()
    session.query.return_value = query
    policy = SimpleNamespace(generated_eml_retention_days=1)

    result = _apply_eml_retention(
        session,
        dry_run=False,
        now=datetime.now(timezone.utc),
        policy_for_campaign_id=lambda _campaign_id: policy,
    )

    assert result["skipped_not_final"] == 1
    assert eml_path.exists()
    session.add.assert_not_called()