Compare commits
4 Commits
6fda123fc3
...
4774a8025c
| Author | SHA1 | Date | |
|---|---|---|---|
| 4774a8025c | |||
| 689dc1fd6b | |||
| 90677348ff | |||
| 06786e86ef |
@@ -9,3 +9,5 @@ GOVOPLAN_MAIL_TEST_IMAP_PORT=3143
|
|||||||
GOVOPLAN_MAIL_TEST_SENT_FOLDER=Sent
|
GOVOPLAN_MAIL_TEST_SENT_FOLDER=Sent
|
||||||
GOVOPLAN_MAIL_TEST_ZIP_PASSWORD=zip-test-password
|
GOVOPLAN_MAIL_TEST_ZIP_PASSWORD=zip-test-password
|
||||||
GOVOPLAN_MAIL_TEST_READY_TIMEOUT_SECONDS=45
|
GOVOPLAN_MAIL_TEST_READY_TIMEOUT_SECONDS=45
|
||||||
|
GOVOPLAN_CAMPAIGN_TEST_REDIS_PORT=36379
|
||||||
|
GOVOPLAN_CAMPAIGN_TEST_REDIS_VISIBILITY_TIMEOUT_SECONDS=3
|
||||||
|
|||||||
@@ -103,6 +103,47 @@ proof, not proof of a target provider's behavior or Redis/Celery broker
|
|||||||
redelivery. Use `--success-only` only when testing the success journey without
|
redelivery. Use `--success-only` only when testing the success journey without
|
||||||
the local failure endpoints.
|
the local failure endpoints.
|
||||||
|
|
||||||
|
## Redis/Celery Redelivery Acceptance
|
||||||
|
|
||||||
|
Run the maintained broker/worker-loss acceptance separately from the GreenMail
|
||||||
|
journey:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /mnt/DATA/git/govoplan-campaign/dev/mail-testbed
|
||||||
|
set -a
|
||||||
|
. ./.env
|
||||||
|
set +a
|
||||||
|
/mnt/DATA/git/govoplan/.venv/bin/python run_celery_redelivery_acceptance.py \
|
||||||
|
--evidence /tmp/govoplan-campaign-celery-redelivery-evidence.json
|
||||||
|
```
|
||||||
|
|
||||||
|
The runner creates a unique Compose project, starts only its loopback-bound,
|
||||||
|
AOF-enabled Redis service, creates an isolated temporary GovOPlaN database,
|
||||||
|
and starts a real Celery worker subscribed to `send_email`. A controlled SMTP
|
||||||
|
server holds the transaction after complete DATA and before the final response.
|
||||||
|
The runner kills that solo worker with the task still unacknowledged and starts
|
||||||
|
a replacement worker. After the configured Redis visibility timeout, the same Celery task identity must be redelivered.
|
||||||
|
The replacement must turn the durable
|
||||||
|
unfinished attempt into `outcome_unknown`, acknowledge the task, drain the
|
||||||
|
broker queue/unacked records, and leave the SMTP endpoint at exactly one
|
||||||
|
connection and one DATA transaction.
|
||||||
|
|
||||||
|
Only bounded counts, classifications, and booleans are retained. Worker logs,
|
||||||
|
task IDs, database identifiers, endpoints, credentials, and raw diagnostics are
|
||||||
|
kept in the temporary runtime and deleted. The evidence proves the local Redis
|
||||||
|
transport, real Celery process boundary, runner-supervised replacement, and
|
||||||
|
Campaign's duplicate-effect guard. It deliberately keeps production daemon supervision,
|
||||||
|
target-provider behavior, and source-artifact provenance false.
|
||||||
|
It does not claim that systemd, Kubernetes, another container orchestrator, or
|
||||||
|
an institution's Redis/SMTP deployment behaves identically.
|
||||||
|
|
||||||
|
The default run requires Docker CLI/Compose/daemon access and permission to
|
||||||
|
pull `redis:7-alpine`; the Celery workers execute from the current Python
|
||||||
|
environment. The isolated Compose project and volume are removed on exit.
|
||||||
|
`GOVOPLAN_CAMPAIGN_TEST_REDIS_VISIBILITY_TIMEOUT_SECONDS` defaults to three
|
||||||
|
seconds only to make this destructive local drill finish promptly; it is not a
|
||||||
|
production recommendation.
|
||||||
|
|
||||||
Starting the maintained test bed requires a working Docker CLI, Compose plugin,
|
Starting the maintained test bed requires a working Docker CLI, Compose plugin,
|
||||||
daemon/socket access, and permission to pull `greenmail/standalone:2.1.9`. The
|
daemon/socket access, and permission to pull `greenmail/standalone:2.1.9`. The
|
||||||
Campaign runner needs only the already-running loopback endpoints; it neither
|
Campaign runner needs only the already-running loopback endpoints; it neither
|
||||||
|
|||||||
@@ -15,3 +15,19 @@ services:
|
|||||||
- "${GOVOPLAN_MAIL_TEST_SMTP_PORT:-3025}:3025"
|
- "${GOVOPLAN_MAIL_TEST_SMTP_PORT:-3025}:3025"
|
||||||
- "${GOVOPLAN_MAIL_TEST_IMAP_PORT:-3143}:3143"
|
- "${GOVOPLAN_MAIL_TEST_IMAP_PORT:-3143}:3143"
|
||||||
- "127.0.0.1:38080:8080"
|
- "127.0.0.1:38080:8080"
|
||||||
|
|
||||||
|
redis:
|
||||||
|
image: redis:7-alpine
|
||||||
|
command: ["redis-server", "--appendonly", "yes"]
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${GOVOPLAN_CAMPAIGN_TEST_REDIS_PORT:-36379}:6379"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "redis-cli", "ping"]
|
||||||
|
interval: 1s
|
||||||
|
timeout: 1s
|
||||||
|
retries: 30
|
||||||
|
volumes:
|
||||||
|
- campaign-redis-data:/data
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
campaign-redis-data:
|
||||||
|
|||||||
856
dev/mail-testbed/run_celery_redelivery_acceptance.py
Normal file
856
dev/mail-testbed/run_celery_redelivery_acceptance.py
Normal file
@@ -0,0 +1,856 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Prove Redis/Celery redelivery does not repeat an ambiguous SMTP effect.
|
||||||
|
|
||||||
|
The default run starts an isolated Redis Compose service, two successive real
|
||||||
|
Celery worker processes, and a controlled loopback SMTP endpoint. It kills the
|
||||||
|
first worker after complete DATA but before a final SMTP response. The same
|
||||||
|
unacknowledged broker task must be delivered to the replacement worker, which
|
||||||
|
must freeze the unfinished durable attempt as ``outcome_unknown`` without a
|
||||||
|
second SMTP connection or DATA transaction.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from dataclasses import dataclass, replace
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import socket
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import time
|
||||||
|
from typing import Any, Callable, Iterator, Mapping
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from redis import Redis
|
||||||
|
from redis.exceptions import RedisError
|
||||||
|
|
||||||
|
|
||||||
|
SCRIPT_ROOT = Path(__file__).resolve().parent
|
||||||
|
REPOSITORY_ROOT = SCRIPT_ROOT.parents[1]
|
||||||
|
if str(SCRIPT_ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_ROOT))
|
||||||
|
|
||||||
|
from run_campaign_acceptance import ( # noqa: E402
|
||||||
|
AcceptanceError,
|
||||||
|
DEFAULT_FIXTURE,
|
||||||
|
EXPECTED_AUDIT_ACTIONS,
|
||||||
|
TestbedSettings,
|
||||||
|
_assert_evidence_safe,
|
||||||
|
_core_package_version,
|
||||||
|
_durable_state_evidence,
|
||||||
|
_expect,
|
||||||
|
_report_evidence,
|
||||||
|
create_mail_profile,
|
||||||
|
prepare_campaign_scenario,
|
||||||
|
required_composition_versions,
|
||||||
|
smtp_fault_endpoint,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
EVIDENCE_SCHEMA = "govoplan.campaign.celery-redelivery-acceptance.v1"
|
||||||
|
MAX_EVIDENCE_BYTES = 128 * 1024
|
||||||
|
DEFAULT_COMPOSE_FILE = SCRIPT_ROOT / "docker-compose.yml"
|
||||||
|
TASK_RECEIVED_PATTERN = re.compile(
|
||||||
|
r"Task govoplan[.]campaigns[.]send_email\[([0-9a-f-]{36})\] received",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
TASK_SUCCEEDED_PATTERN = re.compile(
|
||||||
|
r"Task govoplan[.]campaigns[.]send_email\[([0-9a-f-]{36})\] succeeded",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
WORKER_BOOTSTRAP = r"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from govoplan_core.celery_app import celery
|
||||||
|
|
||||||
|
visibility_timeout = int(os.environ["GOVOPLAN_CAMPAIGN_TEST_REDIS_VISIBILITY_TIMEOUT_SECONDS"])
|
||||||
|
celery.conf.broker_transport_options = {
|
||||||
|
**dict(celery.conf.broker_transport_options or {}),
|
||||||
|
"polling_interval": 0.25,
|
||||||
|
"visibility_timeout": visibility_timeout,
|
||||||
|
}
|
||||||
|
celery.worker_main(
|
||||||
|
[
|
||||||
|
"worker",
|
||||||
|
"--loglevel=INFO",
|
||||||
|
"--pool=solo",
|
||||||
|
"--concurrency=1",
|
||||||
|
"--queues=send_email",
|
||||||
|
f"--hostname={sys.argv[1]}@%h",
|
||||||
|
"--without-gossip",
|
||||||
|
"--without-mingle",
|
||||||
|
"--without-heartbeat",
|
||||||
|
]
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(slots=True)
|
||||||
|
class WorkerProcess:
|
||||||
|
process: subprocess.Popen[bytes]
|
||||||
|
log_path: Path
|
||||||
|
log_handle: Any
|
||||||
|
|
||||||
|
def text(self) -> str:
|
||||||
|
self.log_handle.flush()
|
||||||
|
try:
|
||||||
|
return self.log_path.read_text(encoding="utf-8", errors="replace")
|
||||||
|
except OSError as exc:
|
||||||
|
raise AcceptanceError("Celery worker evidence log could not be read") from exc
|
||||||
|
|
||||||
|
def received_task_ids(self) -> tuple[str, ...]:
|
||||||
|
return tuple(TASK_RECEIVED_PATTERN.findall(self.text()))
|
||||||
|
|
||||||
|
def succeeded_task_ids(self) -> tuple[str, ...]:
|
||||||
|
return tuple(TASK_SUCCEEDED_PATTERN.findall(self.text()))
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True, slots=True)
|
||||||
|
class RedisBrokerState:
|
||||||
|
queue_depth: int
|
||||||
|
unacked_hash_count: int
|
||||||
|
unacked_index_count: int
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, int]:
|
||||||
|
return {
|
||||||
|
"queue_depth": self.queue_depth,
|
||||||
|
"unacked_hash_count": self.unacked_hash_count,
|
||||||
|
"unacked_index_count": self.unacked_index_count,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _positive_int(value: str, *, label: str) -> int:
|
||||||
|
try:
|
||||||
|
parsed = int(value)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise argparse.ArgumentTypeError(f"{label} must be a positive integer") from exc
|
||||||
|
if parsed <= 0:
|
||||||
|
raise argparse.ArgumentTypeError(f"{label} must be a positive integer")
|
||||||
|
return parsed
|
||||||
|
|
||||||
|
|
||||||
|
def _unused_loopback_port() -> int:
|
||||||
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
||||||
|
probe.bind(("127.0.0.1", 0))
|
||||||
|
return int(probe.getsockname()[1])
|
||||||
|
|
||||||
|
|
||||||
|
def _compose_command(
|
||||||
|
*, compose_file: Path, project_name: str, operation: str
|
||||||
|
) -> list[str]:
|
||||||
|
prefix = [
|
||||||
|
"docker",
|
||||||
|
"compose",
|
||||||
|
"--file",
|
||||||
|
str(compose_file),
|
||||||
|
"--project-name",
|
||||||
|
project_name,
|
||||||
|
]
|
||||||
|
if operation == "up":
|
||||||
|
return [*prefix, "up", "--detach", "redis"]
|
||||||
|
if operation == "down":
|
||||||
|
return [*prefix, "down", "--volumes", "--remove-orphans"]
|
||||||
|
raise AcceptanceError("Unsupported Redis Compose operation")
|
||||||
|
|
||||||
|
|
||||||
|
def _run_compose(
|
||||||
|
command: list[str],
|
||||||
|
*,
|
||||||
|
environment: Mapping[str, str],
|
||||||
|
timeout_seconds: int,
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
completed = subprocess.run(
|
||||||
|
command,
|
||||||
|
env=dict(environment),
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
timeout=timeout_seconds,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
||||||
|
raise AcceptanceError("Redis Compose lifecycle command failed") from exc
|
||||||
|
if completed.returncode != 0:
|
||||||
|
raise AcceptanceError("Redis Compose lifecycle command failed")
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_redis(redis_url: str, *, timeout_seconds: int) -> None:
|
||||||
|
deadline = time.monotonic() + timeout_seconds
|
||||||
|
client = Redis.from_url(
|
||||||
|
redis_url,
|
||||||
|
socket_connect_timeout=1,
|
||||||
|
socket_timeout=1,
|
||||||
|
decode_responses=False,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
try:
|
||||||
|
if client.ping() is True:
|
||||||
|
return
|
||||||
|
except RedisError:
|
||||||
|
pass
|
||||||
|
time.sleep(0.25)
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
raise AcceptanceError("Isolated Redis broker did not become ready")
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def isolated_redis_broker(
|
||||||
|
*,
|
||||||
|
compose_file: Path,
|
||||||
|
timeout_seconds: int,
|
||||||
|
requested_port: int | None = None,
|
||||||
|
) -> Iterator[str]:
|
||||||
|
if shutil.which("docker") is None:
|
||||||
|
raise AcceptanceError("Docker CLI is required to start the isolated Redis broker")
|
||||||
|
if not compose_file.is_file():
|
||||||
|
raise AcceptanceError("Redis Compose definition is unavailable")
|
||||||
|
port = requested_port or _unused_loopback_port()
|
||||||
|
if port <= 0 or port > 65_535:
|
||||||
|
raise AcceptanceError("Redis test port is invalid")
|
||||||
|
project_name = f"govoplan-campaign-redelivery-{uuid4().hex[:12]}"
|
||||||
|
environment = {
|
||||||
|
**os.environ,
|
||||||
|
"GOVOPLAN_CAMPAIGN_TEST_REDIS_PORT": str(port),
|
||||||
|
}
|
||||||
|
lifecycle_attempted = False
|
||||||
|
try:
|
||||||
|
lifecycle_attempted = True
|
||||||
|
_run_compose(
|
||||||
|
_compose_command(
|
||||||
|
compose_file=compose_file,
|
||||||
|
project_name=project_name,
|
||||||
|
operation="up",
|
||||||
|
),
|
||||||
|
environment=environment,
|
||||||
|
timeout_seconds=timeout_seconds,
|
||||||
|
)
|
||||||
|
redis_url = f"redis://127.0.0.1:{port}/0"
|
||||||
|
_wait_for_redis(redis_url, timeout_seconds=timeout_seconds)
|
||||||
|
yield redis_url
|
||||||
|
finally:
|
||||||
|
if lifecycle_attempted:
|
||||||
|
_run_compose(
|
||||||
|
_compose_command(
|
||||||
|
compose_file=compose_file,
|
||||||
|
project_name=project_name,
|
||||||
|
operation="down",
|
||||||
|
),
|
||||||
|
environment=environment,
|
||||||
|
timeout_seconds=timeout_seconds,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _start_worker(runtime_root: Path, *, label: str) -> WorkerProcess:
|
||||||
|
log_path = runtime_root / f"{label}.log"
|
||||||
|
log_handle = log_path.open("wb")
|
||||||
|
environment = {**os.environ, "PYTHONUNBUFFERED": "1"}
|
||||||
|
try:
|
||||||
|
process = subprocess.Popen(
|
||||||
|
[sys.executable, "-c", WORKER_BOOTSTRAP, label],
|
||||||
|
env=environment,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=log_handle,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
close_fds=True,
|
||||||
|
)
|
||||||
|
except Exception:
|
||||||
|
log_handle.close()
|
||||||
|
raise
|
||||||
|
return WorkerProcess(process=process, log_path=log_path, log_handle=log_handle)
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_worker_ready(worker: WorkerProcess, *, timeout_seconds: int) -> None:
|
||||||
|
deadline = time.monotonic() + timeout_seconds
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
if worker.process.poll() is not None:
|
||||||
|
raise AcceptanceError("Celery worker exited before becoming ready")
|
||||||
|
if " ready." in worker.text():
|
||||||
|
return
|
||||||
|
time.sleep(0.2)
|
||||||
|
raise AcceptanceError("Celery worker did not become ready")
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_received_task(
|
||||||
|
worker: WorkerProcess,
|
||||||
|
*,
|
||||||
|
timeout_seconds: int,
|
||||||
|
expected_task_id: str | None = None,
|
||||||
|
) -> str:
|
||||||
|
deadline = time.monotonic() + timeout_seconds
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
received = worker.received_task_ids()
|
||||||
|
if received:
|
||||||
|
if len(set(received)) != 1:
|
||||||
|
raise AcceptanceError("Celery worker received more than one task identity")
|
||||||
|
task_id = received[0]
|
||||||
|
if expected_task_id is not None and task_id != expected_task_id:
|
||||||
|
raise AcceptanceError("Replacement worker received a different broker task")
|
||||||
|
return task_id
|
||||||
|
if worker.process.poll() is not None:
|
||||||
|
raise AcceptanceError("Celery worker exited before receiving the task")
|
||||||
|
time.sleep(0.2)
|
||||||
|
raise AcceptanceError("Celery worker did not receive the broker task")
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_task_success(
|
||||||
|
worker: WorkerProcess,
|
||||||
|
*,
|
||||||
|
task_id: str,
|
||||||
|
timeout_seconds: int,
|
||||||
|
) -> None:
|
||||||
|
deadline = time.monotonic() + timeout_seconds
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
succeeded = worker.succeeded_task_ids()
|
||||||
|
if task_id in succeeded:
|
||||||
|
return
|
||||||
|
if worker.process.poll() is not None:
|
||||||
|
raise AcceptanceError("Replacement Celery worker exited before task success")
|
||||||
|
time.sleep(0.2)
|
||||||
|
raise AcceptanceError("Redelivered Celery task did not complete")
|
||||||
|
|
||||||
|
|
||||||
|
def _kill_worker(worker: WorkerProcess, *, timeout_seconds: int) -> int:
|
||||||
|
if worker.process.poll() is not None:
|
||||||
|
raise AcceptanceError("Celery worker exited before controlled termination")
|
||||||
|
worker.process.kill()
|
||||||
|
try:
|
||||||
|
return_code = worker.process.wait(timeout=timeout_seconds)
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
raise AcceptanceError("Celery worker could not be killed") from exc
|
||||||
|
if return_code == 0:
|
||||||
|
raise AcceptanceError("Celery worker termination was not forced")
|
||||||
|
return return_code
|
||||||
|
|
||||||
|
|
||||||
|
def _stop_worker(worker: WorkerProcess, *, timeout_seconds: int) -> None:
|
||||||
|
if worker.process.poll() is None:
|
||||||
|
worker.process.terminate()
|
||||||
|
try:
|
||||||
|
worker.process.wait(timeout=timeout_seconds)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
worker.process.kill()
|
||||||
|
worker.process.wait(timeout=timeout_seconds)
|
||||||
|
worker.log_handle.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _broker_state(redis_url: str) -> RedisBrokerState:
|
||||||
|
client = Redis.from_url(
|
||||||
|
redis_url,
|
||||||
|
socket_connect_timeout=2,
|
||||||
|
socket_timeout=2,
|
||||||
|
decode_responses=False,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return RedisBrokerState(
|
||||||
|
queue_depth=int(client.llen("send_email")),
|
||||||
|
unacked_hash_count=int(client.hlen("unacked")),
|
||||||
|
unacked_index_count=int(client.zcard("unacked_index")),
|
||||||
|
)
|
||||||
|
except RedisError as exc:
|
||||||
|
raise AcceptanceError("Redis broker state could not be inspected") from exc
|
||||||
|
finally:
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_broker_drained(
|
||||||
|
redis_url: str,
|
||||||
|
*,
|
||||||
|
timeout_seconds: int,
|
||||||
|
) -> RedisBrokerState:
|
||||||
|
deadline = time.monotonic() + timeout_seconds
|
||||||
|
last = RedisBrokerState(0, 0, 0)
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
last = _broker_state(redis_url)
|
||||||
|
if last == RedisBrokerState(0, 0, 0):
|
||||||
|
return last
|
||||||
|
time.sleep(0.2)
|
||||||
|
raise AcceptanceError("Redis broker retained delivery state after recovery")
|
||||||
|
|
||||||
|
|
||||||
|
def _queue_evidence(payload: Mapping[str, Any]) -> dict[str, Any]:
|
||||||
|
expected = {
|
||||||
|
"queued_count": 1,
|
||||||
|
"skipped_count": 0,
|
||||||
|
"blocked_count": 0,
|
||||||
|
"enqueued_count": 1,
|
||||||
|
"delivery_mode": "worker_queue",
|
||||||
|
"worker_queue_available": True,
|
||||||
|
"dry_run": False,
|
||||||
|
}
|
||||||
|
evidence = {key: payload.get(key) for key in expected}
|
||||||
|
if evidence != expected:
|
||||||
|
raise AcceptanceError("Campaign was not durably queued to one Celery task")
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def execute_redelivery_scenario(
|
||||||
|
client: Any,
|
||||||
|
headers: Mapping[str, str],
|
||||||
|
*,
|
||||||
|
fixture_path: Path,
|
||||||
|
settings: TestbedSettings,
|
||||||
|
endpoint: Any,
|
||||||
|
redis_url: str,
|
||||||
|
runtime_root: Path,
|
||||||
|
snapshot_probe: Callable[[str], tuple[Mapping[str, Any], Mapping[str, Any]]],
|
||||||
|
audit_probe: Callable[[str, str], Mapping[str, int]],
|
||||||
|
delivery_probe: Callable[[str, str], Mapping[str, Any]],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
profile_id = create_mail_profile(
|
||||||
|
client,
|
||||||
|
headers,
|
||||||
|
settings,
|
||||||
|
name="Campaign Redis Celery redelivery drill",
|
||||||
|
smtp_host=endpoint.host,
|
||||||
|
smtp_port=endpoint.port,
|
||||||
|
)
|
||||||
|
prepared = prepare_campaign_scenario(
|
||||||
|
client,
|
||||||
|
headers,
|
||||||
|
fixture_path=fixture_path,
|
||||||
|
profile_id=profile_id,
|
||||||
|
settings=settings,
|
||||||
|
scenario="celery_broker_redelivery",
|
||||||
|
snapshot_probe=snapshot_probe,
|
||||||
|
)
|
||||||
|
|
||||||
|
first_worker = _start_worker(runtime_root, label="first-worker")
|
||||||
|
replacement_worker: WorkerProcess | None = None
|
||||||
|
try:
|
||||||
|
_wait_for_worker_ready(
|
||||||
|
first_worker,
|
||||||
|
timeout_seconds=settings.provider_timeout_seconds,
|
||||||
|
)
|
||||||
|
queued = _expect(
|
||||||
|
client.post(
|
||||||
|
f"/api/v1/campaigns/{prepared.campaign_id}/queue",
|
||||||
|
headers=dict(headers),
|
||||||
|
json={
|
||||||
|
"version_id": prepared.version_id,
|
||||||
|
"include_warnings": True,
|
||||||
|
"enqueue_celery": True,
|
||||||
|
"dry_run": False,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
200,
|
||||||
|
"Celery-redelivery Campaign queue",
|
||||||
|
)
|
||||||
|
queue_evidence = _queue_evidence(queued)
|
||||||
|
first_task_id = _wait_for_received_task(
|
||||||
|
first_worker,
|
||||||
|
timeout_seconds=settings.provider_timeout_seconds,
|
||||||
|
)
|
||||||
|
if not endpoint.wait_for_data(settings.provider_timeout_seconds):
|
||||||
|
raise AcceptanceError("Celery worker did not reach complete SMTP DATA")
|
||||||
|
first_exit_code = _kill_worker(
|
||||||
|
first_worker,
|
||||||
|
timeout_seconds=settings.provider_timeout_seconds,
|
||||||
|
)
|
||||||
|
endpoint.release_held_connection()
|
||||||
|
|
||||||
|
interrupted_state = _durable_state_evidence(
|
||||||
|
delivery_probe(prepared.campaign_id, prepared.version_id)
|
||||||
|
)
|
||||||
|
expected_interrupted = {
|
||||||
|
"job_count": 1,
|
||||||
|
"send_status_counts": {"sending": 1},
|
||||||
|
"attempt_status_counts": {"smtp_in_progress": 1},
|
||||||
|
"unfinished_attempt_count": 1,
|
||||||
|
}
|
||||||
|
if interrupted_state != expected_interrupted:
|
||||||
|
raise AcceptanceError("Killed worker state was not durably SMTP-in-progress")
|
||||||
|
|
||||||
|
replacement_worker = _start_worker(runtime_root, label="replacement-worker")
|
||||||
|
_wait_for_worker_ready(
|
||||||
|
replacement_worker,
|
||||||
|
timeout_seconds=settings.provider_timeout_seconds,
|
||||||
|
)
|
||||||
|
redelivered_task_id = _wait_for_received_task(
|
||||||
|
replacement_worker,
|
||||||
|
timeout_seconds=settings.provider_timeout_seconds,
|
||||||
|
expected_task_id=first_task_id,
|
||||||
|
)
|
||||||
|
_wait_for_task_success(
|
||||||
|
replacement_worker,
|
||||||
|
task_id=redelivered_task_id,
|
||||||
|
timeout_seconds=settings.provider_timeout_seconds,
|
||||||
|
)
|
||||||
|
recovered_state = _durable_state_evidence(
|
||||||
|
delivery_probe(prepared.campaign_id, prepared.version_id)
|
||||||
|
)
|
||||||
|
expected_recovered = {
|
||||||
|
"job_count": 1,
|
||||||
|
"send_status_counts": {"outcome_unknown": 1},
|
||||||
|
"attempt_status_counts": {"outcome_unknown": 1},
|
||||||
|
"unfinished_attempt_count": 0,
|
||||||
|
}
|
||||||
|
if recovered_state != expected_recovered:
|
||||||
|
raise AcceptanceError("Redelivered task did not freeze the unfinished attempt")
|
||||||
|
|
||||||
|
protocol = endpoint.evidence()
|
||||||
|
expected_protocol = {
|
||||||
|
"connection_count": 1,
|
||||||
|
"accepted_rcpt_commands": 1,
|
||||||
|
"refused_rcpt_commands": 0,
|
||||||
|
"data_transactions": 1,
|
||||||
|
}
|
||||||
|
if protocol != expected_protocol:
|
||||||
|
raise AcceptanceError("Broker redelivery caused an unexpected SMTP transaction")
|
||||||
|
broker_after = _wait_for_broker_drained(
|
||||||
|
redis_url,
|
||||||
|
timeout_seconds=settings.provider_timeout_seconds,
|
||||||
|
)
|
||||||
|
first_received = first_worker.received_task_ids()
|
||||||
|
replacement_received = replacement_worker.received_task_ids()
|
||||||
|
if first_received != (first_task_id,) or replacement_received != (
|
||||||
|
redelivered_task_id,
|
||||||
|
):
|
||||||
|
raise AcceptanceError(
|
||||||
|
"Celery workers did not each receive the broker task exactly once"
|
||||||
|
)
|
||||||
|
report = _report_evidence(
|
||||||
|
_expect(
|
||||||
|
client.get(
|
||||||
|
f"/api/v1/campaigns/{prepared.campaign_id}/report",
|
||||||
|
headers=dict(headers),
|
||||||
|
params={"version_id": prepared.version_id},
|
||||||
|
),
|
||||||
|
200,
|
||||||
|
"Celery-redelivery Campaign report",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if report["send_status_counts"] != {"outcome_unknown": 1}:
|
||||||
|
raise AcceptanceError("Campaign report did not retain outcome_unknown")
|
||||||
|
audit_actions = dict(
|
||||||
|
sorted(audit_probe(prepared.campaign_id, prepared.version_id).items())
|
||||||
|
)
|
||||||
|
if not {
|
||||||
|
"campaign.created",
|
||||||
|
"campaign.validated",
|
||||||
|
"campaign.messages_built",
|
||||||
|
"campaign.queued",
|
||||||
|
}.issubset(audit_actions):
|
||||||
|
raise AcceptanceError("Celery-redelivery Campaign audit evidence is incomplete")
|
||||||
|
|
||||||
|
return {
|
||||||
|
**prepared.public_evidence(),
|
||||||
|
"queue": queue_evidence,
|
||||||
|
"interrupted_durable_state": interrupted_state,
|
||||||
|
"recovered_durable_state": recovered_state,
|
||||||
|
"protocol": protocol,
|
||||||
|
"report": report,
|
||||||
|
"audit_actions": audit_actions,
|
||||||
|
"broker": {
|
||||||
|
"transport": "redis",
|
||||||
|
"same_task_identity_redelivered": first_task_id
|
||||||
|
== redelivered_task_id,
|
||||||
|
"first_worker_received_count": len(first_received),
|
||||||
|
"replacement_worker_received_count": len(replacement_received),
|
||||||
|
**broker_after.as_dict(),
|
||||||
|
},
|
||||||
|
"supervision": {
|
||||||
|
"first_worker_killed_after_complete_data": True,
|
||||||
|
"first_worker_forced_exit": first_exit_code != 0,
|
||||||
|
"replacement_worker_started": True,
|
||||||
|
"replacement_worker_completed_redelivery": True,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
finally:
|
||||||
|
endpoint.release_held_connection()
|
||||||
|
_stop_worker(first_worker, timeout_seconds=5)
|
||||||
|
if replacement_worker is not None:
|
||||||
|
_stop_worker(replacement_worker, timeout_seconds=5)
|
||||||
|
|
||||||
|
|
||||||
|
def _runtime_module_versions(registry: Any) -> dict[str, str]:
|
||||||
|
versions = {"core": _core_package_version()}
|
||||||
|
versions.update(
|
||||||
|
{
|
||||||
|
manifest.id: manifest.version
|
||||||
|
for manifest in registry.manifests()
|
||||||
|
if manifest.id in {"access", "audit", "campaigns", "mail"}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return versions
|
||||||
|
|
||||||
|
|
||||||
|
def _create_runtime_root() -> Path:
|
||||||
|
"""Create the isolated runtime under the platform-selected temp root."""
|
||||||
|
|
||||||
|
return Path(tempfile.mkdtemp(prefix="govoplan-campaign-celery-redelivery-"))
|
||||||
|
|
||||||
|
|
||||||
|
def _bootstrap_and_run(
|
||||||
|
*,
|
||||||
|
settings: TestbedSettings,
|
||||||
|
fixture_path: Path,
|
||||||
|
redis_url: str,
|
||||||
|
visibility_timeout_seconds: int,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
runtime_root = _create_runtime_root()
|
||||||
|
database = None
|
||||||
|
try:
|
||||||
|
os.environ.update(
|
||||||
|
{
|
||||||
|
"APP_ENV": "test",
|
||||||
|
"DATABASE_URL": f"sqlite:///{runtime_root / 'acceptance.db'}",
|
||||||
|
"FILE_STORAGE_BACKEND": "local",
|
||||||
|
"FILE_STORAGE_LOCAL_ROOT": str(runtime_root / "files"),
|
||||||
|
"MOCK_MAILBOX_DIR": str(runtime_root / "mock-mailbox"),
|
||||||
|
"DEV_BOOTSTRAP_ENABLED": "false",
|
||||||
|
"CELERY_ENABLED": "true",
|
||||||
|
"REDIS_URL": redis_url,
|
||||||
|
"GOVOPLAN_CAMPAIGN_TEST_REDIS_VISIBILITY_TIMEOUT_SECONDS": str(
|
||||||
|
visibility_timeout_seconds
|
||||||
|
),
|
||||||
|
"GOVOPLAN_CONNECTOR_ALLOW_PRIVATE_NETWORKS": "true",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from govoplan_core.db.base import Base
|
||||||
|
from govoplan_core.db.bootstrap import bootstrap_dev_data
|
||||||
|
from govoplan_core.db.session import configure_database, set_database
|
||||||
|
from govoplan_core.settings import Settings, settings as core_settings
|
||||||
|
from govoplan_core.tenancy.scope import create_scope_tables
|
||||||
|
|
||||||
|
isolated_settings = Settings()
|
||||||
|
for field_name in Settings.model_fields:
|
||||||
|
setattr(core_settings, field_name, getattr(isolated_settings, field_name))
|
||||||
|
database = configure_database(os.environ["DATABASE_URL"])
|
||||||
|
set_database(database)
|
||||||
|
|
||||||
|
from govoplan_core.server.app import app
|
||||||
|
|
||||||
|
create_scope_tables(database.engine)
|
||||||
|
Base.metadata.create_all(bind=database.engine)
|
||||||
|
with database.SessionLocal() as session:
|
||||||
|
bootstrap_dev_data(
|
||||||
|
session,
|
||||||
|
api_key_secret="celery-redelivery-unused-api-key",
|
||||||
|
user_password="celery-redelivery-admin",
|
||||||
|
)
|
||||||
|
|
||||||
|
def snapshot_probe(
|
||||||
|
version_id: str,
|
||||||
|
) -> tuple[Mapping[str, Any], Mapping[str, Any]]:
|
||||||
|
from govoplan_campaign.backend.db.models import CampaignVersion
|
||||||
|
|
||||||
|
with database.SessionLocal() as session:
|
||||||
|
version = session.get(CampaignVersion, version_id)
|
||||||
|
if version is None:
|
||||||
|
raise AcceptanceError("Campaign execution snapshot is unavailable")
|
||||||
|
raw = version.raw_json if isinstance(version.raw_json, dict) else {}
|
||||||
|
snapshot = (
|
||||||
|
version.execution_snapshot
|
||||||
|
if isinstance(version.execution_snapshot, dict)
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
return raw, snapshot
|
||||||
|
|
||||||
|
def audit_probe(campaign_id: str, version_id: str) -> Mapping[str, int]:
|
||||||
|
from govoplan_audit.backend.db.models import AuditLog
|
||||||
|
|
||||||
|
with database.SessionLocal() as session:
|
||||||
|
actions = [
|
||||||
|
row[0]
|
||||||
|
for row in session.query(AuditLog.action)
|
||||||
|
.filter(AuditLog.object_id.in_([campaign_id, version_id]))
|
||||||
|
.all()
|
||||||
|
if row[0] in EXPECTED_AUDIT_ACTIONS
|
||||||
|
]
|
||||||
|
return dict(Counter(actions))
|
||||||
|
|
||||||
|
def delivery_probe(campaign_id: str, version_id: str) -> Mapping[str, Any]:
|
||||||
|
from govoplan_campaign.backend.db.models import CampaignJob, SendAttempt
|
||||||
|
|
||||||
|
with database.SessionLocal() as session:
|
||||||
|
jobs = (
|
||||||
|
session.query(CampaignJob)
|
||||||
|
.filter(
|
||||||
|
CampaignJob.campaign_id == campaign_id,
|
||||||
|
CampaignJob.campaign_version_id == version_id,
|
||||||
|
)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
job_ids = [job.id for job in jobs]
|
||||||
|
attempts = (
|
||||||
|
session.query(SendAttempt)
|
||||||
|
.filter(SendAttempt.job_id.in_(job_ids))
|
||||||
|
.all()
|
||||||
|
if job_ids
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"job_count": len(jobs),
|
||||||
|
"send_status_counts": dict(
|
||||||
|
Counter(job.send_status for job in jobs)
|
||||||
|
),
|
||||||
|
"attempt_status_counts": dict(
|
||||||
|
Counter(attempt.status for attempt in attempts)
|
||||||
|
),
|
||||||
|
"unfinished_attempt_count": sum(
|
||||||
|
1 for attempt in attempts if attempt.finished_at is None
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
with TestClient(app) as client:
|
||||||
|
login = _expect(
|
||||||
|
client.post(
|
||||||
|
"/api/v1/auth/login",
|
||||||
|
json={
|
||||||
|
"email": "admin@example.local",
|
||||||
|
"password": "celery-redelivery-admin",
|
||||||
|
},
|
||||||
|
),
|
||||||
|
200,
|
||||||
|
"Acceptance login",
|
||||||
|
)
|
||||||
|
access_token = str(login.get("access_token") or "")
|
||||||
|
if not access_token:
|
||||||
|
raise AcceptanceError("Acceptance login returned no access token")
|
||||||
|
from govoplan_core.core.runtime import get_registry
|
||||||
|
|
||||||
|
registry = get_registry()
|
||||||
|
if registry is None:
|
||||||
|
raise AcceptanceError("The GovOPlaN module registry is unavailable")
|
||||||
|
composition_versions = required_composition_versions(
|
||||||
|
fixture_path,
|
||||||
|
_runtime_module_versions(registry),
|
||||||
|
)
|
||||||
|
with smtp_fault_endpoint("post_data_hold") as endpoint:
|
||||||
|
scenario = execute_redelivery_scenario(
|
||||||
|
client,
|
||||||
|
{"Authorization": f"Bearer {access_token}"},
|
||||||
|
fixture_path=fixture_path,
|
||||||
|
settings=settings,
|
||||||
|
endpoint=endpoint,
|
||||||
|
redis_url=redis_url,
|
||||||
|
runtime_root=runtime_root,
|
||||||
|
snapshot_probe=snapshot_probe,
|
||||||
|
audit_probe=audit_probe,
|
||||||
|
delivery_probe=delivery_probe,
|
||||||
|
)
|
||||||
|
|
||||||
|
evidence = {
|
||||||
|
"schema_version": EVIDENCE_SCHEMA,
|
||||||
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"fixture_sha256": hashlib.sha256(fixture_path.read_bytes()).hexdigest(),
|
||||||
|
"declared_module_versions": composition_versions,
|
||||||
|
"target": {
|
||||||
|
"kind": "local_redis_celery_controlled_smtp",
|
||||||
|
"isolated_temporary_database": True,
|
||||||
|
"redis_started_by_runner": True,
|
||||||
|
"worker_pool": "solo",
|
||||||
|
"worker_prefetch_multiplier": 1,
|
||||||
|
"task_acks_late": True,
|
||||||
|
"task_reject_on_worker_lost": True,
|
||||||
|
"visibility_timeout_seconds": visibility_timeout_seconds,
|
||||||
|
},
|
||||||
|
"scenario": scenario,
|
||||||
|
"coverage": {
|
||||||
|
"redis_broker_delivery": True,
|
||||||
|
"celery_worker_processes": True,
|
||||||
|
"forced_worker_loss_after_complete_data": True,
|
||||||
|
"same_task_broker_redelivery": True,
|
||||||
|
"durable_outcome_unknown_recovery": True,
|
||||||
|
"duplicate_smtp_transaction_prevented": True,
|
||||||
|
"production_daemon_supervisor": False,
|
||||||
|
"target_provider": False,
|
||||||
|
"source_artifact_provenance": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
_assert_evidence_safe(evidence, settings=settings)
|
||||||
|
rendered = json.dumps(
|
||||||
|
evidence,
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
sort_keys=True,
|
||||||
|
).encode("utf-8") + b"\n"
|
||||||
|
if len(rendered) > MAX_EVIDENCE_BYTES:
|
||||||
|
raise AcceptanceError("Celery-redelivery evidence exceeds its size limit")
|
||||||
|
return evidence
|
||||||
|
finally:
|
||||||
|
if database is not None:
|
||||||
|
database.engine.dispose()
|
||||||
|
shutil.rmtree(runtime_root, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv: list[str] | None = None) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
|
parser.add_argument("--fixture", type=Path, default=DEFAULT_FIXTURE)
|
||||||
|
parser.add_argument("--compose-file", type=Path, default=DEFAULT_COMPOSE_FILE)
|
||||||
|
parser.add_argument("--redis-port", type=int)
|
||||||
|
parser.add_argument(
|
||||||
|
"--visibility-timeout-seconds",
|
||||||
|
type=lambda value: _positive_int(value, label="visibility timeout"),
|
||||||
|
default=os.environ.get(
|
||||||
|
"GOVOPLAN_CAMPAIGN_TEST_REDIS_VISIBILITY_TIMEOUT_SECONDS",
|
||||||
|
"3",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--timeout-seconds",
|
||||||
|
type=lambda value: _positive_int(value, label="timeout"),
|
||||||
|
default=60,
|
||||||
|
)
|
||||||
|
parser.add_argument("--evidence", type=Path)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
|
||||||
|
try:
|
||||||
|
settings = TestbedSettings.from_environment()
|
||||||
|
settings.assert_local_testbed()
|
||||||
|
settings = replace(
|
||||||
|
settings,
|
||||||
|
provider_timeout_seconds=args.timeout_seconds,
|
||||||
|
)
|
||||||
|
with isolated_redis_broker(
|
||||||
|
compose_file=args.compose_file.resolve(),
|
||||||
|
timeout_seconds=args.timeout_seconds,
|
||||||
|
requested_port=args.redis_port,
|
||||||
|
) as redis_url:
|
||||||
|
evidence = _bootstrap_and_run(
|
||||||
|
settings=settings,
|
||||||
|
fixture_path=args.fixture.resolve(),
|
||||||
|
redis_url=redis_url,
|
||||||
|
visibility_timeout_seconds=args.visibility_timeout_seconds,
|
||||||
|
)
|
||||||
|
rendered = json.dumps(
|
||||||
|
evidence,
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
sort_keys=True,
|
||||||
|
) + "\n"
|
||||||
|
if args.evidence:
|
||||||
|
args.evidence.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.evidence.write_text(rendered, encoding="utf-8")
|
||||||
|
else:
|
||||||
|
sys.stdout.write(rendered)
|
||||||
|
return 0
|
||||||
|
except AcceptanceError as exc:
|
||||||
|
print(f"Campaign Celery-redelivery acceptance failed: {exc}", file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
except Exception as exc:
|
||||||
|
print(
|
||||||
|
"Campaign Celery-redelivery acceptance failed unexpectedly "
|
||||||
|
f"({type(exc).__name__}); inspect local service logs.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -118,9 +118,21 @@ The runner also terminates a dedicated OS process executing the registered
|
|||||||
Campaign send task after complete DATA, then invokes the task in a fresh
|
Campaign send task after complete DATA, then invokes the task in a fresh
|
||||||
process. The unfinished durable attempt must become `outcome_unknown` and the
|
process. The unfinished durable attempt must become `outcome_unknown` and the
|
||||||
endpoint must observe no second connection or DATA transaction. This covers
|
endpoint must observe no second connection or DATA transaction. This covers
|
||||||
the worker task/process boundary. It does not cover Redis/Celery delivery,
|
the worker task/process boundary but not a broker or daemon.
|
||||||
acknowledgement, broker redelivery, or daemon supervision; repeat the drill in
|
|
||||||
that infrastructure before approving a production worker deployment.
|
Run `dev/mail-testbed/run_celery_redelivery_acceptance.py` for the maintained
|
||||||
|
Redis/Celery delivery and broker redelivery boundary. It starts an isolated
|
||||||
|
Redis Compose service and real Celery workers, kills the first solo worker after complete DATA while the
|
||||||
|
late-ack task is unacknowledged, and requires the same task identity to reach a
|
||||||
|
replacement worker after Redis visibility recovery. Passing evidence also
|
||||||
|
requires durable `outcome_unknown`, an empty broker queue/unacked set, and
|
||||||
|
exactly one SMTP connection and DATA transaction. Raw worker logs and task,
|
||||||
|
database, endpoint, and credential identifiers are never retained.
|
||||||
|
|
||||||
|
That second runner proves local runner-supervised process replacement, not the
|
||||||
|
production process manager. Repeat the worker-loss drill under the selected
|
||||||
|
systemd, container, Kubernetes, or other production supervisor and the target
|
||||||
|
Redis/SMTP infrastructure before deployment approval.
|
||||||
|
|
||||||
## Reporting Checks
|
## Reporting Checks
|
||||||
|
|
||||||
|
|||||||
@@ -65,6 +65,13 @@ Before tagging a campaign release:
|
|||||||
- Treat the task-process restart proof separately from the still-open
|
- Treat the task-process restart proof separately from the still-open
|
||||||
Redis/Celery broker redelivery and daemon-supervision check; the coverage
|
Redis/Celery broker redelivery and daemon-supervision check; the coverage
|
||||||
projection must keep `celery_broker_redelivery` false.
|
projection must keep `celery_broker_redelivery` false.
|
||||||
|
- Run `dev/mail-testbed/run_celery_redelivery_acceptance.py` as a separate
|
||||||
|
destructive worker-loss check. Retain its bounded evidence only when the same
|
||||||
|
broker task is observed at both workers, the durable state is
|
||||||
|
`outcome_unknown`, broker queue/unacked counts are zero, and the controlled
|
||||||
|
SMTP endpoint observed one connection and one DATA transaction. This closes
|
||||||
|
local Redis/Celery redelivery coverage, while production supervisor and
|
||||||
|
target-provider coverage remain false until separately tested.
|
||||||
- Confirm reusable mail profile selection is revalidated after campaign owner
|
- Confirm reusable mail profile selection is revalidated after campaign owner
|
||||||
transfer.
|
transfer.
|
||||||
- Confirm every inline SMTP/IMAP field is rejected on import/write, omitted
|
- Confirm every inline SMTP/IMAP field is rejected on import/write, omitted
|
||||||
|
|||||||
@@ -3608,6 +3608,10 @@ def send_campaign_now_endpoint(
|
|||||||
)
|
)
|
||||||
return SendCampaignNowResponse(result=response_result)
|
return SendCampaignNowResponse(result=response_result)
|
||||||
except SynchronousSendRejected as exc:
|
except SynchronousSendRejected as exc:
|
||||||
|
# A synchronous request stages queue state before the all-message
|
||||||
|
# preflight can run. Rejecting that preflight must not leave work
|
||||||
|
# eligible for a background worker when no provider effect occurred.
|
||||||
|
session.rollback()
|
||||||
audit_from_principal(
|
audit_from_principal(
|
||||||
session,
|
session,
|
||||||
principal,
|
principal,
|
||||||
|
|||||||
@@ -622,6 +622,7 @@ def _persist_campaign_queue(
|
|||||||
version: CampaignVersion,
|
version: CampaignVersion,
|
||||||
queued: list[CampaignJob],
|
queued: list[CampaignJob],
|
||||||
delivery_mode: str,
|
delivery_mode: str,
|
||||||
|
commit: bool = True,
|
||||||
) -> None:
|
) -> None:
|
||||||
if queued:
|
if queued:
|
||||||
previous_status = campaign.status
|
previous_status = campaign.status
|
||||||
@@ -640,7 +641,8 @@ def _persist_campaign_queue(
|
|||||||
version_id=version.id,
|
version_id=version.id,
|
||||||
)
|
)
|
||||||
session.add(campaign)
|
session.add(campaign)
|
||||||
session.commit()
|
if commit:
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
|
||||||
def _enqueue_campaign_jobs(queued: list[CampaignJob], *, enabled: bool) -> int:
|
def _enqueue_campaign_jobs(queued: list[CampaignJob], *, enabled: bool) -> int:
|
||||||
@@ -661,6 +663,7 @@ def queue_campaign_jobs(
|
|||||||
include_warnings: bool = True,
|
include_warnings: bool = True,
|
||||||
dry_run: bool = False,
|
dry_run: bool = False,
|
||||||
delivery_mode: str | None = None,
|
delivery_mode: str | None = None,
|
||||||
|
commit_queue: bool = True,
|
||||||
) -> QueueCampaignResult:
|
) -> QueueCampaignResult:
|
||||||
"""Move queueable DB jobs to QUEUED and optionally enqueue Celery tasks."""
|
"""Move queueable DB jobs to QUEUED and optionally enqueue Celery tasks."""
|
||||||
|
|
||||||
@@ -694,6 +697,7 @@ def queue_campaign_jobs(
|
|||||||
version=version,
|
version=version,
|
||||||
queued=queued,
|
queued=queued,
|
||||||
delivery_mode=selected_delivery_mode,
|
delivery_mode=selected_delivery_mode,
|
||||||
|
commit=commit_queue,
|
||||||
)
|
)
|
||||||
enqueued_count = _enqueue_campaign_jobs(
|
enqueued_count = _enqueue_campaign_jobs(
|
||||||
queued,
|
queued,
|
||||||
@@ -766,6 +770,7 @@ def send_campaign_now(
|
|||||||
enqueue_celery=False,
|
enqueue_celery=False,
|
||||||
dry_run=dry_run,
|
dry_run=dry_run,
|
||||||
delivery_mode=DELIVERY_MODE_SYNCHRONOUS,
|
delivery_mode=DELIVERY_MODE_SYNCHRONOUS,
|
||||||
|
commit_queue=False,
|
||||||
)
|
)
|
||||||
if dry_run:
|
if dry_run:
|
||||||
return SendCampaignNowResult(
|
return SendCampaignNowResult(
|
||||||
@@ -802,6 +807,11 @@ def send_campaign_now(
|
|||||||
jobs=jobs,
|
jobs=jobs,
|
||||||
policy=synchronous_policy,
|
policy=synchronous_policy,
|
||||||
)
|
)
|
||||||
|
# Queue state and its inbox notification become durable only after every
|
||||||
|
# message and the selected transport revision have passed preflight. This
|
||||||
|
# preserves late-ack recovery without leaving rejected work eligible for a
|
||||||
|
# background worker.
|
||||||
|
session.commit()
|
||||||
|
|
||||||
results: list[dict[str, Any]] = []
|
results: list[dict[str, Any]] = []
|
||||||
sent_count = 0
|
sent_count = 0
|
||||||
|
|||||||
314
tests/test_celery_redelivery_acceptance.py
Normal file
314
tests/test_celery_redelivery_acceptance.py
Normal file
@@ -0,0 +1,314 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
RUNNER_PATH = (
|
||||||
|
REPOSITORY_ROOT
|
||||||
|
/ "dev"
|
||||||
|
/ "mail-testbed"
|
||||||
|
/ "run_celery_redelivery_acceptance.py"
|
||||||
|
)
|
||||||
|
COMPOSE_PATH = REPOSITORY_ROOT / "dev" / "mail-testbed" / "docker-compose.yml"
|
||||||
|
FIXTURE_PATH = REPOSITORY_ROOT / "examples" / "greenmail-delivery" / "campaign.json"
|
||||||
|
TASK_ID = "12345678-1234-4234-8234-123456789abc"
|
||||||
|
|
||||||
|
|
||||||
|
def _load_runner():
|
||||||
|
spec = importlib.util.spec_from_file_location(
|
||||||
|
"govoplan_campaign_celery_redelivery_acceptance",
|
||||||
|
RUNNER_PATH,
|
||||||
|
)
|
||||||
|
assert spec is not None and spec.loader is not None
|
||||||
|
module = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules[spec.name] = module
|
||||||
|
spec.loader.exec_module(module)
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
runner = _load_runner()
|
||||||
|
|
||||||
|
|
||||||
|
class _Response:
|
||||||
|
def __init__(self, status_code: int, payload: dict) -> None:
|
||||||
|
self.status_code = status_code
|
||||||
|
self._payload = payload
|
||||||
|
|
||||||
|
def json(self) -> dict:
|
||||||
|
return self._payload
|
||||||
|
|
||||||
|
|
||||||
|
class _Client:
|
||||||
|
def post(self, path: str, **_kwargs) -> _Response:
|
||||||
|
assert path.endswith("/queue")
|
||||||
|
return _Response(
|
||||||
|
200,
|
||||||
|
{
|
||||||
|
"queued_count": 1,
|
||||||
|
"skipped_count": 0,
|
||||||
|
"blocked_count": 0,
|
||||||
|
"enqueued_count": 1,
|
||||||
|
"delivery_mode": "worker_queue",
|
||||||
|
"worker_queue_available": True,
|
||||||
|
"dry_run": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def get(self, path: str, **_kwargs) -> _Response:
|
||||||
|
assert path.endswith("/report")
|
||||||
|
return _Response(
|
||||||
|
200,
|
||||||
|
{
|
||||||
|
"cards": {
|
||||||
|
"jobs_total": 1,
|
||||||
|
"outcome_unknown": 1,
|
||||||
|
"needs_attention": 1,
|
||||||
|
},
|
||||||
|
"status_counts": {
|
||||||
|
"send": {"outcome_unknown": 1},
|
||||||
|
"imap": {"pending": 1},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Endpoint:
|
||||||
|
host = "127.0.0.1"
|
||||||
|
port = 4025
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.release_count = 0
|
||||||
|
|
||||||
|
def wait_for_data(self, _timeout_seconds: int) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
def release_held_connection(self) -> None:
|
||||||
|
self.release_count += 1
|
||||||
|
|
||||||
|
def evidence(self) -> dict[str, int]:
|
||||||
|
return {
|
||||||
|
"connection_count": 1,
|
||||||
|
"accepted_rcpt_commands": 1,
|
||||||
|
"refused_rcpt_commands": 0,
|
||||||
|
"data_transactions": 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _settings():
|
||||||
|
return runner.TestbedSettings(
|
||||||
|
smtp_host="127.0.0.1",
|
||||||
|
smtp_port=3025,
|
||||||
|
imap_host="127.0.0.1",
|
||||||
|
imap_port=3143,
|
||||||
|
username="campaign-test@govoplan.test",
|
||||||
|
password="local-test-password",
|
||||||
|
sender="campaign-test@govoplan.test",
|
||||||
|
recipient="campaign-test@govoplan.test",
|
||||||
|
sent_folder="Sent",
|
||||||
|
provider_timeout_seconds=5,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_compose_redis_is_isolated_durable_and_health_checked() -> None:
|
||||||
|
compose = COMPOSE_PATH.read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "redis:7-alpine" in compose
|
||||||
|
assert '"--appendonly", "yes"' in compose
|
||||||
|
assert "127.0.0.1:${GOVOPLAN_CAMPAIGN_TEST_REDIS_PORT:-36379}:6379" in compose
|
||||||
|
assert 'test: ["CMD", "redis-cli", "ping"]' in compose
|
||||||
|
assert "campaign-redis-data:/data" in compose
|
||||||
|
|
||||||
|
|
||||||
|
def test_runbook_keeps_local_redelivery_distinct_from_production_supervision() -> None:
|
||||||
|
testbed = (REPOSITORY_ROOT / "dev" / "mail-testbed" / "README.md").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
runbook = (REPOSITORY_ROOT / "docs" / "CAMPAIGN_DELIVERY_RUNBOOK.md").read_text(
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "run_celery_redelivery_acceptance.py" in testbed
|
||||||
|
assert "same Celery task identity must be redelivered" in testbed
|
||||||
|
assert "production daemon supervision" in testbed
|
||||||
|
assert "empty broker queue/unacked set" in runbook
|
||||||
|
assert "production process manager" in runbook
|
||||||
|
|
||||||
|
|
||||||
|
def test_compose_lifecycle_targets_only_isolated_redis_service() -> None:
|
||||||
|
up = runner._compose_command(
|
||||||
|
compose_file=COMPOSE_PATH,
|
||||||
|
project_name="govoplan-campaign-redelivery-test",
|
||||||
|
operation="up",
|
||||||
|
)
|
||||||
|
down = runner._compose_command(
|
||||||
|
compose_file=COMPOSE_PATH,
|
||||||
|
project_name="govoplan-campaign-redelivery-test",
|
||||||
|
operation="down",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert up[-3:] == ["up", "--detach", "redis"]
|
||||||
|
assert down[-3:] == ["down", "--volumes", "--remove-orphans"]
|
||||||
|
assert "greenmail" not in up
|
||||||
|
assert "--project-name" in up
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_bootstrap_uses_real_late_ack_solo_celery_worker() -> None:
|
||||||
|
source = runner.WORKER_BOOTSTRAP
|
||||||
|
|
||||||
|
assert "celery.worker_main" in source
|
||||||
|
assert '"--pool=solo"' in source
|
||||||
|
assert '"--queues=send_email"' in source
|
||||||
|
assert '"visibility_timeout"' in source
|
||||||
|
assert '"polling_interval"' in source
|
||||||
|
assert "send_email.run" not in source
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_root_uses_platform_temp_selection() -> None:
|
||||||
|
with mock.patch(
|
||||||
|
"govoplan_campaign_celery_redelivery_acceptance.tempfile.mkdtemp",
|
||||||
|
return_value="/selected-temp/govoplan-campaign-celery-redelivery-test",
|
||||||
|
) as mkdtemp:
|
||||||
|
runtime_root = runner._create_runtime_root()
|
||||||
|
|
||||||
|
assert runtime_root == Path(
|
||||||
|
"/selected-temp/govoplan-campaign-celery-redelivery-test"
|
||||||
|
)
|
||||||
|
mkdtemp.assert_called_once_with(prefix="govoplan-campaign-celery-redelivery-")
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_log_projection_matches_redelivered_task_without_retaining_id() -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||||
|
log_path = Path(temporary_directory) / "worker.log"
|
||||||
|
log_path.write_text(
|
||||||
|
"\n".join(
|
||||||
|
[
|
||||||
|
f"Task govoplan.campaigns.send_email[{TASK_ID}] received",
|
||||||
|
f"Task govoplan.campaigns.send_email[{TASK_ID}] succeeded in 0.1s",
|
||||||
|
]
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
with log_path.open("ab") as handle:
|
||||||
|
worker = runner.WorkerProcess(
|
||||||
|
process=SimpleNamespace(),
|
||||||
|
log_path=log_path,
|
||||||
|
log_handle=handle,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert worker.received_task_ids() == (TASK_ID,)
|
||||||
|
assert worker.succeeded_task_ids() == (TASK_ID,)
|
||||||
|
|
||||||
|
|
||||||
|
def test_queue_projection_fails_closed_if_no_task_was_published() -> None:
|
||||||
|
with pytest.raises(runner.AcceptanceError, match="one Celery task"):
|
||||||
|
runner._queue_evidence(
|
||||||
|
{
|
||||||
|
"queued_count": 1,
|
||||||
|
"skipped_count": 0,
|
||||||
|
"blocked_count": 0,
|
||||||
|
"enqueued_count": 0,
|
||||||
|
"delivery_mode": "database_queue",
|
||||||
|
"worker_queue_available": False,
|
||||||
|
"dry_run": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_redelivery_orchestration_requires_same_task_and_no_second_smtp_effect(
|
||||||
|
monkeypatch,
|
||||||
|
) -> None:
|
||||||
|
first_worker = mock.Mock()
|
||||||
|
first_worker.received_task_ids.return_value = (TASK_ID,)
|
||||||
|
replacement_worker = mock.Mock()
|
||||||
|
replacement_worker.received_task_ids.return_value = (TASK_ID,)
|
||||||
|
workers = iter([first_worker, replacement_worker])
|
||||||
|
endpoint = _Endpoint()
|
||||||
|
durable_states = iter(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"job_count": 1,
|
||||||
|
"send_status_counts": {"sending": 1},
|
||||||
|
"attempt_status_counts": {"smtp_in_progress": 1},
|
||||||
|
"unfinished_attempt_count": 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"job_count": 1,
|
||||||
|
"send_status_counts": {"outcome_unknown": 1},
|
||||||
|
"attempt_status_counts": {"outcome_unknown": 1},
|
||||||
|
"unfinished_attempt_count": 0,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
prepared = SimpleNamespace(
|
||||||
|
campaign_id="campaign-internal",
|
||||||
|
version_id="version-internal",
|
||||||
|
public_evidence=lambda: {
|
||||||
|
"validation": {"ok": True},
|
||||||
|
"build": {"built_count": 1},
|
||||||
|
"campaign_mail_boundary": {
|
||||||
|
"profile_reference_only": True,
|
||||||
|
"smtp_revision_frozen": True,
|
||||||
|
"imap_revision_frozen": True,
|
||||||
|
"resolved_transport_material_present": False,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(runner, "create_mail_profile", lambda *args, **kwargs: "profile-internal")
|
||||||
|
monkeypatch.setattr(runner, "prepare_campaign_scenario", lambda *args, **kwargs: prepared)
|
||||||
|
monkeypatch.setattr(runner, "_start_worker", lambda *args, **kwargs: next(workers))
|
||||||
|
monkeypatch.setattr(runner, "_wait_for_worker_ready", lambda *args, **kwargs: None)
|
||||||
|
received = iter([TASK_ID, TASK_ID])
|
||||||
|
monkeypatch.setattr(runner, "_wait_for_received_task", lambda *args, **kwargs: next(received))
|
||||||
|
monkeypatch.setattr(runner, "_wait_for_task_success", lambda *args, **kwargs: None)
|
||||||
|
monkeypatch.setattr(runner, "_kill_worker", lambda *args, **kwargs: -9)
|
||||||
|
monkeypatch.setattr(runner, "_stop_worker", lambda *args, **kwargs: None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
runner,
|
||||||
|
"_wait_for_broker_drained",
|
||||||
|
lambda *args, **kwargs: runner.RedisBrokerState(0, 0, 0),
|
||||||
|
)
|
||||||
|
|
||||||
|
evidence = runner.execute_redelivery_scenario(
|
||||||
|
_Client(),
|
||||||
|
{"Authorization": "not-retained"},
|
||||||
|
fixture_path=FIXTURE_PATH,
|
||||||
|
settings=_settings(),
|
||||||
|
endpoint=endpoint,
|
||||||
|
redis_url="redis://127.0.0.1:36379/0",
|
||||||
|
runtime_root=Path("/not-used"),
|
||||||
|
snapshot_probe=lambda _version_id: ({}, {}),
|
||||||
|
audit_probe=lambda _campaign_id, _version_id: {
|
||||||
|
"campaign.created": 1,
|
||||||
|
"campaign.validated": 1,
|
||||||
|
"campaign.messages_built": 1,
|
||||||
|
"campaign.queued": 1,
|
||||||
|
},
|
||||||
|
delivery_probe=lambda _campaign_id, _version_id: next(durable_states),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert evidence["broker"] == {
|
||||||
|
"transport": "redis",
|
||||||
|
"same_task_identity_redelivered": True,
|
||||||
|
"first_worker_received_count": 1,
|
||||||
|
"replacement_worker_received_count": 1,
|
||||||
|
"queue_depth": 0,
|
||||||
|
"unacked_hash_count": 0,
|
||||||
|
"unacked_index_count": 0,
|
||||||
|
}
|
||||||
|
assert evidence["protocol"]["connection_count"] == 1
|
||||||
|
assert evidence["protocol"]["data_transactions"] == 1
|
||||||
|
assert evidence["recovered_durable_state"]["send_status_counts"] == {
|
||||||
|
"outcome_unknown": 1
|
||||||
|
}
|
||||||
|
assert TASK_ID not in json.dumps(evidence, sort_keys=True)
|
||||||
|
assert endpoint.release_count >= 1
|
||||||
@@ -195,7 +195,10 @@ def test_post_queue_growth_is_rejected_before_batch_or_provider_preflight() -> N
|
|||||||
patch("govoplan_campaign.backend.sending.jobs._ensure_campaign_execution_snapshot"),
|
patch("govoplan_campaign.backend.sending.jobs._ensure_campaign_execution_snapshot"),
|
||||||
patch("govoplan_campaign.backend.sending.jobs.effective_synchronous_send_policy", return_value=policy),
|
patch("govoplan_campaign.backend.sending.jobs.effective_synchronous_send_policy", return_value=policy),
|
||||||
patch("govoplan_campaign.backend.sending.jobs._campaign_jobs_for_queue", return_value=initial_jobs),
|
patch("govoplan_campaign.backend.sending.jobs._campaign_jobs_for_queue", return_value=initial_jobs),
|
||||||
patch("govoplan_campaign.backend.sending.jobs.queue_campaign_jobs", return_value=queued),
|
patch(
|
||||||
|
"govoplan_campaign.backend.sending.jobs.queue_campaign_jobs",
|
||||||
|
return_value=queued,
|
||||||
|
) as queue,
|
||||||
patch("govoplan_campaign.backend.sending.jobs._campaign_jobs_for_version", return_value=post_queue_jobs),
|
patch("govoplan_campaign.backend.sending.jobs._campaign_jobs_for_version", return_value=post_queue_jobs),
|
||||||
patch("govoplan_campaign.backend.sending.jobs._preflight_synchronous_send_batch") as batch_preflight,
|
patch("govoplan_campaign.backend.sending.jobs._preflight_synchronous_send_batch") as batch_preflight,
|
||||||
):
|
):
|
||||||
@@ -207,6 +210,7 @@ def test_post_queue_growth_is_rejected_before_batch_or_provider_preflight() -> N
|
|||||||
)
|
)
|
||||||
|
|
||||||
assert rejected.value.eligible_count == 3
|
assert rejected.value.eligible_count == 3
|
||||||
|
assert queue.call_args.kwargs["commit_queue"] is False
|
||||||
batch_preflight.assert_not_called()
|
batch_preflight.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
@@ -251,6 +255,7 @@ def test_asynchronous_mode_matches_actual_worker_availability(
|
|||||||
assert result.worker_queue_available is workers_available
|
assert result.worker_queue_available is workers_available
|
||||||
assert result.enqueued_count == expected_enqueued
|
assert result.enqueued_count == expected_enqueued
|
||||||
assert persist.call_args.kwargs["delivery_mode"] == expected_mode
|
assert persist.call_args.kwargs["delivery_mode"] == expected_mode
|
||||||
|
assert persist.call_args.kwargs["commit"] is True
|
||||||
assert enqueue.call_args.kwargs["enabled"] is workers_available
|
assert enqueue.call_args.kwargs["enabled"] is workers_available
|
||||||
|
|
||||||
|
|
||||||
@@ -356,3 +361,45 @@ def test_batch_preflight_checks_every_message_before_provider_effects() -> None:
|
|||||||
assert state_preflight.call_count == 2
|
assert state_preflight.call_count == 2
|
||||||
assert input_preflight.call_count == 2
|
assert input_preflight.call_count == 2
|
||||||
provider.send_campaign_email_bytes.assert_not_called()
|
provider.send_campaign_email_bytes.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_rejected_synchronous_preflight_rolls_back_staged_queue_before_audit() -> None:
|
||||||
|
session = Mock()
|
||||||
|
campaign = SimpleNamespace(id="campaign-1", current_version_id="version-1")
|
||||||
|
version = SimpleNamespace(
|
||||||
|
id="version-1",
|
||||||
|
raw_json={},
|
||||||
|
locked_at=object(),
|
||||||
|
validation_summary={"ok": True},
|
||||||
|
build_summary={"built_count": 1},
|
||||||
|
)
|
||||||
|
rejection = SynchronousSendRejected(
|
||||||
|
"Preflight rejected the staged send.",
|
||||||
|
reason="batch_preflight_failed",
|
||||||
|
eligible_count=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
with (
|
||||||
|
patch.object(router, "_get_campaign_for_principal"),
|
||||||
|
patch.object(router, "_require_permission"),
|
||||||
|
patch.object(router, "_get_campaign_for_tenant", return_value=campaign),
|
||||||
|
patch.object(router, "_get_version_for_tenant", return_value=version),
|
||||||
|
patch.object(router, "_require_mail_profile_use_if_needed"),
|
||||||
|
patch.object(router, "is_user_locked_version", return_value=False),
|
||||||
|
patch.object(router, "send_campaign_now", side_effect=rejection),
|
||||||
|
patch.object(router, "audit_from_principal") as audit,
|
||||||
|
pytest.raises(HTTPException) as rejected,
|
||||||
|
):
|
||||||
|
router.send_campaign_now_endpoint(
|
||||||
|
"campaign-1",
|
||||||
|
session=session,
|
||||||
|
principal=_Principal(
|
||||||
|
"campaigns:campaign:send", "campaigns:recipient:read"
|
||||||
|
), # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert rejected.value.status_code == 422
|
||||||
|
session.rollback.assert_called_once_with()
|
||||||
|
audit.assert_called_once()
|
||||||
|
assert audit.call_args.kwargs["action"] == "campaign.send_now_rejected"
|
||||||
|
assert audit.call_args.kwargs["commit"] is True
|
||||||
|
|||||||
Reference in New Issue
Block a user