GovOPlaN/govoplan-campaign
1
0
Fork 0
Code Issues 46 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(campaign): query reports before pagination

Browse Source
This commit is contained in:
Albrecht Degering
2026-07-22 08:31:43 +02:00
parent 60efd1cb5d
commit aa4ec66b7b
9 changed files with 603 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -333,6 +333,7 @@ webui/.policy-test-build/
webui/.template-preview-test-build/
webui/.import-test-build/
webui/.review-preview-test-build/
webui/.report-grid-test-build/
# GovOPlaN shared ignore rules from govoplan-core
# Local WebUI test/build scratch directories

222
src/govoplan_campaign/backend/router.py
View File

@@ -2,9 +2,10 @@ from __future__ import annotations
import copy
import dataclasses
import json
import logging
from collections.abc import Callable
from typing import Any
from typing import Any, Literal
from fastapi import APIRouter, Depends, HTTPException, Query, Response, status
from sqlalchemy import and_, exists, func, or_
@@ -99,6 +100,7 @@ from govoplan_campaign.backend.db.models import (
JobImapStatus,
JobQueueStatus,
JobSendStatus,
JobValidationStatus,
RecipientImportMappingProfile,
SendAttempt,
)
@@ -2252,6 +2254,125 @@ def _status_counts(session: Session, filters: list[object]) -> dict[str, dict[st
return result
CAMPAIGN_JOB_GRID_SORT_COLUMNS = {
"number": CampaignJob.entry_index,
"recipient": func.lower(func.coalesce(CampaignJob.recipient_email, "")),
"subject": func.lower(func.coalesce(CampaignJob.subject, "")),
"validation": CampaignJob.validation_status,
"queue": CampaignJob.queue_status,
"send": CampaignJob.send_status,
"imap": CampaignJob.imap_status,
"attempts": CampaignJob.attempt_count,
"updated": CampaignJob.updated_at,
}
CAMPAIGN_JOB_GRID_LIST_FILTERS = {
"validation": (CampaignJob.validation_status, {item.value for item in JobValidationStatus}),
"queue": (CampaignJob.queue_status, {item.value for item in JobQueueStatus}),
"send": (CampaignJob.send_status, {item.value for item in JobSendStatus}),
"imap": (CampaignJob.imap_status, {item.value for item in JobImapStatus}),
}
def _campaign_jobs_grid_filter_expressions(grid_filters: dict[str, str] | None) -> list[object]:
values = grid_filters or {}
expressions: list[object] = []
recipient = values.get("recipient", "").strip()
if recipient:
pattern = _contains_pattern(recipient)
expressions.append(or_(
CampaignJob.recipient_email.ilike(pattern, escape="\\"),
CampaignJob.entry_id.ilike(pattern, escape="\\"),
))
subject = values.get("subject", "").strip()
if subject:
expressions.append(CampaignJob.subject.ilike(_contains_pattern(subject), escape="\\"))
evidence = values.get("evidence", "").strip()
if evidence:
pattern = _contains_pattern(evidence)
expressions.append(or_(
CampaignJob.message_id_header.ilike(pattern, escape="\\"),
CampaignJob.eml_sha256.ilike(pattern, escape="\\"),
))
attempts = values.get("attempts", "").strip()
if attempts:
expressions.append(_campaign_jobs_integer_filter(CampaignJob.attempt_count, attempts, column_id="attempts"))
for column_id, (column, allowed_values) in CAMPAIGN_JOB_GRID_LIST_FILTERS.items():
raw_value = values.get(column_id, "").strip()
if not raw_value:
continue
selected = _campaign_jobs_list_filter(raw_value, column_id=column_id, allowed_values=allowed_values)
expressions.append(column.in_(selected))
return expressions
def _campaign_jobs_list_filter(raw_value: str, *, column_id: str, allowed_values: set[str]) -> list[str]:
if raw_value.startswith("list:"):
try:
parsed = json.loads(raw_value[5:])
except json.JSONDecodeError as exc:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=f"Invalid {column_id} list filter",
) from exc
if not isinstance(parsed, list) or any(not isinstance(value, str) for value in parsed):
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=f"Invalid {column_id} list filter",
)
selected = list(dict.fromkeys(value.strip() for value in parsed if value.strip()))
else:
selected = list(dict.fromkeys(value.strip() for value in raw_value.split(",") if value.strip()))
if len(selected) > 50 or any(value not in allowed_values for value in selected):
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=f"Invalid {column_id} list filter",
)
return selected
def _campaign_jobs_integer_filter(column: object, raw_value: str, *, column_id: str):
operator, separator, value = raw_value.partition(":")
if not separator:
operator, value = "eq", operator
if operator not in {"eq", "gt", "gte", "lt", "lte"}:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=f"Invalid {column_id} filter operator",
)
try:
expected = int(value)
except ValueError as exc:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail=f"Invalid {column_id} filter value",
) from exc
if operator == "gt":
return column > expected
if operator == "gte":
return column >= expected
if operator == "lt":
return column < expected
if operator == "lte":
return column <= expected
return column == expected
def _contains_pattern(value: str) -> str:
escaped = value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
return f"%{escaped}%"
def _campaign_jobs_ordering(sort_by: str, sort_direction: str) -> list[object]:
column = CAMPAIGN_JOB_GRID_SORT_COLUMNS.get(sort_by)
if column is None:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_CONTENT,
detail="Unsupported Campaign job sort column",
)
primary = column.desc() if sort_direction == "desc" else column.asc()
return [primary, CampaignJob.id.asc()]
def _campaign_jobs_query_context(
session: Session,
principal: ApiPrincipal,
@@ -2262,6 +2383,7 @@ def _campaign_jobs_query_context(
validation_status: list[str] | None,
imap_status: list[str] | None,
query_text: str | None,
grid_filters: dict[str, str] | None = None,
) -> tuple[Campaign, list[object], list[object], dict[str, object], set[str]]:
_get_campaign_for_principal(session, campaign_id, principal)
_require_permission(principal, "campaigns:recipient:read")
@@ -2290,6 +2412,7 @@ def _campaign_jobs_query_context(
CampaignJob.subject.ilike(pattern),
CampaignJob.entry_id.ilike(pattern),
))
filtered.extend(_campaign_jobs_grid_filter_expressions(grid_filters))
return campaign, base_filters, filtered, review_metadata, reviewed_keys
@@ -2308,6 +2431,9 @@ def _campaign_jobs_page_response(
validation_status: list[str] | None = None,
imap_status: list[str] | None = None,
query_text: str | None = None,
grid_filters: dict[str, str] | None = None,
sort_by: str = "number",
sort_direction: str = "asc",
cursor: str | None = None,
changed_job_ids: set[str] | None = None,
) -> CampaignJobsResponse:
@@ -2322,11 +2448,20 @@ def _campaign_jobs_page_response(
validation_status=validation_status,
imap_status=imap_status,
query_text=query_text,
grid_filters=grid_filters,
sort_by=sort_by,
sort_direction=sort_direction,
)
ordered_query = session.query(CampaignJob).filter(*filtered).order_by(CampaignJob.entry_index.asc(), CampaignJob.id.asc())
ordering = _campaign_jobs_ordering(sort_by, sort_direction)
ordered_query = session.query(CampaignJob).filter(*filtered).order_by(*ordering)
start_cursor: str | None = None
effective_offset = 0
if cursor:
if sort_by != "number" or sort_direction != "asc":
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="Campaign job cursors require number ascending order",
)
try:
cursor_values = decode_keyset_cursor(CAMPAIGN_JOBS_CURSOR_SCOPE, cursor, fingerprint=fingerprint)
if cursor_values is None:
@@ -2338,14 +2473,14 @@ def _campaign_jobs_page_response(
else:
effective_offset = (page - 1) * page_size
page_query = session.query(CampaignJob).filter(*filtered)
if effective_offset > 0:
if effective_offset > 0 and sort_by == "number" and sort_direction == "asc":
previous_row = ordered_query.offset(effective_offset - 1).limit(1).first()
if previous_row is not None:
start_cursor = _campaign_jobs_cursor_for_row(previous_row, fingerprint=fingerprint)
rows_plus_one = (
page_query
.order_by(CampaignJob.entry_index.asc(), CampaignJob.id.asc())
.order_by(*ordering)
.offset(effective_offset)
.limit(page_size + 1)
.all()
@@ -2353,7 +2488,12 @@ def _campaign_jobs_page_response(
jobs = rows_plus_one[:page_size]
next_cursor = (
_campaign_jobs_cursor_for_row(jobs[-1], fingerprint=fingerprint)
if changed_job_ids is None and len(rows_plus_one) > page_size and jobs else None
if changed_job_ids is None
and sort_by == "number"
and sort_direction == "asc"
and len(rows_plus_one) > page_size
and jobs
else None
)
if changed_job_ids is not None:
jobs = [job for job in jobs if job.id in changed_job_ids]
@@ -2381,6 +2521,9 @@ def _campaign_jobs_cursor_fingerprint(
validation_status: list[str] | None,
imap_status: list[str] | None,
query_text: str | None,
grid_filters: dict[str, str] | None,
sort_by: str,
sort_direction: str,
) -> str:
return keyset_query_fingerprint(
CAMPAIGN_JOBS_CURSOR_SCOPE,
@@ -2392,7 +2535,8 @@ def _campaign_jobs_cursor_fingerprint(
"validation_status": sorted(validation_status or []),
"imap_status": sorted(imap_status or []),
"query": (query_text or "").strip(),
"order": ["entry_index:asc", "id:asc"],
"grid_filters": sorted((grid_filters or {}).items()),
"order": [f"{sort_by}:{sort_direction}", "id:asc"],
},
)
@@ -2439,6 +2583,16 @@ class CampaignJobsQuery:
validation_status: list[str] | None = Query(default=None),
imap_status: list[str] | None = Query(default=None),
query_text: str | None = Query(default=None, alias="q", max_length=200),
sort_by: Literal["number", "recipient", "subject", "validation", "queue", "send", "imap", "attempts", "updated"] = Query(default="number"),
sort_direction: Literal["asc", "desc"] = Query(default="asc"),
filter_recipient: str | None = Query(default=None, max_length=500),
filter_subject: str | None = Query(default=None, max_length=1000),
filter_validation: str | None = Query(default=None, max_length=1000),
filter_queue: str | None = Query(default=None, max_length=1000),
filter_send: str | None = Query(default=None, max_length=1000),
filter_imap: str | None = Query(default=None, max_length=1000),
filter_attempts: str | None = Query(default=None, max_length=100),
filter_evidence: str | None = Query(default=None, max_length=500),
) -> None:
self.version_id = version_id
self.page = page
@@ -2448,6 +2602,22 @@ class CampaignJobsQuery:
self.validation_status = validation_status
self.imap_status = imap_status
self.query_text = query_text
self.sort_by = sort_by
self.sort_direction = sort_direction
self.grid_filters = {
column_id: value
for column_id, value in {
"recipient": filter_recipient,
"subject": filter_subject,
"validation": filter_validation,
"queue": filter_queue,
"send": filter_send,
"imap": filter_imap,
"attempts": filter_attempts,
"evidence": filter_evidence,
}.items()
if value is not None and value.strip()
}
@router.get("/{campaign_id}/jobs", response_model=CampaignJobsResponse)
@@ -2472,6 +2642,7 @@ def list_jobs(
validation_status=filters.validation_status,
imap_status=filters.imap_status,
query_text=filters.query_text,
grid_filters=filters.grid_filters,
)
return _campaign_jobs_page_response(
session,
@@ -2487,6 +2658,9 @@ def list_jobs(
validation_status=filters.validation_status,
imap_status=filters.imap_status,
query_text=filters.query_text,
grid_filters=filters.grid_filters,
sort_by=filters.sort_by,
sort_direction=filters.sort_direction,
cursor=filters.cursor,
)
@@ -2503,6 +2677,9 @@ def _campaign_jobs_full_delta_response(
validation_status: list[str] | None,
imap_status: list[str] | None,
query_text: str | None,
grid_filters: dict[str, str] | None,
sort_by: str,
sort_direction: str,
cursor: str | None = None,
) -> CampaignJobsDeltaResponse:
_campaign, base_filters, filtered, review_metadata, reviewed_keys = _campaign_jobs_query_context(
@@ -2514,6 +2691,7 @@ def _campaign_jobs_full_delta_response(
validation_status=validation_status,
imap_status=imap_status,
query_text=query_text,
grid_filters=grid_filters,
)
payload = _campaign_jobs_page_response(
session,
@@ -2529,6 +2707,9 @@ def _campaign_jobs_full_delta_response(
validation_status=validation_status,
imap_status=imap_status,
query_text=query_text,
grid_filters=grid_filters,
sort_by=sort_by,
sort_direction=sort_direction,
cursor=cursor,
)
return CampaignJobsDeltaResponse(
@@ -2546,8 +2727,19 @@ def _job_filter_membership_can_shift(
validation_status: list[str] | None,
imap_status: list[str] | None,
query_text: str | None,
grid_filters: dict[str, str] | None,
sort_by: str,
sort_direction: str,
) -> bool:
return bool(send_status or validation_status or imap_status or (query_text and query_text.strip()))
return bool(
send_status
or validation_status
or imap_status
or (query_text and query_text.strip())
or grid_filters
or sort_by != "number"
or sort_direction != "asc"
)
@router.get("/{campaign_id}/jobs/delta", response_model=CampaignJobsDeltaResponse)
@@ -2571,6 +2763,9 @@ def list_jobs_delta(
validation_status=filters.validation_status,
imap_status=filters.imap_status,
query_text=filters.query_text,
grid_filters=filters.grid_filters,
sort_by=filters.sort_by,
sort_direction=filters.sort_direction,
cursor=filters.cursor,
)
@@ -2583,6 +2778,7 @@ def list_jobs_delta(
validation_status=filters.validation_status,
imap_status=filters.imap_status,
query_text=filters.query_text,
grid_filters=filters.grid_filters,
)
try:
since_sequence = decode_sequence_watermark(since)
@@ -2606,6 +2802,9 @@ def list_jobs_delta(
validation_status=filters.validation_status,
imap_status=filters.imap_status,
query_text=filters.query_text,
grid_filters=filters.grid_filters,
sort_by=filters.sort_by,
sort_direction=filters.sort_direction,
cursor=filters.cursor,
)
@@ -2632,6 +2831,9 @@ def list_jobs_delta(
validation_status=filters.validation_status,
imap_status=filters.imap_status,
query_text=filters.query_text,
grid_filters=filters.grid_filters,
sort_by=filters.sort_by,
sort_direction=filters.sort_direction,
)
or any(entry.operation in {"created", "deleted"} for entry in relevant_entries)
):
@@ -2646,6 +2848,9 @@ def list_jobs_delta(
validation_status=filters.validation_status,
imap_status=filters.imap_status,
query_text=filters.query_text,
grid_filters=filters.grid_filters,
sort_by=filters.sort_by,
sort_direction=filters.sort_direction,
cursor=filters.cursor,
)
@@ -2668,6 +2873,9 @@ def list_jobs_delta(
validation_status=filters.validation_status,
imap_status=filters.imap_status,
query_text=filters.query_text,
grid_filters=filters.grid_filters,
sort_by=filters.sort_by,
sort_direction=filters.sort_direction,
cursor=filters.cursor,
changed_job_ids=changed_job_ids,
)

123
tests/test_job_list_query.py Normal file
View File

@@ -0,0 +1,123 @@
from __future__ import annotations
import unittest
from fastapi import HTTPException
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from govoplan_access.backend.db.models import Account, Group, User
from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignVersion
from govoplan_campaign.backend.router import (
_campaign_jobs_grid_filter_expressions,
_campaign_jobs_ordering,
_campaign_jobs_page_response,
)
from govoplan_core.core.change_sequence import ChangeSequenceEntry
from govoplan_core.db.base import Base
class CampaignJobListQueryTests(unittest.TestCase):
def setUp(self) -> None:
self.engine = create_engine("sqlite+pysqlite:///:memory:")
Base.metadata.create_all(
self.engine,
tables=[
Account.__table__,
User.__table__,
Group.__table__,
Campaign.__table__,
CampaignVersion.__table__,
CampaignJob.__table__,
ChangeSequenceEntry.__table__,
],
)
self.session = Session(self.engine)
rows = [
(0, "ordinary-0@example.test", "General notice", "ready", "draft", "not_queued", "not_requested", 0),
(1, "ordinary-1@example.test", "General notice", "ready", "draft", "not_queued", "not_requested", 1),
(2, "target-c@example.test", "Target notice C", "warning", "queued", "queued", "pending", 2),
(3, "ordinary-3@example.test", "General notice", "blocked", "draft", "failed_permanent", "failed", 3),
(4, "target-b@example.test", "Target notice B", "ready", "draft", "failed_temporary", "not_requested", 4),
(5, "target-a@example.test", "Target notice A", "ready", "draft", "outcome_unknown", "outcome_unknown", 5),
]
for entry_index, recipient, subject, validation, queue, send, imap, attempts in rows:
self.session.add(CampaignJob(
id=f"job-{entry_index}",
tenant_id="tenant-1",
campaign_id="campaign-1",
campaign_version_id="version-1",
entry_index=entry_index,
entry_id=f"entry-{entry_index}",
recipient_email=recipient,
subject=subject,
message_id_header=f"<message-{entry_index}@example.test>",
eml_sha256=f"sha-{entry_index}",
build_status="built",
validation_status=validation,
queue_status=queue,
send_status=send,
imap_status=imap,
attempt_count=attempts,
resolved_attachments=[],
issues_snapshot=[],
))
self.session.commit()
def tearDown(self) -> None:
self.session.close()
self.engine.dispose()
def test_grid_filters_apply_before_pagination_and_report_filtered_totals(self) -> None:
base_filters = [CampaignJob.tenant_id == "tenant-1", CampaignJob.campaign_id == "campaign-1"]
grid_filters = {"recipient": "target"}
filtered = [*base_filters, *_campaign_jobs_grid_filter_expressions(grid_filters)]
page = _campaign_jobs_page_response(
self.session,
campaign_id="campaign-1",
version_id="version-1",
base_filters=base_filters,
filtered=filtered,
reviewed_keys=set(),
review_metadata={},
page=1,
page_size=2,
grid_filters=grid_filters,
sort_by="recipient",
sort_direction="asc",
)
self.assertEqual(page.total, 3)
self.assertEqual(page.total_unfiltered, 6)
self.assertEqual(page.pages, 2)
self.assertEqual(
[row["recipient_email"] for row in page.jobs],
["target-a@example.test", "target-b@example.test"],
)
self.assertIsNone(page.next_cursor)
def test_list_and_integer_filters_share_the_full_backend_query(self) -> None:
expressions = _campaign_jobs_grid_filter_expressions({
"send": 'list:["failed_temporary","outcome_unknown"]',
"attempts": "gte:4",
})
rows = (
self.session.query(CampaignJob)
.filter(*expressions)
.order_by(*_campaign_jobs_ordering("attempts", "desc"))
.all()
)
self.assertEqual([row.id for row in rows], ["job-5", "job-4"])
def test_invalid_list_filters_fail_closed(self) -> None:
with self.assertRaises(HTTPException) as raised:
_campaign_jobs_grid_filter_expressions({"send": 'list:["not-a-status"]'})
self.assertEqual(raised.exception.status_code, 422)
if __name__ == "__main__":
unittest.main()

3
webui/package.json
View File

@@ -17,7 +17,7 @@
"read-excel-file": "9.2.0"
},
"peerDependencies": {
"@govoplan/core-webui": "^0.1.9",
"@govoplan/core-webui": "^0.1.11",
"lucide-react": "^1.23.0",
"react": "^19.0.0",
"react-dom": "^19.0.0",
@@ -27,6 +27,7 @@
"test:policy-ui": "rm -rf .policy-test-build && mkdir -p .policy-test-build && printf '{\"type\":\"commonjs\"}\\n' > .policy-test-build/package.json && tsc -p tsconfig.policy-tests.json && node .policy-test-build/tests/policy-ui.test.js",
"test:template-preview": "rm -rf .template-preview-test-build && mkdir -p .template-preview-test-build && printf '{\"type\":\"commonjs\"}\\n' > .template-preview-test-build/package.json && tsc -p tsconfig.template-preview-tests.json && node .template-preview-test-build/tests/template-preview-draft.test.js",
"test:import-utils": "rm -rf .import-test-build && mkdir -p .import-test-build && printf '{\"type\":\"commonjs\"}\\n' > .import-test-build/package.json && tsc -p tsconfig.import-tests.json && node .import-test-build/tests/import-utils.test.js",
"test:report-grid": "rm -rf .report-grid-test-build && mkdir -p .report-grid-test-build && printf '{\"type\":\"commonjs\"}\\n' > .report-grid-test-build/package.json && tsc -p tsconfig.report-grid-tests.json && node .report-grid-test-build/tests/report-grid-query.test.js",
"test:review-preview-ui": "rm -rf .review-preview-test-build && mkdir -p .review-preview-test-build && printf '{\"type\":\"commonjs\"}\\n' > .review-preview-test-build/package.json && tsc -p tsconfig.review-preview-tests.json && node .review-preview-test-build/tests/review-preview-ui.test.js && node tests/delivery-mode-ui-structure.test.mjs"
},
"devDependencies": {

34
webui/src/api/campaigns.ts
View File

@@ -1,5 +1,6 @@
import type { ApiSettings, CampaignListItem, DeltaDeletedItem } from "../types";
import { apiDownload, apiFetch } from "./client";
import { campaignJobsQueryParams, type CampaignJobsQueryParameters } from "../features/campaigns/utils/jobListQuery";
export { fetchResourceAccessExplanation } from "@govoplan/core-webui";
export type {
AccessDecisionProvenanceItem,
@@ -404,16 +405,7 @@ export type CampaignSendJobPayload = {
};
export type CampaignJobsQuery = {
versionId?: string;
page?: number;
pageSize?: number;
cursor?: string | null;
sendStatus?: string[];
validationStatus?: string[];
imapStatus?: string[];
query?: string;
};
export type CampaignJobsQuery = Omit<CampaignJobsQueryParameters, "since" | "limit">;
export type CampaignJobsResponse = {
jobs: Record<string, unknown>[];
@@ -801,15 +793,7 @@ settings: ApiSettings,
campaignId: string,
options: CampaignJobsQuery = {})
: Promise<CampaignJobsResponse> {
const params = new URLSearchParams();
if (options.versionId) params.set("version_id", options.versionId);
if (options.page) params.set("page", String(options.page));
if (options.pageSize) params.set("page_size", String(options.pageSize));
if (options.cursor) params.set("cursor", options.cursor);
for (const value of options.sendStatus ?? []) params.append("send_status", value);
for (const value of options.validationStatus ?? []) params.append("validation_status", value);
for (const value of options.imapStatus ?? []) params.append("imap_status", value);
if (options.query?.trim()) params.set("q", options.query.trim());
const params = campaignJobsQueryParams(options);
const suffix = params.size > 0 ? `?${params.toString()}` : "";
return apiFetch<CampaignJobsResponse>(settings, `/api/v1/campaigns/${campaignId}/jobs${suffix}`);
}
@@ -819,17 +803,7 @@ settings: ApiSettings,
campaignId: string,
options: CampaignJobsQuery & {since?: string | null;limit?: number;} = {})
: Promise<CampaignJobsDeltaResponse> {
const params = new URLSearchParams();
if (options.versionId) params.set("version_id", options.versionId);
if (options.page) params.set("page", String(options.page));
if (options.pageSize) params.set("page_size", String(options.pageSize));
if (options.cursor) params.set("cursor", options.cursor);
for (const value of options.sendStatus ?? []) params.append("send_status", value);
for (const value of options.validationStatus ?? []) params.append("validation_status", value);
for (const value of options.imapStatus ?? []) params.append("imap_status", value);
if (options.query?.trim()) params.set("q", options.query.trim());
if (options.since) params.set("since", options.since);
if (options.limit) params.set("limit", String(options.limit));
const params = campaignJobsQueryParams(options);
const suffix = params.size > 0 ? `?${params.toString()}` : "";
return apiFetch<CampaignJobsDeltaResponse>(settings, `/api/v1/campaigns/${campaignId}/jobs/delta${suffix}`);
}

230
webui/src/features/campaigns/CampaignReportPage.tsx
View File

@@ -5,7 +5,7 @@ import {
downloadCampaignJobsCsv,
emailCampaignReport,
getCampaignJobDetail,
getCampaignJobsDelta,
getCampaignJobs,
resolveCampaignJobOutcome,
retryCampaignJobs,
sendCampaignJob,
@@ -16,16 +16,18 @@ import {
import { Card } from "@govoplan/core-webui";
import { Button } from "@govoplan/core-webui";
import { ConfirmDialog } from "@govoplan/core-webui";
import { DataGrid, type DataGridColumn, type DataGridListOption } from "@govoplan/core-webui";
import { DataGrid, type DataGridColumn, type DataGridListOption, type DataGridQueryState } from "@govoplan/core-webui";
import { Dialog } from "@govoplan/core-webui";
import { DismissibleAlert } from "@govoplan/core-webui";
import { FormField } from "@govoplan/core-webui";
import { PageTitle } from "@govoplan/core-webui";
import { StatusBadge } from "@govoplan/core-webui";
import VersionLine from "./components/VersionLine";
import { LoadingFrame, TableActionGroup, ToggleSwitch, i18nMessage, useDeltaWatermarks } from "@govoplan/core-webui";
import { LoadingFrame, TableActionGroup, ToggleSwitch, i18nMessage } from "@govoplan/core-webui";
import { useCampaignWorkspaceData } from "./hooks/useCampaignWorkspaceData";
import { asRecord, formatDateTime, humanize } from "./utils/campaignView";
import { emptyCampaignJobsResponse, mergeCampaignJobsDelta } from "./utils/jobDeltas";
import { emptyCampaignJobsResponse } from "./utils/jobDeltas";
import type { CampaignJobSortColumn } from "./utils/jobListQuery";
const SEND_STATUS_OPTIONS: DataGridListOption[] = [
"not_queued",
@@ -43,16 +45,37 @@ map((value) => ({ value, label: humanize(value) }));
const IMAP_STATUS_OPTIONS: DataGridListOption[] = [
"not_requested",
"pending",
"appending",
"appended",
"outcome_unknown",
"failed",
"skipped"].
map((value) => ({ value, label: humanize(value) }));
const VALIDATION_STATUS_OPTIONS: DataGridListOption[] = [
"ready",
"warning",
"needs_review",
"blocked",
"excluded",
"inactive"].
map((value) => ({ value, label: humanize(value) }));
const QUEUE_STATUS_OPTIONS: DataGridListOption[] = [
"draft",
"queued",
"sending",
"paused",
"cancelled"].
map((value) => ({ value, label: humanize(value) }));
const DEFAULT_JOB_GRID_SORT = { columnId: "number", direction: "asc" as const };
const JOB_GRID_QUERY_DELAY_MS = 300;
type ReconcileRequest = {jobId: string;decision: "smtp_accepted" | "not_sent";} | null;
export default function CampaignReportPage({ settings, campaignId }: {settings: ApiSettings;campaignId: string;}) {
const { data, loading, error, reload } = useCampaignWorkspaceData(settings, campaignId, { includeSummary: true });
const { getDeltaWatermark, setDeltaWatermark, resetDeltaWatermark } = useDeltaWatermarks();
const version = data.currentVersion;
const cards = data.summary?.cards;
const delivery = asRecord(data.summary?.delivery);
@@ -60,14 +83,19 @@ export default function CampaignReportPage({ settings, campaignId }: {settings:
const imapPolicy = asRecord(delivery.imap_append_sent);
const [jobs, setJobs] = useState<CampaignJobsResponse>(() => emptyCampaignJobsResponse());
const jobsRef = useRef<CampaignJobsResponse>(emptyCampaignJobsResponse());
const jobPageCursorsRef = useRef<Record<number, string | null>>({ 1: null });
const jobsRequestRef = useRef(0);
const [jobsLoading, setJobsLoading] = useState(false);
const [page, setPage] = useState(1);
const [sendStatus, setSendStatus] = useState("");
const [imapStatus, setImapStatus] = useState("");
const [query, setQuery] = useState("");
const [appliedQuery, setAppliedQuery] = useState("");
const [pageSize, setPageSize] = useState(50);
const [initialGridFilters] = useState<Record<string, string | string[]>>(() => initialReportGridFilters());
const initialGridQuery = useMemo<DataGridQueryState>(() => ({
sort: DEFAULT_JOB_GRID_SORT,
filters: serializeInitialGridFilters(initialGridFilters)
}), [initialGridFilters]);
const [jobGridQuery, setJobGridQuery] = useState<DataGridQueryState>(initialGridQuery);
const [appliedJobGridQuery, setAppliedJobGridQuery] = useState<DataGridQueryState>(initialGridQuery);
const [query, setQuery] = useState(() => initialReportQuery());
const [appliedQuery, setAppliedQuery] = useState(query.trim());
const [actionMessage, setActionMessage] = useState("");
const [actionError, setActionError] = useState("");
const [busyAction, setBusyAction] = useState("");
@@ -81,86 +109,46 @@ export default function CampaignReportPage({ settings, campaignId }: {settings:
useEffect(() => {
const handle = window.setTimeout(() => {
setAppliedQuery(query.trim());
setAppliedJobGridQuery((current) => dataGridQueriesEqual(current, jobGridQuery) ? current : jobGridQuery);
setPage(1);
}, 350);
}, JOB_GRID_QUERY_DELAY_MS);
return () => window.clearTimeout(handle);
}, [query]);
}, [query, jobGridQuery]);
const jobsQueryKey = useMemo(
() => JSON.stringify({
campaignId,
versionId: version?.id ?? null,
page,
pageSize: 50,
sendStatus,
imapStatus,
appliedQuery,
apiBaseUrl: settings.apiBaseUrl,
apiKey: settings.apiKey,
accessToken: settings.accessToken
}),
[campaignId, version?.id, page, sendStatus, imapStatus, appliedQuery, settings.apiBaseUrl, settings.apiKey, settings.accessToken]
);
useEffect(() => {
jobPageCursorsRef.current = { 1: null };
}, [campaignId, version?.id, sendStatus, imapStatus, appliedQuery, settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
const handleJobGridQuery = useCallback((next: DataGridQueryState) => {
setJobGridQuery((current) => dataGridQueriesEqual(current, next) ? current : next);
}, []);
const loadJobs = useCallback(async () => {
if (!campaignId) return;
const requestId = ++jobsRequestRef.current;
setJobsLoading(true);
setActionError("");
try {
let nextWatermark = getDeltaWatermark(jobsQueryKey);
let merged = jobsRef.current;
let hasMore = false;
const pageCursor = page === 1 ? null : jobPageCursorsRef.current[page];
do {
const response = await getCampaignJobsDelta(settings, campaignId, {
versionId: version?.id,
page,
pageSize: 50,
cursor: pageCursor,
sendStatus: sendStatus ? [sendStatus] : undefined,
imapStatus: imapStatus ? [imapStatus] : undefined,
query: appliedQuery || undefined,
since: nextWatermark
});
merged = mergeCampaignJobsDelta(merged, response);
if (response.cursor !== undefined) jobPageCursorsRef.current[page] = response.cursor ?? null;
if (response.next_cursor !== undefined) {
if (response.next_cursor) jobPageCursorsRef.current[page + 1] = response.next_cursor;
else delete jobPageCursorsRef.current[page + 1];
}
nextWatermark = response.watermark ?? null;
hasMore = response.has_more;
} while (hasMore);
setDeltaWatermark(jobsQueryKey, nextWatermark);
jobsRef.current = merged;
setJobs(merged);
if (merged.pages > 0 && page > merged.pages) setPage(merged.pages);
const response = await getCampaignJobs(settings, campaignId, {
versionId: version?.id,
page,
pageSize,
query: appliedQuery || undefined,
sortBy: campaignJobSortColumn(appliedJobGridQuery.sort?.columnId),
sortDirection: appliedJobGridQuery.sort?.direction ?? "asc",
filters: appliedJobGridQuery.filters
});
if (requestId !== jobsRequestRef.current) return;
setJobs(response);
if (response.pages > 0 && page > response.pages) setPage(response.pages);
} catch (err) {
setActionError(err instanceof Error ? err.message : String(err));
if (requestId === jobsRequestRef.current) setActionError(err instanceof Error ? err.message : String(err));
} finally {
setJobsLoading(false);
if (requestId === jobsRequestRef.current) setJobsLoading(false);
}
}, [settings, campaignId, version?.id, page, sendStatus, imapStatus, appliedQuery, jobsQueryKey, getDeltaWatermark, setDeltaWatermark]);
useEffect(() => {
resetDeltaWatermark(jobsQueryKey);
jobsRef.current = emptyCampaignJobsResponse();
setJobs(emptyCampaignJobsResponse());
}, [jobsQueryKey, resetDeltaWatermark]);
}, [settings, campaignId, version?.id, page, pageSize, appliedQuery, appliedJobGridQuery]);
useEffect(() => {
void loadJobs();
}, [loadJobs]);
async function reloadAll() {
resetDeltaWatermark(jobsQueryKey);
jobPageCursorsRef.current = { 1: null };
jobsRef.current = emptyCampaignJobsResponse();
setJobs(emptyCampaignJobsResponse());
await Promise.all([reload({ force: true }), loadJobs()]);
}
@@ -327,10 +315,10 @@ export default function CampaignReportPage({ settings, campaignId }: {settings:
value: (row) => String(row.recipient_email ?? "—")
},
{ id: "subject", header: "i18n:govoplan-campaign.subject.8d183dbd", width: "minmax(260px, 1fr)", resizable: true, sortable: true, filterable: true, value: (row) => String(row.subject ?? "—") },
{ id: "validation", header: "i18n:govoplan-campaign.validation.dd74d182", width: 145, sortable: true, filterable: true, render: (row) => <StatusBadge status={String(row.validation_status ?? "unknown")} />, value: (row) => String(row.validation_status ?? "unknown") },
{ id: "queue", header: "i18n:govoplan-campaign.queue.d325fcd9", width: 130, sortable: true, filterable: true, render: (row) => <StatusBadge status={String(row.queue_status ?? "unknown")} />, value: (row) => String(row.queue_status ?? "unknown") },
{ id: "validation", header: "i18n:govoplan-campaign.validation.dd74d182", width: 145, sortable: true, filterable: true, columnType: "from-list", list: { options: VALIDATION_STATUS_OPTIONS, display: "pill" }, render: (row) => <StatusBadge status={String(row.validation_status ?? "unknown")} />, value: (row) => String(row.validation_status ?? "unknown") },
{ id: "queue", header: "i18n:govoplan-campaign.queue.d325fcd9", width: 130, sortable: true, filterable: true, columnType: "from-list", list: { options: QUEUE_STATUS_OPTIONS, display: "pill" }, render: (row) => <StatusBadge status={String(row.queue_status ?? "unknown")} />, value: (row) => String(row.queue_status ?? "unknown") },
{ id: "send", header: "i18n:govoplan-campaign.smtp.efff9cca", width: 160, sortable: true, filterable: true, columnType: "from-list", list: { options: SEND_STATUS_OPTIONS, display: "pill" }, render: (row) => <StatusBadge status={String(row.send_status ?? "unknown")} />, value: (row) => String(row.send_status ?? "unknown") },
{ id: "imap", header: "i18n:govoplan-campaign.imap.271f9ef2", width: 130, sortable: true, filterable: true, render: (row) => <StatusBadge status={String(row.imap_status ?? "unknown")} />, value: (row) => String(row.imap_status ?? "unknown") },
{ id: "imap", header: "i18n:govoplan-campaign.imap.271f9ef2", width: 130, sortable: true, filterable: true, columnType: "from-list", list: { options: IMAP_STATUS_OPTIONS, display: "pill" }, render: (row) => <StatusBadge status={String(row.imap_status ?? "unknown")} />, value: (row) => String(row.imap_status ?? "unknown") },
{ id: "attempts", header: "i18n:govoplan-campaign.attempts.5a29585e", width: 105, align: "right", sortable: true, filterType: "integer", value: (row) => Number(row.attempt_count ?? 0) },
{
id: "evidence",
@@ -351,7 +339,6 @@ export default function CampaignReportPage({ settings, campaignId }: {settings:
header: "i18n:govoplan-campaign.last_result.110b888b",
width: "minmax(220px, 1fr)",
resizable: true,
filterable: true,
render: (row) => <span className={row.last_error ? "recipient-outcome-error" : "muted"} title={String(row.last_error ?? "")}>{String(row.last_error ?? "—")}</span>,
value: (row) => String(row.last_error ?? "—")
},
@@ -428,32 +415,37 @@ export default function CampaignReportPage({ settings, campaignId }: {settings:
<Card title="i18n:govoplan-campaign.recipient_delivery_jobs.52492608">
<div className="page-heading split">
<div className="button-row compact-actions">
<input value={query} onChange={(event) => setQuery(event.target.value)} placeholder="i18n:govoplan-campaign.search_recipient_subject_or_entry_id.6d6544f5" />
<select value={sendStatus} onChange={(event) => {setSendStatus(event.target.value);setPage(1);}}>
<option value="">i18n:govoplan-campaign.all_smtp_states.739597b1</option>
{SEND_STATUS_OPTIONS.map((option) => <option key={option.value} value={option.value}>{option.label}</option>)}
</select>
<select value={imapStatus} onChange={(event) => {setImapStatus(event.target.value);setPage(1);}}>
<option value="">i18n:govoplan-campaign.all_imap_states.8546b84c</option>
{IMAP_STATUS_OPTIONS.map((option) => <option key={option.value} value={option.value}>{option.label}</option>)}
</select>
<FormField label="i18n:govoplan-campaign.search_recipient_subject_or_entry_id.6d6544f5">
<input value={query} onChange={(event) => setQuery(event.target.value)} />
</FormField>
</div>
<span className="muted">{jobs.total} i18n:govoplan-campaign.matching_of.66a3778e {jobs.total_unfiltered} i18n:govoplan-campaign.total_job_s.c94b7d20</span>
</div>
<LoadingFrame loading={jobsLoading} label="i18n:govoplan-campaign.loading_delivery_jobs.20ecc37e">
<DataGrid<Record<string, unknown>>
id={`campaign-report-jobs-${campaignId}`}
id={`campaign-report-jobs-v2-${campaignId}`}
rows={jobs.jobs}
columns={columns}
getRowKey={(row: Record<string, unknown>) => String(row.id ?? "")}
emptyText="i18n:govoplan-campaign.no_jobs_match_the_current_filters.b1501ff5" />
emptyText="i18n:govoplan-campaign.no_jobs_match_the_current_filters.b1501ff5"
initialFilters={initialGridFilters}
initialSort={DEFAULT_JOB_GRID_SORT}
pagination={{
mode: "server",
page,
pageSize,
totalRows: jobs.total,
pageSizeOptions: [25, 50, 100, 200],
disabled: jobsLoading,
onPageChange: setPage,
onPageSizeChange: (nextPageSize) => {
setPageSize(nextPageSize);
setPage(1);
}
}}
onQueryChange={handleJobGridQuery} />
</LoadingFrame>
<div className="button-row compact-actions">
<Button onClick={() => setPage((value) => Math.max(1, value - 1))} disabled={page <= 1 || jobsLoading}>i18n:govoplan-campaign.previous.50f94286</Button>
<span>i18n:govoplan-campaign.page.fb06270f {jobs.pages === 0 ? 0 : jobs.page} of {jobs.pages}</span>
<Button onClick={() => setPage((value) => Math.min(jobs.pages || 1, value + 1))} disabled={page >= jobs.pages || jobsLoading}>i18n:govoplan-campaign.next.bc981983</Button>
</div>
</Card>
</LoadingFrame>
@@ -545,6 +537,58 @@ function AttemptHistoryTable({ kind, rows }: {kind: "smtp" | "imap";rows: Record
}
function initialReportGridFilters(): Record<string, string | string[]> {
if (typeof window === "undefined") return {};
const params = new URLSearchParams(window.location.search);
const result: Record<string, string | string[]> = {};
const send = statusParameters(params, "send_status", SEND_STATUS_OPTIONS);
const imap = statusParameters(params, "imap_status", IMAP_STATUS_OPTIONS);
const validation = statusParameters(params, "validation_status", VALIDATION_STATUS_OPTIONS);
if (send.length > 0) result.send = send;
if (imap.length > 0) result.imap = imap;
if (validation.length > 0) result.validation = validation;
return result;
}
function initialReportQuery(): string {
if (typeof window === "undefined") return "";
return new URLSearchParams(window.location.search).get("q")?.trim() ?? "";
}
function statusParameters(params: URLSearchParams, name: string, options: DataGridListOption[]): string[] {
const allowed = new Set(options.map((option) => option.value));
return [...new Set(
params.getAll(name).
flatMap((value) => value.split(",")).
map((value) => value.trim()).
filter((value) => allowed.has(value))
)];
}
function serializeInitialGridFilters(filters: Record<string, string | string[]>): Record<string, string> {
return Object.fromEntries(Object.entries(filters).map(([columnId, value]) => [
columnId,
Array.isArray(value) ? `list:${JSON.stringify([...new Set(value)])}` : value
]));
}
function campaignJobSortColumn(value?: string): CampaignJobSortColumn {
if (value === "recipient" || value === "subject" || value === "validation" || value === "queue" || value === "send" || value === "imap" || value === "attempts" || value === "updated") {
return value;
}
return "number";
}
function dataGridQueriesEqual(left: DataGridQueryState, right: DataGridQueryState): boolean {
if ((left.sort?.columnId ?? "") !== (right.sort?.columnId ?? "")) return false;
if ((left.sort?.direction ?? "") !== (right.sort?.direction ?? "")) return false;
const keys = new Set([...Object.keys(left.filters), ...Object.keys(right.filters)]);
for (const key of keys) {
if ((left.filters[key] ?? "") !== (right.filters[key] ?? "")) return false;
}
return true;
}
function retryableFailedStatus(status: string): boolean {
return status === "failed_temporary" || status === "failed_permanent";
}

58
webui/src/features/campaigns/utils/jobListQuery.ts Normal file
View File

@@ -0,0 +1,58 @@
export type CampaignJobSortColumn =
| "number"
| "recipient"
| "subject"
| "validation"
| "queue"
| "send"
| "imap"
| "attempts"
| "updated";
export type CampaignJobsQueryParameters = {
versionId?: string;
page?: number;
pageSize?: number;
cursor?: string | null;
sendStatus?: string[];
validationStatus?: string[];
imapStatus?: string[];
query?: string;
sortBy?: CampaignJobSortColumn;
sortDirection?: "asc" | "desc";
filters?: Record<string, string>;
since?: string | null;
limit?: number;
};
const FILTER_PARAMETERS: Record<string, string> = {
recipient: "filter_recipient",
subject: "filter_subject",
validation: "filter_validation",
queue: "filter_queue",
send: "filter_send",
imap: "filter_imap",
attempts: "filter_attempts",
evidence: "filter_evidence"
};
export function campaignJobsQueryParams(options: CampaignJobsQueryParameters = {}): URLSearchParams {
const params = new URLSearchParams();
if (options.versionId) params.set("version_id", options.versionId);
if (options.page) params.set("page", String(options.page));
if (options.pageSize) params.set("page_size", String(options.pageSize));
if (options.cursor) params.set("cursor", options.cursor);
for (const value of options.sendStatus ?? []) params.append("send_status", value);
for (const value of options.validationStatus ?? []) params.append("validation_status", value);
for (const value of options.imapStatus ?? []) params.append("imap_status", value);
if (options.query?.trim()) params.set("q", options.query.trim());
if (options.sortBy) params.set("sort_by", options.sortBy);
if (options.sortDirection) params.set("sort_direction", options.sortDirection);
for (const [columnId, value] of Object.entries(options.filters ?? {})) {
const parameter = FILTER_PARAMETERS[columnId];
if (parameter && value.trim()) params.set(parameter, value);
}
if (options.since) params.set("since", options.since);
if (options.limit) params.set("limit", String(options.limit));
return params;
}

43
webui/tests/report-grid-query.test.ts Normal file
View File

@@ -0,0 +1,43 @@
import { campaignJobsQueryParams } from "../src/features/campaigns/utils/jobListQuery";
declare function require(name: string): {
readFileSync(path: string, encoding: string): string;
};
const { readFileSync } = require("node:fs");
function assert(condition: unknown, message: string): void {
if (!condition) throw new Error(message);
}
const params = campaignJobsQueryParams({
versionId: "version-1",
page: 2,
pageSize: 25,
query: " recipient ",
sortBy: "recipient",
sortDirection: "desc",
filters: {
recipient: "target",
send: 'list:["failed_temporary","outcome_unknown"]',
attempts: "gte:2",
unsupported: "must-not-leak"
}
});
assert(params.get("version_id") === "version-1", "the selected Campaign version is retained");
assert(params.get("page") === "2" && params.get("page_size") === "25", "server pagination is serialized");
assert(params.get("q") === "recipient", "the deep-link search query is normalized");
assert(params.get("sort_by") === "recipient" && params.get("sort_direction") === "desc", "DataGrid sorting is server-owned");
assert(params.get("filter_recipient") === "target", "text filters are sent to the backend");
assert(params.get("filter_send") === 'list:["failed_temporary","outcome_unknown"]', "stable list selections are sent intact");
assert(params.get("filter_attempts") === "gte:2", "typed number filters retain their operator");
assert(!params.toString().includes("unsupported") && !params.toString().includes("must-not-leak"), "only the declared backend filter contract is serialized");
const reportSource = readFileSync("src/features/campaigns/CampaignReportPage.tsx", "utf8");
assert(reportSource.includes('mode: "server"'), "the report DataGrid declares server query ownership");
assert(reportSource.includes("totalRows: jobs.total"), "the shared pagination count uses the filtered backend total");
assert(reportSource.includes("onQueryChange={handleJobGridQuery}"), "header sort and filter changes drive the backend query");
assert(reportSource.includes("initialReportGridFilters()"), "status deep links initialize the DataGrid filters");
assert(reportSource.includes("initialReportQuery()"), "q deep links initialize the report search");
assert(!reportSource.includes("setPage((value) => Math.max(1, value - 1))"), "the one-off report pager is removed in favor of the central DataGrid pager");

20
webui/tsconfig.report-grid-tests.json Normal file
View File

@@ -0,0 +1,20 @@
{
"compilerOptions": {
"target": "ES2020",
"lib": ["ES2020", "DOM"],
"strict": true,
"skipLibCheck": true,
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"forceConsistentCasingInFileNames": true,
"module": "CommonJS",
"moduleResolution": "Node",
"noEmit": false,
"outDir": ".report-grid-test-build",
"rootDir": "."
},
"include": [
"tests/report-grid-query.test.ts",
"src/features/campaigns/utils/jobListQuery.ts"
]
}