diff --git a/.gitignore b/.gitignore index 7f51d0d..a3c86d8 100644 --- a/.gitignore +++ b/.gitignore @@ -333,6 +333,7 @@ webui/.policy-test-build/ webui/.template-preview-test-build/ webui/.import-test-build/ webui/.review-preview-test-build/ +webui/.report-grid-test-build/ # GovOPlaN shared ignore rules from govoplan-core # Local WebUI test/build scratch directories diff --git a/src/govoplan_campaign/backend/router.py b/src/govoplan_campaign/backend/router.py index 4d40079..3eb3abb 100644 --- a/src/govoplan_campaign/backend/router.py +++ b/src/govoplan_campaign/backend/router.py @@ -2,9 +2,10 @@ from __future__ import annotations import copy import dataclasses +import json import logging from collections.abc import Callable -from typing import Any +from typing import Any, Literal from fastapi import APIRouter, Depends, HTTPException, Query, Response, status from sqlalchemy import and_, exists, func, or_ @@ -99,6 +100,7 @@ from govoplan_campaign.backend.db.models import ( JobImapStatus, JobQueueStatus, JobSendStatus, + JobValidationStatus, RecipientImportMappingProfile, SendAttempt, ) @@ -2252,6 +2254,125 @@ def _status_counts(session: Session, filters: list[object]) -> dict[str, dict[st return result +CAMPAIGN_JOB_GRID_SORT_COLUMNS = { + "number": CampaignJob.entry_index, + "recipient": func.lower(func.coalesce(CampaignJob.recipient_email, "")), + "subject": func.lower(func.coalesce(CampaignJob.subject, "")), + "validation": CampaignJob.validation_status, + "queue": CampaignJob.queue_status, + "send": CampaignJob.send_status, + "imap": CampaignJob.imap_status, + "attempts": CampaignJob.attempt_count, + "updated": CampaignJob.updated_at, +} +CAMPAIGN_JOB_GRID_LIST_FILTERS = { + "validation": (CampaignJob.validation_status, {item.value for item in JobValidationStatus}), + "queue": (CampaignJob.queue_status, {item.value for item in JobQueueStatus}), + "send": (CampaignJob.send_status, {item.value for item in JobSendStatus}), + "imap": (CampaignJob.imap_status, {item.value for item in JobImapStatus}), +} + + +def _campaign_jobs_grid_filter_expressions(grid_filters: dict[str, str] | None) -> list[object]: + values = grid_filters or {} + expressions: list[object] = [] + recipient = values.get("recipient", "").strip() + if recipient: + pattern = _contains_pattern(recipient) + expressions.append(or_( + CampaignJob.recipient_email.ilike(pattern, escape="\\"), + CampaignJob.entry_id.ilike(pattern, escape="\\"), + )) + subject = values.get("subject", "").strip() + if subject: + expressions.append(CampaignJob.subject.ilike(_contains_pattern(subject), escape="\\")) + evidence = values.get("evidence", "").strip() + if evidence: + pattern = _contains_pattern(evidence) + expressions.append(or_( + CampaignJob.message_id_header.ilike(pattern, escape="\\"), + CampaignJob.eml_sha256.ilike(pattern, escape="\\"), + )) + attempts = values.get("attempts", "").strip() + if attempts: + expressions.append(_campaign_jobs_integer_filter(CampaignJob.attempt_count, attempts, column_id="attempts")) + for column_id, (column, allowed_values) in CAMPAIGN_JOB_GRID_LIST_FILTERS.items(): + raw_value = values.get(column_id, "").strip() + if not raw_value: + continue + selected = _campaign_jobs_list_filter(raw_value, column_id=column_id, allowed_values=allowed_values) + expressions.append(column.in_(selected)) + return expressions + + +def _campaign_jobs_list_filter(raw_value: str, *, column_id: str, allowed_values: set[str]) -> list[str]: + if raw_value.startswith("list:"): + try: + parsed = json.loads(raw_value[5:]) + except json.JSONDecodeError as exc: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail=f"Invalid {column_id} list filter", + ) from exc + if not isinstance(parsed, list) or any(not isinstance(value, str) for value in parsed): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail=f"Invalid {column_id} list filter", + ) + selected = list(dict.fromkeys(value.strip() for value in parsed if value.strip())) + else: + selected = list(dict.fromkeys(value.strip() for value in raw_value.split(",") if value.strip())) + if len(selected) > 50 or any(value not in allowed_values for value in selected): + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail=f"Invalid {column_id} list filter", + ) + return selected + + +def _campaign_jobs_integer_filter(column: object, raw_value: str, *, column_id: str): + operator, separator, value = raw_value.partition(":") + if not separator: + operator, value = "eq", operator + if operator not in {"eq", "gt", "gte", "lt", "lte"}: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail=f"Invalid {column_id} filter operator", + ) + try: + expected = int(value) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail=f"Invalid {column_id} filter value", + ) from exc + if operator == "gt": + return column > expected + if operator == "gte": + return column >= expected + if operator == "lt": + return column < expected + if operator == "lte": + return column <= expected + return column == expected + + +def _contains_pattern(value: str) -> str: + escaped = value.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + return f"%{escaped}%" + + +def _campaign_jobs_ordering(sort_by: str, sort_direction: str) -> list[object]: + column = CAMPAIGN_JOB_GRID_SORT_COLUMNS.get(sort_by) + if column is None: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, + detail="Unsupported Campaign job sort column", + ) + primary = column.desc() if sort_direction == "desc" else column.asc() + return [primary, CampaignJob.id.asc()] + + def _campaign_jobs_query_context( session: Session, principal: ApiPrincipal, @@ -2262,6 +2383,7 @@ def _campaign_jobs_query_context( validation_status: list[str] | None, imap_status: list[str] | None, query_text: str | None, + grid_filters: dict[str, str] | None = None, ) -> tuple[Campaign, list[object], list[object], dict[str, object], set[str]]: _get_campaign_for_principal(session, campaign_id, principal) _require_permission(principal, "campaigns:recipient:read") @@ -2290,6 +2412,7 @@ def _campaign_jobs_query_context( CampaignJob.subject.ilike(pattern), CampaignJob.entry_id.ilike(pattern), )) + filtered.extend(_campaign_jobs_grid_filter_expressions(grid_filters)) return campaign, base_filters, filtered, review_metadata, reviewed_keys @@ -2308,6 +2431,9 @@ def _campaign_jobs_page_response( validation_status: list[str] | None = None, imap_status: list[str] | None = None, query_text: str | None = None, + grid_filters: dict[str, str] | None = None, + sort_by: str = "number", + sort_direction: str = "asc", cursor: str | None = None, changed_job_ids: set[str] | None = None, ) -> CampaignJobsResponse: @@ -2322,11 +2448,20 @@ def _campaign_jobs_page_response( validation_status=validation_status, imap_status=imap_status, query_text=query_text, + grid_filters=grid_filters, + sort_by=sort_by, + sort_direction=sort_direction, ) - ordered_query = session.query(CampaignJob).filter(*filtered).order_by(CampaignJob.entry_index.asc(), CampaignJob.id.asc()) + ordering = _campaign_jobs_ordering(sort_by, sort_direction) + ordered_query = session.query(CampaignJob).filter(*filtered).order_by(*ordering) start_cursor: str | None = None effective_offset = 0 if cursor: + if sort_by != "number" or sort_direction != "asc": + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="Campaign job cursors require number ascending order", + ) try: cursor_values = decode_keyset_cursor(CAMPAIGN_JOBS_CURSOR_SCOPE, cursor, fingerprint=fingerprint) if cursor_values is None: @@ -2338,14 +2473,14 @@ def _campaign_jobs_page_response( else: effective_offset = (page - 1) * page_size page_query = session.query(CampaignJob).filter(*filtered) - if effective_offset > 0: + if effective_offset > 0 and sort_by == "number" and sort_direction == "asc": previous_row = ordered_query.offset(effective_offset - 1).limit(1).first() if previous_row is not None: start_cursor = _campaign_jobs_cursor_for_row(previous_row, fingerprint=fingerprint) rows_plus_one = ( page_query - .order_by(CampaignJob.entry_index.asc(), CampaignJob.id.asc()) + .order_by(*ordering) .offset(effective_offset) .limit(page_size + 1) .all() @@ -2353,7 +2488,12 @@ def _campaign_jobs_page_response( jobs = rows_plus_one[:page_size] next_cursor = ( _campaign_jobs_cursor_for_row(jobs[-1], fingerprint=fingerprint) - if changed_job_ids is None and len(rows_plus_one) > page_size and jobs else None + if changed_job_ids is None + and sort_by == "number" + and sort_direction == "asc" + and len(rows_plus_one) > page_size + and jobs + else None ) if changed_job_ids is not None: jobs = [job for job in jobs if job.id in changed_job_ids] @@ -2381,6 +2521,9 @@ def _campaign_jobs_cursor_fingerprint( validation_status: list[str] | None, imap_status: list[str] | None, query_text: str | None, + grid_filters: dict[str, str] | None, + sort_by: str, + sort_direction: str, ) -> str: return keyset_query_fingerprint( CAMPAIGN_JOBS_CURSOR_SCOPE, @@ -2392,7 +2535,8 @@ def _campaign_jobs_cursor_fingerprint( "validation_status": sorted(validation_status or []), "imap_status": sorted(imap_status or []), "query": (query_text or "").strip(), - "order": ["entry_index:asc", "id:asc"], + "grid_filters": sorted((grid_filters or {}).items()), + "order": [f"{sort_by}:{sort_direction}", "id:asc"], }, ) @@ -2439,6 +2583,16 @@ class CampaignJobsQuery: validation_status: list[str] | None = Query(default=None), imap_status: list[str] | None = Query(default=None), query_text: str | None = Query(default=None, alias="q", max_length=200), + sort_by: Literal["number", "recipient", "subject", "validation", "queue", "send", "imap", "attempts", "updated"] = Query(default="number"), + sort_direction: Literal["asc", "desc"] = Query(default="asc"), + filter_recipient: str | None = Query(default=None, max_length=500), + filter_subject: str | None = Query(default=None, max_length=1000), + filter_validation: str | None = Query(default=None, max_length=1000), + filter_queue: str | None = Query(default=None, max_length=1000), + filter_send: str | None = Query(default=None, max_length=1000), + filter_imap: str | None = Query(default=None, max_length=1000), + filter_attempts: str | None = Query(default=None, max_length=100), + filter_evidence: str | None = Query(default=None, max_length=500), ) -> None: self.version_id = version_id self.page = page @@ -2448,6 +2602,22 @@ class CampaignJobsQuery: self.validation_status = validation_status self.imap_status = imap_status self.query_text = query_text + self.sort_by = sort_by + self.sort_direction = sort_direction + self.grid_filters = { + column_id: value + for column_id, value in { + "recipient": filter_recipient, + "subject": filter_subject, + "validation": filter_validation, + "queue": filter_queue, + "send": filter_send, + "imap": filter_imap, + "attempts": filter_attempts, + "evidence": filter_evidence, + }.items() + if value is not None and value.strip() + } @router.get("/{campaign_id}/jobs", response_model=CampaignJobsResponse) @@ -2472,6 +2642,7 @@ def list_jobs( validation_status=filters.validation_status, imap_status=filters.imap_status, query_text=filters.query_text, + grid_filters=filters.grid_filters, ) return _campaign_jobs_page_response( session, @@ -2487,6 +2658,9 @@ def list_jobs( validation_status=filters.validation_status, imap_status=filters.imap_status, query_text=filters.query_text, + grid_filters=filters.grid_filters, + sort_by=filters.sort_by, + sort_direction=filters.sort_direction, cursor=filters.cursor, ) @@ -2503,6 +2677,9 @@ def _campaign_jobs_full_delta_response( validation_status: list[str] | None, imap_status: list[str] | None, query_text: str | None, + grid_filters: dict[str, str] | None, + sort_by: str, + sort_direction: str, cursor: str | None = None, ) -> CampaignJobsDeltaResponse: _campaign, base_filters, filtered, review_metadata, reviewed_keys = _campaign_jobs_query_context( @@ -2514,6 +2691,7 @@ def _campaign_jobs_full_delta_response( validation_status=validation_status, imap_status=imap_status, query_text=query_text, + grid_filters=grid_filters, ) payload = _campaign_jobs_page_response( session, @@ -2529,6 +2707,9 @@ def _campaign_jobs_full_delta_response( validation_status=validation_status, imap_status=imap_status, query_text=query_text, + grid_filters=grid_filters, + sort_by=sort_by, + sort_direction=sort_direction, cursor=cursor, ) return CampaignJobsDeltaResponse( @@ -2546,8 +2727,19 @@ def _job_filter_membership_can_shift( validation_status: list[str] | None, imap_status: list[str] | None, query_text: str | None, + grid_filters: dict[str, str] | None, + sort_by: str, + sort_direction: str, ) -> bool: - return bool(send_status or validation_status or imap_status or (query_text and query_text.strip())) + return bool( + send_status + or validation_status + or imap_status + or (query_text and query_text.strip()) + or grid_filters + or sort_by != "number" + or sort_direction != "asc" + ) @router.get("/{campaign_id}/jobs/delta", response_model=CampaignJobsDeltaResponse) @@ -2571,6 +2763,9 @@ def list_jobs_delta( validation_status=filters.validation_status, imap_status=filters.imap_status, query_text=filters.query_text, + grid_filters=filters.grid_filters, + sort_by=filters.sort_by, + sort_direction=filters.sort_direction, cursor=filters.cursor, ) @@ -2583,6 +2778,7 @@ def list_jobs_delta( validation_status=filters.validation_status, imap_status=filters.imap_status, query_text=filters.query_text, + grid_filters=filters.grid_filters, ) try: since_sequence = decode_sequence_watermark(since) @@ -2606,6 +2802,9 @@ def list_jobs_delta( validation_status=filters.validation_status, imap_status=filters.imap_status, query_text=filters.query_text, + grid_filters=filters.grid_filters, + sort_by=filters.sort_by, + sort_direction=filters.sort_direction, cursor=filters.cursor, ) @@ -2632,6 +2831,9 @@ def list_jobs_delta( validation_status=filters.validation_status, imap_status=filters.imap_status, query_text=filters.query_text, + grid_filters=filters.grid_filters, + sort_by=filters.sort_by, + sort_direction=filters.sort_direction, ) or any(entry.operation in {"created", "deleted"} for entry in relevant_entries) ): @@ -2646,6 +2848,9 @@ def list_jobs_delta( validation_status=filters.validation_status, imap_status=filters.imap_status, query_text=filters.query_text, + grid_filters=filters.grid_filters, + sort_by=filters.sort_by, + sort_direction=filters.sort_direction, cursor=filters.cursor, ) @@ -2668,6 +2873,9 @@ def list_jobs_delta( validation_status=filters.validation_status, imap_status=filters.imap_status, query_text=filters.query_text, + grid_filters=filters.grid_filters, + sort_by=filters.sort_by, + sort_direction=filters.sort_direction, cursor=filters.cursor, changed_job_ids=changed_job_ids, ) diff --git a/tests/test_job_list_query.py b/tests/test_job_list_query.py new file mode 100644 index 0000000..e6bfc40 --- /dev/null +++ b/tests/test_job_list_query.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import unittest + +from fastapi import HTTPException +from sqlalchemy import create_engine +from sqlalchemy.orm import Session + +from govoplan_access.backend.db.models import Account, Group, User +from govoplan_campaign.backend.db.models import Campaign, CampaignJob, CampaignVersion +from govoplan_campaign.backend.router import ( + _campaign_jobs_grid_filter_expressions, + _campaign_jobs_ordering, + _campaign_jobs_page_response, +) +from govoplan_core.core.change_sequence import ChangeSequenceEntry +from govoplan_core.db.base import Base + + +class CampaignJobListQueryTests(unittest.TestCase): + def setUp(self) -> None: + self.engine = create_engine("sqlite+pysqlite:///:memory:") + Base.metadata.create_all( + self.engine, + tables=[ + Account.__table__, + User.__table__, + Group.__table__, + Campaign.__table__, + CampaignVersion.__table__, + CampaignJob.__table__, + ChangeSequenceEntry.__table__, + ], + ) + self.session = Session(self.engine) + rows = [ + (0, "ordinary-0@example.test", "General notice", "ready", "draft", "not_queued", "not_requested", 0), + (1, "ordinary-1@example.test", "General notice", "ready", "draft", "not_queued", "not_requested", 1), + (2, "target-c@example.test", "Target notice C", "warning", "queued", "queued", "pending", 2), + (3, "ordinary-3@example.test", "General notice", "blocked", "draft", "failed_permanent", "failed", 3), + (4, "target-b@example.test", "Target notice B", "ready", "draft", "failed_temporary", "not_requested", 4), + (5, "target-a@example.test", "Target notice A", "ready", "draft", "outcome_unknown", "outcome_unknown", 5), + ] + for entry_index, recipient, subject, validation, queue, send, imap, attempts in rows: + self.session.add(CampaignJob( + id=f"job-{entry_index}", + tenant_id="tenant-1", + campaign_id="campaign-1", + campaign_version_id="version-1", + entry_index=entry_index, + entry_id=f"entry-{entry_index}", + recipient_email=recipient, + subject=subject, + message_id_header=f"", + eml_sha256=f"sha-{entry_index}", + build_status="built", + validation_status=validation, + queue_status=queue, + send_status=send, + imap_status=imap, + attempt_count=attempts, + resolved_attachments=[], + issues_snapshot=[], + )) + self.session.commit() + + def tearDown(self) -> None: + self.session.close() + self.engine.dispose() + + def test_grid_filters_apply_before_pagination_and_report_filtered_totals(self) -> None: + base_filters = [CampaignJob.tenant_id == "tenant-1", CampaignJob.campaign_id == "campaign-1"] + grid_filters = {"recipient": "target"} + filtered = [*base_filters, *_campaign_jobs_grid_filter_expressions(grid_filters)] + + page = _campaign_jobs_page_response( + self.session, + campaign_id="campaign-1", + version_id="version-1", + base_filters=base_filters, + filtered=filtered, + reviewed_keys=set(), + review_metadata={}, + page=1, + page_size=2, + grid_filters=grid_filters, + sort_by="recipient", + sort_direction="asc", + ) + + self.assertEqual(page.total, 3) + self.assertEqual(page.total_unfiltered, 6) + self.assertEqual(page.pages, 2) + self.assertEqual( + [row["recipient_email"] for row in page.jobs], + ["target-a@example.test", "target-b@example.test"], + ) + self.assertIsNone(page.next_cursor) + + def test_list_and_integer_filters_share_the_full_backend_query(self) -> None: + expressions = _campaign_jobs_grid_filter_expressions({ + "send": 'list:["failed_temporary","outcome_unknown"]', + "attempts": "gte:4", + }) + + rows = ( + self.session.query(CampaignJob) + .filter(*expressions) + .order_by(*_campaign_jobs_ordering("attempts", "desc")) + .all() + ) + + self.assertEqual([row.id for row in rows], ["job-5", "job-4"]) + + def test_invalid_list_filters_fail_closed(self) -> None: + with self.assertRaises(HTTPException) as raised: + _campaign_jobs_grid_filter_expressions({"send": 'list:["not-a-status"]'}) + + self.assertEqual(raised.exception.status_code, 422) + + +if __name__ == "__main__": + unittest.main() diff --git a/webui/package.json b/webui/package.json index 3853476..c545811 100644 --- a/webui/package.json +++ b/webui/package.json @@ -17,7 +17,7 @@ "read-excel-file": "9.2.0" }, "peerDependencies": { - "@govoplan/core-webui": "^0.1.9", + "@govoplan/core-webui": "^0.1.11", "lucide-react": "^1.23.0", "react": "^19.0.0", "react-dom": "^19.0.0", @@ -27,6 +27,7 @@ "test:policy-ui": "rm -rf .policy-test-build && mkdir -p .policy-test-build && printf '{\"type\":\"commonjs\"}\\n' > .policy-test-build/package.json && tsc -p tsconfig.policy-tests.json && node .policy-test-build/tests/policy-ui.test.js", "test:template-preview": "rm -rf .template-preview-test-build && mkdir -p .template-preview-test-build && printf '{\"type\":\"commonjs\"}\\n' > .template-preview-test-build/package.json && tsc -p tsconfig.template-preview-tests.json && node .template-preview-test-build/tests/template-preview-draft.test.js", "test:import-utils": "rm -rf .import-test-build && mkdir -p .import-test-build && printf '{\"type\":\"commonjs\"}\\n' > .import-test-build/package.json && tsc -p tsconfig.import-tests.json && node .import-test-build/tests/import-utils.test.js", + "test:report-grid": "rm -rf .report-grid-test-build && mkdir -p .report-grid-test-build && printf '{\"type\":\"commonjs\"}\\n' > .report-grid-test-build/package.json && tsc -p tsconfig.report-grid-tests.json && node .report-grid-test-build/tests/report-grid-query.test.js", "test:review-preview-ui": "rm -rf .review-preview-test-build && mkdir -p .review-preview-test-build && printf '{\"type\":\"commonjs\"}\\n' > .review-preview-test-build/package.json && tsc -p tsconfig.review-preview-tests.json && node .review-preview-test-build/tests/review-preview-ui.test.js && node tests/delivery-mode-ui-structure.test.mjs" }, "devDependencies": { diff --git a/webui/src/api/campaigns.ts b/webui/src/api/campaigns.ts index 3849b63..e0f21ea 100644 --- a/webui/src/api/campaigns.ts +++ b/webui/src/api/campaigns.ts @@ -1,5 +1,6 @@ import type { ApiSettings, CampaignListItem, DeltaDeletedItem } from "../types"; import { apiDownload, apiFetch } from "./client"; +import { campaignJobsQueryParams, type CampaignJobsQueryParameters } from "../features/campaigns/utils/jobListQuery"; export { fetchResourceAccessExplanation } from "@govoplan/core-webui"; export type { AccessDecisionProvenanceItem, @@ -404,16 +405,7 @@ export type CampaignSendJobPayload = { }; -export type CampaignJobsQuery = { - versionId?: string; - page?: number; - pageSize?: number; - cursor?: string | null; - sendStatus?: string[]; - validationStatus?: string[]; - imapStatus?: string[]; - query?: string; -}; +export type CampaignJobsQuery = Omit; export type CampaignJobsResponse = { jobs: Record[]; @@ -801,15 +793,7 @@ settings: ApiSettings, campaignId: string, options: CampaignJobsQuery = {}) : Promise { - const params = new URLSearchParams(); - if (options.versionId) params.set("version_id", options.versionId); - if (options.page) params.set("page", String(options.page)); - if (options.pageSize) params.set("page_size", String(options.pageSize)); - if (options.cursor) params.set("cursor", options.cursor); - for (const value of options.sendStatus ?? []) params.append("send_status", value); - for (const value of options.validationStatus ?? []) params.append("validation_status", value); - for (const value of options.imapStatus ?? []) params.append("imap_status", value); - if (options.query?.trim()) params.set("q", options.query.trim()); + const params = campaignJobsQueryParams(options); const suffix = params.size > 0 ? `?${params.toString()}` : ""; return apiFetch(settings, `/api/v1/campaigns/${campaignId}/jobs${suffix}`); } @@ -819,17 +803,7 @@ settings: ApiSettings, campaignId: string, options: CampaignJobsQuery & {since?: string | null;limit?: number;} = {}) : Promise { - const params = new URLSearchParams(); - if (options.versionId) params.set("version_id", options.versionId); - if (options.page) params.set("page", String(options.page)); - if (options.pageSize) params.set("page_size", String(options.pageSize)); - if (options.cursor) params.set("cursor", options.cursor); - for (const value of options.sendStatus ?? []) params.append("send_status", value); - for (const value of options.validationStatus ?? []) params.append("validation_status", value); - for (const value of options.imapStatus ?? []) params.append("imap_status", value); - if (options.query?.trim()) params.set("q", options.query.trim()); - if (options.since) params.set("since", options.since); - if (options.limit) params.set("limit", String(options.limit)); + const params = campaignJobsQueryParams(options); const suffix = params.size > 0 ? `?${params.toString()}` : ""; return apiFetch(settings, `/api/v1/campaigns/${campaignId}/jobs/delta${suffix}`); } diff --git a/webui/src/features/campaigns/CampaignReportPage.tsx b/webui/src/features/campaigns/CampaignReportPage.tsx index 8de0ccc..a0fc123 100644 --- a/webui/src/features/campaigns/CampaignReportPage.tsx +++ b/webui/src/features/campaigns/CampaignReportPage.tsx @@ -5,7 +5,7 @@ import { downloadCampaignJobsCsv, emailCampaignReport, getCampaignJobDetail, - getCampaignJobsDelta, + getCampaignJobs, resolveCampaignJobOutcome, retryCampaignJobs, sendCampaignJob, @@ -16,16 +16,18 @@ import { import { Card } from "@govoplan/core-webui"; import { Button } from "@govoplan/core-webui"; import { ConfirmDialog } from "@govoplan/core-webui"; -import { DataGrid, type DataGridColumn, type DataGridListOption } from "@govoplan/core-webui"; +import { DataGrid, type DataGridColumn, type DataGridListOption, type DataGridQueryState } from "@govoplan/core-webui"; import { Dialog } from "@govoplan/core-webui"; import { DismissibleAlert } from "@govoplan/core-webui"; +import { FormField } from "@govoplan/core-webui"; import { PageTitle } from "@govoplan/core-webui"; import { StatusBadge } from "@govoplan/core-webui"; import VersionLine from "./components/VersionLine"; -import { LoadingFrame, TableActionGroup, ToggleSwitch, i18nMessage, useDeltaWatermarks } from "@govoplan/core-webui"; +import { LoadingFrame, TableActionGroup, ToggleSwitch, i18nMessage } from "@govoplan/core-webui"; import { useCampaignWorkspaceData } from "./hooks/useCampaignWorkspaceData"; import { asRecord, formatDateTime, humanize } from "./utils/campaignView"; -import { emptyCampaignJobsResponse, mergeCampaignJobsDelta } from "./utils/jobDeltas"; +import { emptyCampaignJobsResponse } from "./utils/jobDeltas"; +import type { CampaignJobSortColumn } from "./utils/jobListQuery"; const SEND_STATUS_OPTIONS: DataGridListOption[] = [ "not_queued", @@ -43,16 +45,37 @@ map((value) => ({ value, label: humanize(value) })); const IMAP_STATUS_OPTIONS: DataGridListOption[] = [ "not_requested", "pending", +"appending", "appended", +"outcome_unknown", "failed", "skipped"]. map((value) => ({ value, label: humanize(value) })); +const VALIDATION_STATUS_OPTIONS: DataGridListOption[] = [ +"ready", +"warning", +"needs_review", +"blocked", +"excluded", +"inactive"]. +map((value) => ({ value, label: humanize(value) })); + +const QUEUE_STATUS_OPTIONS: DataGridListOption[] = [ +"draft", +"queued", +"sending", +"paused", +"cancelled"]. +map((value) => ({ value, label: humanize(value) })); + +const DEFAULT_JOB_GRID_SORT = { columnId: "number", direction: "asc" as const }; +const JOB_GRID_QUERY_DELAY_MS = 300; + type ReconcileRequest = {jobId: string;decision: "smtp_accepted" | "not_sent";} | null; export default function CampaignReportPage({ settings, campaignId }: {settings: ApiSettings;campaignId: string;}) { const { data, loading, error, reload } = useCampaignWorkspaceData(settings, campaignId, { includeSummary: true }); - const { getDeltaWatermark, setDeltaWatermark, resetDeltaWatermark } = useDeltaWatermarks(); const version = data.currentVersion; const cards = data.summary?.cards; const delivery = asRecord(data.summary?.delivery); @@ -60,14 +83,19 @@ export default function CampaignReportPage({ settings, campaignId }: {settings: const imapPolicy = asRecord(delivery.imap_append_sent); const [jobs, setJobs] = useState(() => emptyCampaignJobsResponse()); - const jobsRef = useRef(emptyCampaignJobsResponse()); - const jobPageCursorsRef = useRef>({ 1: null }); + const jobsRequestRef = useRef(0); const [jobsLoading, setJobsLoading] = useState(false); const [page, setPage] = useState(1); - const [sendStatus, setSendStatus] = useState(""); - const [imapStatus, setImapStatus] = useState(""); - const [query, setQuery] = useState(""); - const [appliedQuery, setAppliedQuery] = useState(""); + const [pageSize, setPageSize] = useState(50); + const [initialGridFilters] = useState>(() => initialReportGridFilters()); + const initialGridQuery = useMemo(() => ({ + sort: DEFAULT_JOB_GRID_SORT, + filters: serializeInitialGridFilters(initialGridFilters) + }), [initialGridFilters]); + const [jobGridQuery, setJobGridQuery] = useState(initialGridQuery); + const [appliedJobGridQuery, setAppliedJobGridQuery] = useState(initialGridQuery); + const [query, setQuery] = useState(() => initialReportQuery()); + const [appliedQuery, setAppliedQuery] = useState(query.trim()); const [actionMessage, setActionMessage] = useState(""); const [actionError, setActionError] = useState(""); const [busyAction, setBusyAction] = useState(""); @@ -81,86 +109,46 @@ export default function CampaignReportPage({ settings, campaignId }: {settings: useEffect(() => { const handle = window.setTimeout(() => { setAppliedQuery(query.trim()); + setAppliedJobGridQuery((current) => dataGridQueriesEqual(current, jobGridQuery) ? current : jobGridQuery); setPage(1); - }, 350); + }, JOB_GRID_QUERY_DELAY_MS); return () => window.clearTimeout(handle); - }, [query]); + }, [query, jobGridQuery]); - const jobsQueryKey = useMemo( - () => JSON.stringify({ - campaignId, - versionId: version?.id ?? null, - page, - pageSize: 50, - sendStatus, - imapStatus, - appliedQuery, - apiBaseUrl: settings.apiBaseUrl, - apiKey: settings.apiKey, - accessToken: settings.accessToken - }), - [campaignId, version?.id, page, sendStatus, imapStatus, appliedQuery, settings.apiBaseUrl, settings.apiKey, settings.accessToken] - ); - - useEffect(() => { - jobPageCursorsRef.current = { 1: null }; - }, [campaignId, version?.id, sendStatus, imapStatus, appliedQuery, settings.apiBaseUrl, settings.apiKey, settings.accessToken]); + const handleJobGridQuery = useCallback((next: DataGridQueryState) => { + setJobGridQuery((current) => dataGridQueriesEqual(current, next) ? current : next); + }, []); const loadJobs = useCallback(async () => { if (!campaignId) return; + const requestId = ++jobsRequestRef.current; setJobsLoading(true); setActionError(""); try { - let nextWatermark = getDeltaWatermark(jobsQueryKey); - let merged = jobsRef.current; - let hasMore = false; - const pageCursor = page === 1 ? null : jobPageCursorsRef.current[page]; - do { - const response = await getCampaignJobsDelta(settings, campaignId, { - versionId: version?.id, - page, - pageSize: 50, - cursor: pageCursor, - sendStatus: sendStatus ? [sendStatus] : undefined, - imapStatus: imapStatus ? [imapStatus] : undefined, - query: appliedQuery || undefined, - since: nextWatermark - }); - merged = mergeCampaignJobsDelta(merged, response); - if (response.cursor !== undefined) jobPageCursorsRef.current[page] = response.cursor ?? null; - if (response.next_cursor !== undefined) { - if (response.next_cursor) jobPageCursorsRef.current[page + 1] = response.next_cursor; - else delete jobPageCursorsRef.current[page + 1]; - } - nextWatermark = response.watermark ?? null; - hasMore = response.has_more; - } while (hasMore); - setDeltaWatermark(jobsQueryKey, nextWatermark); - jobsRef.current = merged; - setJobs(merged); - if (merged.pages > 0 && page > merged.pages) setPage(merged.pages); + const response = await getCampaignJobs(settings, campaignId, { + versionId: version?.id, + page, + pageSize, + query: appliedQuery || undefined, + sortBy: campaignJobSortColumn(appliedJobGridQuery.sort?.columnId), + sortDirection: appliedJobGridQuery.sort?.direction ?? "asc", + filters: appliedJobGridQuery.filters + }); + if (requestId !== jobsRequestRef.current) return; + setJobs(response); + if (response.pages > 0 && page > response.pages) setPage(response.pages); } catch (err) { - setActionError(err instanceof Error ? err.message : String(err)); + if (requestId === jobsRequestRef.current) setActionError(err instanceof Error ? err.message : String(err)); } finally { - setJobsLoading(false); + if (requestId === jobsRequestRef.current) setJobsLoading(false); } - }, [settings, campaignId, version?.id, page, sendStatus, imapStatus, appliedQuery, jobsQueryKey, getDeltaWatermark, setDeltaWatermark]); - - useEffect(() => { - resetDeltaWatermark(jobsQueryKey); - jobsRef.current = emptyCampaignJobsResponse(); - setJobs(emptyCampaignJobsResponse()); - }, [jobsQueryKey, resetDeltaWatermark]); + }, [settings, campaignId, version?.id, page, pageSize, appliedQuery, appliedJobGridQuery]); useEffect(() => { void loadJobs(); }, [loadJobs]); async function reloadAll() { - resetDeltaWatermark(jobsQueryKey); - jobPageCursorsRef.current = { 1: null }; - jobsRef.current = emptyCampaignJobsResponse(); - setJobs(emptyCampaignJobsResponse()); await Promise.all([reload({ force: true }), loadJobs()]); } @@ -327,10 +315,10 @@ export default function CampaignReportPage({ settings, campaignId }: {settings: value: (row) => String(row.recipient_email ?? "—") }, { id: "subject", header: "i18n:govoplan-campaign.subject.8d183dbd", width: "minmax(260px, 1fr)", resizable: true, sortable: true, filterable: true, value: (row) => String(row.subject ?? "—") }, - { id: "validation", header: "i18n:govoplan-campaign.validation.dd74d182", width: 145, sortable: true, filterable: true, render: (row) => , value: (row) => String(row.validation_status ?? "unknown") }, - { id: "queue", header: "i18n:govoplan-campaign.queue.d325fcd9", width: 130, sortable: true, filterable: true, render: (row) => , value: (row) => String(row.queue_status ?? "unknown") }, + { id: "validation", header: "i18n:govoplan-campaign.validation.dd74d182", width: 145, sortable: true, filterable: true, columnType: "from-list", list: { options: VALIDATION_STATUS_OPTIONS, display: "pill" }, render: (row) => , value: (row) => String(row.validation_status ?? "unknown") }, + { id: "queue", header: "i18n:govoplan-campaign.queue.d325fcd9", width: 130, sortable: true, filterable: true, columnType: "from-list", list: { options: QUEUE_STATUS_OPTIONS, display: "pill" }, render: (row) => , value: (row) => String(row.queue_status ?? "unknown") }, { id: "send", header: "i18n:govoplan-campaign.smtp.efff9cca", width: 160, sortable: true, filterable: true, columnType: "from-list", list: { options: SEND_STATUS_OPTIONS, display: "pill" }, render: (row) => , value: (row) => String(row.send_status ?? "unknown") }, - { id: "imap", header: "i18n:govoplan-campaign.imap.271f9ef2", width: 130, sortable: true, filterable: true, render: (row) => , value: (row) => String(row.imap_status ?? "unknown") }, + { id: "imap", header: "i18n:govoplan-campaign.imap.271f9ef2", width: 130, sortable: true, filterable: true, columnType: "from-list", list: { options: IMAP_STATUS_OPTIONS, display: "pill" }, render: (row) => , value: (row) => String(row.imap_status ?? "unknown") }, { id: "attempts", header: "i18n:govoplan-campaign.attempts.5a29585e", width: 105, align: "right", sortable: true, filterType: "integer", value: (row) => Number(row.attempt_count ?? 0) }, { id: "evidence", @@ -351,7 +339,6 @@ export default function CampaignReportPage({ settings, campaignId }: {settings: header: "i18n:govoplan-campaign.last_result.110b888b", width: "minmax(220px, 1fr)", resizable: true, - filterable: true, render: (row) => {String(row.last_error ?? "—")}, value: (row) => String(row.last_error ?? "—") }, @@ -428,32 +415,37 @@ export default function CampaignReportPage({ settings, campaignId }: {settings:
- setQuery(event.target.value)} placeholder="i18n:govoplan-campaign.search_recipient_subject_or_entry_id.6d6544f5" /> - - + + setQuery(event.target.value)} /> +
{jobs.total} i18n:govoplan-campaign.matching_of.66a3778e {jobs.total_unfiltered} i18n:govoplan-campaign.total_job_s.c94b7d20
> - id={`campaign-report-jobs-${campaignId}`} + id={`campaign-report-jobs-v2-${campaignId}`} rows={jobs.jobs} columns={columns} getRowKey={(row: Record) => String(row.id ?? "")} - emptyText="i18n:govoplan-campaign.no_jobs_match_the_current_filters.b1501ff5" /> + emptyText="i18n:govoplan-campaign.no_jobs_match_the_current_filters.b1501ff5" + initialFilters={initialGridFilters} + initialSort={DEFAULT_JOB_GRID_SORT} + pagination={{ + mode: "server", + page, + pageSize, + totalRows: jobs.total, + pageSizeOptions: [25, 50, 100, 200], + disabled: jobsLoading, + onPageChange: setPage, + onPageSizeChange: (nextPageSize) => { + setPageSize(nextPageSize); + setPage(1); + } + }} + onQueryChange={handleJobGridQuery} /> -
- - i18n:govoplan-campaign.page.fb06270f {jobs.pages === 0 ? 0 : jobs.page} of {jobs.pages} - -
@@ -545,6 +537,58 @@ function AttemptHistoryTable({ kind, rows }: {kind: "smtp" | "imap";rows: Record } +function initialReportGridFilters(): Record { + if (typeof window === "undefined") return {}; + const params = new URLSearchParams(window.location.search); + const result: Record = {}; + const send = statusParameters(params, "send_status", SEND_STATUS_OPTIONS); + const imap = statusParameters(params, "imap_status", IMAP_STATUS_OPTIONS); + const validation = statusParameters(params, "validation_status", VALIDATION_STATUS_OPTIONS); + if (send.length > 0) result.send = send; + if (imap.length > 0) result.imap = imap; + if (validation.length > 0) result.validation = validation; + return result; +} + +function initialReportQuery(): string { + if (typeof window === "undefined") return ""; + return new URLSearchParams(window.location.search).get("q")?.trim() ?? ""; +} + +function statusParameters(params: URLSearchParams, name: string, options: DataGridListOption[]): string[] { + const allowed = new Set(options.map((option) => option.value)); + return [...new Set( + params.getAll(name). + flatMap((value) => value.split(",")). + map((value) => value.trim()). + filter((value) => allowed.has(value)) + )]; +} + +function serializeInitialGridFilters(filters: Record): Record { + return Object.fromEntries(Object.entries(filters).map(([columnId, value]) => [ + columnId, + Array.isArray(value) ? `list:${JSON.stringify([...new Set(value)])}` : value + ])); +} + +function campaignJobSortColumn(value?: string): CampaignJobSortColumn { + if (value === "recipient" || value === "subject" || value === "validation" || value === "queue" || value === "send" || value === "imap" || value === "attempts" || value === "updated") { + return value; + } + return "number"; +} + +function dataGridQueriesEqual(left: DataGridQueryState, right: DataGridQueryState): boolean { + if ((left.sort?.columnId ?? "") !== (right.sort?.columnId ?? "")) return false; + if ((left.sort?.direction ?? "") !== (right.sort?.direction ?? "")) return false; + const keys = new Set([...Object.keys(left.filters), ...Object.keys(right.filters)]); + for (const key of keys) { + if ((left.filters[key] ?? "") !== (right.filters[key] ?? "")) return false; + } + return true; +} + function retryableFailedStatus(status: string): boolean { return status === "failed_temporary" || status === "failed_permanent"; } diff --git a/webui/src/features/campaigns/utils/jobListQuery.ts b/webui/src/features/campaigns/utils/jobListQuery.ts new file mode 100644 index 0000000..4e7c68f --- /dev/null +++ b/webui/src/features/campaigns/utils/jobListQuery.ts @@ -0,0 +1,58 @@ +export type CampaignJobSortColumn = + | "number" + | "recipient" + | "subject" + | "validation" + | "queue" + | "send" + | "imap" + | "attempts" + | "updated"; + +export type CampaignJobsQueryParameters = { + versionId?: string; + page?: number; + pageSize?: number; + cursor?: string | null; + sendStatus?: string[]; + validationStatus?: string[]; + imapStatus?: string[]; + query?: string; + sortBy?: CampaignJobSortColumn; + sortDirection?: "asc" | "desc"; + filters?: Record; + since?: string | null; + limit?: number; +}; + +const FILTER_PARAMETERS: Record = { + recipient: "filter_recipient", + subject: "filter_subject", + validation: "filter_validation", + queue: "filter_queue", + send: "filter_send", + imap: "filter_imap", + attempts: "filter_attempts", + evidence: "filter_evidence" +}; + +export function campaignJobsQueryParams(options: CampaignJobsQueryParameters = {}): URLSearchParams { + const params = new URLSearchParams(); + if (options.versionId) params.set("version_id", options.versionId); + if (options.page) params.set("page", String(options.page)); + if (options.pageSize) params.set("page_size", String(options.pageSize)); + if (options.cursor) params.set("cursor", options.cursor); + for (const value of options.sendStatus ?? []) params.append("send_status", value); + for (const value of options.validationStatus ?? []) params.append("validation_status", value); + for (const value of options.imapStatus ?? []) params.append("imap_status", value); + if (options.query?.trim()) params.set("q", options.query.trim()); + if (options.sortBy) params.set("sort_by", options.sortBy); + if (options.sortDirection) params.set("sort_direction", options.sortDirection); + for (const [columnId, value] of Object.entries(options.filters ?? {})) { + const parameter = FILTER_PARAMETERS[columnId]; + if (parameter && value.trim()) params.set(parameter, value); + } + if (options.since) params.set("since", options.since); + if (options.limit) params.set("limit", String(options.limit)); + return params; +} diff --git a/webui/tests/report-grid-query.test.ts b/webui/tests/report-grid-query.test.ts new file mode 100644 index 0000000..f08b3db --- /dev/null +++ b/webui/tests/report-grid-query.test.ts @@ -0,0 +1,43 @@ +import { campaignJobsQueryParams } from "../src/features/campaigns/utils/jobListQuery"; + +declare function require(name: string): { + readFileSync(path: string, encoding: string): string; +}; + +const { readFileSync } = require("node:fs"); + +function assert(condition: unknown, message: string): void { + if (!condition) throw new Error(message); +} + +const params = campaignJobsQueryParams({ + versionId: "version-1", + page: 2, + pageSize: 25, + query: " recipient ", + sortBy: "recipient", + sortDirection: "desc", + filters: { + recipient: "target", + send: 'list:["failed_temporary","outcome_unknown"]', + attempts: "gte:2", + unsupported: "must-not-leak" + } +}); + +assert(params.get("version_id") === "version-1", "the selected Campaign version is retained"); +assert(params.get("page") === "2" && params.get("page_size") === "25", "server pagination is serialized"); +assert(params.get("q") === "recipient", "the deep-link search query is normalized"); +assert(params.get("sort_by") === "recipient" && params.get("sort_direction") === "desc", "DataGrid sorting is server-owned"); +assert(params.get("filter_recipient") === "target", "text filters are sent to the backend"); +assert(params.get("filter_send") === 'list:["failed_temporary","outcome_unknown"]', "stable list selections are sent intact"); +assert(params.get("filter_attempts") === "gte:2", "typed number filters retain their operator"); +assert(!params.toString().includes("unsupported") && !params.toString().includes("must-not-leak"), "only the declared backend filter contract is serialized"); + +const reportSource = readFileSync("src/features/campaigns/CampaignReportPage.tsx", "utf8"); +assert(reportSource.includes('mode: "server"'), "the report DataGrid declares server query ownership"); +assert(reportSource.includes("totalRows: jobs.total"), "the shared pagination count uses the filtered backend total"); +assert(reportSource.includes("onQueryChange={handleJobGridQuery}"), "header sort and filter changes drive the backend query"); +assert(reportSource.includes("initialReportGridFilters()"), "status deep links initialize the DataGrid filters"); +assert(reportSource.includes("initialReportQuery()"), "q deep links initialize the report search"); +assert(!reportSource.includes("setPage((value) => Math.max(1, value - 1))"), "the one-off report pager is removed in favor of the central DataGrid pager"); diff --git a/webui/tsconfig.report-grid-tests.json b/webui/tsconfig.report-grid-tests.json new file mode 100644 index 0000000..9b24221 --- /dev/null +++ b/webui/tsconfig.report-grid-tests.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2020", + "lib": ["ES2020", "DOM"], + "strict": true, + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "forceConsistentCasingInFileNames": true, + "module": "CommonJS", + "moduleResolution": "Node", + "noEmit": false, + "outDir": ".report-grid-test-build", + "rootDir": "." + }, + "include": [ + "tests/report-grid-query.test.ts", + "src/features/campaigns/utils/jobListQuery.ts" + ] +}