diff --git a/README.md b/README.md index 871512c..000e6ed 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,9 @@ Permission-aware global and contextual search for GovOPlaN. +The route, overlay, state, accessibility, and consequence mapping is recorded in +[`docs/INTERFACE_PATTERN_MIGRATION.md`](docs/INTERFACE_PATTERN_MIGRATION.md). + The module works with its built-in database index and no external search service. PostgreSQL uses native full-text search; SQLite provides a bounded development fallback. Other modules may: @@ -22,17 +25,26 @@ Source modules remain responsible for defining visibility and authorization. Source modules register a versioned `search_sources` provider. A provider declares its resource types and index version, returns bounded resumable backfill pages, and batch-rechecks current authorization for sensitive -resources. Incremental writes use `SearchIndexChange` and -`search.index_writer.enqueue_change()` so change IDs are durable and -idempotent in the same database transaction as the caller. +resources. A source may additionally implement the event-source extension to +translate committed platform events into `SearchIndexChange` records. The +platform event worker queues those records idempotently and applies them to the +derived index; Search never imports a source module or invents its ACL. Direct +capability callers may still use `search.index_writer.enqueue_change()` when +they already own a suitable transactional boundary. The built-in backend exposes opaque cursor pagination and does not return pre-authorization totals. PostgreSQL uses full-text search and will add trigram indexes when `pg_trgm` is already installed; SQLite remains a bounded development fallback. -Tenant search administrators can inspect `/api/v1/search/admin/diagnostics`, +Tenant search administrators can use **Administration > Search index** or the +equivalent `/api/v1/search/admin/*` endpoints to inspect source coverage, reconcile disabled modules, process queued changes, and start or continue -resumable provider rebuilds. Rows requiring a source authorization recheck -are omitted when their provider is unavailable, stale, or fails to return an -explicit allow decision. +bounded provider rebuilds. Quarantined changes and provider errors stay +visible. Rows requiring a source authorization recheck are omitted when their +provider is unavailable, stale, or fails to return an explicit allow decision. + +Files, Campaign, Calendar, Mail, IDM, and Postbox provide native source +adapters. Mail indexes only its bounded read-only cache, and Postbox never +indexes ciphertext or key material. All six recheck current source-owned +authorization when results are returned. diff --git a/docs/INTERFACE_PATTERN_MIGRATION.md b/docs/INTERFACE_PATTERN_MIGRATION.md new file mode 100644 index 0000000..13b7aad --- /dev/null +++ b/docs/INTERFACE_PATTERN_MIGRATION.md @@ -0,0 +1,28 @@ +# Search Interface Pattern Migration + +This is the bounded interface-pattern evidence for GovOPlaN Search. Search owns +presentation and aggregation; source modules continue to own result data and +authorization, and optional external engines remain provider capabilities. + +| Surface | Task and archetype | Consequence and state contract | +| --- | --- | --- | +| Title-bar search and anchored overlay | Global or context-sensitive focused lookup | F3 and Ctrl/Cmd+K open the same focus-contained Core dialog. Arrow keys move through the listbox, Enter opens the selected result, Escape closes it and restores focus. | +| Overlay filters | Progressive-disclosure filter popover | Module and resource filters only narrow authorized results. Active filters stay visible and removable by keyboard. | +| `/search` | Full-page search/results fallback | Query and filters are URL-stable. Loading, empty, provider-partial, failed, and paged states remain inside the result region. | +| Result entries | Permission-filtered list-detail destinations | A source module supplies the title, safe summary, breadcrumbs, and destination. Search does not infer or bypass source authorization. | +| Administration > Search index | Operator table and bounded recovery actions | Operators inspect source coverage and queue health, process pending changes, reconcile module activation, and advance one bounded rebuild page at a time. Quarantined work remains visible. | + +All surfaces use Core buttons, icon buttons, dialog, alerts, loading, scrolling, +and documentation-help primitives. Provider diagnostics are partial-state +evidence: safe results remain usable and a failing provider is identified +without exposing source data. Search has no destructive action. The responsive +layout collapses filters and results at narrow widths, and result activation is +available without pointer interaction. + +Verification: + +- `npm run test:search-overlay` +- `npm run test:interface-pattern` +- the Core TypeScript graph, structural localization audit, theme check, module + permutations, and full-product bundle budget +- Search backend and manifest tests diff --git a/src/govoplan_search/backend/manifest.py b/src/govoplan_search/backend/manifest.py index 886d140..ce84c7c 100644 --- a/src/govoplan_search/backend/manifest.py +++ b/src/govoplan_search/backend/manifest.py @@ -11,6 +11,7 @@ from govoplan_core.core.module_guards import ( persistent_table_uninstall_guard, ) from govoplan_core.core.modules import ( + DocumentationLink, DocumentationTopic, FrontendModule, FrontendRoute, @@ -152,6 +153,13 @@ manifest = ModuleManifest( label="Search results", order=20, ), + ViewSurface( + id="search.admin.index", + module_id=MODULE_ID, + kind="section", + label="Search index administration", + order=30, + ), ), ), migration_spec=MigrationSpec( @@ -201,12 +209,26 @@ manifest = ModuleManifest( "Search works with the built-in database index and can aggregate " "optional providers. Source modules announce searchable types, " "context scopes, and ACL-aware index entries. External engines " - "remain optional adapters." + "remain optional adapters. F3 or the title-bar field opens the " + "keyboard-navigable search overlay; filters never broaden the " + "current principal's source permissions. Provider failures are " + "shown as partial diagnostics without discarding safe results." + " Search administrators can inspect native source coverage, " + "process queued changes, reconcile enabled modules, and run " + "bounded source rebuilds from Administration. Quarantined " + "changes remain visible until repaired and reconciled." ), layer="available", documentation_types=("admin", "user"), audience=("administrator", "user"), related_modules=("connectors", "views"), + links=( + DocumentationLink( + label="Search interface pattern audit", + href="govoplan-search/docs/INTERFACE_PATTERN_MIGRATION.md", + kind="repository", + ), + ), order=12, ), ), diff --git a/src/govoplan_search/backend/service.py b/src/govoplan_search/backend/service.py index fcf31e1..9924524 100644 --- a/src/govoplan_search/backend/service.py +++ b/src/govoplan_search/backend/service.py @@ -27,10 +27,12 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from govoplan_core.core.external_references import ExternalObjectReference +from govoplan_core.core.events import PlatformEvent from govoplan_core.core.search import ( SearchAuthorizationRequest, SearchBackfillRequest, SearchDocument, + SearchEventSourceProvider, SearchIndexChange, SearchQuery, SearchResourceReference, @@ -231,6 +233,74 @@ class SearchIndexService: return False return True + def ingest_event( + self, + session: object, + *, + event: PlatformEvent, + delivery_key: str, + ) -> dict[str, int]: + """Queue source-owned index deltas from one committed platform event.""" + + db = _session(session) + result = { + "sources": 0, + "changes": 0, + "queued": 0, + "duplicates": 0, + } + for registered, provider in _search_sources(self.registry): + if not isinstance(provider, SearchEventSourceProvider): + continue + changes = tuple( + provider.index_changes_for_event( + db, + event=event, + delivery_key=delivery_key, + ) + ) + if not changes: + continue + result["sources"] += 1 + descriptors = { + descriptor.resource_type: descriptor + for descriptor in provider.resource_types() + if descriptor.provider_id == registered.registration.id + } + for change in changes: + descriptor = descriptors.get( + change.reference.resource_type + ) + if ( + change.provider_id != registered.registration.id + or change.reference.module_id != registered.module_id + or descriptor is None + or descriptor.module_id != registered.module_id + ): + raise ValueError( + "Search event source returned a change outside its " + "registered provider boundary." + ) + if ( + event.tenant is not None + and change.reference.tenant_id != event.tenant.id + ): + raise ValueError( + "Search event source returned a cross-tenant change." + ) + if change.document is not None: + _validate_backfill_document( + change.document, + descriptor=descriptor, + tenant_id=change.reference.tenant_id, + ) + result["changes"] += 1 + if self.enqueue_change(db, change=change): + result["queued"] += 1 + else: + result["duplicates"] += 1 + return result + def process_changes( self, session: object, diff --git a/tests/test_search_service.py b/tests/test_search_service.py index 1b1b6cf..b0c4627 100644 --- a/tests/test_search_service.py +++ b/tests/test_search_service.py @@ -6,6 +6,7 @@ from types import SimpleNamespace from sqlalchemy import create_engine from sqlalchemy.orm import Session +from govoplan_core.core.events import EventObjectRef, EventTenantRef, PlatformEvent from govoplan_core.core.search import ( SearchBackfillPage, SearchDocument, @@ -45,6 +46,7 @@ class _Registry: return ( ( SimpleNamespace( + module_id="cases", registration=SimpleNamespace( id="cases.records", order=25, @@ -86,6 +88,45 @@ class _Source: } +class _EventSource(_Source): + def index_changes_for_event(self, session, *, event, delivery_key): + del session + if ( + event.module_id != "cases" + or event.tenant is None + or event.resource is None + or event.resource.type != "case" + or event.resource.id is None + ): + return () + reference = _source_document(event.resource.id).reference + document = SearchDocument( + tenant_id=event.tenant.id, + module_id="cases", + resource_type="case", + resource_id=event.resource.id, + title=f"Permit {event.resource.id}", + url=f"/cases/{event.resource.id}", + acl_tokens=("account:account-1",), + provider_id="cases.records", + source_revision="event-1", + change_cursor=event.event_id, + requires_authorization_recheck=True, + ) + return ( + SearchIndexChange( + change_id=f"{delivery_key}:cases.records", + provider_id="cases.records", + kind="upsert", + reference=reference, + source_revision=document.source_revision, + cursor=event.event_id, + document=document, + occurred_at=event.occurred_at, + ), + ) + + class _ResultProvider: def search(self, session, principal, *, query): del session, principal @@ -382,6 +423,33 @@ class SearchServiceTests(unittest.TestCase): self.session.query(SearchIndexDocument).count(), ) + def test_committed_event_ingestion_is_source_owned_and_idempotent(self) -> None: + service = SearchIndexService(_Registry(_EventSource())) + event = PlatformEvent( + type="cases.case.updated", + module_id="cases", + tenant=EventTenantRef(id="tenant-1"), + resource=EventObjectRef(type="case", id="case-1"), + ) + + first = service.ingest_event( + self.session, + event=event, + delivery_key="delivery-1", + ) + second = service.ingest_event( + self.session, + event=event, + delivery_key="delivery-1", + ) + + self.assertEqual(1, first["queued"]) + self.assertEqual(1, second["duplicates"]) + self.assertEqual(1, service.process_changes(self.session)["applied"]) + indexed = self.session.query(SearchIndexDocument).one() + self.assertEqual("tenant-1", indexed.tenant_id) + self.assertEqual("case-1", indexed.resource_id) + def test_rebuild_resumes_and_removes_stale_documents(self) -> None: stale = _source_document("stale-case") self.service.upsert_document( diff --git a/webui/package.json b/webui/package.json index 396ad8f..db64a88 100644 --- a/webui/package.json +++ b/webui/package.json @@ -14,7 +14,8 @@ "./styles/search.css": "./src/styles/search.css" }, "scripts": { - "test:search-overlay": "node scripts/test-search-overlay-structure.mjs" + "test:search-overlay": "node scripts/test-search-overlay-structure.mjs", + "test:interface-pattern": "node scripts/test-interface-pattern.mjs" }, "peerDependencies": { "@govoplan/core-webui": "^0.1.14", diff --git a/webui/scripts/test-interface-pattern.mjs b/webui/scripts/test-interface-pattern.mjs new file mode 100644 index 0000000..f65751f --- /dev/null +++ b/webui/scripts/test-interface-pattern.mjs @@ -0,0 +1,27 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; + +const page = fs.readFileSync("src/features/search/SearchPage.tsx", "utf8"); +const overlay = fs.readFileSync("src/components/GlobalSearch.tsx", "utf8"); +const admin = fs.readFileSync("src/features/search/SearchAdminPanel.tsx", "utf8"); +const styles = fs.readFileSync("src/styles/search.css", "utf8"); + +for (const source of [page, overlay]) { + assert.ok(source.includes("DocumentationHelpLink"), "Search surfaces expose configured-system help"); + assert.ok(source.includes("DismissibleAlert"), "Search failures use the shared alert contract"); + assert.ok(!source.includes("window.alert("), "Search must not use browser alerts"); + assert.ok(!/<(div|span|li|tr)\b[^>]*\bonClick\s*=/.test(source), "Search uses semantic interactive elements"); +} + +assert.ok(page.includes("PageScrollViewport"), "The full search route owns bounded result scrolling"); +assert.ok(overlay.includes("; + queue_oldest_age_seconds?: number | null; + states: SearchIndexState[]; +}; + +export type SearchChangeDispatch = { + selected: number; + applied: number; + retrying: number; + quarantined: number; +}; + +export type SearchModuleReconcile = { + disabled_documents: number; + enabled_documents: number; +}; + export type SearchRequest = { query: string; modules?: string[]; @@ -107,3 +144,63 @@ export function listSearchProviders( { signal } ); } + +export function getSearchDiagnostics( + settings: ApiSettings, + signal?: AbortSignal +): Promise { + return apiFetch( + settings, + "/api/v1/search/admin/diagnostics", + { signal } + ); +} + +export function reconcileSearchModules( + settings: ApiSettings +): Promise { + return apiFetch( + settings, + "/api/v1/search/admin/reconcile-modules", + { method: "POST" } + ); +} + +export function processSearchChanges( + settings: ApiSettings, + limit = 100 +): Promise { + return apiFetch( + settings, + apiPath("/api/v1/search/admin/changes/process", { limit }), + { method: "POST" } + ); +} + +export function startSearchRebuild( + settings: ApiSettings, + providerId: string, + resourceType: string +): Promise<{ state: SearchIndexState }> { + return apiFetch<{ state: SearchIndexState }>( + settings, + `/api/v1/search/admin/rebuilds/${encodeURIComponent(providerId)}/${encodeURIComponent(resourceType)}/start`, + { method: "POST" } + ); +} + +export function continueSearchRebuild( + settings: ApiSettings, + providerId: string, + resourceType: string, + limit = 100 +): Promise<{ state: SearchIndexState }> { + return apiFetch<{ state: SearchIndexState }>( + settings, + apiPath( + `/api/v1/search/admin/rebuilds/${encodeURIComponent(providerId)}/${encodeURIComponent(resourceType)}/continue`, + { limit } + ), + { method: "POST" } + ); +} diff --git a/webui/src/components/GlobalSearch.tsx b/webui/src/components/GlobalSearch.tsx index f77dc3f..5e8cf67 100644 --- a/webui/src/components/GlobalSearch.tsx +++ b/webui/src/components/GlobalSearch.tsx @@ -13,6 +13,7 @@ import { useLocation } from "react-router"; import { Button, Dialog, + DocumentationHelpLink, DismissibleAlert, IconButton, LoadingIndicator, @@ -588,6 +589,10 @@ export default function GlobalSearch({ settings }: GlobalSearchProps) { } {loading && } + } diff --git a/webui/src/features/search/SearchAdminPanel.tsx b/webui/src/features/search/SearchAdminPanel.tsx new file mode 100644 index 0000000..0ce41e2 --- /dev/null +++ b/webui/src/features/search/SearchAdminPanel.tsx @@ -0,0 +1,291 @@ +import { useEffect, useMemo, useState } from "react"; +import { + AdminPageLayout, + adminErrorMessage, + Button, + Card, + DataGrid, + DismissibleAlert, + DocumentationHelpLink, + MetricCard, + StatusBadge, + TableActionGroup, + type ApiSettings, + type DataGridColumn +} from "@govoplan/core-webui"; +import { Play, RefreshCw, RotateCw } from "lucide-react"; +import { + continueSearchRebuild, + getSearchDiagnostics, + listSearchProviders, + processSearchChanges, + reconcileSearchModules, + startSearchRebuild, + type SearchDiagnostics, + type SearchIndexState, + type SearchResourceType +} from "../../api/search"; + +type Props = { + settings: ApiSettings; +}; + +type ResourceRow = SearchResourceType & { + state?: SearchIndexState; +}; + +const DOCUMENTATION = { + topicId: "search.global-and-contextual", + documentationType: "admin" as const +}; + +export default function SearchAdminPanel({ settings }: Props) { + const [diagnostics, setDiagnostics] = useState(null); + const [resources, setResources] = useState([]); + const [loading, setLoading] = useState(true); + const [busyKey, setBusyKey] = useState(""); + const [error, setError] = useState(""); + const [success, setSuccess] = useState(""); + + useEffect(() => { + void load(); + }, [settings.accessToken, settings.apiBaseUrl, settings.apiKey]); + + async function load() { + setLoading(true); + setError(""); + try { + const [nextDiagnostics, catalogue] = await Promise.all([ + getSearchDiagnostics(settings), + listSearchProviders(settings) + ]); + setDiagnostics(nextDiagnostics); + setResources(catalogue.resources); + } catch (err) { + setError(adminErrorMessage(err)); + } finally { + setLoading(false); + } + } + + async function runAction(key: string, action: () => Promise) { + setBusyKey(key); + setError(""); + setSuccess(""); + try { + setSuccess(await action()); + await load(); + } catch (err) { + setError(adminErrorMessage(err)); + } finally { + setBusyKey(""); + } + } + + const rows = useMemo(() => { + const states = new Map( + (diagnostics?.states ?? []).map((state) => [ + `${state.provider_id}:${state.resource_type}`, + state + ]) + ); + return resources.map((resource) => ({ + ...resource, + state: states.get(`${resource.provider_id}:${resource.resource_type}`) + })); + }, [diagnostics, resources]); + + const columns = useMemo[]>(() => [ + { + id: "resource", + header: "Search source", + minWidth: 250, + resizable: true, + sortable: true, + filterable: true, + value: (row) => row.label, + render: (row) => ( +
+ {row.label} +
{row.module_id} / {row.provider_id}
+
+ ) + }, + { + id: "status", + header: "State", + width: 135, + minWidth: 115, + sortable: true, + filterable: true, + filterType: "list", + value: (row) => row.state?.status ?? "not built", + render: (row) => ( + + ) + }, + { + id: "documents", + header: "Documents", + width: 125, + minWidth: 105, + align: "right", + value: (row) => row.state?.indexed_documents ?? 0 + }, + { + id: "rejected", + header: "Rejected", + width: 105, + minWidth: 90, + align: "right", + value: (row) => row.state?.rejected_documents ?? 0 + }, + { + id: "lastSuccess", + header: "Last success", + width: 190, + minWidth: 165, + sortable: true, + value: (row) => row.state?.last_success_at ?? "", + render: (row) => formatDateTime(row.state?.last_success_at) + }, + { + id: "actions", + header: "Actions", + width: 80, + minWidth: 80, + sticky: "end", + align: "right", + render: (row) => { + const key = `${row.provider_id}:${row.resource_type}`; + const continuing = row.state?.status === "backfilling"; + return ( + : , + disabled: Boolean(busyKey), + onClick: () => void runAction(key, async () => { + const response = continuing + ? await continueSearchRebuild(settings, row.provider_id, row.resource_type) + : await startSearchRebuild(settings, row.provider_id, row.resource_type); + return response.state.status === "backfilling" + ? `${row.label} rebuild advanced to the next checkpoint.` + : `${row.label} rebuild completed.`; + }) + }]} + /> + ); + } + } + ], [busyKey, settings]); + + const queue = diagnostics?.queue ?? {}; + const pending = (queue.queued ?? 0) + (queue.retrying ?? 0); + const quarantined = queue.quarantined ?? 0; + + return ( + + + + + + + )} + > +
+ + + + +
+ + {quarantined > 0 && ( + + Quarantined changes require source or contract repair followed by a source rebuild. They are never silently discarded. + + )} + + +
+ `${row.provider_id}:${row.resource_type}`} + initialFit="container" + emptyText="No active modules announce searchable resource types." + /> +
+ + + {rows.some((row) => row.state?.last_error) && ( + +
+ {rows.filter((row) => row.state?.last_error).map((row) => ( + + {row.label}: {row.state?.last_error} + + ))} +
+ + )} + + ); +} + +function statusTone(status?: string): string { + if (["ready", "idle"].includes(status ?? "")) return "success"; + if (["failed", "quarantined"].includes(status ?? "")) return "danger"; + if (["backfilling", "stale"].includes(status ?? "")) return "warning"; + return "neutral"; +} + +function formatDateTime(value?: string | null): string { + if (!value) return "-"; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString(); +} diff --git a/webui/src/features/search/SearchPage.tsx b/webui/src/features/search/SearchPage.tsx index c511822..88ef270 100644 --- a/webui/src/features/search/SearchPage.tsx +++ b/webui/src/features/search/SearchPage.tsx @@ -9,7 +9,10 @@ import { } from "react"; import { useSearchParams } from "react-router"; import { + Button, + DocumentationHelpLink, DismissibleAlert, + IconButton, LoadingIndicator, PageScrollViewport, useGuardedNavigate, @@ -216,12 +219,12 @@ export default function SearchPage({ settings }: PlatformRouteContext) { placeholder="Search" autoFocus /> - +
- + {filtersOpen &&
Filter results - + } + variant="ghost" + onClick={() => setFiltersOpen(false)} + />
Modules @@ -282,18 +284,22 @@ export default function SearchPage({ settings }: PlatformRouteContext) {
- +
} {loading && } + {activeFilterCount > 0 &&
{modules.map((moduleId) => @@ -359,14 +365,15 @@ export default function SearchPage({ settings }: PlatformRouteContext) { )}
{response?.next_cursor && - + } diff --git a/webui/src/module.ts b/webui/src/module.ts index 4505ceb..6b92abf 100644 --- a/webui/src/module.ts +++ b/webui/src/module.ts @@ -1,5 +1,6 @@ import { createElement, lazy } from "react"; import type { + AdminSectionsUiCapability, PlatformWebModule, SearchRuntimeUiCapability } from "@govoplan/core-webui"; @@ -8,12 +9,29 @@ import "./styles/search.css"; const SearchPage = lazy(() => import("./features/search/SearchPage")); +const SearchAdminPanel = lazy(() => import("./features/search/SearchAdminPanel")); const searchRuntime: SearchRuntimeUiCapability = { GlobalSearch, anyOf: ["search:result:read"] }; +const searchAdminSections: AdminSectionsUiCapability = { + sections: [ + { + id: "tenant-search-index", + moduleId: "search", + kind: "operations", + surfaceId: "search.admin.index", + label: "Search index", + group: "TENANT", + order: 69, + allOf: ["search:index:admin"], + render: ({ settings }) => createElement(SearchAdminPanel, { settings }) + } + ] +}; + export const searchModule: PlatformWebModule = { id: "search", label: "Search", @@ -50,10 +68,18 @@ export const searchModule: PlatformWebModule = { kind: "route", label: "Search results", order: 20 + }, + { + id: "search.admin.index", + moduleId: "search", + kind: "section", + label: "Search index administration", + order: 30 } ], uiCapabilities: { - "search.runtime": searchRuntime + "search.runtime": searchRuntime, + "admin.sections": searchAdminSections } };