feat(ops): show worker and queue readiness

webui/src/api/ops.ts

@@ -169,6 +169,25 @@ export type OpsRuntimeCluster = {
   };
 };
 
+export type OpsRuntimeWorkStatus = {
+  provider_id: string;
+  label: string;
+  backend: string;
+  enabled: boolean | null;
+  configured: boolean | null;
+  state: "disabled" | "unconfigured" | "starting" | "healthy" | "idle" | "busy" | "degraded" | "stale" | "unreachable" | string;
+  detail: string;
+  observed_at: string;
+  active_workers?: number | null;
+  last_heartbeat_at?: string | null;
+  queue_depths: Record<string, number | null>;
+  active_work?: number | null;
+  reserved_work?: number | null;
+  failures?: number | null;
+  stale_after_seconds?: number | null;
+  guidance: string;
+};
+
 export type OpsStatus = {
   summary: {
     app_env: string;
@@ -195,12 +214,15 @@ export type OpsStatus = {
     };
     backup_state?: string;
     worker_metrics: {
-      workers?: number;
-      active_tasks?: number;
+      state?: string;
+      workers?: number | null;
+      active_tasks?: number | null;
+      reserved_tasks?: number | null;
+      failures?: number | null;
       expected_queues?: string[];
       active_queues?: string[];
       missing_queues?: string[];
-      queue_depths?: Record<string, number>;
+      queue_depths?: Record<string, number | null>;
     };
     operational_probe_count: number;
     runtime_node_count: number;
@@ -244,6 +266,7 @@ export type OpsStatus = {
   deployment_profiles: OpsDeploymentProfile[];
   sizing: OpsSizingAssumption[];
   runtime_cluster: OpsRuntimeCluster;
+  runtime_work: OpsRuntimeWorkStatus[];
   infrastructure: {
     configured: boolean;
     available: boolean;

webui/src/features/ops/OpsHealthWidget.tsx

@@ -12,6 +12,7 @@ import {
 } from "@govoplan/core-webui";
 import { fetchOpsStatus, type OpsStatus } from "../../api/ops";
 import { OPS_DOCUMENTATION } from "./interfacePatterns";
+import { knownMetric, knownQueueDepthTotal, runtimeWorkTone } from "./runtimeStatus";
 
 export default function OpsHealthWidget({ settings, refreshKey }: { settings: ApiSettings; refreshKey: number }) {
   const [status, setStatus] = useState<OpsStatus | null>(null);
@@ -35,8 +36,7 @@ export default function OpsHealthWidget({ settings, refreshKey }: { settings: Ap
   const ready = status?.readiness.ready ?? false;
   const workerMetrics = status?.summary.worker_metrics;
   const queueDepths = workerMetrics?.queue_depths ?? {};
-  const queuedTasks = Object.values(queueDepths).reduce((sum, value) => sum + value, 0);
-  const missingQueues = workerMetrics?.missing_queues ?? [];
+  const queuedTasks = knownQueueDepthTotal(queueDepths);
   const storageUsage = status?.summary.storage_metrics?.capacity_used_percent;
 
   return (
@@ -47,9 +47,9 @@ export default function OpsHealthWidget({ settings, refreshKey }: { settings: Ap
       </div>
       <MetricGrid columns={3} spacing="none">
         <MetricCard label="Readiness" value={ready ? "ready" : "blocked"} tone={ready ? "good" : "danger"} detail={status?.readiness.profile ?? "-"} />
-        <MetricCard label="Workers" value={status?.summary.celery_enabled ? workerMetrics?.workers ?? 0 : "off"} tone={status?.summary.celery_enabled && !missingQueues.length ? "good" : "warning"} detail={missingQueues.length ? `${missingQueues.length} queue(s) without a consumer` : status?.summary.celery_enabled ? "All configured queues covered" : "Single-process mode"} />
-        <MetricCard label="Active tasks" value={workerMetrics?.active_tasks ?? 0} tone="info" detail={status?.summary.celery_enabled ? "Reported by live workers" : "Workers disabled"} />
-        <MetricCard label="Queued tasks" value={queuedTasks} tone={queuedTasks ? "warning" : "neutral"} detail={Object.keys(queueDepths).length ? `${Object.keys(queueDepths).length} measured queue(s)` : "Queue depth unavailable"} />
+        <MetricCard label="Workers" value={knownMetric(workerMetrics?.workers)} tone={runtimeWorkTone(workerMetrics?.state ?? "unreachable")} detail={workerMetrics?.state ?? "unavailable"} />
+        <MetricCard label="Active tasks" value={knownMetric(workerMetrics?.active_tasks)} tone="info" detail={workerMetrics?.active_tasks == null ? "Metric unavailable" : "Reported by the runtime provider"} />
+        <MetricCard label="Queued tasks" value={queuedTasks ?? "unavailable"} tone={queuedTasks === null ? "neutral" : queuedTasks ? "warning" : "good"} detail={queuedTasks === null ? "Queue depth unavailable" : `${Object.values(queueDepths).filter((value) => typeof value === "number").length} measured queue(s)`} />
         <MetricCard label="Storage" value={storageUsage === undefined ? status?.summary.file_storage_backend ?? "-" : `${storageUsage}%`} tone={storageUsage !== undefined && storageUsage >= 90 ? "danger" : storageUsage !== undefined && storageUsage >= 75 ? "warning" : "neutral"} detail="Managed Files backend capacity" />
         <MetricCard label="Failed" value={status?.summary.failed_operation_count ?? 0} tone={status?.summary.failed_operation_count ? "danger" : "good"} detail="Recovery-ledger operations" />
         <MetricCard label="Unknown" value={status?.summary.outcome_unknown_count ?? 0} tone={status?.summary.outcome_unknown_count ? "danger" : "good"} detail="Outcome-unknown operations" />

webui/src/features/ops/OpsPage.tsx

@@ -1,6 +1,6 @@
 import { DescriptionList } from "@govoplan/core-webui";
 import { MetricGrid } from "@govoplan/core-webui";
-import { useEffect, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import { PauseCircle, PlayCircle, RefreshCw } from "lucide-react";
 import { ContentGrid,
   ActionBlockerHint,
@@ -33,6 +33,7 @@ import {
   type OpsInfrastructureCapability,
   type OpsRecoveryOperation,
   type OpsRuntimeNode,
+  type OpsRuntimeWorkStatus,
   type OpsSizingAssumption,
   type OpsStatus
 } from "../../api/ops";
@@ -41,6 +42,14 @@ import {
   OPS_I18N,
   OPS_RECOVERY_DOCUMENTATION,
 } from "./interfacePatterns";
+import {
+  heartbeatAgeLabel,
+  heartbeatAgeSeconds,
+  knownMetric,
+  knownQueueDepthTotal,
+  runtimeWorkTone,
+  shouldPollRuntimeStatus
+} from "./runtimeStatus";
 
 export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth: AuthInfo;}) {
   const [status, setStatus] = useState<OpsStatus | null>(null);
@@ -49,18 +58,27 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
   const [error, setError] = useState("");
   const [drainTarget, setDrainTarget] = useState<OpsRuntimeNode | null>(null);
   const [nodeActionId, setNodeActionId] = useState("");
+  const loadInFlight = useRef<Promise<void> | null>(null);
 
-  async function load() {
-    setLoading(true);
-    setError("");
-    try {
-      setStatus(await fetchOpsStatus(settings));
-    } catch (err) {
-      setError(adminErrorMessage(err));
-    } finally {
-      setLoading(false);
-    }
-  }
+  const load = useCallback((background = false): Promise<void> => {
+    if (loadInFlight.current) return loadInFlight.current;
+    const request = (async () => {
+      if (!background) setLoading(true);
+      setError("");
+      try {
+        setStatus(await fetchOpsStatus(settings));
+      } catch (err) {
+        setError(adminErrorMessage(err));
+      } finally {
+        if (!background) setLoading(false);
+      }
+    })();
+    loadInFlight.current = request;
+    void request.finally(() => {
+      if (loadInFlight.current === request) loadInFlight.current = null;
+    });
+    return request;
+  }, [settings.accessToken, settings.apiBaseUrl, settings.apiKey]);
 
   async function runChecks() {
     setRunningProbes(true);
@@ -104,7 +122,37 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
     }
   }
 
-  useEffect(() => {void load();}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
+  useEffect(() => {
+    let intervalId: number | null = null;
+    const stopPolling = () => {
+      if (intervalId !== null) window.clearInterval(intervalId);
+      intervalId = null;
+    };
+    const startPolling = () => {
+      stopPolling();
+      if (document.visibilityState === "hidden") return;
+      intervalId = window.setInterval(() => {
+        if (shouldPollRuntimeStatus(document.visibilityState === "hidden", Boolean(loadInFlight.current))) {
+          void load(true);
+        }
+      }, 15_000);
+    };
+    const handleVisibility = () => {
+      if (document.visibilityState === "hidden") {
+        stopPolling();
+        return;
+      }
+      if (shouldPollRuntimeStatus(false, Boolean(loadInFlight.current))) void load(true);
+      startPolling();
+    };
+    void load();
+    startPolling();
+    document.addEventListener("visibilitychange", handleVisibility);
+    return () => {
+      stopPolling();
+      document.removeEventListener("visibilitychange", handleVisibility);
+    };
+  }, [load]);
 
   const checks = status?.checks ?? [];
   const warningCount = checks.filter((item) => item.state === "warning").length;
@@ -119,7 +167,7 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
         ? OPS_I18N.runPermissionRequired
         : undefined;
   const queueDepths = status?.summary.worker_metrics.queue_depths ?? {};
-  const queuedTasks = Object.values(queueDepths).reduce((total, value) => total + value, 0);
+  const queuedTasks = knownQueueDepthTotal(queueDepths);
   const storageUsage = status?.summary.storage_metrics?.capacity_used_percent;
 
   return (
@@ -156,8 +204,8 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
           <MetricCard label="i18n:govoplan-ops.modules.04e9462c" value={status?.summary.module_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.enabled_in_the_runtime_registry.d2c6142d" />
           <MetricCard label="i18n:govoplan-ops.permissions.842c35eb" value={status?.governance.summary.permission_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.declared_governance_permissions.d08d3bf1" />
           <MetricCard label="i18n:govoplan-ops.policies.e7800f56" value={status?.governance.summary.policy_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.registered_policy_capabilities.112a2b64" />
-          <MetricCard label="i18n:govoplan-ops.workers.b6ef3acd" value={status?.summary.celery_enabled ? status.summary.worker_metrics.workers ?? 0 : "off"} tone={status?.summary.celery_enabled && !(status.summary.worker_metrics.missing_queues?.length) ? "good" : "warning"} detail={workerMetricDetail(status)} />
-          <MetricCard label="Queued tasks" value={queuedTasks} tone={queuedTasks ? "warning" : "good"} detail={Object.keys(queueDepths).length ? `${Object.keys(queueDepths).length} measured queue(s)` : "Queue depth unavailable"} />
+          <MetricCard label="i18n:govoplan-ops.workers.b6ef3acd" value={knownMetric(status?.summary.worker_metrics.workers)} tone={runtimeWorkTone(status?.summary.worker_metrics.state ?? "unreachable")} detail={workerMetricDetail(status)} />
+          <MetricCard label="Queued tasks" value={queuedTasks ?? "unavailable"} tone={queuedTasks === null ? "neutral" : queuedTasks ? "warning" : "good"} detail={queuedTasks === null ? "Queue depth unavailable" : `${Object.values(queueDepths).filter((value) => typeof value === "number").length} measured queue(s)`} />
           <MetricCard label="Storage" value={storageUsage === undefined ? status?.summary.file_storage_backend ?? "-" : `${storageUsage}%`} tone={storageUsage !== undefined && storageUsage >= 90 ? "danger" : storageUsage !== undefined && storageUsage >= 75 ? "warning" : "neutral"} detail={storageMetricDetail(status)} />
           <MetricCard label="Backup evidence" value={status?.summary.backup_state ?? "unknown"} tone={status?.summary.backup_state === "ok" ? "good" : "warning"} detail="Latest coordinated backup and restore-drill evidence" />
           <MetricCard label="Failed operations" value={status?.summary.failed_operation_count ?? 0} tone={status?.summary.failed_operation_count ? "danger" : "good"} detail="Terminal failures or manual intervention" />
@@ -186,6 +234,10 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
             />
           </Card>
 
+          <Card title="Worker and queue readiness">
+            <RuntimeWorkTable items={status?.runtime_work ?? []} />
+          </Card>
+
           <Card title="Recovery evidence">
             <RecoveryTable operations={status?.runtime_cluster.recovery.operations ?? []} />
           </Card>
@@ -226,6 +278,70 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
 
 }
 
+function RuntimeWorkTable({ items }: { items: OpsRuntimeWorkStatus[] }) {
+  const columns: DataGridColumn<OpsRuntimeWorkStatus>[] = [
+    {
+      id: "provider",
+      header: "Backend",
+      width: "minmax(200px, 1fr)",
+      minWidth: 180,
+      resizable: true,
+      sortable: true,
+      filterable: true,
+      value: (item) => `${item.label} ${item.backend}`,
+      render: (item) => <div><strong>{item.label}</strong><span className="muted block">{item.backend} · {item.provider_id}</span></div>
+    },
+    {
+      id: "state",
+      header: "State",
+      width: 150,
+      sortable: true,
+      filterable: true,
+      value: (item) => item.state,
+      render: (item) => <StatusBadge status={stateTone(item.state)} label={item.state} />
+    },
+    {
+      id: "activity",
+      header: "Activity",
+      width: "minmax(180px, .7fr)",
+      minWidth: 170,
+      value: (item) => `${item.active_workers ?? ""} ${item.active_work ?? ""} ${item.reserved_work ?? ""}`,
+      render: (item) => `${knownMetric(item.active_workers)} workers · ${knownMetric(item.active_work)} active · ${knownMetric(item.reserved_work)} reserved`
+    },
+    {
+      id: "queues",
+      header: "Queue depth",
+      width: "minmax(220px, 1fr)",
+      minWidth: 190,
+      resizable: true,
+      value: (item) => Object.entries(item.queue_depths).map(([queue, depth]) => `${queue}:${depth ?? "unavailable"}`).join(" "),
+      render: (item) => runtimeQueueSummary(item.queue_depths)
+    },
+    {
+      id: "heartbeat",
+      header: "Last heartbeat",
+      width: "minmax(180px, .7fr)",
+      minWidth: 170,
+      sortable: true,
+      value: (item) => item.last_heartbeat_at ?? "",
+      render: (item) => {
+        const age = heartbeatAgeSeconds(Date.now(), item.last_heartbeat_at);
+        return <div>{heartbeatAgeLabel(age)}<span className="muted block">stale after {item.stale_after_seconds ?? "unavailable"}s</span></div>;
+      }
+    },
+    {
+      id: "guidance",
+      header: "Guidance",
+      width: "minmax(260px, 1.3fr)",
+      minWidth: 220,
+      resizable: true,
+      value: (item) => `${item.detail} ${item.guidance}`,
+      render: (item) => <div>{item.detail}<span className="muted block">{item.guidance}</span></div>
+    }
+  ];
+  return <DataGrid id="ops-runtime-work-status" rows={items} columns={columns} getRowKey={(item) => item.provider_id} emptyText="Worker and queue status unavailable." />;
+}
+
 function RuntimeNodeTable({
   nodes,
   canManage,
@@ -268,7 +384,7 @@ function RuntimeNodeTable({
       resizable: true,
       sortable: true,
       value: (node) => node.last_heartbeat_at,
-      render: (node) => new Date(node.last_heartbeat_at).toLocaleString()
+      render: (node) => <div>{heartbeatAgeLabel(heartbeatAgeSeconds(Date.now(), node.last_heartbeat_at))}<span className="muted block">{new Date(node.last_heartbeat_at).toLocaleString()}</span></div>
     },
     {
       id: "queues",
@@ -545,9 +661,10 @@ function capabilityTone(state: OpsInfrastructureCapability["state"]): string {
 }
 
 function stateTone(state: string): string {
-  if (state === "ok") return "success";
-  if (state === "warning") return "warning";
-  if (state === "error") return "error";
+  if (["ok", "healthy", "idle"].includes(state)) return "success";
+  if (state === "busy") return "info";
+  if (["warning", "starting", "degraded", "unconfigured"].includes(state)) return "warning";
+  if (["error", "stale", "unreachable"].includes(state)) return "error";
   return "inactive";
 }
 
@@ -586,11 +703,17 @@ function databaseCapacityTone(status: OpsStatus | null): "good" | "warning" | "d
 }
 
 function workerMetricDetail(status: OpsStatus | null): string {
-  if (!status?.summary.celery_enabled) return "i18n:govoplan-ops.celery_worker_setting.323d7737";
+  if (!status) return "Worker status unavailable";
   const metrics = status.summary.worker_metrics;
-  if (metrics.missing_queues?.length) return `Missing queues: ${metrics.missing_queues.join(", ")}`;
-  const queued = Object.values(metrics.queue_depths ?? {}).reduce((total, value) => total + value, 0);
-  return `${metrics.active_tasks ?? 0} active · ${queued} queued`;
+  if (metrics.state === "disabled") return "Workers intentionally disabled";
+  const queued = knownQueueDepthTotal(metrics.queue_depths ?? {});
+  return `${knownMetric(metrics.active_tasks)} active · ${knownMetric(metrics.reserved_tasks)} reserved · ${queued ?? "unavailable"} queued`;
+}
+
+function runtimeQueueSummary(depths: Record<string, number | null>): string {
+  const entries = Object.entries(depths);
+  if (!entries.length) return "unavailable";
+  return entries.map(([queue, depth]) => `${queue}: ${depth ?? "unavailable"}`).join(" · ");
 }
 
 function storageMetricDetail(status: OpsStatus | null): string {

webui/src/features/ops/runtimeStatus.ts

@@ -0,0 +1,34 @@
+export function knownMetric(value: number | null | undefined): number | "unavailable" {
+  return typeof value === "number" && Number.isFinite(value) ? value : "unavailable";
+}
+
+export function knownQueueDepthTotal(depths: Record<string, number | null | undefined>): number | null {
+  const values = Object.values(depths).filter((value): value is number => typeof value === "number" && Number.isFinite(value));
+  return values.length ? values.reduce((total, value) => total + value, 0) : null;
+}
+
+export function runtimeWorkTone(state: string): "good" | "info" | "warning" | "danger" | "neutral" {
+  if (state === "healthy" || state === "idle") return "good";
+  if (state === "busy") return "info";
+  if (state === "starting" || state === "degraded" || state === "unconfigured") return "warning";
+  if (state === "stale" || state === "unreachable") return "danger";
+  return "neutral";
+}
+
+export function heartbeatAgeSeconds(now: number, value?: string | null): number | null {
+  if (!value) return null;
+  const timestamp = Date.parse(value);
+  if (!Number.isFinite(timestamp)) return null;
+  return Math.max(0, Math.floor((now - timestamp) / 1000));
+}
+
+export function heartbeatAgeLabel(ageSeconds: number | null): string {
+  if (ageSeconds === null) return "unavailable";
+  if (ageSeconds < 60) return `${ageSeconds}s ago`;
+  if (ageSeconds < 3600) return `${Math.floor(ageSeconds / 60)}m ago`;
+  return `${Math.floor(ageSeconds / 3600)}h ago`;
+}
+
+export function shouldPollRuntimeStatus(hidden: boolean, requestInFlight: boolean): boolean {
+  return !hidden && !requestInFlight;
+}