feat(ops): show worker and queue readiness

2026-08-19 20:52:44 +02:00
parent f84097224b
commit 6ecb94c99e
12 changed files with 559 additions and 161 deletions
@@ -1,6 +1,6 @@
 import { DescriptionList } from "@govoplan/core-webui";
 import { MetricGrid } from "@govoplan/core-webui";
-import { useEffect, useState } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import { PauseCircle, PlayCircle, RefreshCw } from "lucide-react";
 import { ContentGrid,
  ActionBlockerHint,
@@ -33,6 +33,7 @@ import {
  type OpsInfrastructureCapability,
  type OpsRecoveryOperation,
  type OpsRuntimeNode,
+  type OpsRuntimeWorkStatus,
  type OpsSizingAssumption,
  type OpsStatus
 } from "../../api/ops";
@@ -41,6 +42,14 @@ import {
  OPS_I18N,
  OPS_RECOVERY_DOCUMENTATION,
 } from "./interfacePatterns";
+import {
+  heartbeatAgeLabel,
+  heartbeatAgeSeconds,
+  knownMetric,
+  knownQueueDepthTotal,
+  runtimeWorkTone,
+  shouldPollRuntimeStatus
+} from "./runtimeStatus";

 export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth: AuthInfo;}) {
  const [status, setStatus] = useState<OpsStatus | null>(null);
@@ -49,18 +58,27 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
  const [error, setError] = useState("");
  const [drainTarget, setDrainTarget] = useState<OpsRuntimeNode | null>(null);
  const [nodeActionId, setNodeActionId] = useState("");
+  const loadInFlight = useRef<Promise<void> | null>(null);

-  async function load() {
-    setLoading(true);
-    setError("");
-    try {
-      setStatus(await fetchOpsStatus(settings));
-    } catch (err) {
-      setError(adminErrorMessage(err));
-    } finally {
-      setLoading(false);
-    }
-  }
+  const load = useCallback((background = false): Promise<void> => {
+    if (loadInFlight.current) return loadInFlight.current;
+    const request = (async () => {
+      if (!background) setLoading(true);
+      setError("");
+      try {
+        setStatus(await fetchOpsStatus(settings));
+      } catch (err) {
+        setError(adminErrorMessage(err));
+      } finally {
+        if (!background) setLoading(false);
+      }
+    })();
+    loadInFlight.current = request;
+    void request.finally(() => {
+      if (loadInFlight.current === request) loadInFlight.current = null;
+    });
+    return request;
+  }, [settings.accessToken, settings.apiBaseUrl, settings.apiKey]);

  async function runChecks() {
    setRunningProbes(true);
@@ -104,7 +122,37 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
    }
  }

-  useEffect(() => {void load();}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
+  useEffect(() => {
+    let intervalId: number | null = null;
+    const stopPolling = () => {
+      if (intervalId !== null) window.clearInterval(intervalId);
+      intervalId = null;
+    };
+    const startPolling = () => {
+      stopPolling();
+      if (document.visibilityState === "hidden") return;
+      intervalId = window.setInterval(() => {
+        if (shouldPollRuntimeStatus(document.visibilityState === "hidden", Boolean(loadInFlight.current))) {
+          void load(true);
+        }
+      }, 15_000);
+    };
+    const handleVisibility = () => {
+      if (document.visibilityState === "hidden") {
+        stopPolling();
+        return;
+      }
+      if (shouldPollRuntimeStatus(false, Boolean(loadInFlight.current))) void load(true);
+      startPolling();
+    };
+    void load();
+    startPolling();
+    document.addEventListener("visibilitychange", handleVisibility);
+    return () => {
+      stopPolling();
+      document.removeEventListener("visibilitychange", handleVisibility);
+    };
+  }, [load]);

  const checks = status?.checks ?? [];
  const warningCount = checks.filter((item) => item.state === "warning").length;
@@ -119,7 +167,7 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
        ? OPS_I18N.runPermissionRequired
        : undefined;
  const queueDepths = status?.summary.worker_metrics.queue_depths ?? {};
-  const queuedTasks = Object.values(queueDepths).reduce((total, value) => total + value, 0);
+  const queuedTasks = knownQueueDepthTotal(queueDepths);
  const storageUsage = status?.summary.storage_metrics?.capacity_used_percent;

  return (
@@ -156,8 +204,8 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
          <MetricCard label="i18n:govoplan-ops.modules.04e9462c" value={status?.summary.module_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.enabled_in_the_runtime_registry.d2c6142d" />
          <MetricCard label="i18n:govoplan-ops.permissions.842c35eb" value={status?.governance.summary.permission_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.declared_governance_permissions.d08d3bf1" />
          <MetricCard label="i18n:govoplan-ops.policies.e7800f56" value={status?.governance.summary.policy_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.registered_policy_capabilities.112a2b64" />
-          <MetricCard label="i18n:govoplan-ops.workers.b6ef3acd" value={status?.summary.celery_enabled ? status.summary.worker_metrics.workers ?? 0 : "off"} tone={status?.summary.celery_enabled && !(status.summary.worker_metrics.missing_queues?.length) ? "good" : "warning"} detail={workerMetricDetail(status)} />
-          <MetricCard label="Queued tasks" value={queuedTasks} tone={queuedTasks ? "warning" : "good"} detail={Object.keys(queueDepths).length ? `${Object.keys(queueDepths).length} measured queue(s)` : "Queue depth unavailable"} />
+          <MetricCard label="i18n:govoplan-ops.workers.b6ef3acd" value={knownMetric(status?.summary.worker_metrics.workers)} tone={runtimeWorkTone(status?.summary.worker_metrics.state ?? "unreachable")} detail={workerMetricDetail(status)} />
+          <MetricCard label="Queued tasks" value={queuedTasks ?? "unavailable"} tone={queuedTasks === null ? "neutral" : queuedTasks ? "warning" : "good"} detail={queuedTasks === null ? "Queue depth unavailable" : `${Object.values(queueDepths).filter((value) => typeof value === "number").length} measured queue(s)`} />
          <MetricCard label="Storage" value={storageUsage === undefined ? status?.summary.file_storage_backend ?? "-" : `${storageUsage}%`} tone={storageUsage !== undefined && storageUsage >= 90 ? "danger" : storageUsage !== undefined && storageUsage >= 75 ? "warning" : "neutral"} detail={storageMetricDetail(status)} />
          <MetricCard label="Backup evidence" value={status?.summary.backup_state ?? "unknown"} tone={status?.summary.backup_state === "ok" ? "good" : "warning"} detail="Latest coordinated backup and restore-drill evidence" />
          <MetricCard label="Failed operations" value={status?.summary.failed_operation_count ?? 0} tone={status?.summary.failed_operation_count ? "danger" : "good"} detail="Terminal failures or manual intervention" />
@@ -186,6 +234,10 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
            />
          </Card>

+          <Card title="Worker and queue readiness">
+            <RuntimeWorkTable items={status?.runtime_work ?? []} />
+          </Card>
+
          <Card title="Recovery evidence">
            <RecoveryTable operations={status?.runtime_cluster.recovery.operations ?? []} />
          </Card>
@@ -226,6 +278,70 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:

 }

+function RuntimeWorkTable({ items }: { items: OpsRuntimeWorkStatus[] }) {
+  const columns: DataGridColumn<OpsRuntimeWorkStatus>[] = [
+    {
+      id: "provider",
+      header: "Backend",
+      width: "minmax(200px, 1fr)",
+      minWidth: 180,
+      resizable: true,
+      sortable: true,
+      filterable: true,
+      value: (item) => `${item.label} ${item.backend}`,
+      render: (item) => <div><strong>{item.label}</strong><span className="muted block">{item.backend} · {item.provider_id}</span></div>
+    },
+    {
+      id: "state",
+      header: "State",
+      width: 150,
+      sortable: true,
+      filterable: true,
+      value: (item) => item.state,
+      render: (item) => <StatusBadge status={stateTone(item.state)} label={item.state} />
+    },
+    {
+      id: "activity",
+      header: "Activity",
+      width: "minmax(180px, .7fr)",
+      minWidth: 170,
+      value: (item) => `${item.active_workers ?? ""} ${item.active_work ?? ""} ${item.reserved_work ?? ""}`,
+      render: (item) => `${knownMetric(item.active_workers)} workers · ${knownMetric(item.active_work)} active · ${knownMetric(item.reserved_work)} reserved`
+    },
+    {
+      id: "queues",
+      header: "Queue depth",
+      width: "minmax(220px, 1fr)",
+      minWidth: 190,
+      resizable: true,
+      value: (item) => Object.entries(item.queue_depths).map(([queue, depth]) => `${queue}:${depth ?? "unavailable"}`).join(" "),
+      render: (item) => runtimeQueueSummary(item.queue_depths)
+    },
+    {
+      id: "heartbeat",
+      header: "Last heartbeat",
+      width: "minmax(180px, .7fr)",
+      minWidth: 170,
+      sortable: true,
+      value: (item) => item.last_heartbeat_at ?? "",
+      render: (item) => {
+        const age = heartbeatAgeSeconds(Date.now(), item.last_heartbeat_at);
+        return <div>{heartbeatAgeLabel(age)}<span className="muted block">stale after {item.stale_after_seconds ?? "unavailable"}s</span></div>;
+      }
+    },
+    {
+      id: "guidance",
+      header: "Guidance",
+      width: "minmax(260px, 1.3fr)",
+      minWidth: 220,
+      resizable: true,
+      value: (item) => `${item.detail} ${item.guidance}`,
+      render: (item) => <div>{item.detail}<span className="muted block">{item.guidance}</span></div>
+    }
+  ];
+  return <DataGrid id="ops-runtime-work-status" rows={items} columns={columns} getRowKey={(item) => item.provider_id} emptyText="Worker and queue status unavailable." />;
+}
+
 function RuntimeNodeTable({
  nodes,
  canManage,
@@ -268,7 +384,7 @@ function RuntimeNodeTable({
      resizable: true,
      sortable: true,
      value: (node) => node.last_heartbeat_at,
-      render: (node) => new Date(node.last_heartbeat_at).toLocaleString()
+      render: (node) => <div>{heartbeatAgeLabel(heartbeatAgeSeconds(Date.now(), node.last_heartbeat_at))}<span className="muted block">{new Date(node.last_heartbeat_at).toLocaleString()}</span></div>
    },
    {
      id: "queues",
@@ -545,9 +661,10 @@ function capabilityTone(state: OpsInfrastructureCapability["state"]): string {
 }

 function stateTone(state: string): string {
-  if (state === "ok") return "success";
-  if (state === "warning") return "warning";
-  if (state === "error") return "error";
+  if (["ok", "healthy", "idle"].includes(state)) return "success";
+  if (state === "busy") return "info";
+  if (["warning", "starting", "degraded", "unconfigured"].includes(state)) return "warning";
+  if (["error", "stale", "unreachable"].includes(state)) return "error";
  return "inactive";
 }

@@ -586,11 +703,17 @@ function databaseCapacityTone(status: OpsStatus | null): "good" | "warning" | "d
 }

 function workerMetricDetail(status: OpsStatus | null): string {
-  if (!status?.summary.celery_enabled) return "i18n:govoplan-ops.celery_worker_setting.323d7737";
+  if (!status) return "Worker status unavailable";
  const metrics = status.summary.worker_metrics;
-  if (metrics.missing_queues?.length) return `Missing queues: ${metrics.missing_queues.join(", ")}`;
-  const queued = Object.values(metrics.queue_depths ?? {}).reduce((total, value) => total + value, 0);
-  return `${metrics.active_tasks ?? 0} active · ${queued} queued`;
+  if (metrics.state === "disabled") return "Workers intentionally disabled";
+  const queued = knownQueueDepthTotal(metrics.queue_depths ?? {});
+  return `${knownMetric(metrics.active_tasks)} active · ${knownMetric(metrics.reserved_tasks)} reserved · ${queued ?? "unavailable"} queued`;
+}
+
+function runtimeQueueSummary(depths: Record<string, number | null>): string {
+  const entries = Object.entries(depths);
+  if (!entries.length) return "unavailable";
+  return entries.map(([queue, depth]) => `${queue}: ${depth ?? "unavailable"}`).join(" · ");
 }

 function storageMetricDetail(status: OpsStatus | null): string {