Add module probes and worker telemetry

This commit is contained in:
2026-07-31 22:48:07 +02:00
parent b84cab1aa4
commit 05ce4dc8ec
7 changed files with 422 additions and 14 deletions
+14
View File
@@ -6,6 +6,7 @@ export type OpsCheck = {
state: "ok" | "warning" | "error" | string;
detail: string;
readiness_critical?: boolean;
metrics?: Record<string, unknown>;
};
export type OpsDeploymentProfile = {
@@ -52,6 +53,15 @@ export type OpsStatus = {
};
database_url: string;
file_storage_backend: string;
worker_metrics: {
workers?: number;
active_tasks?: number;
expected_queues?: string[];
active_queues?: string[];
missing_queues?: string[];
queue_depths?: Record<string, number>;
};
operational_probe_count: number;
};
readiness: {
ready: boolean;
@@ -85,3 +95,7 @@ export type OpsStatus = {
export function fetchOpsStatus(settings: ApiSettings): Promise<OpsStatus> {
return apiFetch(settings, "/api/v1/ops/status");
}
export function runOpsChecks(settings: ApiSettings): Promise<OpsStatus> {
return apiFetch(settings, "/api/v1/ops/checks/run", { method: "POST" });
}
+9 -2
View File
@@ -29,13 +29,21 @@ export default function OpsHealthWidget({ settings, refreshKey }: { settings: Ap
const warningCount = checks.filter((item) => item.state === "warning").length;
const errorCount = checks.filter((item) => item.state === "error").length;
const ready = status?.readiness.ready ?? false;
const workerMetrics = status?.summary.worker_metrics;
const queueDepths = workerMetrics?.queue_depths ?? {};
const queuedTasks = Object.values(queueDepths).reduce((sum, value) => sum + value, 0);
const missingQueues = workerMetrics?.missing_queues ?? [];
return (
<LoadingFrame loading={loading} label="Loading operations status">
{error && <DismissibleAlert tone="warning" resetKey={error}>{error}</DismissibleAlert>}
<div className="metric-grid inside dashboard-widget-metrics">
<MetricCard label="Readiness" value={ready ? "ready" : "blocked"} tone={ready ? "good" : "danger"} detail={status?.readiness.profile ?? "-"} />
<MetricCard label="Workers" value={status?.summary.celery_enabled ? "split" : "off"} tone={status?.summary.celery_enabled ? "good" : "warning"} detail={status?.summary.celery_queues?.length ? status.summary.celery_queues.join(", ") : "single-process"} />
<MetricCard label="Workers" value={status?.summary.celery_enabled ? workerMetrics?.workers ?? 0 : "off"} tone={status?.summary.celery_enabled && !missingQueues.length ? "good" : "warning"} detail={missingQueues.length ? `${missingQueues.length} queue(s) without a consumer` : status?.summary.celery_enabled ? "All configured queues covered" : "Single-process mode"} />
<MetricCard label="Active tasks" value={workerMetrics?.active_tasks ?? 0} tone="info" detail={status?.summary.celery_enabled ? "Reported by live workers" : "Workers disabled"} />
<MetricCard label="Queued tasks" value={queuedTasks} tone={queuedTasks ? "warning" : "neutral"} detail={Object.keys(queueDepths).length ? `${Object.keys(queueDepths).length} measured queue(s)` : "Queue depth unavailable"} />
<MetricCard label="Storage" value={status?.summary.file_storage_backend ?? "-"} tone="neutral" detail="Managed Files backend" />
<MetricCard label="Probes" value={status?.summary.operational_probe_count ?? 0} tone="neutral" detail="Module-owned operational checks" />
<MetricCard label="Warnings" value={warningCount + errorCount} tone={errorCount ? "danger" : warningCount ? "warning" : "good"} detail="Current health checks" />
</div>
{status?.readiness.blockers.length ?
@@ -51,4 +59,3 @@ export default function OpsHealthWidget({ settings, refreshKey }: { settings: Ap
}
</LoadingFrame>);
}
+27 -2
View File
@@ -11,11 +11,14 @@ import {
PageTitle,
StatusBadge,
adminErrorMessage,
hasAnyScope,
type ApiSettings,
type AuthInfo,
type DataGridColumn } from
"@govoplan/core-webui";
import {
fetchOpsStatus,
runOpsChecks,
type OpsCheck,
type OpsDeploymentProfile,
type OpsGovernanceModule,
@@ -23,7 +26,7 @@ import {
type OpsStatus
} from "../../api/ops";
export default function OpsPage({ settings }: {settings: ApiSettings;}) {
export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth: AuthInfo;}) {
const [status, setStatus] = useState<OpsStatus | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState("");
@@ -40,12 +43,25 @@ export default function OpsPage({ settings }: {settings: ApiSettings;}) {
}
}
async function runChecks() {
setLoading(true);
setError("");
try {
setStatus(await runOpsChecks(settings));
} catch (err) {
setError(adminErrorMessage(err));
} finally {
setLoading(false);
}
}
useEffect(() => {void load();}, [settings.apiBaseUrl, settings.apiKey, settings.accessToken]);
const checks = status?.checks ?? [];
const warningCount = checks.filter((item) => item.state === "warning").length;
const errorCount = checks.filter((item) => item.state === "error").length;
const ready = status?.readiness.ready ?? false;
const canRunChecks = hasAnyScope(auth, ["ops:operations:run", "system:settings:write"]);
return (
<PageScrollViewport>
@@ -56,6 +72,7 @@ export default function OpsPage({ settings }: {settings: ApiSettings;}) {
<p>i18n:govoplan-ops.runtime_health_deployment_profile_worker_split_a.55340156</p>
</div>
<div className="button-row compact-actions">
{canRunChecks ? <Button variant="primary" onClick={() => void runChecks()} disabled={loading}>Run probes</Button> : null}
<Button onClick={() => void load()} disabled={loading}><RefreshCw size={16} /> i18n:govoplan-ops.reload.cce71553</Button>
</div>
</div>
@@ -69,7 +86,7 @@ export default function OpsPage({ settings }: {settings: ApiSettings;}) {
<MetricCard label="i18n:govoplan-ops.modules.04e9462c" value={status?.summary.module_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.enabled_in_the_runtime_registry.d2c6142d" />
<MetricCard label="i18n:govoplan-ops.permissions.842c35eb" value={status?.governance.summary.permission_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.declared_governance_permissions.d08d3bf1" />
<MetricCard label="i18n:govoplan-ops.policies.e7800f56" value={status?.governance.summary.policy_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.registered_policy_capabilities.112a2b64" />
<MetricCard label="i18n:govoplan-ops.workers.b6ef3acd" value={status?.summary.celery_enabled ? "split" : "off"} tone={status?.summary.celery_enabled ? "good" : "warning"} detail={status?.summary.celery_queues?.length ? status.summary.celery_queues.join(", ") : "i18n:govoplan-ops.celery_worker_setting.323d7737"} />
<MetricCard label="i18n:govoplan-ops.workers.b6ef3acd" value={status?.summary.celery_enabled ? status.summary.worker_metrics.workers ?? 0 : "off"} tone={status?.summary.celery_enabled && !(status.summary.worker_metrics.missing_queues?.length) ? "good" : "warning"} detail={workerMetricDetail(status)} />
<MetricCard label="i18n:govoplan-ops.redis.5eaa1f2f" value={status?.summary.redis_url ? "configured" : "-"} tone={status?.summary.celery_enabled ? "info" : "neutral"} detail={status?.summary.redis_url ?? "-"} />
<MetricCard label="i18n:govoplan-ops.warnings.1430f976" value={warningCount + errorCount} tone={errorCount ? "danger" : warningCount ? "warning" : "good"} detail="i18n:govoplan-ops.current_health_checks.7830bccf" />
</div>
@@ -209,3 +226,11 @@ function stateTone(state: string): string {
if (state === "error") return "error";
return "inactive";
}
function workerMetricDetail(status: OpsStatus | null): string {
if (!status?.summary.celery_enabled) return "i18n:govoplan-ops.celery_worker_setting.323d7737";
const metrics = status.summary.worker_metrics;
if (metrics.missing_queues?.length) return `Missing queues: ${metrics.missing_queues.join(", ")}`;
const queued = Object.values(metrics.queue_depths ?? {}).reduce((total, value) => total + value, 0);
return `${metrics.active_tasks ?? 0} active · ${queued} queued`;
}
+1 -1
View File
@@ -40,7 +40,7 @@ export const opsModule: PlatformWebModule = {
],
navItems: [{ to: "/ops", label: "i18n:govoplan-ops.ops.907a54c2", iconName: "activity", anyOf: opsReadScopes, order: 890 }],
routes: [
{ path: "/ops", anyOf: opsReadScopes, order: 890, render: ({ settings }) => createElement(OpsPage, { settings }) }],
{ path: "/ops", anyOf: opsReadScopes, order: 890, render: ({ settings, auth }) => createElement(OpsPage, { settings, auth }) }],
uiCapabilities: {
"dashboard.widgets": dashboardWidgets
}