Complete operational monitoring summary
This commit is contained in:
@@ -137,6 +137,13 @@ export type OpsRuntimeCluster = {
|
||||
recovery: {
|
||||
operations: OpsRecoveryOperation[];
|
||||
requires_attention: number;
|
||||
metrics?: {
|
||||
failed?: number;
|
||||
outcome_unknown?: number;
|
||||
recovery_required?: number;
|
||||
active?: number;
|
||||
requires_attention?: number;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
@@ -156,6 +163,15 @@ export type OpsStatus = {
|
||||
database_connection_peak?: number | null;
|
||||
database_connection_available?: number | null;
|
||||
file_storage_backend: string;
|
||||
storage_metrics?: {
|
||||
backend?: string;
|
||||
capacity_observable?: boolean;
|
||||
capacity_total_bytes?: number;
|
||||
capacity_used_bytes?: number;
|
||||
capacity_free_bytes?: number;
|
||||
capacity_used_percent?: number;
|
||||
};
|
||||
backup_state?: string;
|
||||
worker_metrics: {
|
||||
workers?: number;
|
||||
active_tasks?: number;
|
||||
@@ -167,6 +183,9 @@ export type OpsStatus = {
|
||||
operational_probe_count: number;
|
||||
runtime_node_count: number;
|
||||
recovery_required_count: number;
|
||||
failed_operation_count?: number;
|
||||
outcome_unknown_count?: number;
|
||||
active_operation_count?: number;
|
||||
};
|
||||
readiness: {
|
||||
ready: boolean;
|
||||
|
||||
@@ -33,6 +33,7 @@ export default function OpsHealthWidget({ settings, refreshKey }: { settings: Ap
|
||||
const queueDepths = workerMetrics?.queue_depths ?? {};
|
||||
const queuedTasks = Object.values(queueDepths).reduce((sum, value) => sum + value, 0);
|
||||
const missingQueues = workerMetrics?.missing_queues ?? [];
|
||||
const storageUsage = status?.summary.storage_metrics?.capacity_used_percent;
|
||||
|
||||
return (
|
||||
<LoadingFrame loading={loading} label="Loading operations status">
|
||||
@@ -42,7 +43,10 @@ export default function OpsHealthWidget({ settings, refreshKey }: { settings: Ap
|
||||
<MetricCard label="Workers" value={status?.summary.celery_enabled ? workerMetrics?.workers ?? 0 : "off"} tone={status?.summary.celery_enabled && !missingQueues.length ? "good" : "warning"} detail={missingQueues.length ? `${missingQueues.length} queue(s) without a consumer` : status?.summary.celery_enabled ? "All configured queues covered" : "Single-process mode"} />
|
||||
<MetricCard label="Active tasks" value={workerMetrics?.active_tasks ?? 0} tone="info" detail={status?.summary.celery_enabled ? "Reported by live workers" : "Workers disabled"} />
|
||||
<MetricCard label="Queued tasks" value={queuedTasks} tone={queuedTasks ? "warning" : "neutral"} detail={Object.keys(queueDepths).length ? `${Object.keys(queueDepths).length} measured queue(s)` : "Queue depth unavailable"} />
|
||||
<MetricCard label="Storage" value={status?.summary.file_storage_backend ?? "-"} tone="neutral" detail="Managed Files backend" />
|
||||
<MetricCard label="Storage" value={storageUsage === undefined ? status?.summary.file_storage_backend ?? "-" : `${storageUsage}%`} tone={storageUsage !== undefined && storageUsage >= 90 ? "danger" : storageUsage !== undefined && storageUsage >= 75 ? "warning" : "neutral"} detail="Managed Files backend capacity" />
|
||||
<MetricCard label="Failed" value={status?.summary.failed_operation_count ?? 0} tone={status?.summary.failed_operation_count ? "danger" : "good"} detail="Recovery-ledger operations" />
|
||||
<MetricCard label="Unknown" value={status?.summary.outcome_unknown_count ?? 0} tone={status?.summary.outcome_unknown_count ? "danger" : "good"} detail="Outcome-unknown operations" />
|
||||
<MetricCard label="Backup" value={status?.summary.backup_state ?? "unknown"} tone={status?.summary.backup_state === "ok" ? "good" : "warning"} detail="Backup and restore evidence" />
|
||||
<MetricCard label="Probes" value={status?.summary.operational_probe_count ?? 0} tone="neutral" detail="Module-owned operational checks" />
|
||||
<MetricCard label="Warnings" value={warningCount + errorCount} tone={errorCount ? "danger" : warningCount ? "warning" : "good"} detail="Current health checks" />
|
||||
</div>
|
||||
|
||||
@@ -97,6 +97,9 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
|
||||
const errorCount = checks.filter((item) => item.state === "error").length;
|
||||
const ready = status?.readiness.ready ?? false;
|
||||
const canRunChecks = hasAnyScope(auth, ["ops:operations:run", "system:settings:write"]);
|
||||
const queueDepths = status?.summary.worker_metrics.queue_depths ?? {};
|
||||
const queuedTasks = Object.values(queueDepths).reduce((total, value) => total + value, 0);
|
||||
const storageUsage = status?.summary.storage_metrics?.capacity_used_percent;
|
||||
|
||||
return (
|
||||
<PageScrollViewport>
|
||||
@@ -122,6 +125,11 @@ export default function OpsPage({ settings, auth }: {settings: ApiSettings;auth:
|
||||
<MetricCard label="i18n:govoplan-ops.permissions.842c35eb" value={status?.governance.summary.permission_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.declared_governance_permissions.d08d3bf1" />
|
||||
<MetricCard label="i18n:govoplan-ops.policies.e7800f56" value={status?.governance.summary.policy_count ?? 0} tone="neutral" detail="i18n:govoplan-ops.registered_policy_capabilities.112a2b64" />
|
||||
<MetricCard label="i18n:govoplan-ops.workers.b6ef3acd" value={status?.summary.celery_enabled ? status.summary.worker_metrics.workers ?? 0 : "off"} tone={status?.summary.celery_enabled && !(status.summary.worker_metrics.missing_queues?.length) ? "good" : "warning"} detail={workerMetricDetail(status)} />
|
||||
<MetricCard label="Queued tasks" value={queuedTasks} tone={queuedTasks ? "warning" : "good"} detail={Object.keys(queueDepths).length ? `${Object.keys(queueDepths).length} measured queue(s)` : "Queue depth unavailable"} />
|
||||
<MetricCard label="Storage" value={storageUsage === undefined ? status?.summary.file_storage_backend ?? "-" : `${storageUsage}%`} tone={storageUsage !== undefined && storageUsage >= 90 ? "danger" : storageUsage !== undefined && storageUsage >= 75 ? "warning" : "neutral"} detail={storageMetricDetail(status)} />
|
||||
<MetricCard label="Backup evidence" value={status?.summary.backup_state ?? "unknown"} tone={status?.summary.backup_state === "ok" ? "good" : "warning"} detail="Latest coordinated backup and restore-drill evidence" />
|
||||
<MetricCard label="Failed operations" value={status?.summary.failed_operation_count ?? 0} tone={status?.summary.failed_operation_count ? "danger" : "good"} detail="Terminal failures or manual intervention" />
|
||||
<MetricCard label="Unknown outcomes" value={status?.summary.outcome_unknown_count ?? 0} tone={status?.summary.outcome_unknown_count ? "danger" : "good"} detail={`${status?.summary.active_operation_count ?? 0} active recovery-ledger operation(s)`} />
|
||||
<MetricCard label="i18n:govoplan-ops.redis.5eaa1f2f" value={status?.summary.redis_url ? "configured" : "-"} tone={status?.summary.celery_enabled ? "info" : "neutral"} detail={status?.summary.redis_url ?? "-"} />
|
||||
<MetricCard label="i18n:govoplan-ops.warnings.1430f976" value={warningCount + errorCount} tone={errorCount ? "danger" : warningCount ? "warning" : "good"} detail="i18n:govoplan-ops.current_health_checks.7830bccf" />
|
||||
<MetricCard label="Runtime nodes" value={status?.summary.runtime_node_count ?? 0} tone={status?.runtime_cluster.available ? "good" : "danger"} detail={runtimeNodeMetricDetail(status)} />
|
||||
@@ -506,3 +514,18 @@ function workerMetricDetail(status: OpsStatus | null): string {
|
||||
const queued = Object.values(metrics.queue_depths ?? {}).reduce((total, value) => total + value, 0);
|
||||
return `${metrics.active_tasks ?? 0} active · ${queued} queued`;
|
||||
}
|
||||
|
||||
function storageMetricDetail(status: OpsStatus | null): string {
|
||||
const metrics = status?.summary.storage_metrics;
|
||||
if (!metrics?.capacity_observable) {
|
||||
return `${status?.summary.file_storage_backend ?? "Storage"} capacity is provider-managed or unavailable`;
|
||||
}
|
||||
return `${formatBytes(metrics.capacity_used_bytes ?? 0)} used of ${formatBytes(metrics.capacity_total_bytes ?? 0)}`;
|
||||
}
|
||||
|
||||
function formatBytes(value: number): string {
|
||||
if (!Number.isFinite(value) || value <= 0) return "0 B";
|
||||
const units = ["B", "KB", "MB", "GB", "TB", "PB"];
|
||||
const index = Math.min(Math.floor(Math.log(value) / Math.log(1024)), units.length - 1);
|
||||
return `${(value / 1024 ** index).toFixed(index ? 1 : 0)} ${units[index]}`;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user