Add support for system metrics (G-Research#33)

vinayan3 · Aug 28, 2024 · f361332 · f361332
1 parent b82ccdb
commit f361332
Show file tree

Hide file tree

Showing 7 changed files with 159 additions and 29 deletions.
diff --git a/src/src/config/systemMetrics/systemMetrics.ts b/src/src/config/systemMetrics/systemMetrics.ts
@@ -1,12 +1,60 @@
 import { systemMetricsDictType } from 'types/utils/formatSystemMetricName';
 
 export const systemMetricsDict: systemMetricsDictType = {
-  __system__cpu: 'CPU (%)',
-  __system__p_memory_percent: 'Process Memory (%)',
-  __system__memory_percent: 'Memory (%)',
-  __system__disk_percent: 'Disk (%)',
-  __system__gpu: 'GPU (%)',
-  __system__gpu_memory_percent: 'GPU Memory (%)',
-  __system__gpu_power_watts: 'GPU Power (W)',
-  __system__gpu_temp: 'GPU Temperature (°C)',
+  'system/cpu_utilization_percentage': 'CPU (%)',
+  'system/disk_available_megabytes': 'Disk Available (MB)',
+  'system/disk_usage_megabytes': 'Disk (MB)',
+  'system/disk_usage_percentage': 'Disk (%)',
+  'system/gpu_0_memory_usage_megabytes': 'GPU 0 Memory (MB)',
+  'system/gpu_0_memory_usage_percentage': 'GPU 0 Memory (%)',
+  'system/gpu_0_utilization_percentage': 'GPU 0 (%)',
+  'system/gpu_1_memory_usage_megabytes': 'GPU 1 Memory (MB)',
+  'system/gpu_1_memory_usage_percentage': 'GPU 1 Memory (%)',
+  'system/gpu_1_utilization_percentage': 'GPU 1 (%)',
+  'system/gpu_2_memory_usage_megabytes': 'GPU 2 Memory (MB)',
+  'system/gpu_2_memory_usage_percentage': 'GPU 2 Memory (%)',
+  'system/gpu_2_utilization_percentage': 'GPU 2 (%)',
+  'system/gpu_3_memory_usage_megabytes': 'GPU 3 Memory (MB)',
+  'system/gpu_3_memory_usage_percentage': 'GPU 3 Memory (%)',
+  'system/gpu_3_utilization_percentage': 'GPU 3 (%)',
+  'system/gpu_4_memory_usage_megabytes': 'GPU 4 Memory (MB)',
+  'system/gpu_4_memory_usage_percentage': 'GPU 4 Memory (%)',
+  'system/gpu_4_utilization_percentage': 'GPU 4 (%)',
+  'system/gpu_5_memory_usage_megabytes': 'GPU 5 Memory (MB)',
+  'system/gpu_5_memory_usage_percentage': 'GPU 5 Memory (%)',
+  'system/gpu_5_utilization_percentage': 'GPU 5 (%)',
+  'system/gpu_6_memory_usage_megabytes': 'GPU 6 Memory (MB)',
+  'system/gpu_6_memory_usage_percentage': 'GPU 6 Memory (%)',
+  'system/gpu_6_utilization_percentage': 'GPU 6 (%)',
+  'system/gpu_7_memory_usage_megabytes': 'GPU 7 Memory (MB)',
+  'system/gpu_7_memory_usage_percentage': 'GPU 7 Memory (%)',
+  'system/gpu_7_utilization_percentage': 'GPU 7 (%)',
+  'system/gpu_8_memory_usage_megabytes': 'GPU 8 Memory (MB)',
+  'system/gpu_8_memory_usage_percentage': 'GPU 8 Memory (%)',
+  'system/gpu_8_utilization_percentage': 'GPU 8 (%)',
+  'system/gpu_9_memory_usage_megabytes': 'GPU 9 Memory (MB)',
+  'system/gpu_9_memory_usage_percentage': 'GPU 9 Memory (%)',
+  'system/gpu_9_utilization_percentage': 'GPU 9 (%)',
+  'system/gpu_10_memory_usage_megabytes': 'GPU 10 Memory (MB)',
+  'system/gpu_10_memory_usage_percentage': 'GPU 10 Memory (%)',
+  'system/gpu_10_utilization_percentage': 'GPU 10 (%)',
+  'system/gpu_11_memory_usage_megabytes': 'GPU 11 Memory (MB)',
+  'system/gpu_11_memory_usage_percentage': 'GPU 11 Memory (%)',
+  'system/gpu_11_utilization_percentage': 'GPU 11 (%)',
+  'system/gpu_12_memory_usage_megabytes': 'GPU 12 Memory (MB)',
+  'system/gpu_12_memory_usage_percentage': 'GPU 12 Memory (%)',
+  'system/gpu_12_utilization_percentage': 'GPU 12 (%)',
+  'system/gpu_13_memory_usage_megabytes': 'GPU 13 Memory (MB)',
+  'system/gpu_13_memory_usage_percentage': 'GPU 13 Memory (%)',
+  'system/gpu_13_utilization_percentage': 'GPU 13 (%)',
+  'system/gpu_14_memory_usage_megabytes': 'GPU 14 Memory (MB)',
+  'system/gpu_14_memory_usage_percentage': 'GPU 14 Memory (%)',
+  'system/gpu_14_utilization_percentage': 'GPU 14 (%)',
+  'system/gpu_15_memory_usage_megabytes': 'GPU 15 Memory (MB)',
+  'system/gpu_15_memory_usage_percentage': 'GPU 15 Memory (%)',
+  'system/gpu_15_utilization_percentage': 'GPU 15 (%)',
+  'system/network_receive_megabytes': 'Network Rx (MB)',
+  'system/network_transmit_megabytes': 'Network Tx (MB)',
+  'system/system_memory_usage_megabytes': 'Memory (MB)',
+  'system/system_memory_usage_percentage': 'Memory (%)',
 };
diff --git a/src/src/pages/Dashboard/components/ProjectStatistics/ProjectStatistics.tsx b/src/src/pages/Dashboard/components/ProjectStatistics/ProjectStatistics.tsx
@@ -22,6 +22,16 @@ const statisticsInitialMap: Record<string, IProjectStatistic> = {
     iconBgColor: '#7A4CE0',
     navLink: routes.METRICS.path,
   },
+  systemMetrics: {
+    label: 'Sys. metrics',
+    count: 0,
+    icon: 'metrics',
+    iconBgColor: '#AF4EAB',
+    navLink: `${routes.METRICS.path}?select=${encode({
+      advancedQuery: "metric.name.startswith('system/')",
+      advancedMode: true,
+    })}`,
+  },
 };
 
 const runsCountingInitialMap: Record<'archived' | 'runs', IProjectStatistic> = {
@@ -59,7 +69,7 @@ function ProjectStatistics() {
       let systemMetricsCount = 0;
       let sequenceItemsCount = 0;
       for (let [itemKey, itemData] of Object.entries(seqData)) {
-        if (itemKey.startsWith('__system__')) {
+        if (itemKey.startsWith('system/')) {
           systemMetricsCount += itemData.length;
         } else {
           sequenceItemsCount += itemData.length;

diff --git a/src/src/pages/RunDetail/RunDetail.tsx b/src/src/pages/RunDetail/RunDetail.tsx
@@ -66,6 +66,7 @@ const tabs: Record<string, string> = {
   overview: 'Overview',
   run_parameters: 'Run Params',
   metrics: 'Metrics',
+  system: 'System',
   settings: 'Settings',
 };
 
@@ -117,6 +118,15 @@ function RunDetail(): React.FunctionComponentElement<React.ReactNode> {
         isRunBatchLoading={runData?.isRunBatchLoading}
       />
     ),
+    system: (
+      <RunDetailMetricsAndSystemTab
+        runHash={runHash}
+        runTraces={runData?.runTraces}
+        runBatch={runData?.runSystemBatch}
+        isSystem
+        isRunBatchLoading={runData?.isRunBatchLoading}
+      />
+    ),
     settings: (
       <RunDetailSettingsTab
         isArchived={runData?.runInfo?.archived}

diff --git a/src/src/pages/RunDetail/RunOverviewTab/components/RunOverviewSidebar/RunOverviewSidebar.tsx b/src/src/pages/RunDetail/RunOverviewTab/components/RunOverviewSidebar/RunOverviewSidebar.tsx
@@ -52,13 +52,18 @@ function RunOverviewSidebar({
   const insightsList = React.useMemo(() => {
     const path = url.split('/').slice(0, -1).join('/');
     const systemMetricsLength: number =
-      traces.metric.filter((m) => m.name.startsWith('__system__')).length || 0;
+      traces.metric.filter((m) => m.name.startsWith('system/')).length || 0;
     return [
       {
         name: 'Metrics',
         path: `${path}/metrics`,
         value: traces?.metric?.length - systemMetricsLength || 0,
       },
+      {
+        name: 'System',
+        path: `${path}/system`,
+        value: systemMetricsLength,
+      },
     ];
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [traces]);

diff --git a/src/src/pages/Runs/components/RunsTableGrid/RunsTableGrid.tsx b/src/src/pages/Runs/components/RunsTableGrid/RunsTableGrid.tsx
@@ -14,7 +14,6 @@ import { ITableColumn } from 'types/pages/metrics/components/TableColumns/TableC
 import { ITagInfo } from 'types/pages/tags/Tags';
 
 import { formatSystemMetricName } from 'utils/formatSystemMetricName';
-import alphabeticalSortComparator from 'utils/alphabeticalSortComparator';
 import { getMetricHash } from 'utils/app/getMetricHash';
 import { getMetricLabel } from 'utils/app/getMetricLabel';
 import { isSystemMetric } from 'utils/isSystemMetric';
@@ -149,14 +148,24 @@ function getRunsTableColumns(
         };
         isSystem ? systemMetricsList.push(column) : metricsList.push(column);
       });
-      acc = [
-        ...acc,
-        ...metricsList.sort(alphabeticalSortComparator({ orderBy: 'key' })),
-        ...systemMetricsList.sort(
-          alphabeticalSortComparator({ orderBy: 'key' }),
-        ),
-      ];
-      return acc;
+      acc = [...acc, ...metricsList, ...systemMetricsList];
+      return acc.sort((a: ITableColumn, b: ITableColumn) => {
+        const aIsSystem = isSystemMetric(a['key']);
+        const bIsSystem = isSystemMetric(b['key']);
+        if (aIsSystem && !bIsSystem) {
+          return -1;
+        } else if (!aIsSystem && bIsSystem) {
+          return 1;
+        }
+        const aLabel = (a['label'] as string).toUpperCase();
+        const bLabel = (b['label'] as string).toUpperCase();
+        if (aLabel < bLabel) {
+          return -1;
+        } else if (aLabel > bLabel) {
+          return 1;
+        }
+        return 0;
+      });
     }, []),
     runColumns.map((param) => ({
       key: param,

diff --git a/src/src/services/models/runs/runDetailAppModel.ts b/src/src/services/models/runs/runDetailAppModel.ts
@@ -147,7 +147,7 @@ function processRunBatchData(
       }),
       sortKey: `${run.name}${contextName}`,
     };
-    if (run.name.startsWith('__system__')) {
+    if (run.name.startsWith('system/')) {
       runSystemBatch.push(metric);
     } else {
       runMetricsBatch.push(metric);

diff --git a/src/src/types/utils/formatSystemMetricName.d.ts b/src/src/types/utils/formatSystemMetricName.d.ts
@@ -1,10 +1,58 @@
 export type systemMetricsDictType = {
-  __system__cpu: string;
-  __system__p_memory_percent: string;
-  __system__memory_percent: string;
-  __system__disk_percent: string;
-  __system__gpu: string;
-  __system__gpu_memory_percent: string;
-  __system__gpu_power_watts: string;
-  __system__gpu_temp: string;
-};
+  'system/cpu_utilization_percentage': string;
+  'system/disk_available_megabytes': string;
+  'system/disk_usage_megabytes': string;
+  'system/disk_usage_percentage': string;
+  'system/gpu_0_memory_usage_megabytes': string;
+  'system/gpu_0_memory_usage_percentage': string;
+  'system/gpu_0_utilization_percentage': string;
+  'system/gpu_1_memory_usage_megabytes': string;
+  'system/gpu_1_memory_usage_percentage': string;
+  'system/gpu_1_utilization_percentage': string;
+  'system/gpu_10_memory_usage_megabytes': string;
+  'system/gpu_10_memory_usage_percentage': string;
+  'system/gpu_10_utilization_percentage': string;
+  'system/gpu_11_memory_usage_megabytes': string;
+  'system/gpu_11_memory_usage_percentage': string;
+  'system/gpu_11_utilization_percentage': string;
+  'system/gpu_12_memory_usage_megabytes': string;
+  'system/gpu_12_memory_usage_percentage': string;
+  'system/gpu_12_utilization_percentage': string;
+  'system/gpu_13_memory_usage_megabytes': string;
+  'system/gpu_13_memory_usage_percentage': string;
+  'system/gpu_13_utilization_percentage': string;
+  'system/gpu_14_memory_usage_megabytes': string;
+  'system/gpu_14_memory_usage_percentage': string;
+  'system/gpu_14_utilization_percentage': string;
+  'system/gpu_15_memory_usage_megabytes': string;
+  'system/gpu_15_memory_usage_percentage': string;
+  'system/gpu_15_utilization_percentage': string;
+  'system/gpu_2_memory_usage_megabytes': string;
+  'system/gpu_2_memory_usage_percentage': string;
+  'system/gpu_2_utilization_percentage': string;
+  'system/gpu_3_memory_usage_megabytes': string;
+  'system/gpu_3_memory_usage_percentage': string;
+  'system/gpu_3_utilization_percentage': string;
+  'system/gpu_4_memory_usage_megabytes': string;
+  'system/gpu_4_memory_usage_percentage': string;
+  'system/gpu_4_utilization_percentage': string;
+  'system/gpu_5_memory_usage_megabytes': string;
+  'system/gpu_5_memory_usage_percentage': string;
+  'system/gpu_5_utilization_percentage': string;
+  'system/gpu_6_memory_usage_megabytes': string;
+  'system/gpu_6_memory_usage_percentage': string;
+  'system/gpu_6_utilization_percentage': string;
+  'system/gpu_7_memory_usage_megabytes': string;
+  'system/gpu_7_memory_usage_percentage': string;
+  'system/gpu_7_utilization_percentage': string;
+  'system/gpu_8_memory_usage_megabytes': string;
+  'system/gpu_8_memory_usage_percentage': string;
+  'system/gpu_8_utilization_percentage': string;
+  'system/gpu_9_memory_usage_megabytes': string;
+  'system/gpu_9_memory_usage_percentage': string;
+  'system/gpu_9_utilization_percentage': string;
+  'system/network_receive_megabytes': string;
+  'system/network_transmit_megabytes': string;
+  'system/system_memory_usage_megabytes': string;
+  'system/system_memory_usage_percentage': string;
+};