handler regex at backend & memory utilization config for pod

Signed-off-by: adarsh0728 <[email protected]>
numaproj · Jan 16, 2025 · d56b1cd · d56b1cd
1 parent b7b795d
commit d56b1cd
Show file tree

Hide file tree

Showing 7 changed files with 156 additions and 90 deletions.
diff --git a/config/advanced-install/namespaced-numaflow-server.yaml b/config/advanced-install/namespaced-numaflow-server.yaml
@@ -204,29 +204,39 @@ data:
     Mono Vertex Read Processing Rate\n      # set \"Units\" or unset for default behaviour\n
     \     # unit: Units\n      required_filters:\n        - namespace\n        - mvtx_name\n
     \     dimensions:\n        - name: mono-vertex\n        - name: pod\n          filters:\n
-    \           - name: pod\n              required: false\n- name: mono-vertex-pod-cpu\n
-    \ object: mono-vertex\n  title: cpu utilization by pod\n  description: cpu utilization
-    by pod for mono-vertex\n  expr: avg_over_time($metric_name{$filters}[$duration]})\n
+    \           - name: pod\n              required: false\n- name: mono_vertex_cpu_memory_utilization_pod\n
+    \ object: mono-vertex\n  title: cpu-memory utilization by pod\n  description:
+    cpu and memory utilization by pod for mono-vertex\n  expr: avg_over_time($metric_name{$filters}[$duration])\n
     \ params:\n    - name: duration\n      required: true\n    - name: start_time\n
     \     required: false\n    - name: end_time\n      required: false\n  metrics:
-    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU utilization
-    by pod\n      required_filters:\n        - namespace\n        # to get only mono-vertex
-    pods\n        - pod  \n      dimensions:\n        - name: mono-vertex\n          filters:
-    \n            - name: pod \n              required: false\n\n- name: pipeline-pod-cpu\n
-    \ object: vertex\n  title: cpu utilization by pod\n  description: cpu utilization
-    by pod for pipelines vertex\n  expr: avg_over_time($metric_name{$filters}[$duration]})\n
+    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU Utilization
+    per Pod (usage seconds/resource limits)\n      required_filters:\n        - namespace\n
+    \       # to get only mono-vertex pods\n        - pod  \n      dimensions:\n        -
+    name: mono-vertex\n          filters: \n            - name: pod \n              required:
+    false\n    - metric_name: namespace_pod_memory_utilization\n      display_name:
+    Memory Utilization per Pod (usage bytes/resource limits)\n      required_filters:\n
+    \       - namespace\n        # to get only mono-vertex pods\n        - pod  \n
+    \     dimensions:\n        - name: mono-vertex\n          filters: \n            -
+    name: pod \n              required: false\n- name: pipeline_vertex_cpu_memory_utilization_pod\n
+    \ object: vertex\n  title: cpu-memory utilization by pod\n  description: cpu and
+    memory utilization by pod for pipeline's vertex\n  expr: avg_over_time($metric_name{$filters}[$duration])\n
     \ params:\n    - name: duration\n      required: true\n    - name: start_time\n
     \     required: false\n    - name: end_time\n      required: false\n  metrics:
-    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU utilization
-    by pod\n      required_filters:\n        - namespace\n        # to get only vertex
-    pods\n        - pod  \n      dimensions:\n        - name: vertex\n          filters:
-    \n            - name: pod \n              required: false\n# - name: mono-vertex-container-cpu\n#
-    \  object: mono-vertex\n#   title: cpu utilization by container\n#   description:
-    cpu utilization by container for mono-vertex\n#   expr: avg_over_time($metric_name{$filters}[$duration]})\n#
-    \  params:\n#     - name: duration\n#       required: true\n#     - name: start_time\n#
-    \      required: false\n#     - name: end_time\n#       required: false\n#   metrics:
-    \n#     - metric_name: namespace_app_container_cpu_utilization\n#       display_name:
-    CPU utilization by container\n#       required_filters:\n#         - namespace\n#
+    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU Utilization
+    per Pod (usage seconds/resource limits)\n      required_filters:\n        - namespace\n
+    \       # to get only vertex pods\n        - pod  \n      dimensions:\n        -
+    name: vertex\n          filters: \n            - name: pod \n              required:
+    false\n    - metric_name: namespace_pod_memory_utilization\n      display_name:
+    Memory Utilization per Pod (usage bytes/resource limits)\n      required_filters:\n
+    \       - namespace\n        # to get only vertex pods\n        - pod  \n      dimensions:\n
+    \       - name: vertex\n          filters: \n            - name: pod \n              required:
+    false\n# - name: mono-vertex-container-cpu\n#   object: mono-vertex\n#   title:
+    cpu utilization by container\n#   description: cpu utilization by container for
+    mono-vertex\n#   expr: avg_over_time($metric_name{$filters}[$duration]})\n#   params:\n#
+    \    - name: duration\n#       required: true\n#     - name: start_time\n#       required:
+    false\n#     - name: end_time\n#       required: false\n#   metrics: \n#     -
+    metric_name: namespace_app_container_cpu_utilization\n#       display_name: CPU
+    utilization by container\n#       required_filters:\n#         - namespace\n#
     \      dimensions:\n#         - name: container\n#           filters: \n#             -
     name: container\n#               required: false\n"
 kind: ConfigMap

diff --git a/config/advanced-install/numaflow-server.yaml b/config/advanced-install/numaflow-server.yaml
@@ -211,29 +211,39 @@ data:
     Mono Vertex Read Processing Rate\n      # set \"Units\" or unset for default behaviour\n
     \     # unit: Units\n      required_filters:\n        - namespace\n        - mvtx_name\n
     \     dimensions:\n        - name: mono-vertex\n        - name: pod\n          filters:\n
-    \           - name: pod\n              required: false\n- name: mono-vertex-pod-cpu\n
-    \ object: mono-vertex\n  title: cpu utilization by pod\n  description: cpu utilization
-    by pod for mono-vertex\n  expr: avg_over_time($metric_name{$filters}[$duration]})\n
+    \           - name: pod\n              required: false\n- name: mono_vertex_cpu_memory_utilization_pod\n
+    \ object: mono-vertex\n  title: cpu-memory utilization by pod\n  description:
+    cpu and memory utilization by pod for mono-vertex\n  expr: avg_over_time($metric_name{$filters}[$duration])\n
     \ params:\n    - name: duration\n      required: true\n    - name: start_time\n
     \     required: false\n    - name: end_time\n      required: false\n  metrics:
-    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU utilization
-    by pod\n      required_filters:\n        - namespace\n        # to get only mono-vertex
-    pods\n        - pod  \n      dimensions:\n        - name: mono-vertex\n          filters:
-    \n            - name: pod \n              required: false\n\n- name: pipeline-pod-cpu\n
-    \ object: vertex\n  title: cpu utilization by pod\n  description: cpu utilization
-    by pod for pipelines vertex\n  expr: avg_over_time($metric_name{$filters}[$duration]})\n
+    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU Utilization
+    per Pod (usage seconds/resource limits)\n      required_filters:\n        - namespace\n
+    \       # to get only mono-vertex pods\n        - pod  \n      dimensions:\n        -
+    name: mono-vertex\n          filters: \n            - name: pod \n              required:
+    false\n    - metric_name: namespace_pod_memory_utilization\n      display_name:
+    Memory Utilization per Pod (usage bytes/resource limits)\n      required_filters:\n
+    \       - namespace\n        # to get only mono-vertex pods\n        - pod  \n
+    \     dimensions:\n        - name: mono-vertex\n          filters: \n            -
+    name: pod \n              required: false\n- name: pipeline_vertex_cpu_memory_utilization_pod\n
+    \ object: vertex\n  title: cpu-memory utilization by pod\n  description: cpu and
+    memory utilization by pod for pipeline's vertex\n  expr: avg_over_time($metric_name{$filters}[$duration])\n
     \ params:\n    - name: duration\n      required: true\n    - name: start_time\n
     \     required: false\n    - name: end_time\n      required: false\n  metrics:
-    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU utilization
-    by pod\n      required_filters:\n        - namespace\n        # to get only vertex
-    pods\n        - pod  \n      dimensions:\n        - name: vertex\n          filters:
-    \n            - name: pod \n              required: false\n# - name: mono-vertex-container-cpu\n#
-    \  object: mono-vertex\n#   title: cpu utilization by container\n#   description:
-    cpu utilization by container for mono-vertex\n#   expr: avg_over_time($metric_name{$filters}[$duration]})\n#
-    \  params:\n#     - name: duration\n#       required: true\n#     - name: start_time\n#
-    \      required: false\n#     - name: end_time\n#       required: false\n#   metrics:
-    \n#     - metric_name: namespace_app_container_cpu_utilization\n#       display_name:
-    CPU utilization by container\n#       required_filters:\n#         - namespace\n#
+    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU Utilization
+    per Pod (usage seconds/resource limits)\n      required_filters:\n        - namespace\n
+    \       # to get only vertex pods\n        - pod  \n      dimensions:\n        -
+    name: vertex\n          filters: \n            - name: pod \n              required:
+    false\n    - metric_name: namespace_pod_memory_utilization\n      display_name:
+    Memory Utilization per Pod (usage bytes/resource limits)\n      required_filters:\n
+    \       - namespace\n        # to get only vertex pods\n        - pod  \n      dimensions:\n
+    \       - name: vertex\n          filters: \n            - name: pod \n              required:
+    false\n# - name: mono-vertex-container-cpu\n#   object: mono-vertex\n#   title:
+    cpu utilization by container\n#   description: cpu utilization by container for
+    mono-vertex\n#   expr: avg_over_time($metric_name{$filters}[$duration]})\n#   params:\n#
+    \    - name: duration\n#       required: true\n#     - name: start_time\n#       required:
+    false\n#     - name: end_time\n#       required: false\n#   metrics: \n#     -
+    metric_name: namespace_app_container_cpu_utilization\n#       display_name: CPU
+    utilization by container\n#       required_filters:\n#         - namespace\n#
     \      dimensions:\n#         - name: container\n#           filters: \n#             -
     name: container\n#               required: false\n"
 kind: ConfigMap

diff --git a/config/base/numaflow-server/numaflow-server-metrics-proxy-config.yaml b/config/base/numaflow-server/numaflow-server-metrics-proxy-config.yaml
@@ -179,11 +179,11 @@ data:
               filters:
                 - name: pod
                   required: false
-    - name: mono-vertex-pod-cpu
+    - name: mono_vertex_cpu_memory_utilization_pod
       object: mono-vertex
-      title: cpu utilization by pod
-      description: cpu utilization by pod for mono-vertex
-      expr: avg_over_time($metric_name{$filters}[$duration]})
+      title: cpu-memory utilization by pod
+      description: cpu and memory utilization by pod for mono-vertex
+      expr: avg_over_time($metric_name{$filters}[$duration])
       params:
         - name: duration
           required: true
@@ -193,7 +193,7 @@ data:
           required: false
       metrics: 
         - metric_name: namespace_pod_cpu_utilization
-          display_name: CPU utilization by pod
+          display_name: CPU Utilization per Pod (usage seconds/resource limits)
           required_filters:
             - namespace
             # to get only mono-vertex pods
@@ -203,12 +203,22 @@ data:
               filters: 
                 - name: pod 
                   required: false
-
-    - name: pipeline-pod-cpu
+        - metric_name: namespace_pod_memory_utilization
+          display_name: Memory Utilization per Pod (usage bytes/resource limits)
+          required_filters:
+            - namespace
+            # to get only mono-vertex pods
+            - pod  
+          dimensions:
+            - name: mono-vertex
+              filters: 
+                - name: pod 
+                  required: false
+    - name: pipeline_vertex_cpu_memory_utilization_pod
       object: vertex
-      title: cpu utilization by pod
-      description: cpu utilization by pod for pipelines vertex
-      expr: avg_over_time($metric_name{$filters}[$duration]})
+      title: cpu-memory utilization by pod
+      description: cpu and memory utilization by pod for pipeline's vertex
+      expr: avg_over_time($metric_name{$filters}[$duration])
       params:
         - name: duration
           required: true
@@ -218,7 +228,18 @@ data:
           required: false
       metrics: 
         - metric_name: namespace_pod_cpu_utilization
-          display_name: CPU utilization by pod
+          display_name: CPU Utilization per Pod (usage seconds/resource limits)
+          required_filters:
+            - namespace
+            # to get only vertex pods
+            - pod  
+          dimensions:
+            - name: vertex
+              filters: 
+                - name: pod 
+                  required: false
+        - metric_name: namespace_pod_memory_utilization
+          display_name: Memory Utilization per Pod (usage bytes/resource limits)
           required_filters:
             - namespace
             # to get only vertex pods

diff --git a/config/install.yaml b/config/install.yaml
@@ -28939,29 +28939,39 @@ data:
     Mono Vertex Read Processing Rate\n      # set \"Units\" or unset for default behaviour\n
     \     # unit: Units\n      required_filters:\n        - namespace\n        - mvtx_name\n
     \     dimensions:\n        - name: mono-vertex\n        - name: pod\n          filters:\n
-    \           - name: pod\n              required: false\n- name: mono-vertex-pod-cpu\n
-    \ object: mono-vertex\n  title: cpu utilization by pod\n  description: cpu utilization
-    by pod for mono-vertex\n  expr: avg_over_time($metric_name{$filters}[$duration]})\n
+    \           - name: pod\n              required: false\n- name: mono_vertex_cpu_memory_utilization_pod\n
+    \ object: mono-vertex\n  title: cpu-memory utilization by pod\n  description:
+    cpu and memory utilization by pod for mono-vertex\n  expr: avg_over_time($metric_name{$filters}[$duration])\n
     \ params:\n    - name: duration\n      required: true\n    - name: start_time\n
     \     required: false\n    - name: end_time\n      required: false\n  metrics:
-    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU utilization
-    by pod\n      required_filters:\n        - namespace\n        # to get only mono-vertex
-    pods\n        - pod  \n      dimensions:\n        - name: mono-vertex\n          filters:
-    \n            - name: pod \n              required: false\n\n- name: pipeline-pod-cpu\n
-    \ object: vertex\n  title: cpu utilization by pod\n  description: cpu utilization
-    by pod for pipelines vertex\n  expr: avg_over_time($metric_name{$filters}[$duration]})\n
+    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU Utilization
+    per Pod (usage seconds/resource limits)\n      required_filters:\n        - namespace\n
+    \       # to get only mono-vertex pods\n        - pod  \n      dimensions:\n        -
+    name: mono-vertex\n          filters: \n            - name: pod \n              required:
+    false\n    - metric_name: namespace_pod_memory_utilization\n      display_name:
+    Memory Utilization per Pod (usage bytes/resource limits)\n      required_filters:\n
+    \       - namespace\n        # to get only mono-vertex pods\n        - pod  \n
+    \     dimensions:\n        - name: mono-vertex\n          filters: \n            -
+    name: pod \n              required: false\n- name: pipeline_vertex_cpu_memory_utilization_pod\n
+    \ object: vertex\n  title: cpu-memory utilization by pod\n  description: cpu and
+    memory utilization by pod for pipeline's vertex\n  expr: avg_over_time($metric_name{$filters}[$duration])\n
     \ params:\n    - name: duration\n      required: true\n    - name: start_time\n
     \     required: false\n    - name: end_time\n      required: false\n  metrics:
-    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU utilization
-    by pod\n      required_filters:\n        - namespace\n        # to get only vertex
-    pods\n        - pod  \n      dimensions:\n        - name: vertex\n          filters:
-    \n            - name: pod \n              required: false\n# - name: mono-vertex-container-cpu\n#
-    \  object: mono-vertex\n#   title: cpu utilization by container\n#   description:
-    cpu utilization by container for mono-vertex\n#   expr: avg_over_time($metric_name{$filters}[$duration]})\n#
-    \  params:\n#     - name: duration\n#       required: true\n#     - name: start_time\n#
-    \      required: false\n#     - name: end_time\n#       required: false\n#   metrics:
-    \n#     - metric_name: namespace_app_container_cpu_utilization\n#       display_name:
-    CPU utilization by container\n#       required_filters:\n#         - namespace\n#
+    \n    - metric_name: namespace_pod_cpu_utilization\n      display_name: CPU Utilization
+    per Pod (usage seconds/resource limits)\n      required_filters:\n        - namespace\n
+    \       # to get only vertex pods\n        - pod  \n      dimensions:\n        -
+    name: vertex\n          filters: \n            - name: pod \n              required:
+    false\n    - metric_name: namespace_pod_memory_utilization\n      display_name:
+    Memory Utilization per Pod (usage bytes/resource limits)\n      required_filters:\n
+    \       - namespace\n        # to get only vertex pods\n        - pod  \n      dimensions:\n
+    \       - name: vertex\n          filters: \n            - name: pod \n              required:
+    false\n# - name: mono-vertex-container-cpu\n#   object: mono-vertex\n#   title:
+    cpu utilization by container\n#   description: cpu utilization by container for
+    mono-vertex\n#   expr: avg_over_time($metric_name{$filters}[$duration]})\n#   params:\n#
+    \    - name: duration\n#       required: true\n#     - name: start_time\n#       required:
+    false\n#     - name: end_time\n#       required: false\n#   metrics: \n#     -
+    metric_name: namespace_app_container_cpu_utilization\n#       display_name: CPU
+    utilization by container\n#       required_filters:\n#         - namespace\n#
     \      dimensions:\n#         - name: container\n#           filters: \n#             -
     name: container\n#               required: false\n"
 kind: ConfigMap