Added custom name for steps to run same step multiple times and also …

…added the index configuration for multiple hyper parameter values

Signed-off-by: AWS ParallelCluster user <[email protected]>

benchmarks/perf-tool/okpt/test/steps/base.py

@@ -9,6 +9,7 @@
 from typing import Any, Dict, List
 
 from okpt.test import profile
+from okpt.io.config.parsers.util import parse_string_param
 
 
 @dataclass
@@ -23,6 +24,7 @@ class Step:
 
     Attributes:
         label: Name of the step.
+        custom_name: Name of the step you want to give to differentiate the same step label from others
 
     Methods:
         execute: Run the step and return a step response with the label and
@@ -33,6 +35,9 @@ class Step:
 
     def __init__(self, step_config: StepConfig):
         self.step_config = step_config
+        self.custom_name = parse_string_param('custom_name', step_config.config,
+                                              step_config.implicit_config,
+                                              self.label)
 
     def _action(self):
         """Step logic/behavior to be executed and profiled."""
@@ -48,13 +53,17 @@ def execute(self) -> List[Dict[str, Any]]:
         Returns:
             Dict containing step label and various step measures.
         """
+        print(f"************** Starting the step with name: {self.custom_name} **************")
         action = self._action
 
         # profile the action with measure decorators - add if necessary
         action = getattr(profile, 'took')(action)
 
         result = action()
         if isinstance(result, dict):
-            return [{'label': self.label, **result}]
+            res = [{'label': self.label, **result}]
+            print(f"************** Completed the step with name: {self.custom_name} **************")
+            return res
+        print(f"************** Completed the step name: {self.custom_name} **************")
 
         raise ValueError('Invalid return by a step')

benchmarks/perf-tool/okpt/test/steps/steps.py

@@ -67,6 +67,7 @@ def _action(self):
         Returns:
             An OpenSearch index creation response body.
         """
+        print("Creating index")
         self.opensearch.indices.create(index=self.index_name, body=self.body)
         return {}
 
@@ -726,7 +727,7 @@ def warmup_operation(endpoint, port, index):
     Returns:
         number of shards the plugin succeeded and failed to warm up.
     """
-    response = requests.get('http://' + endpoint + ':' + str(port) +
+    response = requests.get('https://' + endpoint + ':' + str(port) +
                                '/_plugins/_knn/warmup/' + index,
                                headers={'content-type': 'application/json'})
     return response.json()

benchmarks/perf-tool/release-configs/nmslib-hnsw/index-256-256.json

@@ -0,0 +1,28 @@
+{
+    "settings": {
+      "index": {
+        "knn": true,
+        "number_of_shards": 1,
+        "number_of_replicas": 0,
+        "knn.algo_param.ef_search": 256
+      }
+    },
+    "mappings": {
+      "properties": {
+        "target_field": {
+            "type": "knn_vector",
+            "dimension": 960,
+              "method": {
+                  "name": "hnsw",
+                  "space_type": "l2",
+                  "engine": "nmslib",
+                  "parameters": {
+                        "ef_construction": 512,
+                        "m": 16
+                  }
+              }
+        }
+      }
+    }
+  }
+

benchmarks/perf-tool/release-configs/nmslib-hnsw/index-512-256.json

@@ -0,0 +1,28 @@
+{
+    "settings": {
+      "index": {
+        "knn": true,
+        "number_of_shards": 1,
+        "number_of_replicas": 0,
+        "knn.algo_param.ef_search": 256
+      }
+    },
+    "mappings": {
+      "properties": {
+        "target_field": {
+            "type": "knn_vector",
+            "dimension": 960,
+              "method": {
+                  "name": "hnsw",
+                  "space_type": "l2",
+                  "engine": "nmslib",
+                  "parameters": {
+                        "ef_construction": 512,
+                        "m": 16
+                  }
+              }
+        }
+      }
+    }
+  }
+

benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json

@@ -2,16 +2,16 @@
   "settings": {
     "index": {
       "knn": true,
-      "number_of_shards": 24,
-      "number_of_replicas": 1,
+      "number_of_shards": 1,
+      "number_of_replicas": 0,
       "knn.algo_param.ef_search": 100
     }
   },
   "mappings": {
     "properties": {
       "target_field": {
           "type": "knn_vector",
-          "dimension": 128,
+          "dimension": 960,
       	  "method": {
         		"name": "hnsw",
         		"space_type": "l2",