From 04b08eb6ff304e2a38aac22939ae43762a5371f1 Mon Sep 17 00:00:00 2001 From: AWS ParallelCluster user Date: Wed, 15 Nov 2023 01:59:26 +0000 Subject: [PATCH] Added custom name for steps to run same step multiple times and also added the index configuration for multiple hyper parameter values Signed-off-by: AWS ParallelCluster user --- benchmarks/perf-tool/okpt/test/steps/base.py | 11 +++++++- benchmarks/perf-tool/okpt/test/steps/steps.py | 3 +- .../nmslib-hnsw/index-256-256.json | 28 +++++++++++++++++++ .../nmslib-hnsw/index-512-256.json | 28 +++++++++++++++++++ .../release-configs/nmslib-hnsw/index.json | 6 ++-- 5 files changed, 71 insertions(+), 5 deletions(-) create mode 100644 benchmarks/perf-tool/release-configs/nmslib-hnsw/index-256-256.json create mode 100644 benchmarks/perf-tool/release-configs/nmslib-hnsw/index-512-256.json diff --git a/benchmarks/perf-tool/okpt/test/steps/base.py b/benchmarks/perf-tool/okpt/test/steps/base.py index 8299804211..b1b422efb1 100644 --- a/benchmarks/perf-tool/okpt/test/steps/base.py +++ b/benchmarks/perf-tool/okpt/test/steps/base.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List from okpt.test import profile +from okpt.io.config.parsers.util import parse_string_param @dataclass @@ -23,6 +24,7 @@ class Step: Attributes: label: Name of the step. + custom_name: Name of the step you want to give to differentiate the same step label from others Methods: execute: Run the step and return a step response with the label and @@ -33,6 +35,9 @@ class Step: def __init__(self, step_config: StepConfig): self.step_config = step_config + self.custom_name = parse_string_param('custom_name', step_config.config, + step_config.implicit_config, + self.label) def _action(self): """Step logic/behavior to be executed and profiled.""" @@ -48,6 +53,7 @@ def execute(self) -> List[Dict[str, Any]]: Returns: Dict containing step label and various step measures. """ + print(f"************** Starting the step with name: {self.custom_name} **************") action = self._action # profile the action with measure decorators - add if necessary @@ -55,6 +61,9 @@ def execute(self) -> List[Dict[str, Any]]: result = action() if isinstance(result, dict): - return [{'label': self.label, **result}] + res = [{'label': self.label, **result}] + print(f"************** Completed the step with name: {self.custom_name} **************") + return res + print(f"************** Completed the step name: {self.custom_name} **************") raise ValueError('Invalid return by a step') diff --git a/benchmarks/perf-tool/okpt/test/steps/steps.py b/benchmarks/perf-tool/okpt/test/steps/steps.py index fe73c6cc6b..a2ac66e6c2 100644 --- a/benchmarks/perf-tool/okpt/test/steps/steps.py +++ b/benchmarks/perf-tool/okpt/test/steps/steps.py @@ -67,6 +67,7 @@ def _action(self): Returns: An OpenSearch index creation response body. """ + print("Creating index") self.opensearch.indices.create(index=self.index_name, body=self.body) return {} @@ -726,7 +727,7 @@ def warmup_operation(endpoint, port, index): Returns: number of shards the plugin succeeded and failed to warm up. """ - response = requests.get('http://' + endpoint + ':' + str(port) + + response = requests.get('https://' + endpoint + ':' + str(port) + '/_plugins/_knn/warmup/' + index, headers={'content-type': 'application/json'}) return response.json() diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index-256-256.json b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index-256-256.json new file mode 100644 index 0000000000..766a5c7bc4 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index-256-256.json @@ -0,0 +1,28 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 1, + "number_of_replicas": 0, + "knn.algo_param.ef_search": 256 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 960, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "nmslib", + "parameters": { + "ef_construction": 512, + "m": 16 + } + } + } + } + } + } + \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index-512-256.json b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index-512-256.json new file mode 100644 index 0000000000..766a5c7bc4 --- /dev/null +++ b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index-512-256.json @@ -0,0 +1,28 @@ +{ + "settings": { + "index": { + "knn": true, + "number_of_shards": 1, + "number_of_replicas": 0, + "knn.algo_param.ef_search": 256 + } + }, + "mappings": { + "properties": { + "target_field": { + "type": "knn_vector", + "dimension": 960, + "method": { + "name": "hnsw", + "space_type": "l2", + "engine": "nmslib", + "parameters": { + "ef_construction": 512, + "m": 16 + } + } + } + } + } + } + \ No newline at end of file diff --git a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json index eb714c5c82..caf6cd7000 100644 --- a/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json +++ b/benchmarks/perf-tool/release-configs/nmslib-hnsw/index.json @@ -2,8 +2,8 @@ "settings": { "index": { "knn": true, - "number_of_shards": 24, - "number_of_replicas": 1, + "number_of_shards": 1, + "number_of_replicas": 0, "knn.algo_param.ef_search": 100 } }, @@ -11,7 +11,7 @@ "properties": { "target_field": { "type": "knn_vector", - "dimension": 128, + "dimension": 960, "method": { "name": "hnsw", "space_type": "l2",