Add new concurrent querying challenge (#538)

elastic/logs/challenges/logging-querying-concurrent-stress.json

@@ -0,0 +1,73 @@
+{% import "rally.helpers" as rally %}
+{
+  "name": "logging-querying-concurrent-stress",
+  "description": "Applies a query workload with emphasis on many concurrent clients. Ensures data streams exist so queries can be run, but does not remove existing data.",
+  "parameters": {
+    "generate-data": {{ true | tojson if bulk_start_date and bulk_end_date else false | tojson }}
+  },
+  "schedule": [
+    {% include "tasks/index-setup.json" %},
+    {
+      "name": "concurrency-settings",
+      "operation": {
+        "operation-type": "raw-request",
+        "method": "PUT",
+        "path": "/_cluster/settings",
+        "body": {
+          "transient": {
+            "search.worker_threads_enabled": {{ p_worker_threads_enabled | tojson }}
+          }
+        }
+      }
+    },
+    {% if bulk_start_date and bulk_end_date %}
+      {
+        "name": "bulk-index",
+        "operation": {
+          "operation-type": "raw-bulk",
+          "param-source": "processed-source",
+          "time-format": "milliseconds",
+          "profile": "fixed_interval",
+          "init-load": true,
+          "bulk-size": {{ p_bulk_size }},
+          "detailed-results": true
+        },
+        "clients": {{ p_bulk_indexing_clients }},
+        "ignore-response-error-level": "{{error_level | default('non-fatal')}}"
+      },
+    {% endif %}
+    {
+      "operation": "disable_query_cache",
+      "tags": ["settings"]
+    },
+    {
+      "name": "logging-queries",
+      "parallel": {
+        "time-period": {{ p_query_time_period }},
+        "warmup-time-period": {{ p_query_warmup_time_period }},
+        "tasks": [
+          {% for workflow in p_query_workflows %}
+            {
+              "name": {{workflow | tojson }},
+              "operation": {
+                "operation-type": "composite",
+                "param-source": "workflow-selector",
+                "workflow": {{workflow | tojson }},
+                "task-offset": {{ loop.index }},
+                "request-params": {{ p_query_request_params | tojson(indent=2) }}
+              },
+              "think-time-interval": {{ p_user_think_time }},
+              "workflow-interval": {{ p_user_workflow_time }},
+              "clients": {{ p_search_clients }},
+              "schedule": "workflow-scheduler"
+            }{{ ", " if not loop.last else "" }}
+          {% endfor %}
+        ]
+      }
+    },
+    {
+      "operation": "enable_query_cache",
+      "tags": ["settings"]
+    }
+  ]
+}

elastic/logs/track.json

@@ -42,6 +42,8 @@
 {% set p_snapshot_metadata = (snapshot_metadata | default({}))%}
 {% set p_snapshot_rename_suffix = (snapshot_rename_suffix | default("") ) %}
 
+{% set p_worker_threads_enabled = (worker_threads_enabled | default(true))  %}
+
 {% set es_version = "7.13.2" %}
 
 {% set p_integration_ratios = (integration_ratios | default({