diff --git a/elastic/logs/README.md b/elastic/logs/README.md index 6ce000e4..3a2b0cec 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -246,6 +246,9 @@ The following parameters are available: * `disable_pipelines` (default: `false`) - Prevent installing ingest node pipelines. This parameter is experimental and is to be used with indexing-only challenges. * `initial_indices_count` (default: 0) - Number of initial indices to create, each containing `100` auditbeat style documents. Parameter is applicable in [many-shards-quantitative challenge](#many-shards-quantitative-many-shards-quantitative) and in [many-shards-snapshots challenge](#many-shards-snapshots-many-shards-snapshots). * `ingest_percentage` (default: 100) - The percentage of data to be ingested. +* `index_sorting` (default: unset): Whether index sorting should be used. Accepted values: `hostname` and `timestamp`. +* `synthetic_source_mode` (default: `false`): Whether to enable synthetic source. +* `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use. Only supported in `logging-querying` track. ### Querying parameters diff --git a/elastic/logs/challenges/logging-querying.json b/elastic/logs/challenges/logging-querying.json index c8cc10ac..3b523855 100644 --- a/elastic/logs/challenges/logging-querying.json +++ b/elastic/logs/challenges/logging-querying.json @@ -22,6 +22,52 @@ "clients": {{ p_bulk_indexing_clients }}, "ignore-response-error-level": "{{error_level | default('non-fatal')}}" } + {%- if force_merge_max_num_segments is defined %}, + { + "name": "refresh-after-index", + "index": "logs-*", + "operation": "refresh" + }, + { + "name": "wait-until-index-merges-fininshes", + "operation": { + "operation-type": "index-stats", + "index": "logs-*", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false + } + }, + { + "operation": { + "operation-type": "force-merge", + "index": "logs-*", + "request-timeout": 36000, + "max-num-segments": {{ force_merge_max_num_segments | tojson }} + } + }, + { + "name": "wait-until-merges-finish", + "operation": { + "operation-type": "index-stats", + "index": "logs-*", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false + } + }, + { + "name": "refresh-after-force-merge", + "index": "logs-*", + "operation": "refresh" + } + {%- endif %} {# non-serverless-index-statistics-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}, { "name": "compression-stats", diff --git a/elastic/logs/templates/component/auditbeat-mappings.json b/elastic/logs/templates/component/auditbeat-mappings.json index 84cfd08f..c63978d1 100644 --- a/elastic/logs/templates/component/auditbeat-mappings.json +++ b/elastic/logs/templates/component/auditbeat-mappings.json @@ -1785,6 +1785,9 @@ "message": { "norms": false, "type": "text" + {% if synthetic_source_mode | default(false) is true %}, + "store": true + {% endif %} }, "stack_trace": { "fields": { @@ -2941,6 +2944,9 @@ "message": { "norms": false, "type": "text" + {% if synthetic_source_mode | default(false) is true %}, + "store": true + {% endif %} }, "network": { "properties": { diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json new file mode 100644 index 00000000..e8ee4027 --- /dev/null +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -0,0 +1,22 @@ +{ + "template": { + "settings": { + "index": { + {% if index_sorting == "hostname" %} + "sort.field": [ "host.name", "@timestamp" ], + "sort.order": [ "asc", "desc" ] + {% elif index_sorting == "timestamp" %} + "sort.field": [ "@timestamp", "host.name" ], + "sort.order": [ "desc", "asc" ] + {% endif %} + } + }, + "mappings": { + {% if synthetic_source_mode | default(false) is true %} + "_source": { + "mode": "synthetic" + } + {% endif %} + } + } + } \ No newline at end of file diff --git a/elastic/logs/templates/composable/auditbeat-frozen.json b/elastic/logs/templates/composable/auditbeat-frozen.json index 80bdf0d3..4bb326a4 100644 --- a/elastic/logs/templates/composable/auditbeat-frozen.json +++ b/elastic/logs/templates/composable/auditbeat-frozen.json @@ -15,7 +15,7 @@ "lifecycle": {} {%- endif -%} }, - "composed_of" : ["auditbeat-mappings"], + "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"], "priority": 1, "data_stream" : { } } diff --git a/elastic/logs/templates/composable/auditbeat-quantitative.json b/elastic/logs/templates/composable/auditbeat-quantitative.json index e8927a8e..ab6816a2 100644 --- a/elastic/logs/templates/composable/auditbeat-quantitative.json +++ b/elastic/logs/templates/composable/auditbeat-quantitative.json @@ -15,7 +15,7 @@ "lifecycle": {} {%- endif -%} }, - "composed_of" : ["auditbeat-mappings"], + "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"], "priority": 1, "data_stream" : { } } diff --git a/elastic/logs/templates/composable/auditbeat.json b/elastic/logs/templates/composable/auditbeat.json index d63433c7..1da39f96 100644 --- a/elastic/logs/templates/composable/auditbeat.json +++ b/elastic/logs/templates/composable/auditbeat.json @@ -15,7 +15,7 @@ "lifecycle": {} {%- endif -%} }, - "composed_of" : ["auditbeat-mappings"], + "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"], "priority": 1, "data_stream" : { } } diff --git a/elastic/logs/templates/composable/logs-apache.access.json b/elastic/logs/templates/composable/logs-apache.access.json index ecade6ed..145355d8 100644 --- a/elastic/logs/templates/composable/logs-apache.access.json +++ b/elastic/logs/templates/composable/logs-apache.access.json @@ -521,7 +521,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-apache.error.json b/elastic/logs/templates/composable/logs-apache.error.json index 010bdfa2..3f589e1b 100644 --- a/elastic/logs/templates/composable/logs-apache.error.json +++ b/elastic/logs/templates/composable/logs-apache.error.json @@ -477,7 +477,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-k8-application.log.json b/elastic/logs/templates/composable/logs-k8-application.log.json index 37e886e9..2c8d8bfe 100644 --- a/elastic/logs/templates/composable/logs-k8-application.log.json +++ b/elastic/logs/templates/composable/logs-k8-application.log.json @@ -2000,7 +2000,8 @@ "composed_of": [ "logs-mappings", "track-custom-mappings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-kafka.log.json b/elastic/logs/templates/composable/logs-kafka.log.json index c670bfaa..883aac4e 100644 --- a/elastic/logs/templates/composable/logs-kafka.log.json +++ b/elastic/logs/templates/composable/logs-kafka.log.json @@ -158,6 +158,9 @@ "properties": { "message": { "type": "text" + {% if synthetic_source_mode | default(false) is true %}, + "store": true + {% endif %} }, "class": { "ignore_above": 1024, @@ -266,11 +269,17 @@ }, "message": { "type": "text" + {% if synthetic_source_mode | default(false) is true %}, + "store": true + {% endif %} }, "error": { "properties": { "message": { "type": "text" + {% if synthetic_source_mode | default(false) is true %}, + "store": true + {% endif %} } } }, @@ -287,7 +296,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-mysql.error.json b/elastic/logs/templates/composable/logs-mysql.error.json index 0f130ef6..510b9230 100644 --- a/elastic/logs/templates/composable/logs-mysql.error.json +++ b/elastic/logs/templates/composable/logs-mysql.error.json @@ -305,7 +305,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-mysql.slowlog.json b/elastic/logs/templates/composable/logs-mysql.slowlog.json index 2a32bef4..ac8cea55 100644 --- a/elastic/logs/templates/composable/logs-mysql.slowlog.json +++ b/elastic/logs/templates/composable/logs-mysql.slowlog.json @@ -450,7 +450,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-nginx.access.json b/elastic/logs/templates/composable/logs-nginx.access.json index 23453a08..6003836d 100644 --- a/elastic/logs/templates/composable/logs-nginx.access.json +++ b/elastic/logs/templates/composable/logs-nginx.access.json @@ -461,7 +461,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-nginx.error.json b/elastic/logs/templates/composable/logs-nginx.error.json index a2f3dea1..e411b4ed 100644 --- a/elastic/logs/templates/composable/logs-nginx.error.json +++ b/elastic/logs/templates/composable/logs-nginx.error.json @@ -296,7 +296,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-postgresql.log.json b/elastic/logs/templates/composable/logs-postgresql.log.json index 6dd5862d..68c54142 100644 --- a/elastic/logs/templates/composable/logs-postgresql.log.json +++ b/elastic/logs/templates/composable/logs-postgresql.log.json @@ -407,7 +407,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-redis.log.json b/elastic/logs/templates/composable/logs-redis.log.json index 62eed0ca..8e9d3417 100644 --- a/elastic/logs/templates/composable/logs-redis.log.json +++ b/elastic/logs/templates/composable/logs-redis.log.json @@ -278,7 +278,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-redis.slowlog.json b/elastic/logs/templates/composable/logs-redis.slowlog.json index 4ec11f1b..2bbd8760 100644 --- a/elastic/logs/templates/composable/logs-redis.slowlog.json +++ b/elastic/logs/templates/composable/logs-redis.slowlog.json @@ -259,7 +259,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-system.auth.json b/elastic/logs/templates/composable/logs-system.auth.json index 51cdbe39..1f795226 100644 --- a/elastic/logs/templates/composable/logs-system.auth.json +++ b/elastic/logs/templates/composable/logs-system.auth.json @@ -473,7 +473,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-system.syslog.json b/elastic/logs/templates/composable/logs-system.syslog.json index 87828803..6585fd64 100644 --- a/elastic/logs/templates/composable/logs-system.syslog.json +++ b/elastic/logs/templates/composable/logs-system.syslog.json @@ -297,7 +297,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/track.json b/elastic/logs/track.json index b751659b..378b62f8 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -303,6 +303,10 @@ { "name": "track-data-stream-lifecycle", "template": "./templates/component/track-data-stream-lifecycle.json" + }, + { + "name": "track-shared-logsdb-mode", + "template": "./templates/component/track-shared-logsdb-mode.json" } ], "composable-templates": [