From 3d5de9c263f61976b3146eac323cd1c45a3e3bad Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 23 Nov 2023 16:50:52 +0100 Subject: [PATCH 1/4] Add index_sorting track parameter to elastic/logs track. This allows to enable indexing sorting, that either sorts by hostname and timestamp or timestamp and hostname. When index sorting is enabled than synthetic source is also enabled. --- elastic/logs/README.md | 1 + .../component/auditbeat-mappings.json | 6 +++++ .../component/track-shared-logsdb-mode.json | 22 +++++++++++++++++++ .../composable/auditbeat-frozen.json | 2 +- .../composable/auditbeat-quantitative.json | 2 +- .../logs/templates/composable/auditbeat.json | 2 +- .../composable/logs-apache.access.json | 3 ++- .../composable/logs-apache.error.json | 3 ++- .../composable/logs-k8-application.log.json | 3 ++- .../templates/composable/logs-kafka.log.json | 12 +++++++++- .../composable/logs-mysql.error.json | 3 ++- .../composable/logs-mysql.slowlog.json | 3 ++- .../composable/logs-nginx.access.json | 3 ++- .../composable/logs-nginx.error.json | 3 ++- .../composable/logs-postgresql.log.json | 3 ++- .../templates/composable/logs-redis.log.json | 3 ++- .../composable/logs-redis.slowlog.json | 3 ++- .../composable/logs-system.auth.json | 3 ++- .../composable/logs-system.syslog.json | 3 ++- elastic/logs/track.json | 7 +++++- 20 files changed, 73 insertions(+), 17 deletions(-) create mode 100644 elastic/logs/templates/component/track-shared-logsdb-mode.json diff --git a/elastic/logs/README.md b/elastic/logs/README.md index efdc0caa..a6dce6a7 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -246,6 +246,7 @@ The following parameters are available: * `disable_pipelines` (default: `false`) - Prevent installing ingest node pipelines. This parameter is experimental and is to be used with indexing-only challenges. * `initial_indices_count` (default: 0) - Number of initial indices to create, each containing `100` auditbeat style documents. Parameter is applicable in [many-shards-quantitative challenge](#many-shards-quantitative-many-shards-quantitative) and in [many-shards-snapshots challenge](#many-shards-snapshots-many-shards-snapshots). * `ingest_percentage` (default: 100) - The percentage of data to be ingested. +* `index_sorting` (default: unset): Whether index sorting should be used. Accepted values: `hostnams` and `timestamp`. Setting index sorting also enabled synthetic source. ### Querying parameters diff --git a/elastic/logs/templates/component/auditbeat-mappings.json b/elastic/logs/templates/component/auditbeat-mappings.json index 84cfd08f..8572ccb1 100644 --- a/elastic/logs/templates/component/auditbeat-mappings.json +++ b/elastic/logs/templates/component/auditbeat-mappings.json @@ -1785,6 +1785,9 @@ "message": { "norms": false, "type": "text" + {% if index_sorting %}, + "store": true + {% endif %} }, "stack_trace": { "fields": { @@ -2941,6 +2944,9 @@ "message": { "norms": false, "type": "text" + {% if index_sorting %}, + "store": true + {% endif %} }, "network": { "properties": { diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json new file mode 100644 index 00000000..5ac5661d --- /dev/null +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -0,0 +1,22 @@ +{ + "template": { + "settings": { + "index": { + {% if index_sorting == "hostname" %} + "sort.field": [ "host.name", "@timestamp" ], + "sort.order": [ "asc", "desc" ] + {%- elif index_sorting == "timestamp" %} + "sort.field": [ "@timestamp", "host.name" ], + "sort.order": [ "desc", "asc" ] + {%- endif %} + } + }, + "mappings": { + {% if index_sorting %} + "_source": { + "mode": "synthetic" + } + {%- endif %} + } + } + } \ No newline at end of file diff --git a/elastic/logs/templates/composable/auditbeat-frozen.json b/elastic/logs/templates/composable/auditbeat-frozen.json index 80bdf0d3..4bb326a4 100644 --- a/elastic/logs/templates/composable/auditbeat-frozen.json +++ b/elastic/logs/templates/composable/auditbeat-frozen.json @@ -15,7 +15,7 @@ "lifecycle": {} {%- endif -%} }, - "composed_of" : ["auditbeat-mappings"], + "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"], "priority": 1, "data_stream" : { } } diff --git a/elastic/logs/templates/composable/auditbeat-quantitative.json b/elastic/logs/templates/composable/auditbeat-quantitative.json index e8927a8e..ab6816a2 100644 --- a/elastic/logs/templates/composable/auditbeat-quantitative.json +++ b/elastic/logs/templates/composable/auditbeat-quantitative.json @@ -15,7 +15,7 @@ "lifecycle": {} {%- endif -%} }, - "composed_of" : ["auditbeat-mappings"], + "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"], "priority": 1, "data_stream" : { } } diff --git a/elastic/logs/templates/composable/auditbeat.json b/elastic/logs/templates/composable/auditbeat.json index d63433c7..1da39f96 100644 --- a/elastic/logs/templates/composable/auditbeat.json +++ b/elastic/logs/templates/composable/auditbeat.json @@ -15,7 +15,7 @@ "lifecycle": {} {%- endif -%} }, - "composed_of" : ["auditbeat-mappings"], + "composed_of" : ["auditbeat-mappings", "track-shared-logsdb-mode"], "priority": 1, "data_stream" : { } } diff --git a/elastic/logs/templates/composable/logs-apache.access.json b/elastic/logs/templates/composable/logs-apache.access.json index ecade6ed..145355d8 100644 --- a/elastic/logs/templates/composable/logs-apache.access.json +++ b/elastic/logs/templates/composable/logs-apache.access.json @@ -521,7 +521,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-apache.error.json b/elastic/logs/templates/composable/logs-apache.error.json index 010bdfa2..3f589e1b 100644 --- a/elastic/logs/templates/composable/logs-apache.error.json +++ b/elastic/logs/templates/composable/logs-apache.error.json @@ -477,7 +477,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-k8-application.log.json b/elastic/logs/templates/composable/logs-k8-application.log.json index 37e886e9..2c8d8bfe 100644 --- a/elastic/logs/templates/composable/logs-k8-application.log.json +++ b/elastic/logs/templates/composable/logs-k8-application.log.json @@ -2000,7 +2000,8 @@ "composed_of": [ "logs-mappings", "track-custom-mappings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-kafka.log.json b/elastic/logs/templates/composable/logs-kafka.log.json index c670bfaa..12a6beb6 100644 --- a/elastic/logs/templates/composable/logs-kafka.log.json +++ b/elastic/logs/templates/composable/logs-kafka.log.json @@ -158,6 +158,9 @@ "properties": { "message": { "type": "text" + {% if index_sorting %}, + "store": true + {% endif %} }, "class": { "ignore_above": 1024, @@ -266,11 +269,17 @@ }, "message": { "type": "text" + {% if index_sorting %}, + "store": true + {% endif %} }, "error": { "properties": { "message": { "type": "text" + {% if index_sorting %}, + "store": true + {% endif %} } } }, @@ -287,7 +296,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-mysql.error.json b/elastic/logs/templates/composable/logs-mysql.error.json index 0f130ef6..510b9230 100644 --- a/elastic/logs/templates/composable/logs-mysql.error.json +++ b/elastic/logs/templates/composable/logs-mysql.error.json @@ -305,7 +305,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-mysql.slowlog.json b/elastic/logs/templates/composable/logs-mysql.slowlog.json index 2a32bef4..ac8cea55 100644 --- a/elastic/logs/templates/composable/logs-mysql.slowlog.json +++ b/elastic/logs/templates/composable/logs-mysql.slowlog.json @@ -450,7 +450,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-nginx.access.json b/elastic/logs/templates/composable/logs-nginx.access.json index 23453a08..6003836d 100644 --- a/elastic/logs/templates/composable/logs-nginx.access.json +++ b/elastic/logs/templates/composable/logs-nginx.access.json @@ -461,7 +461,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-nginx.error.json b/elastic/logs/templates/composable/logs-nginx.error.json index a2f3dea1..e411b4ed 100644 --- a/elastic/logs/templates/composable/logs-nginx.error.json +++ b/elastic/logs/templates/composable/logs-nginx.error.json @@ -296,7 +296,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-postgresql.log.json b/elastic/logs/templates/composable/logs-postgresql.log.json index 6dd5862d..68c54142 100644 --- a/elastic/logs/templates/composable/logs-postgresql.log.json +++ b/elastic/logs/templates/composable/logs-postgresql.log.json @@ -407,7 +407,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-redis.log.json b/elastic/logs/templates/composable/logs-redis.log.json index 62eed0ca..8e9d3417 100644 --- a/elastic/logs/templates/composable/logs-redis.log.json +++ b/elastic/logs/templates/composable/logs-redis.log.json @@ -278,7 +278,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-redis.slowlog.json b/elastic/logs/templates/composable/logs-redis.slowlog.json index 4ec11f1b..2bbd8760 100644 --- a/elastic/logs/templates/composable/logs-redis.slowlog.json +++ b/elastic/logs/templates/composable/logs-redis.slowlog.json @@ -259,7 +259,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-system.auth.json b/elastic/logs/templates/composable/logs-system.auth.json index 51cdbe39..1f795226 100644 --- a/elastic/logs/templates/composable/logs-system.auth.json +++ b/elastic/logs/templates/composable/logs-system.auth.json @@ -473,7 +473,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/templates/composable/logs-system.syslog.json b/elastic/logs/templates/composable/logs-system.syslog.json index 87828803..6585fd64 100644 --- a/elastic/logs/templates/composable/logs-system.syslog.json +++ b/elastic/logs/templates/composable/logs-system.syslog.json @@ -297,7 +297,8 @@ ".fleet_component_template-1", "track-custom-mappings", "track-custom-shared-settings", - "track-data-stream-lifecycle" + "track-data-stream-lifecycle", + "track-shared-logsdb-mode" ], "priority": 200, "_meta": { diff --git a/elastic/logs/track.json b/elastic/logs/track.json index b751659b..995d96c0 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -303,7 +303,12 @@ { "name": "track-data-stream-lifecycle", "template": "./templates/component/track-data-stream-lifecycle.json" - } + }, + + { + "name": "track-shared-logsdb-mode", + "template": "./templates/component/track-shared-logsdb-mode.json" + } ], "composable-templates": [ { From 05e0620ff0c7737c1f743d3fb6d0c41aa157a6c5 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 24 Nov 2023 09:31:53 +0100 Subject: [PATCH 2/4] fix typo Co-authored-by: Adrien Grand --- elastic/logs/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastic/logs/README.md b/elastic/logs/README.md index a6dce6a7..4d5dcc24 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -246,7 +246,7 @@ The following parameters are available: * `disable_pipelines` (default: `false`) - Prevent installing ingest node pipelines. This parameter is experimental and is to be used with indexing-only challenges. * `initial_indices_count` (default: 0) - Number of initial indices to create, each containing `100` auditbeat style documents. Parameter is applicable in [many-shards-quantitative challenge](#many-shards-quantitative-many-shards-quantitative) and in [many-shards-snapshots challenge](#many-shards-snapshots-many-shards-snapshots). * `ingest_percentage` (default: 100) - The percentage of data to be ingested. -* `index_sorting` (default: unset): Whether index sorting should be used. Accepted values: `hostnams` and `timestamp`. Setting index sorting also enabled synthetic source. +* `index_sorting` (default: unset): Whether index sorting should be used. Accepted values: `hostname` and `timestamp`. Setting index sorting also enabled synthetic source. ### Querying parameters From 804adbbc2507833db89a510c1ab0abb9579594da Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Fri, 24 Nov 2023 10:21:38 +0100 Subject: [PATCH 3/4] iter --- elastic/logs/README.md | 3 ++- elastic/logs/templates/component/auditbeat-mappings.json | 4 ++-- .../templates/component/track-shared-logsdb-mode.json | 8 ++++---- elastic/logs/templates/composable/logs-kafka.log.json | 6 +++--- elastic/logs/track.json | 3 +-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/elastic/logs/README.md b/elastic/logs/README.md index 4d5dcc24..028b74c8 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -246,7 +246,8 @@ The following parameters are available: * `disable_pipelines` (default: `false`) - Prevent installing ingest node pipelines. This parameter is experimental and is to be used with indexing-only challenges. * `initial_indices_count` (default: 0) - Number of initial indices to create, each containing `100` auditbeat style documents. Parameter is applicable in [many-shards-quantitative challenge](#many-shards-quantitative-many-shards-quantitative) and in [many-shards-snapshots challenge](#many-shards-snapshots-many-shards-snapshots). * `ingest_percentage` (default: 100) - The percentage of data to be ingested. -* `index_sorting` (default: unset): Whether index sorting should be used. Accepted values: `hostname` and `timestamp`. Setting index sorting also enabled synthetic source. +* `index_sorting` (default: unset): Whether index sorting should be used. Accepted values: `hostname` and `timestamp`. +* `synthetic_source_mode` (default: `false`): Whether to enable synthetic source. ### Querying parameters diff --git a/elastic/logs/templates/component/auditbeat-mappings.json b/elastic/logs/templates/component/auditbeat-mappings.json index 8572ccb1..c63978d1 100644 --- a/elastic/logs/templates/component/auditbeat-mappings.json +++ b/elastic/logs/templates/component/auditbeat-mappings.json @@ -1785,7 +1785,7 @@ "message": { "norms": false, "type": "text" - {% if index_sorting %}, + {% if synthetic_source_mode | default(false) is true %}, "store": true {% endif %} }, @@ -2944,7 +2944,7 @@ "message": { "norms": false, "type": "text" - {% if index_sorting %}, + {% if synthetic_source_mode | default(false) is true %}, "store": true {% endif %} }, diff --git a/elastic/logs/templates/component/track-shared-logsdb-mode.json b/elastic/logs/templates/component/track-shared-logsdb-mode.json index 5ac5661d..e8ee4027 100644 --- a/elastic/logs/templates/component/track-shared-logsdb-mode.json +++ b/elastic/logs/templates/component/track-shared-logsdb-mode.json @@ -5,18 +5,18 @@ {% if index_sorting == "hostname" %} "sort.field": [ "host.name", "@timestamp" ], "sort.order": [ "asc", "desc" ] - {%- elif index_sorting == "timestamp" %} + {% elif index_sorting == "timestamp" %} "sort.field": [ "@timestamp", "host.name" ], "sort.order": [ "desc", "asc" ] - {%- endif %} + {% endif %} } }, "mappings": { - {% if index_sorting %} + {% if synthetic_source_mode | default(false) is true %} "_source": { "mode": "synthetic" } - {%- endif %} + {% endif %} } } } \ No newline at end of file diff --git a/elastic/logs/templates/composable/logs-kafka.log.json b/elastic/logs/templates/composable/logs-kafka.log.json index 12a6beb6..883aac4e 100644 --- a/elastic/logs/templates/composable/logs-kafka.log.json +++ b/elastic/logs/templates/composable/logs-kafka.log.json @@ -158,7 +158,7 @@ "properties": { "message": { "type": "text" - {% if index_sorting %}, + {% if synthetic_source_mode | default(false) is true %}, "store": true {% endif %} }, @@ -269,7 +269,7 @@ }, "message": { "type": "text" - {% if index_sorting %}, + {% if synthetic_source_mode | default(false) is true %}, "store": true {% endif %} }, @@ -277,7 +277,7 @@ "properties": { "message": { "type": "text" - {% if index_sorting %}, + {% if synthetic_source_mode | default(false) is true %}, "store": true {% endif %} } diff --git a/elastic/logs/track.json b/elastic/logs/track.json index 995d96c0..378b62f8 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -304,11 +304,10 @@ "name": "track-data-stream-lifecycle", "template": "./templates/component/track-data-stream-lifecycle.json" }, - { "name": "track-shared-logsdb-mode", "template": "./templates/component/track-shared-logsdb-mode.json" - } + } ], "composable-templates": [ { From c4cf4205ae6a71a06aa89a85a09eabb0d9a01ca0 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 6 Dec 2023 09:34:18 +0100 Subject: [PATCH 4/4] added force_merge_max_num_segments parameter --- elastic/logs/README.md | 1 + elastic/logs/challenges/logging-querying.json | 46 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/elastic/logs/README.md b/elastic/logs/README.md index 028b74c8..1b70895a 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -248,6 +248,7 @@ The following parameters are available: * `ingest_percentage` (default: 100) - The percentage of data to be ingested. * `index_sorting` (default: unset): Whether index sorting should be used. Accepted values: `hostname` and `timestamp`. * `synthetic_source_mode` (default: `false`): Whether to enable synthetic source. +* `force_merge_max_num_segments` (default: unset): An integer specifying the max amount of segments the force-merge operation should use. Only supported in `logging-querying` track. ### Querying parameters diff --git a/elastic/logs/challenges/logging-querying.json b/elastic/logs/challenges/logging-querying.json index 7e72c9c6..a116de9e 100644 --- a/elastic/logs/challenges/logging-querying.json +++ b/elastic/logs/challenges/logging-querying.json @@ -22,6 +22,52 @@ "clients": {{ p_bulk_indexing_clients }}, "ignore-response-error-level": "{{error_level | default('non-fatal')}}" } + {%- if force_merge_max_num_segments is defined %}, + { + "name": "refresh-after-index", + "index": "logs-*", + "operation": "refresh" + }, + { + "name": "wait-until-index-merges-fininshes", + "operation": { + "operation-type": "index-stats", + "index": "logs-*", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false + } + }, + { + "operation": { + "operation-type": "force-merge", + "index": "logs-*", + "request-timeout": 36000, + "max-num-segments": {{ force_merge_max_num_segments | tojson }} + } + }, + { + "name": "wait-until-merges-finish", + "operation": { + "operation-type": "index-stats", + "index": "logs-*", + "condition": { + "path": "_all.total.merges.current", + "expected-value": 0 + }, + "retry-until-success": true, + "include-in-reporting": false + } + }, + { + "name": "refresh-after-force-merge", + "index": "logs-*", + "operation": "refresh" + } + {%- endif %} {# non-serverless-index-statistics-marker-start #}{%- if build_flavor != "serverless" or serverless_operator == true -%}, { "name": "compression-stats",