From c4889890feb16ca1c161a7c03ee084cde5185117 Mon Sep 17 00:00:00 2001 From: Elastic Machine Date: Tue, 12 Dec 2023 12:31:32 +1100 Subject: [PATCH 01/19] [Release] Update test environment to 8.13.0 (#37310) * [Release] update test environment * Update to 8.11.2 --------- Co-authored-by: Craig MacKenzie --- metricbeat/docker-compose.yml | 12 ++++++------ testing/environments/latest.yml | 6 +++--- x-pack/metricbeat/docker-compose.yml | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 7f0200dfb094..03e0b65272e5 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -17,11 +17,11 @@ services: # Used by base tests elasticsearch: - image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-8.10.2}-1 + image: docker.elastic.co/integrations-ci/beats-elasticsearch:${ELASTICSEARCH_VERSION:-8.11.2}-1 build: context: ./module/elasticsearch/_meta args: - ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-8.10.2} + ELASTICSEARCH_VERSION: ${ELASTICSEARCH_VERSION:-8.11.2} environment: - "ES_JAVA_OPTS=-Xms256m -Xmx256m" - "transport.host=127.0.0.1" @@ -38,11 +38,11 @@ services: # Used by base tests kibana: - image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.10.2}-1 + image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.11.2}-1 build: context: ./module/kibana/_meta args: - KIBANA_VERSION: ${KIBANA_VERSION:-8.10.2} + KIBANA_VERSION: ${KIBANA_VERSION:-8.11.2} healthcheck: test: ["CMD-SHELL", "curl -u beats:testing -s http://localhost:5601/api/status?v8format=true | grep -q '\"overall\":{\"level\":\"available\"'"] retries: 600 @@ -53,11 +53,11 @@ services: # Used by base tests metricbeat: - image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-8.10.2}-1 + image: docker.elastic.co/integrations-ci/beats-metricbeat:${BEAT_VERSION:-8.11.2}-1 build: context: ./module/beat/_meta args: - BEAT_VERSION: ${BEAT_VERSION:-8.10.2} + BEAT_VERSION: ${BEAT_VERSION:-8.11.2} command: '-e' ports: - 5066:5066 diff --git a/testing/environments/latest.yml b/testing/environments/latest.yml index ff0074f3a067..2ce1f719210e 100644 --- a/testing/environments/latest.yml +++ b/testing/environments/latest.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.10.2 + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.2 healthcheck: test: ["CMD-SHELL", "curl -s http://localhost:9200/_cat/health?h=status | grep -q green"] retries: 300 @@ -19,7 +19,7 @@ services: - "script.context.template.cache_max_size=2000" logstash: - image: docker.elastic.co/logstash/logstash:8.10.2 + image: docker.elastic.co/logstash/logstash:8.11.2 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 300 @@ -29,7 +29,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:8.10.2 + image: docker.elastic.co/kibana/kibana:8.11.2 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5601"] retries: 300 diff --git a/x-pack/metricbeat/docker-compose.yml b/x-pack/metricbeat/docker-compose.yml index b2b0f50020ac..f1396fdf9e8f 100644 --- a/x-pack/metricbeat/docker-compose.yml +++ b/x-pack/metricbeat/docker-compose.yml @@ -24,11 +24,11 @@ services: kibana: # Copied configuration from OSS metricbeat because services with depends_on # cannot be extended with extends - image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.10.2}-1 + image: docker.elastic.co/integrations-ci/beats-kibana:${KIBANA_VERSION:-8.11.2}-1 build: context: ../../metricbeat/module/kibana/_meta args: - KIBANA_VERSION: ${KIBANA_VERSION:-8.10.2} + KIBANA_VERSION: ${KIBANA_VERSION:-8.11.2} depends_on: - elasticsearch ports: From 2a348121f04c4dc8601b42a9621c05974be97210 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 21:12:28 -0500 Subject: [PATCH 02/19] [Docs] Document memory leak in `8.11.0` and `8.11.1` release notes (#37393) (#37394) * Add memory leak note to 8.11.0 and 8.11.1 release notes * Update CHANGELOG.asciidoc Co-authored-by: Craig MacKenzie * Update CHANGELOG.asciidoc Co-authored-by: Craig MacKenzie --------- Co-authored-by: Craig MacKenzie (cherry picked from commit 0856c8cf54c4ff628d6722cf2eae7c093b1ca56c) Co-authored-by: Shaunak Kashyap --- CHANGELOG.asciidoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index f0fccce54dae..d49f36e16011 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -54,6 +54,8 @@ https://github.com/elastic/beats/compare/v8.11.1\...v8.11.2[View commits] === Beats version 8.11.1 https://github.com/elastic/beats/compare/v8.11.0\...v8.11.1[View commits] +IMPORTANT: Due to a memory leak issue, Windows users running {agent} are recommended to avoid upgrading to this release and instead use the 8.11.2 release in which the issue is resolved. If you've already upgraded to version 8.11.0 or 8.11.1, we recommend upgrading to 8.11.2 as soon as possible. + ==== Added *Affecting all Beats* @@ -65,9 +67,13 @@ https://github.com/elastic/beats/compare/v8.11.0\...v8.11.1[View commits] === Beats version 8.11.0 https://github.com/elastic/beats/compare/v8.10.4\...v8.11.0[View commits] + +IMPORTANT: Due to a memory leak issue, Windows users running {agent} are recommended to avoid upgrading to this release and instead use the 8.11.2 release in which the issue is resolved. If you've already upgraded to version 8.11.0 or 8.11.1, we recommend upgrading to 8.11.2 as soon as possible. + ==== Breaking changes *Affecting all Beats* + - The Elasticsearch output now enables compression by default. This decreases network data usage by an average of 70-80%, in exchange for 20-25% increased CPU use and ~10% increased ingestion time. The previous default can be restored by setting the flag `compression_level: 0` under `output.elasticsearch`. {pull}36681[36681] - The `elastic-agent-autodiscover` library is updated to version 0.6.4, disabling metadata for deployment and cronjob. Pods that will be created from deployments or cronjobs will not have the extra metadata field for `kubernetes.deployment` or `kubernetes.cronjob`, respectively. {pull}36879[36879] From ca6991ec024e48796861d2de5c4daaa252d5f16d Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 12 Dec 2023 14:38:57 +0100 Subject: [PATCH 03/19] ci(action): auto-bump daily and split in two jobs (#37351) --- .github/workflows/bump-golang.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bump-golang.yml b/.github/workflows/bump-golang.yml index 7f1fbc4a208a..393c57f89a07 100644 --- a/.github/workflows/bump-golang.yml +++ b/.github/workflows/bump-golang.yml @@ -4,7 +4,7 @@ name: bump-golang on: workflow_dispatch: schedule: - - cron: "0 20 * * 6" + - cron: "0 20 * * 1-6" permissions: contents: read @@ -13,7 +13,7 @@ env: JOB_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" jobs: - bump: + bump-main: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -27,6 +27,11 @@ jobs: notifySlackChannel: "#ingest-notifications" messageIfFailure: ":traffic_cone: updatecli failed for `${{ github.repository }}@${{ github.ref_name }}`, `@update-me-with-the-slack-team-to-be-poked` please look what's going on <${{ env.JOB_URL }}|here>" + bump-7-17: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: elastic/apm-pipeline-library/.github/actions/updatecli@current with: vaultUrl: ${{ secrets.VAULT_ADDR }} From 06a8c09655b0d1d5d4ec4bd77bc930d18975155e Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 12 Dec 2023 15:06:00 +0100 Subject: [PATCH 04/19] docs: Prepare Changelog for 8.11.3 (#37399) (#37402) * docs: Close changelog for 8.11.3 * Update CHANGELOG.asciidoc --------- Co-authored-by: Pierre HILBERT (cherry picked from commit 33d97e131d989dcc3bb89aef6588d112aa433c36) Co-authored-by: Elastic Machine --- CHANGELOG.asciidoc | 14 ++++++++++++++ CHANGELOG.next.asciidoc | 8 +++----- libbeat/docs/release.asciidoc | 1 + 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d49f36e16011..d017edbd6d0f 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,20 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-8.11.3]] +=== Beats version 8.11.3 +https://github.com/elastic/beats/compare/v8.11.2\...v8.11.3[View commits] + +==== Breaking changes + +*Affecting all Beats* +- `queue.mem.events` is changing from `4096` to `3200`. +- `queue.mem.flush.min_events` is changing from `2048` to `1600`. +- `queue.mem.flush.timeout` is changing from `1s` to `10s`. +- `output.elasticsearch.bulk_max_size` is changing from `50` to `1600`. +- `output.elasticsearch.idle_connection_timeout` is changing from `60s` to `3s`. + + [[release-notes-8.11.2]] === Beats version 8.11.2 https://github.com/elastic/beats/compare/v8.11.1\...v8.11.2[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 41ec75f0fb5b..2add786f8e1e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -9,11 +9,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] ==== Breaking changes *Affecting all Beats* -- `queue.mem.events` is changing from `4096` to `3200`. -- `queue.mem.flush.min_events` is changing from `2048` to `1600`. -- `queue.mem.flush.timeout` is changing from `1s` to `10s`. -- `output.elasticsearch.bulk_max_size` is changing from `50` to `1600`. -- `output.elasticsearch.idle_connection_timeout` is changing from `60s` to `3s`. *Auditbeat* @@ -331,6 +326,9 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d + + + diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index 027c6099923e..3219d84d4007 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <> From dcd09e98bb200780e0dccfd8602df44db7599071 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 12 Dec 2023 12:05:33 -0500 Subject: [PATCH 05/19] Add security notice to Beats 8.11.3 Release Notes (#37404) (#37410) (cherry picked from commit 10e79d9941962b1e64b6bc6401a905ba3cebdad0) Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- CHANGELOG.asciidoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d017edbd6d0f..c0d84b9a6fc6 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -7,9 +7,12 @@ === Beats version 8.11.3 https://github.com/elastic/beats/compare/v8.11.2\...v8.11.3[View commits] +The 8.11.3 patch release contains a fix for a potential security vulnerability. Please see our link:https://discuss.elastic.co/c/announcements/security-announcements/31[security advisory for more details]. + ==== Breaking changes *Affecting all Beats* + - `queue.mem.events` is changing from `4096` to `3200`. - `queue.mem.flush.min_events` is changing from `2048` to `1600`. - `queue.mem.flush.timeout` is changing from `1s` to `10s`. From c7a9d15095cf48370ca55990a677163bde0e0563 Mon Sep 17 00:00:00 2001 From: Abdon Pijpelink Date: Tue, 12 Dec 2023 19:15:18 +0100 Subject: [PATCH 06/19] Add security notice to Beats 8.11.2 release notes (#37407) --- CHANGELOG.asciidoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index c0d84b9a6fc6..90cd1cf93876 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -24,9 +24,12 @@ The 8.11.3 patch release contains a fix for a potential security vulnerability. === Beats version 8.11.2 https://github.com/elastic/beats/compare/v8.11.1\...v8.11.2[View commits] +The 8.11.2 patch release contains a fix for a potential security vulnerability. https://discuss.elastic.co/c/announcements/security-announcements/31[Please see our security advisory for more details]. + ==== Breaking changes *Affecting all Beats* + - Avoid logging fields values when handling Elasticsearch output errors except at the debug log level. The debug log level must now be used to see detailed errors, for example mapping errors and their cause. {pull}37229[37229] ==== Bugfixes From 9216bdfa030745d56f30fdf8c6f12ac84eec3e40 Mon Sep 17 00:00:00 2001 From: Kylie Meli Date: Tue, 12 Dec 2023 16:19:47 -0500 Subject: [PATCH 07/19] [winlogbeat] Adding nanoseconds time unit to docs for `source_lag_time`, `batch_read_period` (#37418) * Specifying metric for lag time is in nanoseconds * adding nanos to another metric --- winlogbeat/docs/metrics-winlogbeat.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/winlogbeat/docs/metrics-winlogbeat.asciidoc b/winlogbeat/docs/metrics-winlogbeat.asciidoc index c74d1130ac53..7c3c9eed943e 100644 --- a/winlogbeat/docs/metrics-winlogbeat.asciidoc +++ b/winlogbeat/docs/metrics-winlogbeat.asciidoc @@ -16,6 +16,6 @@ observe the event log processing activity of Winlogbeat. | `discarded_events_total` | Total number of discarded events. | `errors_total` | Total number of errors. | `received_events_count` | Histogram of the number of events in each non-zero batch. -| `source_lag_time` | Histogram of the difference between timestamped event's creation and reading. -| `batch_read_period` | Histogram of the elapsed time between non-zero batch reads. +| `source_lag_time` | Histogram of the difference in nanoseconds between timestamped event's creation and reading. +| `batch_read_period` | Histogram of the elapsed time in nanoseconds between non-zero batch reads. |======= From fe9b7ce5a9180ba820d1c0719fe12f8cf7738917 Mon Sep 17 00:00:00 2001 From: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> Date: Tue, 12 Dec 2023 16:33:38 -0500 Subject: [PATCH 08/19] Revert 37407 to remove security notice in 8.11.2 RN (#37421) --- CHANGELOG.asciidoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 90cd1cf93876..8291c2aa2a8d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -24,8 +24,6 @@ The 8.11.3 patch release contains a fix for a potential security vulnerability. === Beats version 8.11.2 https://github.com/elastic/beats/compare/v8.11.1\...v8.11.2[View commits] -The 8.11.2 patch release contains a fix for a potential security vulnerability. https://discuss.elastic.co/c/announcements/security-announcements/31[Please see our security advisory for more details]. - ==== Breaking changes *Affecting all Beats* From ef1bce5cfc693a9e1b015b98a96757738eb8aee9 Mon Sep 17 00:00:00 2001 From: Milton Hultgren Date: Wed, 13 Dec 2023 14:28:54 +0100 Subject: [PATCH 09/19] [beat] Set cluster UUID in state metricset correctly (#37419) * [beat] Set cluster UUID in state metricset correctly * Add test * Fix file ending with new line * Fix spacing error --- .../beat/state/_meta/test/uuid_es_output.json | 43 +++++++++ .../test/uuid_es_output_pre_connect.json | 43 +++++++++ .../_meta/test/uuid_monitoring_config.json | 52 +++++++++++ .../_meta/test/uuid_no_monitoring_config.json | 52 +++++++++++ metricbeat/module/beat/state/data.go | 11 +-- metricbeat/module/beat/state/data_test.go | 89 +++++++++++++++++++ 6 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 metricbeat/module/beat/state/_meta/test/uuid_es_output.json create mode 100644 metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json create mode 100644 metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json create mode 100644 metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json diff --git a/metricbeat/module/beat/state/_meta/test/uuid_es_output.json b/metricbeat/module/beat/state/_meta/test/uuid_es_output.json new file mode 100644 index 000000000000..3cabff1ad069 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_es_output.json @@ -0,0 +1,43 @@ +{ + "beat": { + "name": "Shaunaks-MBP-2" + }, + "host": { + "architecture": "x86_64", + "hostname": "Shaunaks-MBP-2", + "id": "EF6274EA-462F-5316-A14A-850E7BFD8126", + "os": { + "build": "18F132", + "family": "darwin", + "kernel": "18.6.0", + "name": "Mac OS X", + "platform": "darwin", + "version": "10.14.5" + } + }, + "management": { + "enabled": false + }, + "module": { + "count": 3, + "names": [ + "system" + ] + }, + "output": { + "name": "elasticsearch" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "uuid_from_es_output" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", + "name": "metricbeat", + "version": "8.0.0" + } +} diff --git a/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json b/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json new file mode 100644 index 000000000000..557b4163d359 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_es_output_pre_connect.json @@ -0,0 +1,43 @@ +{ + "beat": { + "name": "Shaunaks-MBP-2" + }, + "host": { + "architecture": "x86_64", + "hostname": "Shaunaks-MBP-2", + "id": "EF6274EA-462F-5316-A14A-850E7BFD8126", + "os": { + "build": "18F132", + "family": "darwin", + "kernel": "18.6.0", + "name": "Mac OS X", + "platform": "darwin", + "version": "10.14.5" + } + }, + "management": { + "enabled": false + }, + "module": { + "count": 3, + "names": [ + "system" + ] + }, + "output": { + "name": "elasticsearch" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "1f0c187b-f2ef-4950-b9cc-dd6864b9191a", + "name": "metricbeat", + "version": "8.0.0" + } +} diff --git a/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json b/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json new file mode 100644 index 000000000000..1464e65b6402 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_monitoring_config.json @@ -0,0 +1,52 @@ +{ + "beat": { + "name": "MacBook-Pro" + }, + "host": { + "architecture": "arm64", + "hostname": "MacBook-Pro", + "id": "B0C1E948-241E-53FD-A6A3-0D0F352403AF", + "os": { + "build": "23B92", + "family": "darwin", + "kernel": "23.1.0", + "name": "macOS", + "platform": "darwin", + "version": "14.1.2" + } + }, + "input": { + "count": 1, + "names": [ + "log" + ] + }, + "management": { + "enabled": false + }, + "module": { + "count": 0, + "names": [] + }, + "monitoring": { + "cluster_uuid": "uuid_from_monitoring_config" + }, + "output": { + "batch_size": 2048, + "clients": 1, + "name": "logstash" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "8d6af17b-cf55-4029-bd22-64cd538acbd0", + "name": "filebeat", + "version": "8.13.0" + } +} diff --git a/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json b/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json new file mode 100644 index 000000000000..c204e26d2491 --- /dev/null +++ b/metricbeat/module/beat/state/_meta/test/uuid_no_monitoring_config.json @@ -0,0 +1,52 @@ +{ + "beat": { + "name": "MacBook-Pro" + }, + "host": { + "architecture": "arm64", + "hostname": "MacBook-Pro", + "id": "B0C1E948-241E-53FD-A6A3-0D0F352403AF", + "os": { + "build": "23B92", + "family": "darwin", + "kernel": "23.1.0", + "name": "macOS", + "platform": "darwin", + "version": "14.1.2" + } + }, + "input": { + "count": 1, + "names": [ + "log" + ] + }, + "management": { + "enabled": false + }, + "module": { + "count": 0, + "names": [] + }, + "monitoring": { + "cluster_uuid": "" + }, + "output": { + "batch_size": 2048, + "clients": 1, + "name": "logstash" + }, + "outputs": { + "elasticsearch": { + "cluster_uuid": "" + } + }, + "queue": { + "name": "mem" + }, + "service": { + "id": "8d6af17b-cf55-4029-bd22-64cd538acbd0", + "name": "filebeat", + "version": "8.13.0" + } +} diff --git a/metricbeat/module/beat/state/data.go b/metricbeat/module/beat/state/data.go index b555c84bd402..5d8231046043 100644 --- a/metricbeat/module/beat/state/data.go +++ b/metricbeat/module/beat/state/data.go @@ -77,22 +77,19 @@ func eventMapping(r mb.ReporterV2, info beat.Info, content []byte, isXpack bool) return fmt.Errorf("failure parsing Beat's State API response: %w", err) } - event.MetricSetFields, _ = schema.Apply(data) - clusterUUID := getMonitoringClusterUUID(data) if clusterUUID == "" { if isOutputES(data) { clusterUUID = getClusterUUID(data) - if clusterUUID != "" { - event.ModuleFields.Put("elasticsearch.cluster.id", clusterUUID) - if event.MetricSetFields != nil { - event.MetricSetFields.Put("cluster.uuid", clusterUUID) - } + if clusterUUID == "" { + return nil } } } + event.ModuleFields.Put("elasticsearch.cluster.id", clusterUUID) + event.MetricSetFields, _ = schema.Apply(data) if event.MetricSetFields != nil { diff --git a/metricbeat/module/beat/state/data_test.go b/metricbeat/module/beat/state/data_test.go index 6123b7539b7e..4d94a3f24da9 100644 --- a/metricbeat/module/beat/state/data_test.go +++ b/metricbeat/module/beat/state/data_test.go @@ -53,3 +53,92 @@ func TestEventMapping(t *testing.T) { require.Equal(t, 0, len(reporter.GetErrors()), f) } } + +func TestUuidFromEsOutput(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_es_output.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) + require.Equal(t, 0, len(reporter.GetErrors())) + + event := reporter.GetEvents()[0] + + uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id") + require.NoError(t, err) + + require.Equal(t, "uuid_from_es_output", uuid) +} + +func TestNoEventIfEsOutputButNoUuidYet(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_es_output_pre_connect.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.Equal(t, 0, len(reporter.GetEvents())) + require.Equal(t, 0, len(reporter.GetErrors())) +} + +func TestUuidFromMonitoringConfig(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_monitoring_config.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) + require.Equal(t, 0, len(reporter.GetErrors())) + + event := reporter.GetEvents()[0] + + uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id") + require.NoError(t, err) + + require.Equal(t, "uuid_from_monitoring_config", uuid) +} + +func TestNoUuidInMonitoringConfig(t *testing.T) { + reporter := &mbtest.CapturingReporterV2{} + + info := beat.Info{ + UUID: "1234", + Beat: "testbeat", + } + + input, err := ioutil.ReadFile("./_meta/test/uuid_no_monitoring_config.json") + require.NoError(t, err) + + err = eventMapping(reporter, info, input, true) + require.NoError(t, err) + require.True(t, len(reporter.GetEvents()) >= 1) + require.Equal(t, 0, len(reporter.GetErrors())) + + event := reporter.GetEvents()[0] + + uuid, err := event.ModuleFields.GetValue("elasticsearch.cluster.id") + require.NoError(t, err) + + require.Equal(t, "", uuid) +} From 6c5d221596c64c0cd25ecf02cf4387171947b977 Mon Sep 17 00:00:00 2001 From: Vignesh Shanmugam Date: Wed, 13 Dec 2023 07:40:38 -0800 Subject: [PATCH 10/19] fix: adjust formatting for heartbeat monitor and state loader (#37427) * fix: adjust formatting for heartbeat monitor and state loader * reset ES error --- CHANGELOG.next.asciidoc | 1 + heartbeat/monitors/active/icmp/stdloop.go | 2 +- heartbeat/monitors/monitor.go | 2 +- heartbeat/monitors/task.go | 6 +++--- heartbeat/monitors/wrappers/monitorstate/tracker.go | 2 +- x-pack/heartbeat/monitors/browser/sourcejob.go | 2 +- x-pack/heartbeat/monitors/browser/synthexec/synthexec.go | 2 +- x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go | 2 +- 8 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2add786f8e1e..66a2ad9c40c3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -86,6 +86,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] *Heartbeat* - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] +- Added fix for formatting the logs from stateloader properly. {pull}37369[37369] *Metricbeat* diff --git a/heartbeat/monitors/active/icmp/stdloop.go b/heartbeat/monitors/active/icmp/stdloop.go index b49d243ec4c6..f67ae402bc70 100644 --- a/heartbeat/monitors/active/icmp/stdloop.go +++ b/heartbeat/monitors/active/icmp/stdloop.go @@ -165,7 +165,7 @@ func (l *stdICMPLoop) runICMPRecv(conn *icmp.PacketConn, proto int) { bytes := make([]byte, 512) err := conn.SetReadDeadline(time.Now().Add(time.Second)) if err != nil { - logp.L().Error("could not set read deadline for ICMP: %w", err) + logp.L().Errorf("could not set read deadline for ICMP: %w", err) return } _, addr, err := conn.ReadFrom(bytes) diff --git a/heartbeat/monitors/monitor.go b/heartbeat/monitors/monitor.go index 6a16c7d3f303..29e7713145ca 100644 --- a/heartbeat/monitors/monitor.go +++ b/heartbeat/monitors/monitor.go @@ -256,7 +256,7 @@ func (m *Monitor) Stop() { if m.close != nil { err := m.close() if err != nil { - logp.L().Error("error closing monitor %s: %w", m.String(), err) + logp.L().Errorf("error closing monitor %s: %w", m.String(), err) } } diff --git a/heartbeat/monitors/task.go b/heartbeat/monitors/task.go index ee0839fe14e5..a655e1d15467 100644 --- a/heartbeat/monitors/task.go +++ b/heartbeat/monitors/task.go @@ -80,13 +80,13 @@ func (t *configuredJob) Start(pubClient beat.Client) { t.pubClient = pubClient if err != nil { - logp.L().Info("could not start monitor: %v", err) + logp.L().Infof("could not start monitor: %v", err) return } t.cancelFn, err = t.monitor.addTask(t.config.Schedule, t.monitor.stdFields.ID, t.makeSchedulerTaskFunc(), t.config.Type) if err != nil { - logp.L().Info("could not start monitor: %v", err) + logp.L().Infof("could not start monitor: %v", err) } } @@ -107,7 +107,7 @@ func runPublishJob(job jobs.Job, pubClient beat.Client) []scheduler.TaskFunc { conts, err := job(event) if err != nil { - logp.L().Info("Job failed with: %s", err) + logp.L().Infof("Job failed with: %s", err) } hasContinuations := len(conts) > 0 diff --git a/heartbeat/monitors/wrappers/monitorstate/tracker.go b/heartbeat/monitors/wrappers/monitorstate/tracker.go index 03909d55aa83..e350294e46e8 100644 --- a/heartbeat/monitors/wrappers/monitorstate/tracker.go +++ b/heartbeat/monitors/wrappers/monitorstate/tracker.go @@ -104,7 +104,7 @@ func (t *Tracker) GetCurrentState(sf stdfields.StdMonitorFields) (state *State) time.Sleep(sleepFor) } if err != nil { - logp.L().Warn("could not load prior state from elasticsearch after %d attempts, will create new state for monitor: %s", tries, sf.ID) + logp.L().Warnf("could not load prior state from elasticsearch after %d attempts, will create new state for monitor: %s", tries, sf.ID) } if loadedState != nil { diff --git a/x-pack/heartbeat/monitors/browser/sourcejob.go b/x-pack/heartbeat/monitors/browser/sourcejob.go index c62c50b3bb17..697e51abf51a 100644 --- a/x-pack/heartbeat/monitors/browser/sourcejob.go +++ b/x-pack/heartbeat/monitors/browser/sourcejob.go @@ -125,7 +125,7 @@ func (sj *SourceJob) extraArgs(uiOrigin bool) []string { s, err := json.Marshal(sj.browserCfg.PlaywrightOpts) if err != nil { // This should never happen, if it was parsed as a config it should be serializable - logp.L().Warn("could not serialize playwright options '%v': %w", sj.browserCfg.PlaywrightOpts, err) + logp.L().Warnf("could not serialize playwright options '%v': %w", sj.browserCfg.PlaywrightOpts, err) } else { extraArgs = append(extraArgs, "--playwright-options", string(s)) } diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go index fbfb71526cc5..32f127de98c3 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthexec.go @@ -219,7 +219,7 @@ func runCmd( break } if err != nil { - logp.L().Warn("error decoding json for test json results: %w", err) + logp.L().Warnf("error decoding json for test json results: %w", err) } mpx.writeSynthEvent(&se) diff --git a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go index 8555ae448a71..ddd928b216d6 100644 --- a/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go +++ b/x-pack/heartbeat/monitors/browser/synthexec/synthtypes.go @@ -96,7 +96,7 @@ func (se SynthEvent) ToMap() (m mapstr.M) { u, e := url.Parse(se.URL) if e != nil { _, _ = m.Put("url", mapstr.M{"full": se.URL}) - logp.L().Warn("Could not parse synthetics URL '%s': %s", se.URL, e.Error()) + logp.L().Warnf("Could not parse synthetics URL '%s': %s", se.URL, e.Error()) } else { _, _ = m.Put("url", wraputil.URLFields(u)) } From d11eac9452a28598f2afaed47202289a2463922c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Alvarez=20Pi=C3=B1eiro?= <95703246+emilioalvap@users.noreply.github.com> Date: Thu, 14 Dec 2023 14:35:56 +0100 Subject: [PATCH 11/19] [Heartbeat] Remove duplicate seccomp syscalls (#37440) Remove duplicated pwrte64 syscall is preventing the filter from being installed. --- CHANGELOG.next.asciidoc | 1 + heartbeat/security/policy_linux_arm64.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 66a2ad9c40c3..ae767239de47 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -87,6 +87,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix panics when parsing dereferencing invalid parsed url. {pull}34702[34702] - Added fix for formatting the logs from stateloader properly. {pull}37369[37369] +- Remove duplicated syscall from arm seccomp profile. {pull}37440[37440] *Metricbeat* diff --git a/heartbeat/security/policy_linux_arm64.go b/heartbeat/security/policy_linux_arm64.go index 45c8192f5e9a..e198819dbb94 100644 --- a/heartbeat/security/policy_linux_arm64.go +++ b/heartbeat/security/policy_linux_arm64.go @@ -115,7 +115,6 @@ func init() { "prlimit64", "pselect6", "pwrite64", - "pwrite64", "read", "readlinkat", "recvfrom", From 6759826fc71a09c0c1eb65570f71dfd6be4421be Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Fri, 15 Dec 2023 14:37:16 -0500 Subject: [PATCH 12/19] [updatecli] update elastic stack version for testing 8.13.0-6av99u5d (#37312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli --------- Co-authored-by: apmmachine Co-authored-by: Pierre HILBERT Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 3ec6770467f0..4c2c820e6b07 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.12.0-33e8d7e1-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-6av99u5d-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.12.0-33e8d7e1-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.13.0-6av99u5d-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.12.0-33e8d7e1-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.13.0-6av99u5d-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From a633696068edd5cf3a698eeaea0c9390909de2b2 Mon Sep 17 00:00:00 2001 From: Denis Date: Mon, 18 Dec 2023 14:18:41 +0000 Subject: [PATCH 13/19] Add filestream benchmarks for many files case, fix data race (#37345) Before it was only testing for a single file not the tests support many files. During test runs with the race detector a data race was found and fixed. --- filebeat/input/filestream/input.go | 27 +++-- filebeat/input/filestream/input_test.go | 146 ++++++++++++++++-------- 2 files changed, 111 insertions(+), 62 deletions(-) diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go index b935161c1269..6cedd1bd9d16 100644 --- a/filebeat/input/filestream/input.go +++ b/filebeat/input/filestream/input.go @@ -59,7 +59,6 @@ type fileMeta struct { type filestream struct { readerConfig readerConfig encodingFactory encoding.EncodingFactory - encoding encoding.Encoding closerConfig closerConfig parsers parser.Config } @@ -175,7 +174,7 @@ func initState(log *logp.Logger, c loginp.Cursor, s fileSource) state { } func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSource, offset int64) (reader.Reader, error) { - f, err := inp.openFile(log, fs.newPath, offset) + f, encoding, err := inp.openFile(log, fs.newPath, offset) if err != nil { return nil, err } @@ -216,7 +215,7 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo var r reader.Reader r, err = readfile.NewEncodeReader(dbgReader, readfile.Config{ - Codec: inp.encoding, + Codec: encoding, BufferSize: inp.readerConfig.BufferSize, Terminator: inp.readerConfig.LineTerminator, MaxBytes: encReaderMaxBytes, @@ -241,33 +240,33 @@ func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, fs fileSo // or the file cannot be opened because for example of failing read permissions, an error // is returned and the harvester is closed. The file will be picked up again the next time // the file system is scanned -func (inp *filestream) openFile(log *logp.Logger, path string, offset int64) (*os.File, error) { +func (inp *filestream) openFile(log *logp.Logger, path string, offset int64) (*os.File, encoding.Encoding, error) { fi, err := os.Stat(path) if err != nil { - return nil, fmt.Errorf("failed to stat source file %s: %w", path, err) + return nil, nil, fmt.Errorf("failed to stat source file %s: %w", path, err) } // it must be checked if the file is not a named pipe before we try to open it // if it is a named pipe os.OpenFile fails, so there is no need to try opening it. if fi.Mode()&os.ModeNamedPipe != 0 { - return nil, fmt.Errorf("failed to open file %s, named pipes are not supported", fi.Name()) + return nil, nil, fmt.Errorf("failed to open file %s, named pipes are not supported", fi.Name()) } ok := false f, err := file.ReadOpen(path) if err != nil { - return nil, fmt.Errorf("failed opening %s: %w", path, err) + return nil, nil, fmt.Errorf("failed opening %s: %w", path, err) } defer cleanup.IfNot(&ok, cleanup.IgnoreError(f.Close)) fi, err = f.Stat() if err != nil { - return nil, fmt.Errorf("failed to stat source file %s: %w", path, err) + return nil, nil, fmt.Errorf("failed to stat source file %s: %w", path, err) } err = checkFileBeforeOpening(fi) if err != nil { - return nil, err + return nil, nil, err } if fi.Size() < offset { @@ -276,20 +275,20 @@ func (inp *filestream) openFile(log *logp.Logger, path string, offset int64) (*o } err = inp.initFileOffset(f, offset) if err != nil { - return nil, err + return nil, nil, err } - inp.encoding, err = inp.encodingFactory(f) + encoding, err := inp.encodingFactory(f) if err != nil { f.Close() if errors.Is(err, transform.ErrShortSrc) { - return nil, fmt.Errorf("initialising encoding for '%v' failed due to file being too short", f) + return nil, nil, fmt.Errorf("initialising encoding for '%v' failed due to file being too short", f) } - return nil, fmt.Errorf("initialising encoding for '%v' failed: %w", f, err) + return nil, nil, fmt.Errorf("initialising encoding for '%v' failed: %w", f, err) } ok = true - return f, nil + return f, encoding, nil } func checkFileBeforeOpening(fi os.FileInfo) error { diff --git a/filebeat/input/filestream/input_test.go b/filebeat/input/filestream/input_test.go index 55b8d2e7fc66..a1d9729c5aad 100644 --- a/filebeat/input/filestream/input_test.go +++ b/filebeat/input/filestream/input_test.go @@ -21,10 +21,11 @@ import ( "context" "fmt" "os" + "path/filepath" + "sync/atomic" "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" @@ -38,25 +39,26 @@ import ( func BenchmarkFilestream(b *testing.B) { logp.TestingSetup(logp.ToDiscardOutput()) - lineCount := 10000 - filename := generateFile(b, lineCount) - b.ResetTimer() + b.Run("single file", func(b *testing.B) { + lineCount := 10000 + filename := generateFile(b, b.TempDir(), lineCount) + b.ResetTimer() - b.Run("filestream default throughput", func(b *testing.B) { - cfg := ` + b.Run("inode throughput", func(b *testing.B) { + cfg := ` type: filestream prospector.scanner.check_interval: 1s paths: - ` + filename + ` ` - for i := 0; i < b.N; i++ { - runFilestreamBenchmark(b, fmt.Sprintf("default-benchmark-%d", i), cfg, lineCount) - } - }) + for i := 0; i < b.N; i++ { + runFilestreamBenchmark(b, fmt.Sprintf("one-file-inode-benchmark-%d", i), cfg, lineCount) + } + }) - b.Run("filestream fingerprint throughput", func(b *testing.B) { - cfg := ` + b.Run("fingerprint throughput", func(b *testing.B) { + cfg := ` type: filestream prospector.scanner: fingerprint.enabled: true @@ -65,9 +67,51 @@ file_identity.fingerprint: ~ paths: - ` + filename + ` ` - for i := 0; i < b.N; i++ { - runFilestreamBenchmark(b, fmt.Sprintf("fp-benchmark-%d", i), cfg, lineCount) + for i := 0; i < b.N; i++ { + runFilestreamBenchmark(b, fmt.Sprintf("one-file-fp-benchmark-%d", i), cfg, lineCount) + } + }) + }) + + b.Run("many files", func(b *testing.B) { + lineCount := 1000 + fileCount := 100 + dir := b.TempDir() + + for i := 0; i < fileCount; i++ { + _ = generateFile(b, dir, lineCount) } + + ingestPath := filepath.Join(dir, "*") + expEvents := lineCount * fileCount + b.ResetTimer() + + b.Run("inode throughput", func(b *testing.B) { + cfg := ` +type: filestream +prospector.scanner.check_interval: 1s +paths: + - ` + ingestPath + ` +` + for i := 0; i < b.N; i++ { + runFilestreamBenchmark(b, fmt.Sprintf("many-files-inode-benchmark-%d", i), cfg, expEvents) + } + }) + + b.Run("fingerprint throughput", func(b *testing.B) { + cfg := ` +type: filestream +prospector.scanner: + fingerprint.enabled: true + check_interval: 1s +file_identity.fingerprint: ~ +paths: + - ` + ingestPath + ` +` + for i := 0; i < b.N; i++ { + runFilestreamBenchmark(b, fmt.Sprintf("many-files-fp-benchmark-%d", i), cfg, expEvents) + } + }) }) } @@ -76,13 +120,13 @@ paths: // `cfg` must be a valid YAML string containing valid filestream configuration // `expEventCount` is an expected amount of produced events func runFilestreamBenchmark(b *testing.B, testID string, cfg string, expEventCount int) { + b.Helper() // we don't include initialization in the benchmark time b.StopTimer() - runner := createFilestreamTestRunner(b, testID, cfg, expEventCount) + runner := createFilestreamTestRunner(context.Background(), b, testID, cfg, int64(expEventCount), false) // this is where the benchmark actually starts b.StartTimer() - events := runner(b) - require.Len(b, events, expEventCount) + _ = runner(b) } // createFilestreamTestRunner can be used for both benchmarks and regular tests to run a filestream input @@ -90,9 +134,11 @@ func runFilestreamBenchmark(b *testing.B, testID string, cfg string, expEventCou // `testID` must be unique for each test run // `cfg` must be a valid YAML string containing valid filestream configuration // `eventLimit` is an amount of produced events after which the filestream will shutdown +// `collectEvents` if `true` the runner will return a list of all events produced by the filestream input. +// Events should not be collected in benchmarks due to high extra costs of using the channel. // // returns a runner function that returns produced events. -func createFilestreamTestRunner(b testing.TB, testID string, cfg string, eventLimit int) func(t testing.TB) []beat.Event { +func createFilestreamTestRunner(ctx context.Context, b testing.TB, testID string, cfg string, eventLimit int64, collectEvents bool) func(t testing.TB) []beat.Event { logger := logp.L() c, err := conf.NewConfigWithYAML([]byte(cfg), cfg) require.NoError(b, err) @@ -101,41 +147,43 @@ func createFilestreamTestRunner(b testing.TB, testID string, cfg string, eventLi input, err := p.Manager.Create(c) require.NoError(b, err) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(ctx) context := v2.Context{ Logger: logger, ID: testID, Cancelation: ctx, } - events := make([]beat.Event, 0, eventLimit) - connector, eventsDone := newTestPipeline(eventLimit, &events) - done := make(chan struct{}) + connector, events := newTestPipeline(eventLimit, collectEvents) + var out []beat.Event + if collectEvents { + out = make([]beat.Event, 0, eventLimit) + } + go func() { + // even if `collectEvents` is false we need to range the channel + // and wait until it's closed indicating that the input finished its job + for event := range events { + out = append(out, event) + } + cancel() + }() return func(t testing.TB) []beat.Event { - go func() { - err := input.Run(context, connector) - assert.NoError(b, err) - close(done) - }() + err := input.Run(context, connector) + require.NoError(b, err) - <-eventsDone - cancel() - <-done // for more stable results we should wait until the full shutdown - return events + return out } } -func generateFile(t testing.TB, lineCount int) string { +func generateFile(t testing.TB, dir string, lineCount int) string { t.Helper() - dir := t.TempDir() - file, err := os.CreateTemp(dir, "lines.log") + file, err := os.CreateTemp(dir, "*") require.NoError(t, err) - + filename := file.Name() for i := 0; i < lineCount; i++ { - fmt.Fprintf(file, "rather mediocre log line message - %d\n", i) + fmt.Fprintf(file, "rather mediocre log line message in %s - %d\n", filename, i) } - filename := file.Name() err = file.Close() require.NoError(t, err) return filename @@ -161,15 +209,15 @@ func (s *testStore) CleanupInterval() time.Duration { return time.Second } -func newTestPipeline(eventLimit int, out *[]beat.Event) (pc beat.PipelineConnector, done <-chan struct{}) { - ch := make(chan struct{}) - return &testPipeline{limit: eventLimit, done: ch, out: out}, ch +func newTestPipeline(eventLimit int64, collectEvents bool) (pc beat.PipelineConnector, out <-chan beat.Event) { + ch := make(chan beat.Event, eventLimit) + return &testPipeline{limit: eventLimit, out: ch, collect: collectEvents}, ch } type testPipeline struct { - done chan struct{} - limit int - out *[]beat.Event + limit int64 + out chan beat.Event + collect bool } func (p *testPipeline) ConnectWith(beat.ClientConfig) (beat.Client, error) { @@ -184,13 +232,15 @@ type testClient struct { } func (c *testClient) Publish(event beat.Event) { - c.testPipeline.limit-- - if c.testPipeline.limit < 0 { + newLimit := atomic.AddInt64(&c.testPipeline.limit, -1) + if newLimit < 0 { return } - *c.testPipeline.out = append(*c.testPipeline.out, event) - if c.testPipeline.limit == 0 { - close(c.testPipeline.done) + if c.testPipeline.collect { + c.testPipeline.out <- event + } + if newLimit == 0 { + close(c.testPipeline.out) } } From 46174c64e784554de04cb524f37b4bc6bb3f115a Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Mon, 18 Dec 2023 13:32:57 -0500 Subject: [PATCH 14/19] chore: Update snapshot.yml (#37454) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 4c2c820e6b07..03612a9033b3 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-6av99u5d-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-h30gube9-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.13.0-6av99u5d-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.13.0-h30gube9-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.13.0-6av99u5d-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.13.0-h30gube9-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 5fab3e9b1a93305cc84f73504d31df0c796d7047 Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 18 Dec 2023 17:13:32 -0500 Subject: [PATCH 15/19] Update to elastic-agent-client with chunking support. (#37343) * Update to elastic-agent-client with chunking support. * Add changelog entry. * Fix changelog again. * Update to v7.6.0 elastic-agent-client. * Fix tests. * Fix another test. * Fix windows lint. --- CHANGELOG.next.asciidoc | 1 + NOTICE.txt | 77 ++++++++++++++----- go.mod | 22 +++--- go.sum | 42 +++++----- x-pack/libbeat/management/managerV2.go | 6 +- x-pack/libbeat/management/managerV2_test.go | 6 +- .../management/simple_input_config_test.go | 2 +- .../libbeat/management/tests/mock_server.go | 2 +- 8 files changed, 99 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ae767239de47..172fbf334d2a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -53,6 +53,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Upgraded apache arrow library used in x-pack/libbeat/reader/parquet from v11 to v12.0.1 in order to fix cross-compilation issues {pull}35640[35640] - Fix panic when MaxRetryInterval is specified, but RetryInterval is not {pull}35820[35820] - Support build of projects outside of beats directory {pull}36126[36126] +- Support Elastic Agent control protocol chunking support {pull}37343[37343] *Auditbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 0070ee8ff110..cfc6e641dd8a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -12479,11 +12479,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-a -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 -Version: v7.5.0 +Version: v7.6.0 Licence type (autodetected): Elastic -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.5.0/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-client/v7@v7.6.0/LICENSE.txt: ELASTIC LICENSE AGREEMENT @@ -24701,11 +24701,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/crypto -Version: v0.14.0 +Version: v0.16.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.14.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/crypto@v0.16.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24775,11 +24775,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/mod -Version: v0.10.0 +Version: v0.14.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.10.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/mod@v0.14.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24812,11 +24812,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/net -Version: v0.17.0 +Version: v0.19.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.17.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/net@v0.19.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24886,11 +24886,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sync -Version: v0.3.0 +Version: v0.5.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sync@v0.3.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sync@v0.5.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24923,11 +24923,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/sys -Version: v0.13.0 +Version: v0.15.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/sys@v0.15.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -24960,11 +24960,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/text -Version: v0.13.0 +Version: v0.14.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/text@v0.14.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -25034,11 +25034,48 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/tools -Version: v0.9.1 +Version: v0.16.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.16.0/LICENSE: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : golang.org/x/tools/go/vcs +Version: v0.1.0-deprecated Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/tools@v0.9.1/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/tools/go/vcs@v0.1.0-deprecated/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -51250,11 +51287,11 @@ THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : golang.org/x/exp -Version: v0.0.0-20220921023135-46d9e7742f1e +Version: v0.0.0-20231127185646-65229373498e Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20220921023135-46d9e7742f1e/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/exp@v0.0.0-20231127185646-65229373498e/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. @@ -51287,11 +51324,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : golang.org/x/term -Version: v0.13.0 +Version: v0.15.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.13.0/LICENSE: +Contents of probable licence file $GOMODCACHE/golang.org/x/term@v0.15.0/LICENSE: Copyright (c) 2009 The Go Authors. All rights reserved. diff --git a/go.mod b/go.mod index 77aa9c7861ce..bb0ec7a35489 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( github.com/dustin/go-humanize v1.0.1 github.com/eapache/go-resiliency v1.2.0 github.com/eclipse/paho.mqtt.golang v1.3.5 - github.com/elastic/elastic-agent-client/v7 v7.5.0 + github.com/elastic/elastic-agent-client/v7 v7.6.0 github.com/elastic/go-concert v0.2.0 github.com/elastic/go-libaudit/v2 v2.4.0 github.com/elastic/go-licenser v0.4.1 @@ -152,16 +152,16 @@ require ( go.uber.org/atomic v1.11.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.16.0 golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 - golang.org/x/mod v0.10.0 - golang.org/x/net v0.17.0 + golang.org/x/mod v0.14.0 + golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.10.0 - golang.org/x/sync v0.3.0 - golang.org/x/sys v0.13.0 - golang.org/x/text v0.13.0 + golang.org/x/sync v0.5.0 + golang.org/x/sys v0.15.0 + golang.org/x/text v0.14.0 golang.org/x/time v0.3.0 - golang.org/x/tools v0.9.1 + golang.org/x/tools v0.16.0 google.golang.org/api v0.126.0 google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/grpc v1.58.3 @@ -224,6 +224,7 @@ require ( go.elastic.co/apm/module/apmhttp/v2 v2.4.7 go.elastic.co/apm/v2 v2.4.7 go.mongodb.org/mongo-driver v1.5.1 + golang.org/x/tools/go/vcs v0.1.0-deprecated google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) @@ -366,8 +367,8 @@ require ( go.opentelemetry.io/otel v1.19.0 // indirect go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/otel/trace v1.19.0 // indirect - golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e // indirect - golang.org/x/term v0.13.0 // indirect + golang.org/x/exp v0.0.0-20231127185646-65229373498e // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect @@ -404,6 +405,7 @@ replace ( github.com/docker/go-plugins-helpers => github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f github.com/dop251/goja => github.com/andrewkroh/goja v0.0.0-20190128172624-dd2ac4456e20 github.com/dop251/goja_nodejs => github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 + github.com/fsnotify/fsevents => github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270 github.com/fsnotify/fsnotify => github.com/adriansr/fsnotify v1.4.8-0.20211018144411-a81f2b630e7c github.com/godror/godror => github.com/godror/godror v0.33.2 // updating to v0.24.2 caused a breaking change diff --git a/go.sum b/go.sum index e836a3d92670..5b4d63c39652 100644 --- a/go.sum +++ b/go.sum @@ -654,8 +654,8 @@ github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3 h1:lnDkqiRFKm0rxdljqr github.com/elastic/dhcp v0.0.0-20200227161230-57ec251c7eb3/go.mod h1:aPqzac6AYkipvp4hufTyMj5PDIphF3+At8zr7r51xjY= github.com/elastic/elastic-agent-autodiscover v0.6.5 h1:5DeMpuNc8c/tN6HN0A4A2uOFTNFHSg7xrKApzfhvF1U= github.com/elastic/elastic-agent-autodiscover v0.6.5/go.mod h1:chulyCAyZb/njMHgzkhC/yWnt8v/Y6eCRUhmFVnsA5o= -github.com/elastic/elastic-agent-client/v7 v7.5.0 h1:niI3WQ+01Lnp2r5LxK8SyNhrPJe13vBiOkqrDRK2oTA= -github.com/elastic/elastic-agent-client/v7 v7.5.0/go.mod h1:DYoX95xjC4BW/p2avyu724Qr2+hoUIz9eCU9CVS1d+0= +github.com/elastic/elastic-agent-client/v7 v7.6.0 h1:FEn6FjzynW4TIQo5G096Tr7xYK/P5LY9cSS6wRbXZTc= +github.com/elastic/elastic-agent-client/v7 v7.6.0/go.mod h1:GlUKrbVd/O1CRAZonpBeN3J0RlVqP6VGcrBjFWca+aM= github.com/elastic/elastic-agent-libs v0.7.2 h1:yT0hF0UAxJCdQqhHh6SFpgYrcpB10oFzPj8IaytPS2o= github.com/elastic/elastic-agent-libs v0.7.2/go.mod h1:pVBEElQJUO9mr4WStWNXuQGsJn54lcjAoYAHmsvBLBc= github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3 h1:sb+25XJn/JcC9/VL8HX4r4QXSUq4uTNzGS2kxOE7u1U= @@ -2030,8 +2030,8 @@ golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2047,8 +2047,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= -golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e h1:Ctm9yurWsg7aWwIpH9Bnap/IdSVxixymIb3MhiMEQQA= -golang.org/x/exp v0.0.0-20220921023135-46d9e7742f1e/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= +golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2089,8 +2089,8 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2173,8 +2173,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190130055435-99b60b757ec1/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -2207,8 +2207,8 @@ golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2344,8 +2344,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2354,8 +2354,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2370,8 +2370,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2475,8 +2475,10 @@ golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= +golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools/go/vcs v0.1.0-deprecated h1:cOIJqWBl99H1dH5LWizPa+0ImeeJq3t3cJjaeOWUAL4= +golang.org/x/tools/go/vcs v0.1.0-deprecated/go.mod h1:zUrvATBAvEI9535oC0yWYsLsHIV4Z7g63sNPVMtuBy8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/x-pack/libbeat/management/managerV2.go b/x-pack/libbeat/management/managerV2.go index fd38051efb3a..235325c0cbfc 100644 --- a/x-pack/libbeat/management/managerV2.go +++ b/x-pack/libbeat/management/managerV2.go @@ -35,8 +35,6 @@ import ( "github.com/elastic/beats/v7/libbeat/version" ) -var errStoppingOnOutputChange = errors.New("stopping Beat on output change") - // diagnosticHandler is a wrapper type that's a bit of a hack, the compiler won't let us send the raw unit struct, // since there's a type disagreement with the `client.DiagnosticHook` argument, and due to licensing issues we can't import the agent client types into the reloader type diagnosticHandler struct { @@ -173,7 +171,7 @@ func NewV2AgentManager(config *conf.C, registry *reload.Registry) (lbmanagement. client.VersionInfo{ Name: "beat-v2-client-for-testing", Version: version.GetDefaultVersion(), - }, grpc.WithTransportCredentials(insecure.NewCredentials())) + }, client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) } else { // Normal Elastic-Agent-Client initialisation agentClient, _, err = client.NewV2FromReader(os.Stdin, client.VersionInfo{ @@ -557,7 +555,7 @@ func (cm *BeatV2Manager) reload(units map[unitKey]*client.Unit) { for _, unit := range units { errs := unitErrors[unit.ID()] if len(errs) != 0 { - unit.UpdateState(client.UnitStateFailed, errors.Join(errs...).Error(), nil) + _ = unit.UpdateState(client.UnitStateFailed, errors.Join(errs...).Error(), nil) } } }() diff --git a/x-pack/libbeat/management/managerV2_test.go b/x-pack/libbeat/management/managerV2_test.go index 9fe238605b49..ea67fdd89f40 100644 --- a/x-pack/libbeat/management/managerV2_test.go +++ b/x-pack/libbeat/management/managerV2_test.go @@ -209,7 +209,7 @@ func TestManagerV2(t *testing.T) { Meta: map[string]string{ "key": "value", }, - }, grpc.WithTransportCredentials(insecure.NewCredentials())) + }, client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) m, err := NewV2AgentManagerWithClient(&Config{ Enabled: true, @@ -321,7 +321,7 @@ func TestOutputError(t *testing.T) { fmt.Sprintf(":%d", server.Port), "", client.VersionInfo{}, - grpc.WithTransportCredentials(insecure.NewCredentials())) + client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) m, err := NewV2AgentManagerWithClient( &Config{ @@ -488,7 +488,7 @@ func TestErrorPerUnit(t *testing.T) { fmt.Sprintf(":%d", server.Port), "", client.VersionInfo{}, - grpc.WithTransportCredentials(insecure.NewCredentials())) + client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) m, err := NewV2AgentManagerWithClient( &Config{ diff --git a/x-pack/libbeat/management/simple_input_config_test.go b/x-pack/libbeat/management/simple_input_config_test.go index ef88781a8622..7822d347112b 100644 --- a/x-pack/libbeat/management/simple_input_config_test.go +++ b/x-pack/libbeat/management/simple_input_config_test.go @@ -133,7 +133,7 @@ func TestSimpleInputConfig(t *testing.T) { fmt.Sprintf(":%d", server.Port), "", client.VersionInfo{}, - grpc.WithTransportCredentials(insecure.NewCredentials())) + client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) m, err := NewV2AgentManagerWithClient( &Config{ diff --git a/x-pack/libbeat/management/tests/mock_server.go b/x-pack/libbeat/management/tests/mock_server.go index 84805c1f5cc8..8671b1242339 100644 --- a/x-pack/libbeat/management/tests/mock_server.go +++ b/x-pack/libbeat/management/tests/mock_server.go @@ -103,7 +103,7 @@ func NewMockServer(t *testing.T, canStop func(string) bool, inputConfig *proto.U Meta: map[string]string{ "key": "value", }, - }, grpc.WithTransportCredentials(insecure.NewCredentials())) + }, client.WithGRPCDialOptions(grpc.WithTransportCredentials(insecure.NewCredentials()))) return MockV2Handler{Srv: srv, Client: client} } From 8fe2f53bf1613bf6c0566a7e8e7afda785c51770 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 19 Dec 2023 11:12:37 +0100 Subject: [PATCH 16/19] update codeowners (#37336) --- .github/CODEOWNERS | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ee4de95884df..d8f36cdde15c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -29,19 +29,18 @@ CHANGELOG* /filebeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module. /filebeat/input/syslog/ @elastic/security-external-integrations /filebeat/input/winlog/ @elastic/security-external-integrations -/filebeat/module/ @elastic/integrations /filebeat/module/apache @elastic/obs-infraobs-integrations /filebeat/module/auditd @elastic/security-external-integrations /filebeat/module/elasticsearch/ @elastic/infra-monitoring-ui /filebeat/module/haproxy @elastic/obs-infraobs-integrations -/filebeat/module/icinga @elastic/integrations +/filebeat/module/icinga # TODO: find right team /filebeat/module/iis @elastic/obs-infraobs-integrations /filebeat/module/kafka @elastic/obs-infraobs-integrations /filebeat/module/kibana/ @elastic/infra-monitoring-ui /filebeat/module/logstash/ @elastic/infra-monitoring-ui /filebeat/module/mongodb @elastic/obs-infraobs-integrations /filebeat/module/mysql @elastic/security-external-integrations -/filebeat/module/nats @elastic/integrations +/filebeat/module/nats @elastic/obs-infraobs-integrations /filebeat/module/nginx @elastic/obs-infraobs-integrations /filebeat/module/osquery @elastic/security-external-integrations /filebeat/module/pensando @elastic/security-external-integrations @@ -49,8 +48,8 @@ CHANGELOG* /filebeat/module/redis @elastic/obs-infraobs-integrations /filebeat/module/santa @elastic/security-external-integrations /filebeat/module/system @elastic/elastic-agent-data-plane -/filebeat/module/traefik @elastic/integrations -/heartbeat/ @elastic/hosted-services +/filebeat/module/traefik # TODO: find right team +/heartbeat/ @elastic/obs-ds-hosted-services /journalbeat @elastic/elastic-agent-data-plane /libbeat/ @elastic/elastic-agent-data-plane /libbeat/docs/processors-list.asciidoc @elastic/ingest-docs @@ -68,7 +67,6 @@ CHANGELOG* /metricbeat/ @elastic/elastic-agent-data-plane /metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module. /metricbeat/helper/kubernetes @elastic/obs-cloudnative-monitoring -/metricbeat/module/ @elastic/integrations /metricbeat/module/apache @elastic/obs-infraobs-integrations /metricbeat/module/beat/ @elastic/infra-monitoring-ui /metricbeat/module/ceph @elastic/obs-infraobs-integrations @@ -122,7 +120,6 @@ CHANGELOG* /x-pack/filebeat/input/lumberjack/ @elastic/security-external-integrations /x-pack/filebeat/input/netflow/ @elastic/security-external-integrations /x-pack/filebeat/input/o365audit/ @elastic/security-external-integrations -/x-pack/filebeat/module/ @elastic/integrations /x-pack/filebeat/module/activemq @elastic/obs-infraobs-integrations /x-pack/filebeat/module/aws @elastic/obs-cloud-monitoring /x-pack/filebeat/module/awsfargate @elastic/obs-cloud-monitoring @@ -174,10 +171,9 @@ CHANGELOG* /x-pack/filebeat/module/zscaler @elastic/security-external-integrations /x-pack/filebeat/modules.d/zoom.yml.disabled @elastic/security-external-integrations /x-pack/filebeat/processors/decode_cef/ @elastic/security-external-integrations -/x-pack/heartbeat/ @elastic/hosted-services +/x-pack/heartbeat/ @elastic/obs-ds-hosted-services /x-pack/metricbeat/ @elastic/elastic-agent-data-plane /x-pack/metricbeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module. -/x-pack/metricbeat/module/ @elastic/integrations /x-pack/metricbeat/module/activemq @elastic/obs-infraobs-integrations /x-pack/metricbeat/module/airflow @elastic/obs-infraobs-integrations /x-pack/metricbeat/module/cloudfoundry @elastic/obs-infraobs-integrations From 50aac8069c525c59781f347f66a25ee7534ade34 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:35:07 -0500 Subject: [PATCH 17/19] chore: Update snapshot.yml (#37463) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 03612a9033b3..0e66db12a2a5 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-h30gube9-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-ubdkmnyz-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.13.0-h30gube9-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.13.0-ubdkmnyz-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.13.0-h30gube9-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.13.0-ubdkmnyz-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From b59a8f4769c20c1d076bf7ff8c9b9b175f0f5969 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 20 Dec 2023 11:36:50 +0100 Subject: [PATCH 18/19] Replace EOL modules: github.com/golang/protobuf by google.golang.org/protobuf (#37212) --- NOTICE.txt | 76 +++++++++---------- go.mod | 2 +- .../provider/aws/aws/kinesis_test.go | 7 +- .../aws/aws/transformer/transformer_test.go | 7 +- .../gcp/metrics/cloudsql/metadata_test.go | 10 +-- .../gcp/metrics/compute/metadata_test.go | 10 +-- .../module/gcp/metrics/metrics_requester.go | 11 ++- .../gcp/metrics/metrics_requester_test.go | 12 +-- .../module/gcp/metrics/metricset.go | 6 +- .../module/gcp/metrics/redis/metadata_test.go | 10 +-- 10 files changed, 76 insertions(+), 75 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index cfc6e641dd8a..cca066349fa3 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -17323,44 +17323,6 @@ Contents of probable licence file $GOMODCACHE/github.com/golang/mock@v1.6.0/LICE limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/golang/protobuf -Version: v1.5.3 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/golang/protobuf@v1.5.3/LICENSE: - -Copyright 2010 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - - -------------------------------------------------------------------------------- Dependency : github.com/golang/snappy Version: v0.0.4 @@ -37758,6 +37720,44 @@ third-party archives. limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/golang/protobuf +Version: v1.5.3 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/golang/protobuf@v1.5.3/LICENSE: + +Copyright 2010 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + -------------------------------------------------------------------------------- Dependency : github.com/google/gofuzz Version: v1.2.0 diff --git a/go.mod b/go.mod index bb0ec7a35489..6732fbc60061 100644 --- a/go.mod +++ b/go.mod @@ -95,7 +95,6 @@ require ( github.com/gofrs/uuid v4.4.0+incompatible github.com/gogo/protobuf v1.3.2 github.com/golang/mock v1.6.0 - github.com/golang/protobuf v1.5.3 github.com/golang/snappy v0.0.4 github.com/gomodule/redigo v1.8.3 github.com/google/flatbuffers v23.3.3+incompatible @@ -291,6 +290,7 @@ require ( github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/licenseclassifier v0.0.0-20221004142553-c1ed8fcf4bab // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect diff --git a/x-pack/functionbeat/provider/aws/aws/kinesis_test.go b/x-pack/functionbeat/provider/aws/aws/kinesis_test.go index 242ed63d4a86..20d92ce36060 100644 --- a/x-pack/functionbeat/provider/aws/aws/kinesis_test.go +++ b/x-pack/functionbeat/provider/aws/aws/kinesis_test.go @@ -14,8 +14,9 @@ import ( "github.com/aws/aws-lambda-go/events" "github.com/awslabs/kinesis-aggregation/go/v2/deaggregator" aggRecProto "github.com/awslabs/kinesis-aggregation/go/v2/records" - "github.com/golang/protobuf/proto" //nolint:staticcheck // SA1019 dependency uses deprecated package "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/runtime/protoimpl" "github.com/elastic/beats/v7/x-pack/functionbeat/function/provider" conf "github.com/elastic/elastic-agent-libs/config" @@ -133,14 +134,14 @@ func generateAggregatedKinesisEvent(validRec bool) events.KinesisEvent { partKeyTable = append(partKeyTable, "0") aggRec.PartitionKeyTable = partKeyTable - data, _ := proto.Marshal(aggRec) + data, _ := proto.Marshal(protoimpl.X.ProtoMessageV2Of(aggRec)) md5Hash := md5.Sum(data) aggRecBytes = append(aggRecBytes, data...) aggRecBytes = append(aggRecBytes, md5Hash[:]...) return events.KinesisEvent{ Records: []events.KinesisEventRecord{ - events.KinesisEventRecord{ + { AwsRegion: "east-1", EventID: "1234", EventName: "connect", diff --git a/x-pack/functionbeat/provider/aws/aws/transformer/transformer_test.go b/x-pack/functionbeat/provider/aws/aws/transformer/transformer_test.go index e29bba6cde32..ae0dcb12eee4 100644 --- a/x-pack/functionbeat/provider/aws/aws/transformer/transformer_test.go +++ b/x-pack/functionbeat/provider/aws/aws/transformer/transformer_test.go @@ -14,12 +14,13 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/service/kinesis/types" + "google.golang.org/protobuf/runtime/protoimpl" "github.com/aws/aws-lambda-go/events" "github.com/awslabs/kinesis-aggregation/go/v2/deaggregator" aggRecProto "github.com/awslabs/kinesis-aggregation/go/v2/records" - "github.com/golang/protobuf/proto" //nolint:staticcheck // SA1019 dependency uses deprecated package "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/proto" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/elastic-agent-libs/mapstr" @@ -33,7 +34,7 @@ func TestCloudwatch(t *testing.T) { SubscriptionFilters: []string{"MyFilter"}, MessageType: "DATA_MESSAGE", LogEvents: []events.CloudwatchLogsLogEvent{ - events.CloudwatchLogsLogEvent{ + { ID: "1234567890123456789", Timestamp: 1566908691193, Message: "my interesting message", @@ -393,7 +394,7 @@ func generateKinesisAggregateRecord(numRecords int, valid bool) []byte { } aggRec.PartitionKeyTable = partKeyTable - data, _ := proto.Marshal(aggRec) + data, _ := proto.Marshal(protoimpl.X.ProtoMessageV2Of(aggRec)) md5Hash := md5.Sum(data) aggRecBytes = append(aggRecBytes, data...) aggRecBytes = append(aggRecBytes, md5Hash[:]...) diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go index cb678607c242..b6cd6e228f85 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata_test.go @@ -8,9 +8,9 @@ import ( "testing" monitoring "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/golang/protobuf/ptypes/timestamp" "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/genproto/googleapis/api/monitoredres" + "google.golang.org/protobuf/types/known/timestamppb" "gotest.tools/assert" "github.com/elastic/elastic-agent-libs/mapstr" @@ -43,10 +43,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.0041224284852319215}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, }, @@ -55,10 +55,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.004205757571772513}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, }, diff --git a/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go index 8cfae6675928..17213b8e1b89 100644 --- a/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/compute/metadata_test.go @@ -8,10 +8,10 @@ import ( "testing" monitoring "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/golang/protobuf/ptypes/timestamp" "github.com/stretchr/testify/assert" "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/genproto/googleapis/api/monitoredres" + "google.golang.org/protobuf/types/known/timestamppb" ) var fake = &monitoring.TimeSeries{ @@ -41,10 +41,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.0041224284852319215}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, }, @@ -53,10 +53,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.004205757571772513}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, }, diff --git a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go index a39f320ef447..e7e7f081e526 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go +++ b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go @@ -12,12 +12,11 @@ import ( "sync" "time" - "github.com/golang/protobuf/ptypes/duration" - monitoring "cloud.google.com/go/monitoring/apiv3/v2" "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/golang/protobuf/ptypes/timestamp" "google.golang.org/api/iterator" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" @@ -256,7 +255,7 @@ func (r *metricsRequester) getFilterForMetric(serviceName, m string) string { } // Returns a GCP TimeInterval based on the ingestDelay and samplePeriod from ListMetricDescriptor -func getTimeIntervalAligner(ingestDelay time.Duration, samplePeriod time.Duration, collectionPeriod *duration.Duration, inputAligner string) (*monitoringpb.TimeInterval, string) { +func getTimeIntervalAligner(ingestDelay time.Duration, samplePeriod time.Duration, collectionPeriod *durationpb.Duration, inputAligner string) (*monitoringpb.TimeInterval, string) { var startTime, endTime, currentTime time.Time var needsAggregation bool currentTime = time.Now().UTC() @@ -280,10 +279,10 @@ func getTimeIntervalAligner(ingestDelay time.Duration, samplePeriod time.Duratio } interval := &monitoringpb.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: startTime.Unix(), }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: endTime.Unix(), }, } diff --git a/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go b/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go index 2b058de2beed..658568b66ca5 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/metrics_requester_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - "github.com/golang/protobuf/ptypes/duration" "github.com/stretchr/testify/assert" "go.uber.org/zap/zapcore" + "google.golang.org/protobuf/types/known/durationpb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" @@ -21,7 +21,7 @@ func TestGetTimeIntervalAligner(t *testing.T) { title string ingestDelay time.Duration samplePeriod time.Duration - collectionPeriod *duration.Duration + collectionPeriod *durationpb.Duration inputAligner string expectedAligner string }{ @@ -29,7 +29,7 @@ func TestGetTimeIntervalAligner(t *testing.T) { "test collectionPeriod equals to samplePeriod", time.Duration(240) * time.Second, time.Duration(60) * time.Second, - &duration.Duration{ + &durationpb.Duration{ Seconds: int64(60), }, "", @@ -39,7 +39,7 @@ func TestGetTimeIntervalAligner(t *testing.T) { "test collectionPeriod larger than samplePeriod", time.Duration(240) * time.Second, time.Duration(60) * time.Second, - &duration.Duration{ + &durationpb.Duration{ Seconds: int64(300), }, "ALIGN_MEAN", @@ -49,7 +49,7 @@ func TestGetTimeIntervalAligner(t *testing.T) { "test collectionPeriod smaller than samplePeriod", time.Duration(240) * time.Second, time.Duration(60) * time.Second, - &duration.Duration{ + &durationpb.Duration{ Seconds: int64(30), }, "ALIGN_MAX", @@ -59,7 +59,7 @@ func TestGetTimeIntervalAligner(t *testing.T) { "test collectionPeriod equals to samplePeriod with given aligner", time.Duration(240) * time.Second, time.Duration(60) * time.Second, - &duration.Duration{ + &durationpb.Duration{ Seconds: int64(60), }, "ALIGN_MEAN", diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index ffe5cd788a1f..49cdce5d73a8 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -13,10 +13,10 @@ import ( monitoring "cloud.google.com/go/monitoring/apiv3/v2" "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/golang/protobuf/ptypes/duration" "google.golang.org/api/iterator" "google.golang.org/api/option" "google.golang.org/genproto/googleapis/api/metric" + "google.golang.org/protobuf/types/known/durationpb" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" @@ -107,7 +107,7 @@ type config struct { CredentialsJSON string `config:"credentials_json"` opt []option.ClientOption - period *duration.Duration + period *durationpb.Duration } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -139,7 +139,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return m, fmt.Errorf("no credentials_file_path or credentials_json specified") } - m.config.period = &duration.Duration{ + m.config.period = &durationpb.Duration{ Seconds: int64(m.Module().Config().Period.Seconds()), } diff --git a/x-pack/metricbeat/module/gcp/metrics/redis/metadata_test.go b/x-pack/metricbeat/module/gcp/metrics/redis/metadata_test.go index 9fc790bf9e51..48ac45661ed9 100644 --- a/x-pack/metricbeat/module/gcp/metrics/redis/metadata_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/redis/metadata_test.go @@ -8,10 +8,10 @@ import ( "testing" monitoring "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" - "github.com/golang/protobuf/ptypes/timestamp" "github.com/stretchr/testify/assert" "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/genproto/googleapis/api/monitoredres" + "google.golang.org/protobuf/types/known/timestamppb" ) var fake = &monitoring.TimeSeries{ @@ -41,10 +41,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.0041224284852319215}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932700, }, }, @@ -53,10 +53,10 @@ var fake = &monitoring.TimeSeries{ Value: &monitoring.TypedValue_DoubleValue{DoubleValue: 0.004205757571772513}, }, Interval: &monitoring.TimeInterval{ - StartTime: ×tamp.Timestamp{ + StartTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, - EndTime: ×tamp.Timestamp{ + EndTime: ×tamppb.Timestamp{ Seconds: 1569932640, }, }, From e92dae6d18a17923c721a87a04891e3966877eb8 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Wed, 20 Dec 2023 12:37:07 -0500 Subject: [PATCH 19/19] chore: Update snapshot.yml (#37471) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Made with ❤️️ by updatecli Co-authored-by: apmmachine --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 0e66db12a2a5..7b2595d18d14 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-ubdkmnyz-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.13.0-9amqxdis-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.13.0-ubdkmnyz-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.13.0-9amqxdis-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.13.0-ubdkmnyz-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.13.0-9amqxdis-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing"