Merge branch 'main' into 172558-elasticsearch-mappings-updates

.buildkite/filebeat/filebeat-pipeline.yml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+steps:
+  - label: "Example test"
+    command: echo "Hello!"

.buildkite/metricbeat/pipeline.yml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+steps:
+  - label: "Example test"
+    command: echo "Hello!"

.buildkite/pipeline.yml

@@ -0,0 +1,5 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+
+steps:
+  - label: "Example test"
+    command: echo "Hello!"

.buildkite/pull-requests.json

@@ -0,0 +1,52 @@
+{
+    "jobs": [
+        {
+            "enabled": true,
+            "pipelineSlug": "beats",
+            "allow_org_users": true,
+            "allowed_repo_permissions": ["admin", "write"],
+            "allowed_list": [ ],
+            "set_commit_status": true,
+            "build_on_commit": true,
+            "build_on_comment": true,
+            "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
+            "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test$",
+            "skip_ci_labels": [ ],
+            "skip_target_branches": [ ],
+            "skip_ci_on_only_changed": [ ],
+            "always_require_ci_on_changed": [ ]
+        },
+        {
+            "enabled": true,
+            "pipelineSlug": "filebeat",
+            "allow_org_users": true,
+            "allowed_repo_permissions": ["admin", "write"],
+            "allowed_list": [ ],
+            "set_commit_status": true,
+            "build_on_commit": true,
+            "build_on_comment": true,
+            "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test filebeat$",
+            "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))|^/test filebeat$",
+            "skip_ci_labels": [ ],
+            "skip_target_branches": [ ],
+            "skip_ci_on_only_changed": [ ],
+            "always_require_ci_on_changed": [ ]
+        },
+        {
+            "enabled": true,
+            "pipelineSlug": "beats-metricbeat",
+            "allow_org_users": true,
+            "allowed_repo_permissions": ["admin", "write"],
+            "allowed_list": [ ],
+            "set_commit_status": true,
+            "build_on_commit": true,
+            "build_on_comment": true,
+            "trigger_comment_regex": "^/test metricbeat$",
+            "always_trigger_comment_regex": "^/test metricbeat$",
+            "skip_ci_labels": [  ],
+            "skip_target_branches": [ ],
+            "skip_ci_on_only_changed": [ ],
+            "always_require_ci_on_changed": [ "^metricbeat/.*", ".buildkite/metricbeat/.*", "^go.mod", "^pytest.ini", "^dev-tools/.*", "^libbeat/.*", "^testing/.*"]
+        }
+    ]
+}

.ci/beats-tester.groovy

@@ -24,7 +24,7 @@ pipeline {
   }
   stages {
     stage('Filter build') {
-      agent { label 'ubuntu-20' }
+      agent { label 'ubuntu-22' }
       when {
         beforeAgent true
         anyOf {
@@ -69,8 +69,7 @@ pipeline {
           when { changeRequest() }
           steps {
             runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT",
-                              apm: "https://storage.googleapis.com/apm-ci-artifacts/jobs/pull-requests/pr-${env.CHANGE_ID}",
-                              beats: "https://storage.googleapis.com/beats-ci-artifacts/pull-requests/pr-${env.CHANGE_ID}")
+                              beats: "https://storage.googleapis.com/beats-ci-artifacts/beats/pull-requests/pr-${env.CHANGE_ID}")
           }
         }
         stage('Build release branch') {
@@ -106,7 +105,7 @@ def runBeatsTesterJob(Map args = [:]) {
     def props = readProperties(file: 'beats-tester.properties')
     apm = props.get('APM_URL_BASE', '')
     beats = props.get('BEATS_URL_BASE', '')
-    version = props.get('VERSION', '8.0.0-SNAPSHOT')
+    version = props.get('VERSION', '8.7.0-SNAPSHOT')
   }
   if (apm?.trim() || beats?.trim()) {
     build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false,

.ci/build-docker-images.groovy

@@ -3,7 +3,7 @@
 @Library('apm@current') _
 
 pipeline {
-  agent { label 'ubuntu-20' }
+  agent { label 'ubuntu-22' }
   environment {
     REPO = 'beats'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"

.ci/bump-elastic-stack-snapshot.yml

@@ -0,0 +1,49 @@
+---
+name: Bump elastic-stack to latest snapshot version
+pipelineid: 'bump-elastic-stack-snapshot-{{ requiredEnv "BRANCH" }}'
+
+actions:
+  default:
+    title: '[updatecli] update elastic stack version for testing {{ source "latestVersion" }}'
+    kind: github/pullrequest
+    spec:
+      labels:
+        - automation
+        - dependency
+        - backport-skip
+        - build-monitoring
+        - Team:Beats-On-Call
+      description: |
+        Generated automatically with {{ requiredEnv "JOB_URL" }}
+    scmid: default
+
+scms:
+  default:
+    kind: github
+    spec:
+      user: '{{ requiredEnv "GIT_USER" }}'
+      email: '{{ requiredEnv "GIT_EMAIL" }}'
+      owner: elastic
+      repository: beats
+      token: '{{ requiredEnv "GITHUB_TOKEN" }}'
+      username: '{{ requiredEnv "GIT_USER" }}'
+      branch: '{{ requiredEnv "BRANCH" }}'
+
+sources:
+  latestVersion:
+    name: Get latest snapshot build
+    kind: json
+    spec:
+      file: https://storage.googleapis.com/artifacts-api/snapshots/{{ requiredEnv "BRANCH" }}.json
+      key: .build_id
+
+targets:
+  update-snapshot:
+    name: "Update snapshot.yml"
+    kind: file
+    sourceid: latestVersion
+    scmid: default
+    spec:
+      file: testing/environments/snapshot.yml
+      matchpattern: '(image: docker\.elastic\.co/.*):\d+.\d+.\d+-.*-SNAPSHOT'
+      replacepattern: '$1:{{ source "latestVersion" }}-SNAPSHOT'