Package all beats

elastic · Apr 23, 2024 · 5671e37 · 5671e37
1 parent 0d1da08
commit 5671e37
Showing 1 changed file with 77 additions and 103 deletions.
diff --git a/.buildkite/packaging.pipeline.yml b/.buildkite/packaging.pipeline.yml
@@ -1,5 +1,4 @@
 # yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
-
 # TODO: Pre-cache beats-dev/golang-crossbuild container image
 
 env:
@@ -43,18 +42,12 @@ steps:
     key: packaging-snapshot
     steps:
       # x86
-      - label: ":ubuntu: {{matrix}}/Packaging Linux"
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: true
         command: |
           .buildkite/scripts/packaging/package-dra.sh {{matrix}}
-          echo 'build/distributions/'
-          ls -lah build/distributions/
-          echo '{{matrix}}/build/distributions/'
-          ls -lah {{matrix}}/build/distributions/
-          echo 'build/distributions/{{matrix}}'
-          ls -lah build/distributions/{{matrix}}
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -65,71 +58,63 @@ steps:
           - auditbeat
           - filebeat
           - heartbeat
-          # - metricbeat
-          # - packetbeat
-          # - winlogbeat
+          - metricbeat
+          - packetbeat
+          - winlogbeat
           - x-pack/auditbeat
-          # - x-pack/dockerlogbeat
-          # - x-pack/filebeat
-          # - x-pack/functionbeat
-          # - x-pack/heartbeat
-          # - x-pack/metricbeat
-          # - x-pack/packetbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/functionbeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
           - x-pack/winlogbeat
 
       ## ARM
-      # - label: ":linux: {{matrix}}/Packaging Linux arm64"
-      #   env:
-      #     PLATFORMS: "linux/arm64"
-      #     PACKAGES: "docker"
-      #     SNAPSHOT: true
-      #   command: |
-      #     .buildkite/scripts/packaging/package-dra.sh {{matrix}}
-      #     echo 'build/distributions/'
-      #     ls -lah build/distributions/
-      #     echo '{{matrix}}/build/distributions/'
-      #     ls -lah {{matrix}}/build/distributions/
-      #     echo 'build/distributions/{{matrix}}'
-      #     ls -lah build/distributions/{{matrix}}
-      #     echo '{{matrix}}/build/distributions/{{matrix}}'
-      #     ls -lah {{matrix}}/build/distributions/{{matrix}}
-      #   agents:
-      #     provider: "aws"
-      #     imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
-      #     instanceType: "${AWS_ARM_INSTANCE_TYPE}"
-      #   artifact_paths:
-      #     - build/distributions/**
-      #   matrix:
-      #     - auditbeat
-      #     # - filebeat
-      #     # - heartbeat
-      #     # - metricbeat
-      #     # - packetbeat
-      #     - x-pack/auditbeat
-      #     # - x-pack/dockerlogbeat
-      #     # - x-pack/filebeat
-      #     # - x-pack/heartbeat
-      #     # - x-pack/metricbeat
-      #     - x-pack/packetbeat
+      - label: ":linux: {{matrix}}/Packaging Linux arm64 Snapshot"
+        env:
+          PLATFORMS: "linux/arm64"
+          PACKAGES: "docker"
+          SNAPSHOT: true
+        command: |
+          .buildkite/scripts/packaging/package-dra.sh {{matrix}}
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - auditbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
+          - x-pack/auditbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
 
-      # ## Osquerybeat and agentbeat require msitools to be installed
-      # ## TODO: pre-install msitools to the VM and delete this exclusion
-      # - label: ":ubuntu: {{matrix}}/Packaging Linux"
-      #   env:
-      #     PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-      #     SNAPSHOT: true
-      #   command: |
-      #     .buildkite/scripts/install-msitools.sh
-      #     .buildkite/scripts/packaging/package-dra.sh {{matrix}}
-      #   agents:
-      #     provider: gcp
-      #     image: "${IMAGE_UBUNTU_X86_64}"
-      #     machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-      #   artifact_paths:
-      #     - build/distributions/**
-      #   matrix:
-      #     - x-pack/agentbeat
-      #     - x-pack/osquerybeat
+      ## Osquerybeat and agentbeat require msitools to be installed
+      ## TODO: pre-install msitools to the VM and delete this exclusion
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Snapshot"
+        env:
+          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+          SNAPSHOT: true
+        command: |
+          .buildkite/scripts/install-msitools.sh
+          .buildkite/scripts/packaging/package-dra.sh {{matrix}}
+        agents:
+          provider: gcp
+          image: "${IMAGE_UBUNTU_X86_64}"
+          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+        artifact_paths:
+          - build/distributions/**/*
+        matrix:
+          - x-pack/agentbeat
+          - x-pack/osquerybeat
 
   - group: Packaging staging
 
@@ -138,7 +123,7 @@ steps:
     if: "build.branch =~ /\\d+\\.\\d+/ || build.branch == 'main'"
     steps:
       # x86
-      - label: ":ubuntu: {{matrix}}/Packaging Linux"
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Staging"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: false
@@ -148,35 +133,30 @@ steps:
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
         artifact_paths:
-          - build/distributions/**
+          - build/distributions/**/*
         matrix:
           - auditbeat
           - filebeat
           - heartbeat
-          # - metricbeat
-          # - packetbeat
-          # - winlogbeat
+          - metricbeat
+          - packetbeat
+          - winlogbeat
           - x-pack/auditbeat
-          # - x-pack/dockerlogbeat
-          # - x-pack/filebeat
-          # - x-pack/functionbeat
-          # - x-pack/heartbeat
-          # - x-pack/metricbeat
-          # - x-pack/packetbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/functionbeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
+          - x-pack/packetbeat
           - x-pack/winlogbeat
 
       ## ARM
-      - label: ":linux: {{matrix}}/Packaging Linux arm64"
+      - label: ":linux: {{matrix}}/Packaging Linux arm64 Staging"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
           SNAPSHOT: false
-        command: |
-          WORKSPACE=$$PWD
-          cd {{matrix}}
-          mage package
-          mage ironbank
-          mv build $$WORKSPACE
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
         agents:
           provider: "aws"
           imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
@@ -185,30 +165,24 @@ steps:
           - build/distributions/**
         matrix:
           - auditbeat
-          # - filebeat
-          # - heartbeat
-          # - metricbeat
-          # - packetbeat
+          - filebeat
+          - heartbeat
+          - metricbeat
+          - packetbeat
           - x-pack/auditbeat
-          # - x-pack/dockerlogbeat
-          # - x-pack/filebeat
-          # - x-pack/heartbeat
-          # - x-pack/metricbeat
+          - x-pack/dockerlogbeat
+          - x-pack/filebeat
+          - x-pack/heartbeat
+          - x-pack/metricbeat
           - x-pack/packetbeat
 
         ## Osquerybeat and agentbeat require msitools to be installed
         ## TODO: pre-install msitools to the VM and delete this exclusion
-      - label: ":ubuntu: {{matrix}}/Packaging Linux"
+      - label: ":ubuntu: {{matrix}}/Packaging Linux Staging"
         env:
           PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
           SNAPSHOT: false
-        command: |
-          WORKSPACE=$$PWD
-          .buildkite/scripts/install-msitools.sh
-          cd {{matrix}}
-          mage package
-          mage ironbank        
-          mv build $$WORKSPACE
+        command: ".buildkite/scripts/packaging/package-dra.sh {{matrix}}"
         agents:
           provider: gcp
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -229,7 +203,7 @@ steps:
         depends_on:
           - packaging-snapshot
         command: |
-          buildkite-agent artifact download "build/**" .
+          buildkite-agent artifact download "build/**/*" .
           ls -lah build
           ls -lah build/distributions
           .buildkite/scripts/dra.sh