[CI] Jenkins package pipeline for agentbeat (#38951)

* [CI] Jenkins package pipeline for auditbeat

* Jenkins packaging: install msitools before packaging agentbeat

* Install msiutils

* Removed packaging-arm for agentbeat

* Install msitools for agentbeat PR packaging

* Install msitools for agentbeat PR packaging

* Fix missing close )

* Fix osquerybeat packaging buildkite

* Fix ispackaging

* Improved changesets for agentbeat

* made install-msitools.sh executable

* Fix osquerybeat packaging for buildkite

* Cleanup

* Install tools without sudo

* test: moved packaging before build&tests for check

* test: packaging pipeline

* added sudo

* Cleanup

* Cleanup

---------

Co-authored-by: Blake Rouse <[email protected]>
Co-authored-by: Craig MacKenzie <[email protected]>

.buildkite/scripts/install-msitools.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+sudo apt-get update -y
+DEBIAN_FRONTEND=noninteractive sudo apt-get install --no-install-recommends --yes msitools

.buildkite/x-pack/pipeline.xpack.osquerybeat.yml

@@ -199,6 +199,7 @@ steps:
       - label: ":linux: Packaging Linux"
         key: "packaging-linux"
         command: |
+          .buildkite/scripts/install-msitools.sh
           cd x-pack/osquerybeat
           mage package
         agents:

.ci/packaging.groovy

@@ -204,6 +204,7 @@ def generateSteps() {
     'metricbeat',
     'packetbeat',
     'winlogbeat',
+    'x-pack/agentbeat'
     'x-pack/auditbeat',
     'x-pack/dockerlogbeat',
     'x-pack/filebeat',
@@ -280,6 +281,9 @@ def generateLinuxStep(beat) {
       withEnv(["HOME=${env.WORKSPACE}", "PLATFORMS=${linuxPlatforms()}", "BEATS_FOLDER=${beat}"]) {
         withGithubNotify(context: "Packaging Linux ${beat}") {
           deleteDir()
+          if (beat.equals('x-pack/agentbeat') || beat.equals('x-pack/osquerybeat')) {
+            sh(label: 'install msitools', script: '.buildkite/scripts/install-msitools.sh')
+          }
           release('snapshot')
           dir("${BASE_DIR}"){
             pushCIDockerImages(arch: 'amd64')

Jenkinsfile

@@ -608,6 +608,9 @@ def targetWithoutNode(Map args = [:]) {
         }
       }
       withTools(k8s: installK8s, gcp: withGCP, nodejs: withNodejs) {
+        if (isPackaging && (directory.equals('x-pack/agentbeat') || directory.equals('x-pack/osquerybeat'))) {
+          sh(label: 'install msitools', script: '.buildkite/scripts/install-msitools.sh')
+        }
         // make commands use -C <folder> while mage commands require the dir(folder)
         // let's support this scenario with the location variable.
         dir(isMage ? directory : '') {

Jenkinsfile.yml

@@ -7,6 +7,7 @@ projects:
     - "metricbeat"
     - "packetbeat"
     - "winlogbeat"
+    - "x-pack/agentbeat"
     - "x-pack/auditbeat"
     - "x-pack/dockerlogbeat"
     - "x-pack/filebeat"

x-pack/agentbeat/Jenkinsfile.yml

@@ -0,0 +1,39 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^x-pack/agentbeat/.*"
+        - "^auditbeat/.*"
+        - "^filebeat/.*"
+        - "^heartbeat/.*"
+        - "^metricbeat/.*"
+        - "^osquerybeat/.*"
+        - "^packetbeat/.*"
+        - "@oss"
+        - "@xpack"
+    comments:                  ## when PR comment contains any of those entries
+        - "/test agentbeat"
+    labels:                    ## when PR labels matches any of those entries
+        - "agentbeat"
+    parameters:                ## when parameter was selected in the UI.
+        - "agentbeat"
+    tags: true                 ## for all the tags
+platform: "immutable && ubuntu-22" ## default label for all the stages
+stages:    
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false
+        stage: packaging
+        when:
+            branches: false    ## Only on a PR basis for the time being
+            tags: false        ## packaging on branches/tags is already in place with the downstream build.
+            changeset:         ## when PR contains any of those entries in the changeset
+              - "^x-pack/agentbeat/.*"
+              - "^auditbeat/.*"
+              - "^filebeat/.*"
+              - "^heartbeat/.*"
+              - "^metricbeat/.*"
+              - "^osquerybeat/.*"
+              - "^packetbeat/.*"
+              - "@oss"               ## special token regarding the changeset for the oss
+              - "@xpack"    

x-pack/osquerybeat/internal/distro/distro.go

@@ -178,7 +178,7 @@ var specs = map[OSArch]Spec{
 	{"linux", "arm64"}:   {"_1.linux_aarch64.tar.gz", osqueryDistroLinuxARMSHA256, true},
 	{"darwin", "amd64"}:  {osqueryPkgExt, osqueryDistroDarwinSHA256, true},
 	{"darwin", "arm64"}:  {osqueryPkgExt, osqueryDistroDarwinSHA256, true},
-	{"windows", "amd64"}: {osqueryMSIExt, osqueryDistroWindowsSHA256, false},
+	{"windows", "amd64"}: {osqueryMSIExt, osqueryDistroWindowsSHA256, true},
 }
 
 func GetSpec(osarch OSArch) (spec Spec, err error) {

x-pack/osquerybeat/internal/msiutil/expand.go

@@ -0,0 +1,21 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package msiutil
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/command"
+)
+
+// Expand runs msiextract to extract the MSI.
+func Expand(msiFile, dstDir string) error {
+	output, err := command.Execute(context.Background(), "msiextract", "--directory", dstDir, msiFile)
+	if err != nil {
+		return fmt.Errorf("failed to run msiextract: %w (output: %s)", err, output)
+	}
+	return nil
+}

x-pack/osquerybeat/magefile.go

@@ -7,21 +7,16 @@
 package main
 
 import (
-	"context"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
-	"strings"
 	"time"
 
 	"github.com/magefile/mage/mg"
 
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/build"
-	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/command"
-	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/distro"
 	osquerybeat "github.com/elastic/beats/v7/x-pack/osquerybeat/scripts/mage"
 
 	// mage:import
@@ -92,92 +87,10 @@ func Clean() error {
 	return devtools.Clean(paths)
 }
 
-func extractFromMSI() error {
-	if os.Getenv("GOOS") != "windows" {
-		return nil
-	}
-
-	ctx := context.Background()
-
-	execCommand := func(name string, args ...string) error {
-		ps := strings.Join(append([]string{name}, args...), " ")
-		fmt.Println(ps)
-		output, err := command.Execute(ctx, name, args...)
-		if err != nil {
-			fmt.Println(ps, ", failed: ", err)
-			return err
-		}
-		fmt.Print(output)
-		return err
-	}
-
-	osArchs := osquerybeat.OSArchs(devtools.Platforms)
-
-	for _, osarch := range osArchs {
-		if osarch.OS != "windows" {
-			continue
-		}
-		spec, err := distro.GetSpec(osarch)
-		if err != nil {
-			if errors.Is(err, distro.ErrUnsupportedOS) {
-				continue
-			} else {
-				return err
-			}
-		}
-		dip := distro.GetDataInstallDir(osarch)
-		msiFile := spec.DistroFilepath(dip)
-
-		// MSI extract
-		err = execCommand("msiextract", "--directory", dip, msiFile)
-		if err != nil {
-			return err
-		}
-
-		fmt.Println("copy certs.pem from MSI")
-		err = devtools.Copy(filepath.Join(dip, distro.OsquerydCertsWindowsDistroPath()), distro.OsquerydCertsPath(dip))
-		if err != nil {
-			return err
-		}
-
-		fmt.Println("copy osqueryd.exe from MSI")
-		dp := distro.OsquerydPathForOS(osarch.OS, dip)
-		err = devtools.Copy(filepath.Join(dip, "osquery", "osqueryd", "osqueryd.exe"), dp)
-		if err != nil {
-			fmt.Println("copy osqueryd.exe from MSI failed: ", err)
-			return err
-		}
-		// Chmod set to the same as other executables in the final package
-		if err = os.Chmod(dp, 0755); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 // GolangCrossBuild build the Beat binary inside of the golang-builder.
 // Do not use directly, use crossBuild instead.
 func GolangCrossBuild() error {
-	// This is to fix a defect in the field where msiexec fails to extract the osqueryd.exe
-	// from bundled osquery.msi, with error code 1603
-	// https://docs.microsoft.com/en-us/troubleshoot/windows-server/application-management/msi-installation-error-1603
-	// SDH: https://github.com/elastic/sdh-beats/issues/1575
-	// Currently we can't reproduce this is issue, but here we can eliminate the need for calling msiexec
-	// if extract the osqueryd.exe binary during the build.
-	//
-	// The cross build is currently called for two binaries osquerybeat and osqquery-extension
-	// Only extract osqueryd.exe during osquerybeat build on windows
-	args := devtools.DefaultGolangCrossBuildArgs()
-
-	if !strings.HasPrefix(args.Name, "osquery-extension-") {
-		// Extract osqueryd.exe from MSI
-		if err := extractFromMSI(); err != nil {
-			return err
-		}
-	}
-
-	return devtools.GolangCrossBuild(args)
+	return devtools.GolangCrossBuild(devtools.DefaultGolangCrossBuildArgs())
 }
 
 // BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon).

x-pack/osquerybeat/scripts/mage/distro.go

@@ -18,6 +18,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/fetch"
 	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/fileutil"
 	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/hash"
+	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/msiutil"
 	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/pkgutil"
 	"github.com/elastic/beats/v7/x-pack/osquerybeat/internal/tar"
 )
@@ -139,6 +140,7 @@ func checkCacheAndFetch(osarch distro.OSArch, spec distro.Spec) (fetched bool, e
 const (
 	suffixTarGz = ".tar.gz"
 	suffixPkg   = ".pkg"
+	suffixMsi   = ".msi"
 )
 
 func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error {
@@ -157,7 +159,7 @@ func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error {
 		return devtools.Copy(src, dst)
 	}
 
-	if !strings.HasSuffix(src, suffixTarGz) && !strings.HasSuffix(src, suffixPkg) {
+	if !strings.HasSuffix(src, suffixTarGz) && !strings.HasSuffix(src, suffixPkg) && !strings.HasSuffix(src, suffixMsi) {
 		return fmt.Errorf("unsupported file: %s", src)
 	}
 	tmpdir, err := os.MkdirTemp(distro.DataDir, "")
@@ -188,7 +190,6 @@ func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error {
 			return err
 		}
 	}
-
 	if strings.HasSuffix(src, suffixPkg) {
 		log.Printf("Extract .pkg from %v", src)
 
@@ -204,6 +205,19 @@ func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error {
 			return err
 		}
 	}
+	if strings.HasSuffix(src, suffixMsi) {
+		log.Printf("Extract .msi from %v", src)
+
+		osdp = filepath.Join("osquery", "osqueryd", "osqueryd.exe")
+		osdcp = distro.OsquerydCertsWindowsDistroPath()
+		distp = distro.OsquerydPathForOS(osarch.OS, dir)
+
+		// Msiutil expand full
+		err = msiutil.Expand(src, tmpdir)
+		if err != nil {
+			return err
+		}
+	}
 
 	// Copy over certs directory
 	certsDir := filepath.Dir(distro.OsquerydCertsPath(dir))
@@ -217,14 +231,16 @@ func extractOrCopy(osarch distro.OSArch, spec distro.Spec) error {
 	}
 
 	// Copy over lenses directory
-	lensesDir := distro.OsquerydLensesDir(dir)
-	err = os.MkdirAll(lensesDir, 0750)
-	if err != nil {
-		return err
-	}
-	err = devtools.Copy(filepath.Join(tmpdir, osdlp), lensesDir)
-	if err != nil {
-		return err
+	if osdlp != "" {
+		lensesDir := distro.OsquerydLensesDir(dir)
+		err = os.MkdirAll(lensesDir, 0750)
+		if err != nil {
+			return err
+		}
+		err = devtools.Copy(filepath.Join(tmpdir, osdlp), lensesDir)
+		if err != nil {
+			return err
+		}
 	}
 
 	// Copy over the osqueryd binary or osquery.app dir