Elastic agent core publishing

.buildkite/x-pack/elastic-agent/.buildkite/pipeline.xpack.elastic-agent-binary-dra.yml

@@ -0,0 +1,78 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
+env:
+  # this is required in order to allow the build process to override the default PWD of the BEAT_NAME.
+  BEAT_NAME: "elastic-agent"
+  # after moving elastic-agent out of beats, we should update the URL of the packaging.
+  BEAT_URL: "https://www.elastic.co/elastic-agent"
+  DRA_PROJECT_ID: "elastic-agent-core"
+  DRA_PROJECT_ARTIFACT_ID: "agent-core"
+
+steps:
+  - group: ":beats: DRA Elastic-Agent Core Snapshot :beats:"
+    key: "dra-core-snapshot"
+    if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_SNAPSHOT") == "true"
+    steps:
+      - label: ":package: Build Elastic-Agent Core Snapshot"
+        commands:
+          - .buildkite/x-pack/elastic-agent/scripts/steps/build-agent-core.sh
+        key: "build-dra-snapshot"
+        artifact_paths:
+          - "build/distributions/**/*"
+        agents:
+          provider: "gcp"
+          machineType: "c2-standard-16"
+        env:
+          DRA_WORKFLOW: "snapshot"
+
+      - wait
+
+      - label: ":hammer: DRA Publish Elastic-Agent Core Snapshot"
+        command: |
+          echo "+++ Restoring Artifacts"
+          buildkite-agent artifact download "build/**/*" .
+          echo "+++ Changing permissions for the release manager"
+          sudo chown -R :1000 x-pack/elastic-agent/build/distributions/
+          echo "+++ Running DRA publish step"
+          .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+        key: "publish-dra-snapshot"
+        agents:
+          provider: "gcp"
+          machineType: "c2-standard-16"
+        env:
+          DRA_WORKFLOW: "snapshot"
+
+  - group: ":beats: DRA Elastic-Agent Core Staging :beats:"
+    key: "dra-core-staging"
+    if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true"
+    steps:
+      - label: ":package: Build Elastic-Agent Core staging"
+        commands:
+          - .buildkite/x-pack/elastic-agent/scripts/steps/build-agent-core.sh
+        key: "build-dra-staging"
+        artifact_paths:
+          - "build/distributions/**/*"
+        agents:
+          provider: "gcp"
+          machineType: "c2-standard-16"
+        env:
+          DRA_WORKFLOW: "staging"
+
+      - wait
+
+      - label: ":hammer: DRA Publish Elastic-Agent Core staging"
+        command: |
+          echo "+++ Restoring Artifacts"
+          buildkite-agent artifact download "build/**/*" .
+          echo "+++ Changing permissions for the release manager"
+          sudo chown -R :1000 x-pack/elastic-agent/build/distributions/
+          echo "+++ Running DRA publish step"
+          .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh
+        key: "publish-dra-staging"
+        agents:
+          provider: "gcp"
+          machineType: "c2-standard-16"
+        env:
+          DRA_WORKFLOW: "staging"
+
+notify:
+  - slack: "#ingest-notifications"

.buildkite/x-pack/elastic-agent/scripts/steps/build-agent-core.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common.sh
+
+echo "+++ Build Agent artifacts"
+SNAPSHOT=""
+BEAT_VERSION_FULL=$BEAT_VERSION
+if [ "$DRA_WORKFLOW" == "snapshot" ]; then
+    SNAPSHOT="true"
+    BEAT_VERSION_FULL="${BEAT_VERSION}-SNAPSHOT"
+fi
+
+SNAPSHOT=$SNAPSHOT mage packageAgentCore
+chmod -R 777 build/distributions
+
+echo  "+++ Generate dependencies report"
+./dev-tools/dependencies-report
+mkdir -p build/distributions/reports
+mv dependencies.csv "build/distributions/reports/dependencies-${BEAT_VERSION_FULL}.csv"

dev-tools/mage/pkgspecs.go

@@ -33,6 +33,12 @@ const packageSpecFile = "dev-tools/packaging/packages.yml"
 // executed.
 var Packages []OSPackageArgs
 
+// UseElasticAgentCorePackaging configures the package target to build binary packages
+// for an Elastic Agent.
+func UseElasticAgentCorePackaging() {
+	MustUsePackaging("elastic_agent_core", packageSpecFile)
+}
+
 // UseCommunityBeatPackaging configures the package target to build packages for
 // a community Beat.
 func UseCommunityBeatPackaging() {

dev-tools/mage/pkgtypes.go

@@ -48,7 +48,13 @@ const (
 	packageStagingDir = "build/package"
 
 	// defaultBinaryName specifies the output file for zip and tar.gz.
-	defaultBinaryName = "{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}"
+	defaultBinaryName = "{{.Name}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}"
+
+	// defaultRootDir is the default name of the root directory contained inside of zip and
+	// tar.gz packages.
+	// NOTE: This uses .BeatName instead of .Name because we wanted the internal
+	// directory to not include "-oss".
+	defaultRootDir = "{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}"
 )
 
 // PackageType defines the file format of the package (e.g. zip, rpm, etc).
@@ -88,6 +94,7 @@ type PackageSpec struct {
 	PreInstallScript  string                 `yaml:"pre_install_script,omitempty"`
 	PostInstallScript string                 `yaml:"post_install_script,omitempty"`
 	Files             map[string]PackageFile `yaml:"files"`
+	Qualifier         string                 `yaml:"qualifier,omitempty"`   // Optional
 	OutputFile        string                 `yaml:"output_file,omitempty"` // Optional
 	ExtraVars         map[string]string      `yaml:"extra_vars,omitempty"`  // Optional
 
@@ -530,9 +537,7 @@ func (s PackageSpec) rootDir() string {
 		return filepath.Base(s.OutputFile)
 	}
 
-	// NOTE: This uses .BeatName instead of .Name because we wanted the internal
-	// directory to not include "-oss".
-	return s.MustExpand("{{.BeatName}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}")
+	return s.MustExpand(defaultRootDir)
 }
 
 // PackageZip packages a zip file.
@@ -582,7 +587,10 @@ func PackageZip(spec PackageSpec) error {
 		return nil
 	}
 
-	return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file")
+	if err := CreateSHA512File(spec.OutputFile); err != nil {
+		return fmt.Errorf("failed to create .sha512 file: %w", err)
+	}
+	return nil
 }
 
 // PackageTarGz packages a gzipped tar file.
@@ -661,7 +669,10 @@ func PackageTarGz(spec PackageSpec) error {
 		return nil
 	}
 
-	return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file")
+	if err := CreateSHA512File(spec.OutputFile); err != nil {
+		return fmt.Errorf("failed to create .sha512 file: %w", err)
+	}
+	return nil
 }
 
 // PackageDeb packages a deb file. This requires Docker to execute FPM.
@@ -764,7 +775,10 @@ func runFPM(spec PackageSpec, packageType PackageType) error {
 		return errors.Wrap(err, "failed while running FPM in docker")
 	}
 
-	return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file")
+	if err := CreateSHA512File(spec.OutputFile); err != nil {
+		return fmt.Errorf("failed to create .sha512 file: %w", err)
+	}
+	return nil
 }
 
 func addUidGidEnvArgs(args []string) ([]string, error) {

dev-tools/packaging/packages.yml

@@ -1148,3 +1148,46 @@ specs:
         files:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+  elastic_agent_core:
+    ###
+    # Elastic Licensed Core Packages
+    ###
+    - os: windows
+      types: [zip]
+      spec:
+        <<: *common
+        <<: *elastic_license_for_binaries
+        qualifier: core
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+
+    - os: darwin
+      types: [tgz]
+      spec:
+        <<: *common
+        <<: *elastic_license_for_binaries
+        qualifier: core
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+
+    - os: linux
+      types: [tgz]
+      spec:
+        <<: *common
+        <<: *elastic_license_for_binaries
+        qualifier: core
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+
+    - os: aix
+      types: [tgz]
+      spec:
+        <<: *common
+        <<: *elastic_license_for_binaries
+        qualifier: core
+        files:
+          '{{.BeatName}}{{.BinaryExt}}':
+            source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}

x-pack/elastic-agent/magefile.go

@@ -456,6 +456,19 @@ func CrossBuildGoDaemon() error {
 	return devtools.CrossBuildGoDaemon()
 }
 
+// PackageAgentCore cross-builds and packages distribution artifacts containing
+// only elastic-agent binaries with no extra files or dependencies.
+func PackageAgentCore() {
+	start := time.Now()
+	defer func() { fmt.Println("packageAgentCore ran for", time.Since(start)) }()
+
+	mg.Deps(CrossBuild, CrossBuildGoDaemon)
+
+	devtools.UseElasticAgentCorePackaging()
+
+	mg.Deps(devtools.Package)
+}
+
 // Config generates both the short/reference/docker.
 func Config() {
 	mg.Deps(configYML)