diff --git a/.buildkite/x-pack/elastic-agent/.buildkite/pipeline.xpack.elastic-agent-binary-dra.yml b/.buildkite/x-pack/elastic-agent/.buildkite/pipeline.xpack.elastic-agent-binary-dra.yml new file mode 100644 index 000000000000..61fbe11d6195 --- /dev/null +++ b/.buildkite/x-pack/elastic-agent/.buildkite/pipeline.xpack.elastic-agent-binary-dra.yml @@ -0,0 +1,78 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json +env: + # this is required in order to allow the build process to override the default PWD of the BEAT_NAME. + BEAT_NAME: "elastic-agent" + # after moving elastic-agent out of beats, we should update the URL of the packaging. + BEAT_URL: "https://www.elastic.co/elastic-agent" + DRA_PROJECT_ID: "elastic-agent-core" + DRA_PROJECT_ARTIFACT_ID: "agent-core" + +steps: + - group: ":beats: DRA Elastic-Agent Core Snapshot :beats:" + key: "dra-core-snapshot" + if: build.branch == 'main' || build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_SNAPSHOT") == "true" + steps: + - label: ":package: Build Elastic-Agent Core Snapshot" + commands: + - .buildkite/x-pack/elastic-agent/scripts/steps/build-agent-core.sh + key: "build-dra-snapshot" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + DRA_WORKFLOW: "snapshot" + + - wait + + - label: ":hammer: DRA Publish Elastic-Agent Core Snapshot" + command: | + echo "+++ Restoring Artifacts" + buildkite-agent artifact download "build/**/*" . + echo "+++ Changing permissions for the release manager" + sudo chown -R :1000 x-pack/elastic-agent/build/distributions/ + echo "+++ Running DRA publish step" + .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh + key: "publish-dra-snapshot" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + DRA_WORKFLOW: "snapshot" + + - group: ":beats: DRA Elastic-Agent Core Staging :beats:" + key: "dra-core-staging" + if: build.branch =~ /^[0-9]+\.[0-9]+\$/ || build.env("RUN_STAGING") == "true" + steps: + - label: ":package: Build Elastic-Agent Core staging" + commands: + - .buildkite/x-pack/elastic-agent/scripts/steps/build-agent-core.sh + key: "build-dra-staging" + artifact_paths: + - "build/distributions/**/*" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + DRA_WORKFLOW: "staging" + + - wait + + - label: ":hammer: DRA Publish Elastic-Agent Core staging" + command: | + echo "+++ Restoring Artifacts" + buildkite-agent artifact download "build/**/*" . + echo "+++ Changing permissions for the release manager" + sudo chown -R :1000 x-pack/elastic-agent/build/distributions/ + echo "+++ Running DRA publish step" + .buildkite/x-pack/elastic-agent/scripts/steps/dra-publish.sh + key: "publish-dra-staging" + agents: + provider: "gcp" + machineType: "c2-standard-16" + env: + DRA_WORKFLOW: "staging" + +notify: + - slack: "#ingest-notifications" diff --git a/.buildkite/x-pack/elastic-agent/scripts/steps/build-agent-core.sh b/.buildkite/x-pack/elastic-agent/scripts/steps/build-agent-core.sh new file mode 100755 index 000000000000..b93008809643 --- /dev/null +++ b/.buildkite/x-pack/elastic-agent/scripts/steps/build-agent-core.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +source .buildkite/scripts/common.sh + +echo "+++ Build Agent artifacts" +SNAPSHOT="" +BEAT_VERSION_FULL=$BEAT_VERSION +if [ "$DRA_WORKFLOW" == "snapshot" ]; then + SNAPSHOT="true" + BEAT_VERSION_FULL="${BEAT_VERSION}-SNAPSHOT" +fi + +SNAPSHOT=$SNAPSHOT mage packageAgentCore +chmod -R 777 build/distributions + +echo "+++ Generate dependencies report" +./dev-tools/dependencies-report +mkdir -p build/distributions/reports +mv dependencies.csv "build/distributions/reports/dependencies-${BEAT_VERSION_FULL}.csv" diff --git a/dev-tools/mage/pkgspecs.go b/dev-tools/mage/pkgspecs.go index 558b8e37ae8d..25b0e7533caf 100644 --- a/dev-tools/mage/pkgspecs.go +++ b/dev-tools/mage/pkgspecs.go @@ -33,6 +33,12 @@ const packageSpecFile = "dev-tools/packaging/packages.yml" // executed. var Packages []OSPackageArgs +// UseElasticAgentCorePackaging configures the package target to build binary packages +// for an Elastic Agent. +func UseElasticAgentCorePackaging() { + MustUsePackaging("elastic_agent_core", packageSpecFile) +} + // UseCommunityBeatPackaging configures the package target to build packages for // a community Beat. func UseCommunityBeatPackaging() { diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index c2c454c873d1..91c777a1b9a9 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -48,7 +48,13 @@ const ( packageStagingDir = "build/package" // defaultBinaryName specifies the output file for zip and tar.gz. - defaultBinaryName = "{{.Name}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" + defaultBinaryName = "{{.Name}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" + + // defaultRootDir is the default name of the root directory contained inside of zip and + // tar.gz packages. + // NOTE: This uses .BeatName instead of .Name because we wanted the internal + // directory to not include "-oss". + defaultRootDir = "{{.BeatName}}{{if .Qualifier}}-{{.Qualifier}}{{end}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}" ) // PackageType defines the file format of the package (e.g. zip, rpm, etc). @@ -88,6 +94,7 @@ type PackageSpec struct { PreInstallScript string `yaml:"pre_install_script,omitempty"` PostInstallScript string `yaml:"post_install_script,omitempty"` Files map[string]PackageFile `yaml:"files"` + Qualifier string `yaml:"qualifier,omitempty"` // Optional OutputFile string `yaml:"output_file,omitempty"` // Optional ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional @@ -530,9 +537,7 @@ func (s PackageSpec) rootDir() string { return filepath.Base(s.OutputFile) } - // NOTE: This uses .BeatName instead of .Name because we wanted the internal - // directory to not include "-oss". - return s.MustExpand("{{.BeatName}}-{{.Version}}{{if .Snapshot}}-SNAPSHOT{{end}}{{if .OS}}-{{.OS}}{{end}}{{if .Arch}}-{{.Arch}}{{end}}") + return s.MustExpand(defaultRootDir) } // PackageZip packages a zip file. @@ -582,7 +587,10 @@ func PackageZip(spec PackageSpec) error { return nil } - return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file") + if err := CreateSHA512File(spec.OutputFile); err != nil { + return fmt.Errorf("failed to create .sha512 file: %w", err) + } + return nil } // PackageTarGz packages a gzipped tar file. @@ -661,7 +669,10 @@ func PackageTarGz(spec PackageSpec) error { return nil } - return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file") + if err := CreateSHA512File(spec.OutputFile); err != nil { + return fmt.Errorf("failed to create .sha512 file: %w", err) + } + return nil } // PackageDeb packages a deb file. This requires Docker to execute FPM. @@ -764,7 +775,10 @@ func runFPM(spec PackageSpec, packageType PackageType) error { return errors.Wrap(err, "failed while running FPM in docker") } - return errors.Wrap(CreateSHA512File(spec.OutputFile), "failed to create .sha512 file") + if err := CreateSHA512File(spec.OutputFile); err != nil { + return fmt.Errorf("failed to create .sha512 file: %w", err) + } + return nil } func addUidGidEnvArgs(args []string) ([]string, error) { diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index 921e61134260..a7e44d25fd86 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -1148,3 +1148,46 @@ specs: files: '{{.BeatName}}{{.BinaryExt}}': source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + elastic_agent_core: + ### + # Elastic Licensed Core Packages + ### + - os: windows + types: [zip] + spec: + <<: *common + <<: *elastic_license_for_binaries + qualifier: core + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + - os: darwin + types: [tgz] + spec: + <<: *common + <<: *elastic_license_for_binaries + qualifier: core + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + - os: linux + types: [tgz] + spec: + <<: *common + <<: *elastic_license_for_binaries + qualifier: core + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} + + - os: aix + types: [tgz] + spec: + <<: *common + <<: *elastic_license_for_binaries + qualifier: core + files: + '{{.BeatName}}{{.BinaryExt}}': + source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}} \ No newline at end of file diff --git a/x-pack/elastic-agent/magefile.go b/x-pack/elastic-agent/magefile.go index 5cbb9d9d7bab..f5be9be5935a 100644 --- a/x-pack/elastic-agent/magefile.go +++ b/x-pack/elastic-agent/magefile.go @@ -456,6 +456,19 @@ func CrossBuildGoDaemon() error { return devtools.CrossBuildGoDaemon() } +// PackageAgentCore cross-builds and packages distribution artifacts containing +// only elastic-agent binaries with no extra files or dependencies. +func PackageAgentCore() { + start := time.Now() + defer func() { fmt.Println("packageAgentCore ran for", time.Since(start)) }() + + mg.Deps(CrossBuild, CrossBuildGoDaemon) + + devtools.UseElasticAgentCorePackaging() + + mg.Deps(devtools.Package) +} + // Config generates both the short/reference/docker. func Config() { mg.Deps(configYML)