From 9be503234bcb9629d2922268e7d25fee304d478b Mon Sep 17 00:00:00 2001 From: Oleg Sucharevich Date: Fri, 14 Feb 2025 15:41:13 -0600 Subject: [PATCH 1/2] feat(elastic-agent-services): add another tag to docker image for agentless use p override the name in the template use map copy remove fmt update comments typo --- dev-tools/mage/dockerbuilder.go | 56 +++++++++++++++++++++++++------- dev-tools/mage/pkg_test.go | 7 ++++ dev-tools/mage/pkgtypes.go | 7 ++++ dev-tools/mage/settings.go | 12 +++++++ dev-tools/packaging/packages.yml | 2 ++ 5 files changed, 73 insertions(+), 11 deletions(-) diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 0e0bd2078ca..6a181aca734 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "io/fs" + "maps" "os" "os/exec" "path/filepath" @@ -53,13 +54,13 @@ func (b *dockerBuilder) Build() error { return fmt.Errorf("failed to prepare build: %w", err) } - tag, err := b.dockerBuild() + tag, additionalTags, err := b.dockerBuild() tries := 3 for err != nil && tries != 0 { fmt.Println(">> Building docker images again (after 10 s)") // This sleep is to avoid hitting the docker build issues when resources are not available. time.Sleep(time.Second * 10) - tag, err = b.dockerBuild() + tag, additionalTags, err = b.dockerBuild() tries-- } if err != nil { @@ -70,6 +71,16 @@ func (b *dockerBuilder) Build() error { return fmt.Errorf("failed to save docker as artifact: %w", err) } + // additional tags should not be created with + for _, tag := range additionalTags { + if err := b.dockerSave(tag, map[string]interface{}{ + // effectively override the name used from b.ImageName() to the tag + "Name": strings.ReplaceAll(tag, ":", "-"), + }); err != nil { + return fmt.Errorf("failed to save docker with tag %s as artifact: %w", tag, err) + } + } + return nil } @@ -173,22 +184,41 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in return nil } -func (b *dockerBuilder) dockerBuild() (string, error) { - tag := fmt.Sprintf("%s:%s", b.imageName, b.Version) +// dockerBuild runs "docker build -t t1 -t t2 ... buildDir" +// returns the main tag additional tags if specified as part of extra_tags property +// the extra tags are not push to the registry from b.ExtraVars["repository"] +// returns an error if the command fails +func (b *dockerBuilder) dockerBuild() (string, []string, error) { + mainTag := fmt.Sprintf("%s:%s", b.imageName, b.Version) // For Independent Agent releases, replace the "+" with a "." since the "+" character // currently isn't allowed in a tag in Docker // E.g., 8.13.0+build202402191057 -> 8.13.0.build202402191057 - tag = strings.Replace(tag, "+", ".", 1) + mainTag = strings.Replace(mainTag, "+", ".", 1) if b.Snapshot { - tag = tag + "-SNAPSHOT" + mainTag = mainTag + "-SNAPSHOT" } if repository := b.ExtraVars["repository"]; repository != "" { - tag = fmt.Sprintf("%s/%s", repository, tag) + mainTag = fmt.Sprintf("%s/%s", repository, mainTag) } - return tag, sh.Run("docker", "build", "-t", tag, b.buildDir) + + extraTags := []string{} + for _, tag := range b.ExtraTags { + extraTags = append(extraTags, fmt.Sprintf("%s:%s", b.imageName, tag)) + } + + args := []string{ + "build", + "-t", mainTag, + } + for _, t := range extraTags { + args = append(args, "-t", t) + } + args = append(args, b.buildDir) + + return mainTag, extraTags, sh.Run("docker", args...) } -func (b *dockerBuilder) dockerSave(tag string) error { +func (b *dockerBuilder) dockerSave(tag string, templateExtraArgs ...map[string]interface{}) error { if _, err := os.Stat(distributionsDir); os.IsNotExist(err) { err := os.MkdirAll(distributionsDir, 0750) if err != nil { @@ -198,9 +228,13 @@ func (b *dockerBuilder) dockerSave(tag string) error { // Save the container as artifact outputFile := b.OutputFile if outputFile == "" { - outputTar, err := b.Expand(defaultBinaryName+".docker.tar.gz", map[string]interface{}{ + args := map[string]interface{}{ "Name": b.imageName, - }) + } + for _, extraArgs := range templateExtraArgs { + maps.Copy(args, extraArgs) + } + outputTar, err := b.Expand(defaultBinaryName+".docker.tar.gz", args) if err != nil { return err } diff --git a/dev-tools/mage/pkg_test.go b/dev-tools/mage/pkg_test.go index 9d1b40928b3..dbbc74b6fcd 100644 --- a/dev-tools/mage/pkg_test.go +++ b/dev-tools/mage/pkg_test.go @@ -20,6 +20,9 @@ func testPackageSpec() PackageSpec { Snapshot: true, OS: "windows", Arch: "x86_64", + ExtraTags: []string{ + "git-{{ substring commit 0 12 }}", + }, Files: map[string]PackageFile{ "brewbeat.yml": PackageFile{ Source: "./testdata/config.yml", @@ -66,6 +69,10 @@ func testPackage(t testing.TB, pack func(PackageSpec) error) { readmePath := filepath.ToSlash(filepath.Clean(readme.Source)) assert.True(t, strings.HasPrefix(readmePath, packageStagingDir)) + commit := spec.ExtraTags[0] + expected := "git-" + commitHash[:12] + assert.Equal(t, expected, commit) + if err := pack(spec); err != nil { t.Fatal(err) } diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 5c6b32c0083..33910e892dd 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -104,6 +104,7 @@ type PackageSpec struct { Qualifier string `yaml:"qualifier,omitempty"` // Optional OutputFile string `yaml:"output_file,omitempty"` // Optional ExtraVars map[string]string `yaml:"extra_vars,omitempty"` // Optional + ExtraTags []string `yaml:"extra_tags,omitempty"` // Optional evalContext map[string]interface{} packageDir string @@ -384,6 +385,12 @@ func (s PackageSpec) Evaluate(args ...map[string]interface{}) PackageSpec { s.evalContext[k] = mustExpand(v) } + if s.ExtraTags != nil { + for i, tag := range s.ExtraTags { + s.ExtraTags[i] = mustExpand(tag) + } + } + s.Name = mustExpand(s.Name) s.ServiceName = mustExpand(s.ServiceName) s.OS = mustExpand(s.OS) diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index 3318e361968..b53c51b36b2 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -112,6 +112,7 @@ var ( "title": func(s string) string { return cases.Title(language.English, cases.NoLower).String(s) }, "tolower": strings.ToLower, "contains": strings.Contains, + "substring": Substring, agentPackageVersionMappedFunc: AgentPackageVersion, agentManifestGeneratorMappedFunc: PackageManifest, snapshotSuffix: SnapshotSuffix, @@ -378,6 +379,17 @@ func SnapshotSuffix() string { return GenerateSnapshotSuffix(Snapshot) } +func Substring(s string, start, length int) string { + if start < 0 || start >= len(s) { + return "" + } + end := start + length + if end > len(s) { + end = len(s) + } + return s[start:end] +} + func GenerateSnapshotSuffix(snapshot bool) string { if !snapshot { return "" diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml index abfe28527f0..a8f10c256c6 100644 --- a/dev-tools/packaging/packages.yml +++ b/dev-tools/packaging/packages.yml @@ -288,6 +288,8 @@ shared: # service build is based on previous cloud variant - &agent_docker_service_spec docker_variant: 'service' + extra_tags: + - 'git-{{ substring commit 0 12 }}' files: 'data/service/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip': source: '{{.AgentDropPath}}/archives/{{.GOOS}}-{{.AgentArchName}}.tar.gz/connectors-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}.zip' From 6c9b1f3b8dbec196ee39cf7e9d75bb9167418fe6 Mon Sep 17 00:00:00 2001 From: Oleg Sucharevich Date: Tue, 18 Feb 2025 11:09:09 -0600 Subject: [PATCH 2/2] (ci:fix) linting error on G115 --- dev-tools/mage/pkgtypes.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dev-tools/mage/pkgtypes.go b/dev-tools/mage/pkgtypes.go index 33910e892dd..c9da7ff1ada 100644 --- a/dev-tools/mage/pkgtypes.go +++ b/dev-tools/mage/pkgtypes.go @@ -14,6 +14,7 @@ import ( "io" "io/fs" "log" + "math" "os" "path/filepath" "reflect" @@ -969,7 +970,7 @@ func addFileToTar(ar *tar.Writer, baseDir string, pkgFile PackageFile) error { } if mg.Verbose() { - log.Println("Adding", os.FileMode(header.Mode), header.Name) + log.Println("Adding", os.FileMode(mustConvertToUnit32(header.Mode)), header.Name) } if err := ar.WriteHeader(header); err != nil { return err @@ -1037,7 +1038,7 @@ func addSymlinkToTar(tmpdir string, ar *tar.Writer, baseDir string, pkgFile Pack header.Typeflag = tar.TypeSymlink if mg.Verbose() { - log.Println("Adding", os.FileMode(header.Mode), header.Name) + log.Println("Adding", os.FileMode(mustConvertToUnit32(header.Mode)), header.Name) } if err := ar.WriteHeader(header); err != nil { return err @@ -1059,3 +1060,10 @@ func PackageDocker(spec PackageSpec) error { } return b.Build() } + +func mustConvertToUnit32(i int64) uint32 { + if i > math.MaxUint32 { + panic(fmt.Sprintf("%d is bigger than math.MaxUint32", i)) + } + return uint32(i) // #nosec +}