From 605647a00c17a0513b0c07b74aadff9e2ef430a9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 26 Dec 2024 10:58:20 -0600 Subject: [PATCH 1/2] fix(deps): update module golang.org/x/crypto to v0.31.0 [security] (#1226) --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 81d20dba5..bb76fcd3d 100644 --- a/go.mod +++ b/go.mod @@ -46,9 +46,9 @@ require ( go.opentelemetry.io/otel/sdk v1.30.0 go.opentelemetry.io/otel/trace v1.30.0 go.starlark.net v0.0.0-20240925182052-1207426daebd - golang.org/x/crypto v0.27.0 + golang.org/x/crypto v0.31.0 golang.org/x/oauth2 v0.23.0 - golang.org/x/sync v0.8.0 + golang.org/x/sync v0.10.0 golang.org/x/time v0.6.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/postgres v1.5.9 @@ -147,8 +147,8 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/grpc v1.66.1 // indirect diff --git a/go.sum b/go.sum index fd4582b91..ecbcdaa17 100644 --- a/go.sum +++ b/go.sum @@ -347,8 +347,8 @@ golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -363,8 +363,8 @@ golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -373,12 +373,12 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 72319ae4ce75eb935da7a5bef642bca52294508e Mon Sep 17 00:00:00 2001 From: Easton Crupper <65553218+ecrupper@users.noreply.github.com> Date: Thu, 26 Dec 2024 13:23:18 -0600 Subject: [PATCH 2/2] enhance(yaml): allow for users to set version legacy for buildkite (#1230) Co-authored-by: David May <49894298+wass3rw3rk@users.noreply.github.com> --- compiler/native/compile_test.go | 197 ++++++++++++++++++ compiler/native/expand_test.go | 4 +- compiler/native/parse.go | 13 +- compiler/native/parse_test.go | 78 +++++++ compiler/native/testdata/stages_merged.yml | 38 ++++ .../native/testdata/steps_merge_anchor.yml | 46 ++++ compiler/template/native/render.go | 13 +- compiler/template/starlark/render.go | 15 +- internal/yaml.go | 63 ++++++ internal/yaml_test.go | 88 ++++++++ 10 files changed, 526 insertions(+), 29 deletions(-) create mode 100644 compiler/native/testdata/stages_merged.yml create mode 100644 compiler/native/testdata/steps_merge_anchor.yml create mode 100644 internal/yaml.go create mode 100644 internal/yaml_test.go diff --git a/compiler/native/compile_test.go b/compiler/native/compile_test.go index e2134c9db..3767760c2 100644 --- a/compiler/native/compile_test.go +++ b/compiler/native/compile_test.go @@ -1981,6 +1981,203 @@ func TestNative_Compile_StepsandStages(t *testing.T) { } } +func TestNative_Compile_LegacyMergeAnchor(t *testing.T) { + // setup types + set := flag.NewFlagSet("test", 0) + set.String("clone-image", defaultCloneImage, "doc") + c := cli.NewContext(nil, set, nil) + name := "foo" + author := "author" + event := "push" + number := 1 + + m := &internal.Metadata{ + Database: &internal.Database{ + Driver: "foo", + Host: "foo", + }, + Queue: &internal.Queue{ + Channel: "foo", + Driver: "foo", + Host: "foo", + }, + Source: &internal.Source{ + Driver: "foo", + Host: "foo", + }, + Vela: &internal.Vela{ + Address: "foo", + WebAddress: "foo", + }, + } + + compiler, err := FromCLIContext(c) + if err != nil { + t.Errorf("Creating compiler returned err: %v", err) + } + + compiler.repo = &api.Repo{Name: &author} + compiler.build = &api.Build{Author: &name, Number: &number, Event: &event} + compiler.WithMetadata(m) + + testEnv := environment(&api.Build{Author: &name, Number: &number, Event: &event}, m, &api.Repo{Name: &author}, nil, nil) + + serviceEnv := environment(&api.Build{Author: &name, Number: &number, Event: &event}, m, &api.Repo{Name: &author}, nil, nil) + serviceEnv["REGION"] = "dev" + + alphaEnv := environment(&api.Build{Author: &name, Number: &number, Event: &event}, m, &api.Repo{Name: &author}, nil, nil) + alphaEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo alpha"}) + alphaEnv["HOME"] = "/root" + alphaEnv["SHELL"] = "/bin/sh" + + betaEnv := environment(&api.Build{Author: &name, Number: &number, Event: &event}, m, &api.Repo{Name: &author}, nil, nil) + betaEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo beta"}) + betaEnv["HOME"] = "/root" + betaEnv["SHELL"] = "/bin/sh" + + gammaEnv := environment(&api.Build{Author: &name, Number: &number, Event: &event}, m, &api.Repo{Name: &author}, nil, nil) + gammaEnv["VELA_BUILD_SCRIPT"] = generateScriptPosix([]string{"echo gamma"}) + gammaEnv["HOME"] = "/root" + gammaEnv["SHELL"] = "/bin/sh" + gammaEnv["REGION"] = "dev" + + want := &pipeline.Build{ + Version: "legacy", + ID: "_author_1", + Metadata: pipeline.Metadata{ + Clone: true, + Template: false, + Environment: []string{"steps", "services", "secrets"}, + AutoCancel: &pipeline.CancelOptions{ + Running: false, + Pending: false, + DefaultBranch: false, + }, + }, + Worker: pipeline.Worker{ + Flavor: "", + Platform: "", + }, + Services: pipeline.ContainerSlice{ + &pipeline.Container{ + ID: "service__author_1_service-a", + Detach: true, + Directory: "", + Environment: serviceEnv, + Image: "postgres", + Name: "service-a", + Number: 1, + Pull: "not_present", + Ports: []string{"5432:5432"}, + }, + }, + Steps: pipeline.ContainerSlice{ + &pipeline.Container{ + ID: "step__author_1_init", + Directory: "/vela/src/foo//author", + Environment: testEnv, + Image: "#init", + Name: "init", + Number: 1, + Pull: "not_present", + }, + &pipeline.Container{ + ID: "step__author_1_clone", + Directory: "/vela/src/foo//author", + Environment: testEnv, + Image: defaultCloneImage, + Name: "clone", + Number: 2, + Pull: "not_present", + }, + &pipeline.Container{ + ID: "step__author_1_alpha", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//author", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: alphaEnv, + Image: "alpine:latest", + Name: "alpha", + Number: 3, + Pull: "not_present", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + &pipeline.Container{ + ID: "step__author_1_beta", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//author", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: betaEnv, + Image: "alpine:latest", + Name: "beta", + Number: 4, + Pull: "not_present", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + &pipeline.Container{ + ID: "step__author_1_gamma", + Commands: []string{"echo $VELA_BUILD_SCRIPT | base64 -d | /bin/sh -e"}, + Directory: "/vela/src/foo//author", + Entrypoint: []string{"/bin/sh", "-c"}, + Environment: gammaEnv, + Image: "alpine:latest", + Name: "gamma", + Number: 5, + Pull: "not_present", + Ruleset: pipeline.Ruleset{ + If: pipeline.Rules{ + Event: []string{"push"}, + }, + Matcher: "filepath", + Operator: "and", + }, + }, + }, + } + + // run test on legacy version + yaml, err := os.ReadFile("testdata/steps_merge_anchor.yml") + if err != nil { + t.Errorf("Reading yaml file return err: %v", err) + } + + got, _, err := compiler.Compile(context.Background(), yaml) + if err != nil { + t.Errorf("Compile returned err: %v", err) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Compile() mismatch (-want +got):\n%s", diff) + } + + // run test on current version (should fail) + yaml, err = os.ReadFile("../types/yaml/buildkite/testdata/merge_anchor.yml") // has `version: "1"` instead of `version: "legacy"` + if err != nil { + t.Errorf("Reading yaml file return err: %v", err) + } + + got, _, err = compiler.Compile(context.Background(), yaml) + if err == nil { + t.Errorf("Compile should have returned err") + } + + if got != nil { + t.Errorf("Compile is %v, want %v", got, nil) + } +} + // convertResponse converts the build to the ModifyResponse. func convertResponse(build *yaml.Build) (*ModifyResponse, error) { data, err := yml.Marshal(build) diff --git a/compiler/native/expand_test.go b/compiler/native/expand_test.go index 0ca596338..a47df02ed 100644 --- a/compiler/native/expand_test.go +++ b/compiler/native/expand_test.go @@ -694,7 +694,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { "auth_method": "token", "username": "octocat", "items": []interface{}{ - map[interface{}]interface{}{"path": "docker", "source": "secret/docker"}, + map[string]interface{}{"path": "docker", "source": "secret/docker"}, }, }, }, @@ -715,7 +715,7 @@ func TestNative_ExpandStepsMulti(t *testing.T) { "auth_method": "token", "username": "octocat", "items": []interface{}{ - map[interface{}]interface{}{"path": "docker", "source": "secret/docker"}, + map[string]interface{}{"path": "docker", "source": "secret/docker"}, }, }, }, diff --git a/compiler/native/parse.go b/compiler/native/parse.go index 804b1e015..317c52600 100644 --- a/compiler/native/parse.go +++ b/compiler/native/parse.go @@ -7,14 +7,12 @@ import ( "io" "os" - bkYaml "github.com/buildkite/yaml" - "github.com/go-vela/server/compiler/template/native" "github.com/go-vela/server/compiler/template/starlark" typesRaw "github.com/go-vela/server/compiler/types/raw" - bkYamlTypes "github.com/go-vela/server/compiler/types/yaml/buildkite" "github.com/go-vela/server/compiler/types/yaml/yaml" "github.com/go-vela/server/constants" + "github.com/go-vela/server/internal" ) // ParseRaw converts an object to a string. @@ -114,12 +112,9 @@ func (c *client) Parse(v interface{}, pipelineType string, template *yaml.Templa // ParseBytes converts a byte slice to a yaml configuration. func ParseBytes(data []byte) (*yaml.Build, []byte, error) { - config := new(bkYamlTypes.Build) - - // unmarshal the bytes into the yaml configuration - err := bkYaml.Unmarshal(data, config) + config, err := internal.ParseYAML(data) if err != nil { - return nil, data, fmt.Errorf("unable to unmarshal yaml: %w", err) + return nil, nil, err } // initializing Environment to prevent nil error @@ -129,7 +124,7 @@ func ParseBytes(data []byte) (*yaml.Build, []byte, error) { config.Environment = typesRaw.StringSliceMap{} } - return config.ToYAML(), data, nil + return config, data, nil } // ParseFile converts an os.File into a yaml configuration. diff --git a/compiler/native/parse_test.go b/compiler/native/parse_test.go index ae5b263b7..1ac5953b9 100644 --- a/compiler/native/parse_test.go +++ b/compiler/native/parse_test.go @@ -604,6 +604,84 @@ func TestNative_Parse_Stages(t *testing.T) { } } +func TestNative_Parse_StagesLegacyMergeAnchor(t *testing.T) { + // setup types + client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) + want := &yaml.Build{ + Version: "legacy", + Metadata: yaml.Metadata{ + Environment: []string{"steps", "services", "secrets"}, + }, + Environment: raw.StringSliceMap{}, + Stages: yaml.StageSlice{ + &yaml.Stage{ + Name: "install", + Needs: raw.StringSlice{"clone"}, + Steps: yaml.StepSlice{ + &yaml.Step{ + Commands: []string{"./gradlew downloadDependencies"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "install", + Pull: "always", + }, + }, + }, + &yaml.Stage{ + Name: "test", + Needs: []string{"install", "clone"}, + Steps: yaml.StepSlice{ + &yaml.Step{ + Commands: []string{"./gradlew check"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "test", + Pull: "always", + }, + }, + }, + &yaml.Stage{ + Name: "build", + Needs: []string{"install", "clone"}, + Steps: yaml.StepSlice{ + &yaml.Step{ + Commands: []string{"./gradlew build"}, + Environment: map[string]string{ + "GRADLE_OPTS": "-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false", + "GRADLE_USER_HOME": ".gradle", + }, + Image: "openjdk:latest", + Name: "build", + Pull: "always", + }, + }, + }, + }, + } + + // run test + b, err := os.ReadFile("testdata/stages_merged.yml") + if err != nil { + t.Errorf("Reading file returned err: %v", err) + } + + got, _, err := client.Parse(b, "", new(yaml.Template)) + + if err != nil { + t.Errorf("Parse returned err: %v", err) + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Parse() mismatch (-want +got):\n%s", diff) + } +} + func TestNative_Parse_Steps(t *testing.T) { // setup types client, _ := FromCLIContext(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil)) diff --git a/compiler/native/testdata/stages_merged.yml b/compiler/native/testdata/stages_merged.yml new file mode 100644 index 000000000..6bb17c276 --- /dev/null +++ b/compiler/native/testdata/stages_merged.yml @@ -0,0 +1,38 @@ +version: "legacy" + +stages: + install: + steps: + - name: install + commands: + - ./gradlew downloadDependencies + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + + test: + needs: [ install ] + steps: + - name: test + commands: + - ./gradlew check + environment: + GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + GRADLE_USER_HOME: .gradle + image: openjdk:latest + pull: true + +stages: + build: + needs: [ install ] + steps: + - name: build + commands: + - ./gradlew build + environment: + - GRADLE_OPTS=-Dorg.gradle.daemon=false -Dorg.gradle.workers.max=1 -Dorg.gradle.parallel=false + - GRADLE_USER_HOME=.gradle + image: openjdk:latest + pull: true \ No newline at end of file diff --git a/compiler/native/testdata/steps_merge_anchor.yml b/compiler/native/testdata/steps_merge_anchor.yml new file mode 100644 index 000000000..1d488edd2 --- /dev/null +++ b/compiler/native/testdata/steps_merge_anchor.yml @@ -0,0 +1,46 @@ +# test file that uses the non-standard multiple anchor keys in one step to test custom step unmarshaler + +version: "legacy" + +aliases: + images: + alpine: &alpine-image + image: alpine:latest + postgres: &pg-image + image: postgres + + events: + push: &event-push + ruleset: + event: + - push + env: + dev-env: &dev-environment + environment: + REGION: dev + +services: + - name: service-a + <<: *pg-image + <<: *dev-environment + ports: + - "5432:5432" + +steps: + - name: alpha + <<: *alpine-image + <<: *event-push + commands: + - echo alpha + + - name: beta + <<: [ *alpine-image, *event-push ] + commands: + - echo beta + + - name: gamma + <<: *alpine-image + <<: *event-push + <<: *dev-environment + commands: + - echo gamma \ No newline at end of file diff --git a/compiler/template/native/render.go b/compiler/template/native/render.go index 9fec0fd50..1004d62d1 100644 --- a/compiler/template/native/render.go +++ b/compiler/template/native/render.go @@ -8,17 +8,15 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" - "github.com/buildkite/yaml" "github.com/go-vela/server/compiler/types/raw" - bkTypes "github.com/go-vela/server/compiler/types/yaml/buildkite" types "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/internal" ) // Render combines the template with the step in the yaml pipeline. func Render(tmpl string, name string, tName string, environment raw.StringSliceMap, variables map[string]interface{}) (*types.Build, error) { buffer := new(bytes.Buffer) - config := new(bkTypes.Build) velaFuncs := funcHandler{envs: convertPlatformVars(environment, name)} templateFuncMap := map[string]interface{}{ @@ -48,7 +46,7 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM } // unmarshal the template to the pipeline - err = yaml.Unmarshal(buffer.Bytes(), config) + config, err := internal.ParseYAML(buffer.Bytes()) if err != nil { return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) } @@ -58,13 +56,12 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM config.Steps[index].Name = fmt.Sprintf("%s_%s", name, newStep.Name) } - return &types.Build{Metadata: *config.Metadata.ToYAML(), Steps: *config.Steps.ToYAML(), Secrets: *config.Secrets.ToYAML(), Services: *config.Services.ToYAML(), Environment: config.Environment, Templates: *config.Templates.ToYAML(), Deployment: *config.Deployment.ToYAML()}, nil + return &types.Build{Metadata: config.Metadata, Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment, Templates: config.Templates, Deployment: config.Deployment}, nil } // RenderBuild renders the templated build. func RenderBuild(tmpl string, b string, envs map[string]string, variables map[string]interface{}) (*types.Build, error) { buffer := new(bytes.Buffer) - config := new(bkTypes.Build) velaFuncs := funcHandler{envs: convertPlatformVars(envs, tmpl)} templateFuncMap := map[string]interface{}{ @@ -94,10 +91,10 @@ func RenderBuild(tmpl string, b string, envs map[string]string, variables map[st } // unmarshal the template to the pipeline - err = yaml.Unmarshal(buffer.Bytes(), config) + config, err := internal.ParseYAML(buffer.Bytes()) if err != nil { return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) } - return config.ToYAML(), nil + return config, nil } diff --git a/compiler/template/starlark/render.go b/compiler/template/starlark/render.go index a89d63f99..1abc52319 100644 --- a/compiler/template/starlark/render.go +++ b/compiler/template/starlark/render.go @@ -7,14 +7,13 @@ import ( "errors" "fmt" - "github.com/buildkite/yaml" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" "go.starlark.net/syntax" "github.com/go-vela/server/compiler/types/raw" - bkTypes "github.com/go-vela/server/compiler/types/yaml/buildkite" types "github.com/go-vela/server/compiler/types/yaml/yaml" + "github.com/go-vela/server/internal" ) var ( @@ -33,8 +32,6 @@ var ( // Render combines the template with the step in the yaml pipeline. func Render(tmpl string, name string, tName string, environment raw.StringSliceMap, variables map[string]interface{}, limit int64) (*types.Build, error) { - config := new(bkTypes.Build) - thread := &starlark.Thread{Name: name} if limit < 0 { @@ -125,7 +122,7 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM } // unmarshal the template to the pipeline - err = yaml.Unmarshal(buf.Bytes(), config) + config, err := internal.ParseYAML(buf.Bytes()) if err != nil { return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) } @@ -135,15 +132,13 @@ func Render(tmpl string, name string, tName string, environment raw.StringSliceM config.Steps[index].Name = fmt.Sprintf("%s_%s", name, newStep.Name) } - return &types.Build{Steps: *config.Steps.ToYAML(), Secrets: *config.Secrets.ToYAML(), Services: *config.Services.ToYAML(), Environment: config.Environment}, nil + return &types.Build{Steps: config.Steps, Secrets: config.Secrets, Services: config.Services, Environment: config.Environment}, nil } // RenderBuild renders the templated build. // //nolint:lll // ignore function length due to input args func RenderBuild(tmpl string, b string, envs map[string]string, variables map[string]interface{}, limit int64) (*types.Build, error) { - config := new(bkTypes.Build) - thread := &starlark.Thread{Name: "templated-base"} if limit < 0 { @@ -234,10 +229,10 @@ func RenderBuild(tmpl string, b string, envs map[string]string, variables map[st } // unmarshal the template to the pipeline - err = yaml.Unmarshal(buf.Bytes(), config) + config, err := internal.ParseYAML(buf.Bytes()) if err != nil { return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) } - return config.ToYAML(), nil + return config, nil } diff --git a/internal/yaml.go b/internal/yaml.go new file mode 100644 index 000000000..8267486e8 --- /dev/null +++ b/internal/yaml.go @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: Apache-2.0 + +package internal + +import ( + "fmt" + + bkYaml "github.com/buildkite/yaml" + yaml "gopkg.in/yaml.v3" + + legacyTypes "github.com/go-vela/server/compiler/types/yaml/buildkite" + types "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +// ParseYAML is a helper function for transitioning teams away from legacy buildkite YAML parser. +func ParseYAML(data []byte) (*types.Build, error) { + var ( + rootNode yaml.Node + version string + ) + + err := yaml.Unmarshal(data, &rootNode) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal pipeline version yaml: %w", err) + } + + if len(rootNode.Content) == 0 || rootNode.Content[0].Kind != yaml.MappingNode { + return nil, fmt.Errorf("unable to find pipeline version in yaml") + } + + for i, subNode := range rootNode.Content[0].Content { + if subNode.Kind == yaml.ScalarNode && subNode.Value == "version" { + if len(rootNode.Content[0].Content) > i { + version = rootNode.Content[0].Content[i+1].Value + + break + } + } + } + + config := new(types.Build) + + switch version { + case "legacy": + legacyConfig := new(legacyTypes.Build) + + err := bkYaml.Unmarshal(data, legacyConfig) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal legacy yaml: %w", err) + } + + config = legacyConfig.ToYAML() + + default: + // unmarshal the bytes into the yaml configuration + err := yaml.Unmarshal(data, config) + if err != nil { + return nil, fmt.Errorf("unable to unmarshal yaml: %w", err) + } + } + + return config, nil +} diff --git a/internal/yaml_test.go b/internal/yaml_test.go new file mode 100644 index 000000000..627139229 --- /dev/null +++ b/internal/yaml_test.go @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 + +package internal + +import ( + "os" + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/go-vela/server/compiler/types/yaml/yaml" +) + +func TestInternal_ParseYAML(t *testing.T) { + // wantBuild + wantBuild := &yaml.Build{ + Version: "1", + Metadata: yaml.Metadata{ + Environment: []string{"steps", "services", "secrets"}, + }, + Steps: yaml.StepSlice{ + &yaml.Step{ + Name: "example", + Image: "alpine:latest", + Environment: map[string]string{ + "REGION": "dev", + }, + Pull: "not_present", + Commands: []string{ + "echo $REGION", + }, + }, + }, + } + + // set up tests + tests := []struct { + file string + want *yaml.Build + wantErr bool + }{ + { + file: "testdata/go-yaml.yml", + want: wantBuild, + }, + { + file: "testdata/buildkite.yml", + want: wantBuild, + }, + { + file: "testdata/no_version.yml", + want: wantBuild, + }, + { + file: "testdata/invalid.yml", + want: nil, + wantErr: true, + }, + } + + // run tests + for _, test := range tests { + bytes, err := os.ReadFile(test.file) + if err != nil { + t.Errorf("unable to read file: %v", err) + } + + gotBuild, err := ParseYAML(bytes) + if err != nil && !test.wantErr { + t.Errorf("ParseYAML returned err: %v", err) + } + + if err == nil && test.wantErr { + t.Errorf("ParseYAML returned nil error") + } + + if err != nil && test.wantErr { + continue + } + + // different versions expected + wantBuild.Version = gotBuild.Version + + if diff := cmp.Diff(gotBuild, test.want); diff != "" { + t.Errorf("ParseYAML returned diff (-got +want):\n%s", diff) + } + } +}