From f942ccf798667450d5df1c4fe9e7248e0b11e026 Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Mon, 8 Jul 2024 13:26:00 -0400 Subject: [PATCH 1/2] correct exit code assertion in exec plugin The constructor for assertions in the exec plugin was incorrectly setting the expected exit code equal to the exit code it received from the test spec, resulting in script or command executions returning non-0 exit codes from causing a test.FailNow(). Signed-off-by: Jay Pipes --- plugin/exec/assertions.go | 6 ++- plugin/exec/eval_test.go | 47 +++++++++++++++++++ .../exec/testdata/ls-fail-no-exit-code.yaml | 4 ++ 3 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 plugin/exec/testdata/ls-fail-no-exit-code.yaml diff --git a/plugin/exec/assertions.go b/plugin/exec/assertions.go index defebb0..84fb1f9 100644 --- a/plugin/exec/assertions.go +++ b/plugin/exec/assertions.go @@ -166,9 +166,13 @@ func newAssertions( outPipe *bytes.Buffer, errPipe *bytes.Buffer, ) api.Assertions { + expExitCode := 0 + if e != nil { + expExitCode = e.ExitCode + } a := &assertions{ failures: []error{}, - expExitCode: exitCode, + expExitCode: expExitCode, exitCode: exitCode, } if e != nil { diff --git a/plugin/exec/eval_test.go b/plugin/exec/eval_test.go index 60f40e8..67f339d 100644 --- a/plugin/exec/eval_test.go +++ b/plugin/exec/eval_test.go @@ -9,6 +9,7 @@ import ( "bytes" "context" "flag" + "fmt" "os" "os/exec" "path/filepath" @@ -71,6 +72,52 @@ func TestExitCode(t *testing.T) { require.Nil(err) } +func TestFailExecExitCodeNotSpecified(t *testing.T) { + if !*failFlag { + t.Skip("skipping without -fail flag") + } + require := require.New(t) + + fp := filepath.Join("testdata", "ls-fail-no-exit-code.yaml") + f, err := os.Open(fp) + require.Nil(err) + + s, err := scenario.FromReader( + f, + scenario.WithPath(fp), + ) + require.Nil(err) + require.NotNil(s) + + ctx := gdtcontext.New(gdtcontext.WithDebug()) + err = s.Run(ctx, t) + require.Nil(err) +} + +func TestExecFailExitCodeNotSpecified(t *testing.T) { + require := require.New(t) + target := os.Args[0] + failArgs := []string{ + "-test.v", + "-test.run=FailExecExitCodeNotSpecified", + "-fail", + } + outerr, err := exec.Command(target, failArgs...).CombinedOutput() + + // The test should have failed... + require.NotNil(err) + debugout := string(outerr) + ec := 2 + // Yay, different exit codes for the same not found error... + if runtime.GOOS == "darwin" { + ec = 1 + } + msg := fmt.Sprintf( + "assertion failed: not equal: expected 0 but got %d", ec, + ) + require.Contains(debugout, msg) +} + func TestShellList(t *testing.T) { require := require.New(t) diff --git a/plugin/exec/testdata/ls-fail-no-exit-code.yaml b/plugin/exec/testdata/ls-fail-no-exit-code.yaml new file mode 100644 index 0000000..35f16af --- /dev/null +++ b/plugin/exec/testdata/ls-fail-no-exit-code.yaml @@ -0,0 +1,4 @@ +name: ls-fail-no-exit-code +description: a scenario that runs the `ls` command with a non-0 exit code and no assertion on exit code +tests: + - exec: ls /this/dir/does/not/exist From df4c3d6785276c26edd9492529b85ef3d35d47ae Mon Sep 17 00:00:00 2001 From: Jay Pipes Date: Mon, 8 Jul 2024 13:37:42 -0400 Subject: [PATCH 2/2] update Github Action runners Bring in latest step security and checkout action. Signed-off-by: Jay Pipes --- .github/workflows/fmtcheck.yml | 5 +++-- .github/workflows/lint.yml | 1 + .github/workflows/test.yml | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fmtcheck.yml b/.github/workflows/fmtcheck.yml index 242acc1..11cafcb 100644 --- a/.github/workflows/fmtcheck.yml +++ b/.github/workflows/fmtcheck.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: harden runner - uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 + uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 with: egress-policy: block disable-sudo: true @@ -26,8 +26,9 @@ jobs: raw.githubusercontent.com:443 objects.githubusercontent.com:443 proxy.golang.org:443 + blob.core.windows.net:443 - name: checkout code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: setup go uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 with: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 50c153f..1f5b2e6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,6 +27,7 @@ jobs: raw.githubusercontent.com:443 objects.githubusercontent.com:443 proxy.golang.org:443 + blob.core.windows.net:443 - name: checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: setup go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a727f30..cade935 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: harden runner - uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 + uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 with: egress-policy: block disable-sudo: true @@ -29,8 +29,9 @@ jobs: raw.githubusercontent.com:443 objects.githubusercontent.com:443 proxy.golang.org:443 + blob.core.windows.net:443 - name: checkout code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - name: setup go uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 with: