diff --git a/agent/handlers/task_server_setup.go b/agent/handlers/task_server_setup.go index ec0ec020f23..057de85f435 100644 --- a/agent/handlers/task_server_setup.go +++ b/agent/handlers/task_server_setup.go @@ -37,7 +37,9 @@ import ( tmdsv1 "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v1" tmdsv2 "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v2" tmdsv4 "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v4" + "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper" "github.com/aws/amazon-ecs-agent/ecs-agent/utils/retry" + "github.com/cihub/seelog" "github.com/gorilla/mux" ) @@ -93,7 +95,8 @@ func taskServerSetup( taskProtectionClientFactory, metricsFactory) // TODO: Future PR to pass in TMDS server router once all of the handlers have been implemented. - registerFaultHandlers(nil, tmdsAgentState, metricsFactory) + execWrapper := execwrapper.NewExec() + registerFaultHandlers(nil, tmdsAgentState, metricsFactory, execWrapper) return tmds.NewServer(auditLogger, tmds.WithHandler(muxRouter), @@ -195,8 +198,9 @@ func registerFaultHandlers( muxRouter *mux.Router, agentState *v4.TMDSAgentState, metricsFactory metrics.EntryFactory, + execWrapper execwrapper.Exec, ) { - handler := fault.New(agentState, metricsFactory) + handler := fault.New(agentState, metricsFactory, execWrapper) if muxRouter == nil { return diff --git a/agent/handlers/task_server_setup_test.go b/agent/handlers/task_server_setup_test.go index c9531d9361f..1b2aaed5b9b 100644 --- a/agent/handlers/task_server_setup_test.go +++ b/agent/handlers/task_server_setup_test.go @@ -54,6 +54,8 @@ import ( tmdsv1 "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v1" v2 "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v2" v4 "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v4/state" + mock_execwrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/mocks" + "github.com/gorilla/mux" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" @@ -116,9 +118,10 @@ const ( hostNetworkNamespace = "host" defaultIfname = "eth0" - port = 1234 - protocol = "tcp" - trafficType = "ingress" + port = 1234 + protocol = "tcp" + trafficType = "ingress" + iptablesChainNotFoundError = "iptables: Bad rule (does a matching rule exist in that chain?)." ) var ( @@ -460,6 +463,12 @@ var ( state.EXPECT().PulledContainerMapByArn(taskARN).Return(nil, true), ) } + + happyBlackHolePortReqBody = map[string]interface{}{ + "Port": port, + "Protocol": protocol, + "TrafficType": trafficType, + } ) func standardTask() *apitask.Task { @@ -3718,50 +3727,13 @@ type blackholePortFaultTestCase struct { requestBody interface{} expectedFaultResponse faulttype.NetworkFaultInjectionResponse setStateExpectations func(state *mock_dockerstate.MockTaskEngineState, faultInjectionEnabled bool, networkMode string) + setExecExpectations func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) faultInjectionEnabled bool networkMode string } -func getNetworkBlackHolePortHandlerTestCases(name, fault string, expectedHappyResponseBody string) []blackholePortFaultTestCase { - happyBlackHolePortReqBody := map[string]interface{}{ - "Port": port, - "Protocol": protocol, - "TrafficType": trafficType, - } - +func getNetworkBlackHolePortHandlerTestCases(name, fault string) []blackholePortFaultTestCase { tcs := []blackholePortFaultTestCase{ - { - name: fmt.Sprintf("%s success host mode", name), - expectedStatusCode: 200, - requestBody: happyBlackHolePortReqBody, - expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse(expectedHappyResponseBody), - setStateExpectations: agentStateExpectations, - faultInjectionEnabled: true, - networkMode: apitask.HostNetworkMode, - }, - { - name: fmt.Sprintf("%s success awsvpc mode", name), - expectedStatusCode: 200, - requestBody: happyBlackHolePortReqBody, - expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse(expectedHappyResponseBody), - setStateExpectations: agentStateExpectations, - faultInjectionEnabled: true, - networkMode: apitask.AWSVPCNetworkMode, - }, - { - name: fmt.Sprintf("%s unknown request body", name), - expectedStatusCode: 200, - requestBody: map[string]interface{}{ - "Port": port, - "Protocol": protocol, - "TrafficType": trafficType, - "Unknown": "", - }, - expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse(expectedHappyResponseBody), - setStateExpectations: agentStateExpectations, - faultInjectionEnabled: true, - networkMode: apitask.AWSVPCNetworkMode, - }, { name: fmt.Sprintf("%s malformed request body", name), expectedStatusCode: 400, @@ -3870,18 +3842,209 @@ func getNetworkBlackHolePortHandlerTestCases(name, fault string, expectedHappyRe return tcs } +func getStartNetworkBlackHolePortHandlerTestCases() []blackholePortFaultTestCase { + commonTcs := getNetworkBlackHolePortHandlerTestCases("start blackhole port", faulttype.BlackHolePortFaultType) + tcs := []blackholePortFaultTestCase{ + { + name: "start blackhole port success host mode", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("running"), + setStateExpectations: agentStateExpectations, + faultInjectionEnabled: true, + networkMode: apitask.HostNetworkMode, + }, + { + name: "start blackhole port success awsvpc mode", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("running"), + setStateExpectations: agentStateExpectations, + faultInjectionEnabled: true, + networkMode: apitask.AWSVPCNetworkMode, + }, + { + name: "start blackhole port unknown request body", + expectedStatusCode: 200, + requestBody: map[string]interface{}{ + "Port": port, + "Protocol": protocol, + "TrafficType": trafficType, + "Unknown": "", + }, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("running"), + setStateExpectations: agentStateExpectations, + faultInjectionEnabled: true, + networkMode: apitask.AWSVPCNetworkMode, + }, + } + return append(tcs, commonTcs...) +} + +func getStopNetworkBlackHolePortHandlerTestCases() []blackholePortFaultTestCase { + commonTcs := getNetworkBlackHolePortHandlerTestCases("stop blackhole port", faulttype.BlackHolePortFaultType) + tcs := []blackholePortFaultTestCase{ + { + name: "stop blackhole port success host mode", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("stopped"), + setStateExpectations: agentStateExpectations, + faultInjectionEnabled: true, + networkMode: apitask.HostNetworkMode, + }, + { + name: "stop blackhole port success awsvpc mode", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("stopped"), + setStateExpectations: agentStateExpectations, + faultInjectionEnabled: true, + networkMode: apitask.AWSVPCNetworkMode, + }, + { + name: "stop blackhole port unknown request body", + expectedStatusCode: 200, + requestBody: map[string]interface{}{ + "Port": port, + "Protocol": protocol, + "TrafficType": trafficType, + "Unknown": "", + }, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("stopped"), + setStateExpectations: agentStateExpectations, + faultInjectionEnabled: true, + networkMode: apitask.AWSVPCNetworkMode, + }, + } + return append(tcs, commonTcs...) +} + +func getCheckStatusNetworkBlackHolePortHandlerTestCases() []blackholePortFaultTestCase { + commonTcs := getNetworkBlackHolePortHandlerTestCases("start blackhole port", faulttype.BlackHolePortFaultType) + tcs := []blackholePortFaultTestCase{ + { + name: "check blackhole port success host mode running", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("running"), + setStateExpectations: agentStateExpectations, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + ) + }, + faultInjectionEnabled: true, + networkMode: apitask.HostNetworkMode, + }, + { + name: "check blackhole port success awsvpc mode running", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("running"), + setStateExpectations: agentStateExpectations, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + ) + }, + faultInjectionEnabled: true, + networkMode: apitask.AWSVPCNetworkMode, + }, + { + name: "check blackhole port unknown request body", + expectedStatusCode: 200, + requestBody: map[string]interface{}{ + "Port": port, + "Protocol": protocol, + "TrafficType": trafficType, + "Unknown": "", + }, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("running"), + setStateExpectations: agentStateExpectations, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + ) + }, + faultInjectionEnabled: true, + networkMode: apitask.AWSVPCNetworkMode, + }, + { + name: "check blackhole port success host mode not running", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("not-running"), + setStateExpectations: agentStateExpectations, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(iptablesChainNotFoundError), errors.New("exit 1")), + exec.EXPECT().ConvertToExitError(gomock.Any()).Times(1).Return(nil, true), + exec.EXPECT().GetExitCode(gomock.Any()).Times(1).Return(1), + ) + }, + faultInjectionEnabled: true, + networkMode: apitask.HostNetworkMode, + }, + { + name: "check blackhole port success awsvpc mode not running", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionSuccessResponse("not-running"), + setStateExpectations: agentStateExpectations, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(iptablesChainNotFoundError), errors.New("exit 1")), + exec.EXPECT().ConvertToExitError(gomock.Any()).Times(1).Return(nil, true), + exec.EXPECT().GetExitCode(gomock.Any()).Times(1).Return(1), + ) + }, + faultInjectionEnabled: true, + networkMode: apitask.AWSVPCNetworkMode, + }, + { + name: "check blackhole port fail", + expectedStatusCode: 500, + requestBody: happyBlackHolePortReqBody, + expectedFaultResponse: faulttype.NewNetworkFaultInjectionErrorResponse("internal error"), + setStateExpectations: agentStateExpectations, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte("internal error"), errors.New("exit 1")), + exec.EXPECT().ConvertToExitError(gomock.Any()).Times(1).Return(nil, false), + ) + }, + faultInjectionEnabled: true, + networkMode: apitask.AWSVPCNetworkMode, + }, + } + return append(tcs, commonTcs...) +} + func TestRegisterStartBlackholePortFaultHandler(t *testing.T) { - tcs := getNetworkBlackHolePortHandlerTestCases("start blackhole port", faulttype.BlackHolePortFaultType, "running") + tcs := getStartNetworkBlackHolePortHandlerTestCases() testRegisterFaultHandler(t, tcs, "PUT", faulttype.BlackHolePortFaultType) } func TestRegisterStopBlackholePortFaultHandler(t *testing.T) { - tcs := getNetworkBlackHolePortHandlerTestCases("stop blackhole port", faulttype.BlackHolePortFaultType, "stopped") + tcs := getStopNetworkBlackHolePortHandlerTestCases() testRegisterFaultHandler(t, tcs, "DELETE", faulttype.BlackHolePortFaultType) } func TestRegisterCheckBlackholePortFaultHandler(t *testing.T) { - tcs := getNetworkBlackHolePortHandlerTestCases("start blackhole port", faulttype.BlackHolePortFaultType, "running") + tcs := getCheckStatusNetworkBlackHolePortHandlerTestCases() testRegisterFaultHandler(t, tcs, "GET", faulttype.BlackHolePortFaultType) } @@ -3898,13 +4061,18 @@ func testRegisterFaultHandler(t *testing.T, tcs []blackholePortFaultTestCase, me agentState := agentV4.NewTMDSAgentState(state, statsEngine, ecsClient, clusterName, availabilityzone, vpcID, containerInstanceArn) metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) + execWrapper := mock_execwrapper.NewMockExec(ctrl) if tc.setStateExpectations != nil { tc.setStateExpectations(state, tc.faultInjectionEnabled, tc.networkMode) } + if tc.setExecExpectations != nil { + tc.setExecExpectations(execWrapper, ctrl) + } + router := mux.NewRouter() - registerFaultHandlers(router, agentState, metricsFactory) + registerFaultHandlers(router, agentState, metricsFactory, execWrapper) var requestBody io.Reader if tc.requestBody != "" { reqBodyBytes, err := json.Marshal(tc.requestBody) diff --git a/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/fault/v1/handlers/handlers.go b/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/fault/v1/handlers/handlers.go index 76424ac0ccd..fe78e0857ed 100644 --- a/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/fault/v1/handlers/handlers.go +++ b/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/fault/v1/handlers/handlers.go @@ -15,12 +15,16 @@ package handlers import ( "bytes" + "context" "encoding/json" "errors" "fmt" "io" "net/http" + "strconv" + "strings" "sync" + "time" "github.com/aws/amazon-ecs-agent/ecs-agent/api/ecs/model/ecs" "github.com/aws/amazon-ecs-agent/ecs-agent/logger" @@ -30,6 +34,8 @@ import ( "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/utils" v4 "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v4" state "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v4/state" + "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper" + "github.com/aws/aws-sdk-go/aws" "github.com/gorilla/mux" ) @@ -40,6 +46,13 @@ const ( checkStatusFaultRequestType = "check status %s" invalidNetworkModeError = "%s mode is not supported. Please use either host or awsvpc mode." faultInjectionEnabledError = "enableFaultInjection is not enabled for task: %s" + requestTimedOutError = "%s: request timed out" + requestTimeoutDuration = 5 * time.Second +) + +var ( + iptablesChainExistCmd = "iptables -C %s -p %s --dport %s -j DROP" + nsenterCommandString = "nsenter --net=%s" ) type FaultHandler struct { @@ -49,13 +62,15 @@ type FaultHandler struct { mutexMap sync.Map AgentState state.AgentState MetricsFactory metrics.EntryFactory + osExecWrapper execwrapper.Exec } -func New(agentState state.AgentState, mf metrics.EntryFactory) *FaultHandler { +func New(agentState state.AgentState, mf metrics.EntryFactory, execWrapper execwrapper.Exec) *FaultHandler { return &FaultHandler{ AgentState: agentState, MetricsFactory: mf, mutexMap: sync.Map{}, + osExecWrapper: execWrapper, } } @@ -203,23 +218,94 @@ func (h *FaultHandler) CheckNetworkBlackHolePort() func(http.ResponseWriter, *ht rwMu.RLock() defer rwMu.RUnlock() - // TODO: Check status of current fault injection - // TODO: Return the correct status state - responseBody := types.NewNetworkFaultInjectionSuccessResponse("running") - logger.Info("Successfully checked status for fault", logger.Fields{ - field.RequestType: requestType, - field.Request: request.ToString(), - field.Response: responseBody.ToString(), - }) + ctx := context.Background() + ctxWithTimeout, cancel := context.WithTimeout(ctx, requestTimeoutDuration) + defer cancel() + + var responseBody types.NetworkFaultInjectionResponse + var statusCode int + port := strconv.FormatUint(uint64(aws.Uint16Value(request.Port)), 10) + chainName := fmt.Sprintf("%s-%s-%s", aws.StringValue(request.TrafficType), aws.StringValue(request.Protocol), port) + running, cmdOutput, cmdErr := h.checkNetworkBlackHolePort(ctxWithTimeout, aws.StringValue(request.Protocol), port, chainName, + taskMetadata.TaskNetworkConfig.NetworkMode, taskMetadata.TaskNetworkConfig.NetworkNamespaces[0].Path) + + // We've timed out trying to check if the black hole port fault injection is running + if err := ctx.Err(); err == context.DeadlineExceeded { + logger.Error("Request timed out", logger.Fields{ + field.RequestType: requestType, + field.Request: request.ToString(), + field.Error: err, + }) + statusCode = http.StatusInternalServerError + responseBody = types.NewNetworkFaultInjectionErrorResponse(fmt.Sprintf(requestTimedOutError, requestType)) + } else if cmdErr != nil { + logger.Error("Unknown error encountered for request", logger.Fields{ + field.RequestType: requestType, + field.Request: request.ToString(), + field.Error: cmdErr, + }) + statusCode = http.StatusInternalServerError + responseBody = types.NewNetworkFaultInjectionErrorResponse(cmdOutput) + } else { + statusCode = http.StatusOK + if running { + responseBody = types.NewNetworkFaultInjectionSuccessResponse("running") + } else { + responseBody = types.NewNetworkFaultInjectionSuccessResponse("not-running") + } + logger.Info("[INFO] Successfully checked status for fault", logger.Fields{ + field.RequestType: requestType, + field.Request: request.ToString(), + field.Response: responseBody.ToString(), + }) + } utils.WriteJSONResponse( w, - http.StatusOK, + statusCode, responseBody, requestType, ) } } +// checkNetworkBlackHolePort will check if there's a running black hole port within the task network namespace based on the chain name and the passed in required request fields. +// It does so by calling iptables linux utility tool. +func (h *FaultHandler) checkNetworkBlackHolePort(ctx context.Context, protocol, port, chain, networkMode, netNs string) (bool, string, error) { + cmdString := fmt.Sprintf(iptablesChainExistCmd, chain, protocol, port) + cmdList := strings.Split(cmdString, " ") + + // For host mode, the task network namespace is the host network namespace (i.e. we don't need to run nsenter) + if networkMode != ecs.NetworkModeHost { + cmdList = append(strings.Split(fmt.Sprintf(nsenterCommandString, netNs), " "), cmdList...) + } + + cmdOutput, err := h.runExecCommand(ctx, cmdList) + if err != nil { + if exitErr, eok := h.osExecWrapper.ConvertToExitError(err); eok { + logger.Info("[INFO] Black hole port fault is not running", logger.Fields{ + "netns": netNs, + "command": strings.Join(cmdList, " "), + "output": string(cmdOutput), + "exitCode": h.osExecWrapper.GetExitCode(exitErr), + }) + return false, string(cmdOutput), nil + } + logger.Error("Error: Unable to check status of black hole port fault", logger.Fields{ + "netns": netNs, + "command": strings.Join(cmdList, " "), + "output": string(cmdOutput), + "err": err, + }) + return false, string(cmdOutput), err + } + logger.Info("[INFO] Black hole port fault has been found running", logger.Fields{ + "netns": netNs, + "command": strings.Join(cmdList, " "), + "output": string(cmdOutput), + }) + return true, string(cmdOutput), nil +} + // StartNetworkLatency starts a network latency fault in the associated ENI if no existing same fault. func (h *FaultHandler) StartNetworkLatency() func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { @@ -707,3 +793,10 @@ func validateTaskNetworkConfig(taskNetworkConfig *state.TaskNetworkConfig) error return nil } + +// runExecCommand wraps around the execwrapper, providing a convenient way of running any Linux command +// and getting the result in both stdout and stderr. +func (h *FaultHandler) runExecCommand(ctx context.Context, cmdList []string) ([]byte, error) { + cmdExec := h.osExecWrapper.CommandContext(ctx, cmdList[0], cmdList[1:]...) + return cmdExec.CombinedOutput() +} diff --git a/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/exec.go b/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/exec.go new file mode 100644 index 00000000000..d1f94cb82c5 --- /dev/null +++ b/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/exec.go @@ -0,0 +1,141 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package execwrapper + +import ( + "context" + "io" + "os" + "os/exec" +) + +// Exec acts as a wrapper to functions exposed by the exec package. +// Having this interface enables us to create mock objects we can use +// for testing. +type Exec interface { + CommandContext(ctx context.Context, name string, arg ...string) Cmd + ConvertToExitError(err error) (*exec.ExitError, bool) + GetExitCode(exitErr *exec.ExitError) int +} + +// execWrapper is a placeholder struct which implements the Exec interface. +type execWrapper struct { +} + +func NewExec() Exec { + return &execWrapper{} +} + +// CommandContext essentially acts as a wrapper function for exec.CommandContext function. +func (e *execWrapper) CommandContext(ctx context.Context, name string, arg ...string) Cmd { + return NewCMDContext(ctx, name, arg...) +} + +// ConvertToExitError converts an error object to an exec.ExitError +func (e *execWrapper) ConvertToExitError(err error) (*exec.ExitError, bool) { + exitErr, eok := err.(*exec.ExitError) + return exitErr, eok +} + +// GetExitCode gets the exit code of an exec.ExitError object +func (e *execWrapper) GetExitCode(exitErr *exec.ExitError) int { + return exitErr.ExitCode() +} + +// Cmd acts as a wrapper to functions exposed by the exec.Cmd object. +// Having this interface enables us to create mock objects we can use +// for testing. +type Cmd interface { + Run() error + Start() error + Wait() error + KillProcess() error + AppendExtraFiles(...*os.File) + Args() []string + SetIOStreams(io.Reader, io.Writer, io.Writer) + Output() ([]byte, error) + CombinedOutput() ([]byte, error) +} + +type cmdWrapper struct { + *exec.Cmd +} + +// NewCMDContext returns a new cmdWrapper object which will be used to call standard go os exec calls with a context +func NewCMDContext(ctx context.Context, name string, arg ...string) Cmd { + cmd := exec.CommandContext(ctx, name, arg...) + return &cmdWrapper{Cmd: cmd} +} + +// NewCMDContext returns a new cmdWrapper object which will be used to call standard go os exec calls +func NewCMD(name string, arg ...string) Cmd { + cmd := exec.Command(name, arg...) + return &cmdWrapper{Cmd: cmd} +} + +// Run is a wrapper to existing Run() method of the os/exec go library. +// Run starts the specified command and waits for it to complete. +// Returns nil if the command runs and exits with a zero code. +// Otherwise returns an error +func (c *cmdWrapper) Run() error { + return c.Cmd.Run() +} + +// Start is a wrapper to the existing Start() method of the os/exec go library +// Start starts the specified command but does not wait for it to complete. +func (c *cmdWrapper) Start() error { + return c.Cmd.Start() +} + +// Wait is a wrapper to the existing Wait() method of the os/exec go library +// Wait waits for the command started by a Start() call to exit and waits for any copying to stdin or copying from stdout or stderr to complete. +func (c *cmdWrapper) Wait() error { + return c.Cmd.Wait() +} + +func (c *cmdWrapper) KillProcess() error { + return c.Cmd.Process.Kill() +} + +func (c *cmdWrapper) AppendExtraFiles(ef ...*os.File) { + c.ExtraFiles = append(c.ExtraFiles, ef...) +} + +func (c *cmdWrapper) Args() []string { + return c.Cmd.Args +} + +func (c *cmdWrapper) SetIOStreams(stdin io.Reader, stdout io.Writer, stderr io.Writer) { + if stdin != nil { + c.Stdin = stdin + } + if stdout != nil { + c.Stdout = stdout + } + if stderr != nil { + c.Stderr = stderr + } +} + +// Output is a wrapper to the existing Output() method of the os/exec go library. +// Output runs the command and returns its standard output as well as the standard error output. +func (c *cmdWrapper) Output() ([]byte, error) { + return c.Cmd.Output() +} + +// CombinedOutput is a wrapper to the existing CombinedOutput() method of the os/exec go library. +// CombinedOutput runs the command and returns its combined standard output and standard error. +func (c *cmdWrapper) CombinedOutput() ([]byte, error) { + return c.Cmd.CombinedOutput() +} diff --git a/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/generate_mocks.go b/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/generate_mocks.go new file mode 100644 index 00000000000..0c0f9e9aba1 --- /dev/null +++ b/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/generate_mocks.go @@ -0,0 +1,16 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +//go:generate mockgen -build_flags=--mod=mod -destination=mocks/execwrapper_mocks.go -copyright_file=../../../scripts/copyright_file github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper Cmd,Exec + +package execwrapper diff --git a/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/mocks/execwrapper_mocks.go b/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/mocks/execwrapper_mocks.go new file mode 100644 index 00000000000..25bff9f5241 --- /dev/null +++ b/agent/vendor/github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/mocks/execwrapper_mocks.go @@ -0,0 +1,252 @@ +// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper (interfaces: Cmd,Exec) + +// Package mock_execwrapper is a generated GoMock package. +package mock_execwrapper + +import ( + context "context" + io "io" + os "os" + exec "os/exec" + reflect "reflect" + + execwrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper" + gomock "github.com/golang/mock/gomock" +) + +// MockCmd is a mock of Cmd interface. +type MockCmd struct { + ctrl *gomock.Controller + recorder *MockCmdMockRecorder +} + +// MockCmdMockRecorder is the mock recorder for MockCmd. +type MockCmdMockRecorder struct { + mock *MockCmd +} + +// NewMockCmd creates a new mock instance. +func NewMockCmd(ctrl *gomock.Controller) *MockCmd { + mock := &MockCmd{ctrl: ctrl} + mock.recorder = &MockCmdMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCmd) EXPECT() *MockCmdMockRecorder { + return m.recorder +} + +// AppendExtraFiles mocks base method. +func (m *MockCmd) AppendExtraFiles(arg0 ...*os.File) { + m.ctrl.T.Helper() + varargs := []interface{}{} + for _, a := range arg0 { + varargs = append(varargs, a) + } + m.ctrl.Call(m, "AppendExtraFiles", varargs...) +} + +// AppendExtraFiles indicates an expected call of AppendExtraFiles. +func (mr *MockCmdMockRecorder) AppendExtraFiles(arg0 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendExtraFiles", reflect.TypeOf((*MockCmd)(nil).AppendExtraFiles), arg0...) +} + +// Args mocks base method. +func (m *MockCmd) Args() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Args") + ret0, _ := ret[0].([]string) + return ret0 +} + +// Args indicates an expected call of Args. +func (mr *MockCmdMockRecorder) Args() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Args", reflect.TypeOf((*MockCmd)(nil).Args)) +} + +// CombinedOutput mocks base method. +func (m *MockCmd) CombinedOutput() ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CombinedOutput") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CombinedOutput indicates an expected call of CombinedOutput. +func (mr *MockCmdMockRecorder) CombinedOutput() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CombinedOutput", reflect.TypeOf((*MockCmd)(nil).CombinedOutput)) +} + +// KillProcess mocks base method. +func (m *MockCmd) KillProcess() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "KillProcess") + ret0, _ := ret[0].(error) + return ret0 +} + +// KillProcess indicates an expected call of KillProcess. +func (mr *MockCmdMockRecorder) KillProcess() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KillProcess", reflect.TypeOf((*MockCmd)(nil).KillProcess)) +} + +// Output mocks base method. +func (m *MockCmd) Output() ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Output") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Output indicates an expected call of Output. +func (mr *MockCmdMockRecorder) Output() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Output", reflect.TypeOf((*MockCmd)(nil).Output)) +} + +// Run mocks base method. +func (m *MockCmd) Run() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Run") + ret0, _ := ret[0].(error) + return ret0 +} + +// Run indicates an expected call of Run. +func (mr *MockCmdMockRecorder) Run() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockCmd)(nil).Run)) +} + +// SetIOStreams mocks base method. +func (m *MockCmd) SetIOStreams(arg0 io.Reader, arg1, arg2 io.Writer) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "SetIOStreams", arg0, arg1, arg2) +} + +// SetIOStreams indicates an expected call of SetIOStreams. +func (mr *MockCmdMockRecorder) SetIOStreams(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetIOStreams", reflect.TypeOf((*MockCmd)(nil).SetIOStreams), arg0, arg1, arg2) +} + +// Start mocks base method. +func (m *MockCmd) Start() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Start") + ret0, _ := ret[0].(error) + return ret0 +} + +// Start indicates an expected call of Start. +func (mr *MockCmdMockRecorder) Start() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Start", reflect.TypeOf((*MockCmd)(nil).Start)) +} + +// Wait mocks base method. +func (m *MockCmd) Wait() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Wait") + ret0, _ := ret[0].(error) + return ret0 +} + +// Wait indicates an expected call of Wait. +func (mr *MockCmdMockRecorder) Wait() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Wait", reflect.TypeOf((*MockCmd)(nil).Wait)) +} + +// MockExec is a mock of Exec interface. +type MockExec struct { + ctrl *gomock.Controller + recorder *MockExecMockRecorder +} + +// MockExecMockRecorder is the mock recorder for MockExec. +type MockExecMockRecorder struct { + mock *MockExec +} + +// NewMockExec creates a new mock instance. +func NewMockExec(ctrl *gomock.Controller) *MockExec { + mock := &MockExec{ctrl: ctrl} + mock.recorder = &MockExecMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockExec) EXPECT() *MockExecMockRecorder { + return m.recorder +} + +// CommandContext mocks base method. +func (m *MockExec) CommandContext(arg0 context.Context, arg1 string, arg2 ...string) execwrapper.Cmd { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "CommandContext", varargs...) + ret0, _ := ret[0].(execwrapper.Cmd) + return ret0 +} + +// CommandContext indicates an expected call of CommandContext. +func (mr *MockExecMockRecorder) CommandContext(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CommandContext", reflect.TypeOf((*MockExec)(nil).CommandContext), varargs...) +} + +// ConvertToExitError mocks base method. +func (m *MockExec) ConvertToExitError(arg0 error) (*exec.ExitError, bool) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConvertToExitError", arg0) + ret0, _ := ret[0].(*exec.ExitError) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +// ConvertToExitError indicates an expected call of ConvertToExitError. +func (mr *MockExecMockRecorder) ConvertToExitError(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConvertToExitError", reflect.TypeOf((*MockExec)(nil).ConvertToExitError), arg0) +} + +// GetExitCode mocks base method. +func (m *MockExec) GetExitCode(arg0 *exec.ExitError) int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetExitCode", arg0) + ret0, _ := ret[0].(int) + return ret0 +} + +// GetExitCode indicates an expected call of GetExitCode. +func (mr *MockExecMockRecorder) GetExitCode(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExitCode", reflect.TypeOf((*MockExec)(nil).GetExitCode), arg0) +} diff --git a/agent/vendor/modules.txt b/agent/vendor/modules.txt index 2eb147265f4..777a834f94f 100644 --- a/agent/vendor/modules.txt +++ b/agent/vendor/modules.txt @@ -74,6 +74,8 @@ github.com/aws/amazon-ecs-agent/ecs-agent/tmds/utils/mux github.com/aws/amazon-ecs-agent/ecs-agent/utils github.com/aws/amazon-ecs-agent/ecs-agent/utils/arn github.com/aws/amazon-ecs-agent/ecs-agent/utils/cipher +github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper +github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/mocks github.com/aws/amazon-ecs-agent/ecs-agent/utils/httpproxy github.com/aws/amazon-ecs-agent/ecs-agent/utils/retry github.com/aws/amazon-ecs-agent/ecs-agent/utils/retry/mock diff --git a/ecs-agent/tmds/handlers/fault/v1/handlers/handlers.go b/ecs-agent/tmds/handlers/fault/v1/handlers/handlers.go index 76424ac0ccd..fe78e0857ed 100644 --- a/ecs-agent/tmds/handlers/fault/v1/handlers/handlers.go +++ b/ecs-agent/tmds/handlers/fault/v1/handlers/handlers.go @@ -15,12 +15,16 @@ package handlers import ( "bytes" + "context" "encoding/json" "errors" "fmt" "io" "net/http" + "strconv" + "strings" "sync" + "time" "github.com/aws/amazon-ecs-agent/ecs-agent/api/ecs/model/ecs" "github.com/aws/amazon-ecs-agent/ecs-agent/logger" @@ -30,6 +34,8 @@ import ( "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/utils" v4 "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v4" state "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v4/state" + "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper" + "github.com/aws/aws-sdk-go/aws" "github.com/gorilla/mux" ) @@ -40,6 +46,13 @@ const ( checkStatusFaultRequestType = "check status %s" invalidNetworkModeError = "%s mode is not supported. Please use either host or awsvpc mode." faultInjectionEnabledError = "enableFaultInjection is not enabled for task: %s" + requestTimedOutError = "%s: request timed out" + requestTimeoutDuration = 5 * time.Second +) + +var ( + iptablesChainExistCmd = "iptables -C %s -p %s --dport %s -j DROP" + nsenterCommandString = "nsenter --net=%s" ) type FaultHandler struct { @@ -49,13 +62,15 @@ type FaultHandler struct { mutexMap sync.Map AgentState state.AgentState MetricsFactory metrics.EntryFactory + osExecWrapper execwrapper.Exec } -func New(agentState state.AgentState, mf metrics.EntryFactory) *FaultHandler { +func New(agentState state.AgentState, mf metrics.EntryFactory, execWrapper execwrapper.Exec) *FaultHandler { return &FaultHandler{ AgentState: agentState, MetricsFactory: mf, mutexMap: sync.Map{}, + osExecWrapper: execWrapper, } } @@ -203,23 +218,94 @@ func (h *FaultHandler) CheckNetworkBlackHolePort() func(http.ResponseWriter, *ht rwMu.RLock() defer rwMu.RUnlock() - // TODO: Check status of current fault injection - // TODO: Return the correct status state - responseBody := types.NewNetworkFaultInjectionSuccessResponse("running") - logger.Info("Successfully checked status for fault", logger.Fields{ - field.RequestType: requestType, - field.Request: request.ToString(), - field.Response: responseBody.ToString(), - }) + ctx := context.Background() + ctxWithTimeout, cancel := context.WithTimeout(ctx, requestTimeoutDuration) + defer cancel() + + var responseBody types.NetworkFaultInjectionResponse + var statusCode int + port := strconv.FormatUint(uint64(aws.Uint16Value(request.Port)), 10) + chainName := fmt.Sprintf("%s-%s-%s", aws.StringValue(request.TrafficType), aws.StringValue(request.Protocol), port) + running, cmdOutput, cmdErr := h.checkNetworkBlackHolePort(ctxWithTimeout, aws.StringValue(request.Protocol), port, chainName, + taskMetadata.TaskNetworkConfig.NetworkMode, taskMetadata.TaskNetworkConfig.NetworkNamespaces[0].Path) + + // We've timed out trying to check if the black hole port fault injection is running + if err := ctx.Err(); err == context.DeadlineExceeded { + logger.Error("Request timed out", logger.Fields{ + field.RequestType: requestType, + field.Request: request.ToString(), + field.Error: err, + }) + statusCode = http.StatusInternalServerError + responseBody = types.NewNetworkFaultInjectionErrorResponse(fmt.Sprintf(requestTimedOutError, requestType)) + } else if cmdErr != nil { + logger.Error("Unknown error encountered for request", logger.Fields{ + field.RequestType: requestType, + field.Request: request.ToString(), + field.Error: cmdErr, + }) + statusCode = http.StatusInternalServerError + responseBody = types.NewNetworkFaultInjectionErrorResponse(cmdOutput) + } else { + statusCode = http.StatusOK + if running { + responseBody = types.NewNetworkFaultInjectionSuccessResponse("running") + } else { + responseBody = types.NewNetworkFaultInjectionSuccessResponse("not-running") + } + logger.Info("[INFO] Successfully checked status for fault", logger.Fields{ + field.RequestType: requestType, + field.Request: request.ToString(), + field.Response: responseBody.ToString(), + }) + } utils.WriteJSONResponse( w, - http.StatusOK, + statusCode, responseBody, requestType, ) } } +// checkNetworkBlackHolePort will check if there's a running black hole port within the task network namespace based on the chain name and the passed in required request fields. +// It does so by calling iptables linux utility tool. +func (h *FaultHandler) checkNetworkBlackHolePort(ctx context.Context, protocol, port, chain, networkMode, netNs string) (bool, string, error) { + cmdString := fmt.Sprintf(iptablesChainExistCmd, chain, protocol, port) + cmdList := strings.Split(cmdString, " ") + + // For host mode, the task network namespace is the host network namespace (i.e. we don't need to run nsenter) + if networkMode != ecs.NetworkModeHost { + cmdList = append(strings.Split(fmt.Sprintf(nsenterCommandString, netNs), " "), cmdList...) + } + + cmdOutput, err := h.runExecCommand(ctx, cmdList) + if err != nil { + if exitErr, eok := h.osExecWrapper.ConvertToExitError(err); eok { + logger.Info("[INFO] Black hole port fault is not running", logger.Fields{ + "netns": netNs, + "command": strings.Join(cmdList, " "), + "output": string(cmdOutput), + "exitCode": h.osExecWrapper.GetExitCode(exitErr), + }) + return false, string(cmdOutput), nil + } + logger.Error("Error: Unable to check status of black hole port fault", logger.Fields{ + "netns": netNs, + "command": strings.Join(cmdList, " "), + "output": string(cmdOutput), + "err": err, + }) + return false, string(cmdOutput), err + } + logger.Info("[INFO] Black hole port fault has been found running", logger.Fields{ + "netns": netNs, + "command": strings.Join(cmdList, " "), + "output": string(cmdOutput), + }) + return true, string(cmdOutput), nil +} + // StartNetworkLatency starts a network latency fault in the associated ENI if no existing same fault. func (h *FaultHandler) StartNetworkLatency() func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { @@ -707,3 +793,10 @@ func validateTaskNetworkConfig(taskNetworkConfig *state.TaskNetworkConfig) error return nil } + +// runExecCommand wraps around the execwrapper, providing a convenient way of running any Linux command +// and getting the result in both stdout and stderr. +func (h *FaultHandler) runExecCommand(ctx context.Context, cmdList []string) ([]byte, error) { + cmdExec := h.osExecWrapper.CommandContext(ctx, cmdList[0], cmdList[1:]...) + return cmdExec.CombinedOutput() +} diff --git a/ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go b/ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go index f1761c8182e..0f80eb96321 100644 --- a/ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go +++ b/ecs-agent/tmds/handlers/fault/v1/handlers/handlers_test.go @@ -31,6 +31,7 @@ import ( v2 "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v2" state "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v4/state" mock_state "github.com/aws/amazon-ecs-agent/ecs-agent/tmds/handlers/v4/state/mocks" + mock_execwrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper/mocks" "github.com/golang/mock/gomock" "github.com/gorilla/mux" @@ -39,17 +40,18 @@ import ( ) const ( - endpointId = "endpointId" - port = 1234 - protocol = "tcp" - trafficType = "ingress" - delayMilliseconds = 123456789 - jitterMilliseconds = 4567 - lossPercent = 6 - taskARN = "taskArn" - awsvpcNetworkMode = "awsvpc" - deviceName = "eth0" - invalidNetworkMode = "invalid" + endpointId = "endpointId" + port = 1234 + protocol = "tcp" + trafficType = "ingress" + delayMilliseconds = 123456789 + jitterMilliseconds = 4567 + lossPercent = 6 + taskARN = "taskArn" + awsvpcNetworkMode = "awsvpc" + deviceName = "eth0" + invalidNetworkMode = "invalid" + iptablesChainNotFoundError = "iptables: Bad rule (does a matching rule exist in that chain?)." ) var ( @@ -90,6 +92,12 @@ var ( FaultInjectionEnabled: true, } + happyBlackHolePortReqBody = map[string]interface{}{ + "Port": port, + "Protocol": protocol, + "TrafficType": trafficType, + } + ipSources = []string{"52.95.154.1", "52.95.154.2"} ) @@ -99,6 +107,7 @@ type networkFaultInjectionTestCase struct { requestBody interface{} expectedResponseBody types.NetworkFaultInjectionResponse setAgentStateExpectations func(agentState *mock_state.MockAgentState) + setExecExpectations func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) } // Tests the path for Fault Network Faults API @@ -114,40 +123,8 @@ func TestFaultPacketLossFaultPath(t *testing.T) { assert.Equal(t, "/api/{endpointContainerIDMuxName:[^/]*}/fault/v1/network-packet-loss", NetworkFaultPath(types.PacketLossFaultType)) } -func generateNetworkBlackHolePortTestCases(name string, expectedHappyResponseBody string) []networkFaultInjectionTestCase { - happyBlackHolePortReqBody := map[string]interface{}{ - "Port": port, - "Protocol": protocol, - "TrafficType": trafficType, - } +func generateNetworkBlackHolePortTestCases(name string) []networkFaultInjectionTestCase { tcs := []networkFaultInjectionTestCase{ - { - name: fmt.Sprintf("%s success", name), - expectedStatusCode: 200, - requestBody: happyBlackHolePortReqBody, - expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse(expectedHappyResponseBody), - setAgentStateExpectations: func(agentState *mock_state.MockAgentState) { - agentState.EXPECT().GetTaskMetadata(endpointId). - Return(happyTaskResponse, nil). - Times(1) - }, - }, - { - name: fmt.Sprintf("%s unknown request body", name), - expectedStatusCode: 200, - requestBody: map[string]interface{}{ - "Port": port, - "Protocol": protocol, - "TrafficType": trafficType, - "Unknown": "", - }, - expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse(expectedHappyResponseBody), - setAgentStateExpectations: func(agentState *mock_state.MockAgentState) { - agentState.EXPECT().GetTaskMetadata(endpointId). - Return(happyTaskResponse, nil). - Times(1) - }, - }, { name: fmt.Sprintf("%s no request body", name), expectedStatusCode: 400, @@ -343,7 +320,7 @@ func generateNetworkBlackHolePortTestCases(name string, expectedHappyResponseBod TaskNetworkConfig: &state.TaskNetworkConfig{ NetworkMode: awsvpcNetworkMode, NetworkNamespaces: []*state.NetworkNamespace{ - &state.NetworkNamespace{ + { Path: "/path", NetworkInterfaces: noDeviceNameInNetworkInterfaces, }, @@ -356,8 +333,173 @@ func generateNetworkBlackHolePortTestCases(name string, expectedHappyResponseBod return tcs } +func generateStartBlackHolePortFaultTestCases() []networkFaultInjectionTestCase { + commonTcs := generateNetworkBlackHolePortTestCases("start blackhole port") + tcs := []networkFaultInjectionTestCase{ + { + name: "start blackhole port success running", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("running"), + setAgentStateExpectations: func(agentState *mock_state.MockAgentState) { + agentState.EXPECT().GetTaskMetadata(endpointId). + Return(happyTaskResponse, nil). + Times(1) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + }, + }, + { + name: "start blackhole unknown request body", + expectedStatusCode: 200, + requestBody: map[string]interface{}{ + "Port": port, + "Protocol": protocol, + "TrafficType": trafficType, + "Unknown": "", + }, + expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("running"), + setAgentStateExpectations: func(agentState *mock_state.MockAgentState) { + agentState.EXPECT().GetTaskMetadata(endpointId). + Return(happyTaskResponse, nil). + Times(1) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + }, + }, + } + + return append(tcs, commonTcs...) +} + +func generateStopBlackHolePortFaultTestCases() []networkFaultInjectionTestCase { + commonTcs := generateNetworkBlackHolePortTestCases("stop blackhole port") + tcs := []networkFaultInjectionTestCase{ + { + name: "stop blackhole port success running", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("stopped"), + setAgentStateExpectations: func(agentState *mock_state.MockAgentState) { + agentState.EXPECT().GetTaskMetadata(endpointId). + Return(happyTaskResponse, nil). + Times(1) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + }, + }, + { + name: "stop blackhole unknown request body", + expectedStatusCode: 200, + requestBody: map[string]interface{}{ + "Port": port, + "Protocol": protocol, + "TrafficType": trafficType, + "Unknown": "", + }, + expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("stopped"), + setAgentStateExpectations: func(agentState *mock_state.MockAgentState) { + agentState.EXPECT().GetTaskMetadata(endpointId). + Return(happyTaskResponse, nil). + Times(1) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + }, + }, + } + return append(tcs, commonTcs...) +} + +func generateCheckBlackHolePortFaultStatusTestCases() []networkFaultInjectionTestCase { + commonTcs := generateNetworkBlackHolePortTestCases("check blackhole port") + tcs := []networkFaultInjectionTestCase{ + { + name: "check blackhole port success running", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("running"), + setAgentStateExpectations: func(agentState *mock_state.MockAgentState) { + agentState.EXPECT().GetTaskMetadata(endpointId). + Return(happyTaskResponse, nil). + Times(1) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + ) + }, + }, + { + name: "check blackhole unknown request body", + expectedStatusCode: 200, + requestBody: map[string]interface{}{ + "Port": port, + "Protocol": protocol, + "TrafficType": trafficType, + "Unknown": "", + }, + expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("running"), + setAgentStateExpectations: func(agentState *mock_state.MockAgentState) { + agentState.EXPECT().GetTaskMetadata(endpointId). + Return(happyTaskResponse, nil). + Times(1) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte{}, nil), + ) + }, + }, + { + name: "check blackhole port success not running", + expectedStatusCode: 200, + requestBody: happyBlackHolePortReqBody, + expectedResponseBody: types.NewNetworkFaultInjectionSuccessResponse("not-running"), + setAgentStateExpectations: func(agentState *mock_state.MockAgentState) { + agentState.EXPECT().GetTaskMetadata(endpointId). + Return(happyTaskResponse, nil). + Times(1) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte(iptablesChainNotFoundError), errors.New("exit status 1")), + exec.EXPECT().ConvertToExitError(gomock.Any()).Times(1).Return(nil, true), + exec.EXPECT().GetExitCode(gomock.Any()).Times(1).Return(1), + ) + }, + }, + { + name: "check blackhole port failure", + expectedStatusCode: 500, + requestBody: happyBlackHolePortReqBody, + expectedResponseBody: types.NewNetworkFaultInjectionErrorResponse("internal error"), + setAgentStateExpectations: func(agentState *mock_state.MockAgentState) { + agentState.EXPECT().GetTaskMetadata(endpointId). + Return(happyTaskResponse, nil). + Times(1) + }, + setExecExpectations: func(exec *mock_execwrapper.MockExec, ctrl *gomock.Controller) { + cmdExec := mock_execwrapper.NewMockCmd(ctrl) + gomock.InOrder( + exec.EXPECT().CommandContext(gomock.Any(), gomock.Any(), gomock.Any()).Times(1).Return(cmdExec), + cmdExec.EXPECT().CombinedOutput().Times(1).Return([]byte("internal error"), errors.New("exit 2")), + exec.EXPECT().ConvertToExitError(gomock.Any()).Times(1).Return(nil, false), + ) + }, + }, + } + + return append(tcs, commonTcs...) +} + func TestStartNetworkBlackHolePort(t *testing.T) { - tcs := generateNetworkBlackHolePortTestCases("start blackhole port", "running") + tcs := generateStartBlackHolePortFaultTestCases() for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { @@ -367,13 +509,14 @@ func TestStartNetworkBlackHolePort(t *testing.T) { agentState := mock_state.NewMockAgentState(ctrl) metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) + execWrapper := mock_execwrapper.NewMockExec(ctrl) if tc.setAgentStateExpectations != nil { tc.setAgentStateExpectations(agentState) } router := mux.NewRouter() - handler := New(agentState, metricsFactory) + handler := New(agentState, metricsFactory, execWrapper) router.HandleFunc( NetworkFaultPath(types.BlackHolePortFaultType), handler.StartNetworkBlackholePort(), @@ -406,7 +549,7 @@ func TestStartNetworkBlackHolePort(t *testing.T) { } func TestStopNetworkBlackHolePort(t *testing.T) { - tcs := generateNetworkBlackHolePortTestCases("stop blackhole port", "stopped") + tcs := generateStopBlackHolePortFaultTestCases() for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { // Mocks @@ -415,13 +558,14 @@ func TestStopNetworkBlackHolePort(t *testing.T) { agentState := mock_state.NewMockAgentState(ctrl) metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) + execWrapper := mock_execwrapper.NewMockExec(ctrl) if tc.setAgentStateExpectations != nil { tc.setAgentStateExpectations(agentState) } router := mux.NewRouter() - handler := New(agentState, metricsFactory) + handler := New(agentState, metricsFactory, execWrapper) router.HandleFunc( NetworkFaultPath(types.BlackHolePortFaultType), handler.StopNetworkBlackHolePort(), @@ -454,7 +598,7 @@ func TestStopNetworkBlackHolePort(t *testing.T) { } func TestCheckNetworkBlackHolePort(t *testing.T) { - tcs := generateNetworkBlackHolePortTestCases("check blackhole port", "running") + tcs := generateCheckBlackHolePortFaultStatusTestCases() for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { @@ -464,14 +608,18 @@ func TestCheckNetworkBlackHolePort(t *testing.T) { agentState := mock_state.NewMockAgentState(ctrl) metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) - - router := mux.NewRouter() - handler := New(agentState, metricsFactory) + execWrapper := mock_execwrapper.NewMockExec(ctrl) if tc.setAgentStateExpectations != nil { tc.setAgentStateExpectations(agentState) } + if tc.setExecExpectations != nil { + tc.setExecExpectations(execWrapper, ctrl) + } + + router := mux.NewRouter() + handler := New(agentState, metricsFactory, execWrapper) router.HandleFunc( NetworkFaultPath(types.BlackHolePortFaultType), handler.CheckNetworkBlackHolePort(), @@ -768,13 +916,14 @@ func TestStartNetworkLatency(t *testing.T) { agentState := mock_state.NewMockAgentState(ctrl) metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) + execWrapper := mock_execwrapper.NewMockExec(ctrl) if tc.setAgentStateExpectations != nil { tc.setAgentStateExpectations(agentState) } router := mux.NewRouter() - handler := New(agentState, metricsFactory) + handler := New(agentState, metricsFactory, execWrapper) router.HandleFunc( NetworkFaultPath(types.LatencyFaultType), handler.StartNetworkLatency(), @@ -816,13 +965,14 @@ func TestStopNetworkLatency(t *testing.T) { agentState := mock_state.NewMockAgentState(ctrl) metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) + execWrapper := mock_execwrapper.NewMockExec(ctrl) if tc.setAgentStateExpectations != nil { tc.setAgentStateExpectations(agentState) } router := mux.NewRouter() - handler := New(agentState, metricsFactory) + handler := New(agentState, metricsFactory, execWrapper) router.HandleFunc( NetworkFaultPath(types.LatencyFaultType), handler.StopNetworkLatency(), @@ -865,9 +1015,10 @@ func TestCheckNetworkLatency(t *testing.T) { agentState := mock_state.NewMockAgentState(ctrl) metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) + execWrapper := mock_execwrapper.NewMockExec(ctrl) router := mux.NewRouter() - handler := New(agentState, metricsFactory) + handler := New(agentState, metricsFactory, execWrapper) if tc.setAgentStateExpectations != nil { tc.setAgentStateExpectations(agentState) } @@ -1142,13 +1293,14 @@ func TestStartNetworkPacketLoss(t *testing.T) { agentState := mock_state.NewMockAgentState(ctrl) metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) + execWrapper := mock_execwrapper.NewMockExec(ctrl) if tc.setAgentStateExpectations != nil { tc.setAgentStateExpectations(agentState) } router := mux.NewRouter() - handler := New(agentState, metricsFactory) + handler := New(agentState, metricsFactory, execWrapper) router.HandleFunc( NetworkFaultPath(types.PacketLossFaultType), handler.StartNetworkPacketLoss(), @@ -1190,13 +1342,14 @@ func TestStopNetworkPacketLoss(t *testing.T) { agentState := mock_state.NewMockAgentState(ctrl) metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) + execWrapper := mock_execwrapper.NewMockExec(ctrl) if tc.setAgentStateExpectations != nil { tc.setAgentStateExpectations(agentState) } router := mux.NewRouter() - handler := New(agentState, metricsFactory) + handler := New(agentState, metricsFactory, execWrapper) router.HandleFunc( NetworkFaultPath(types.PacketLossFaultType), handler.StopNetworkPacketLoss(), @@ -1239,13 +1392,14 @@ func TestCheckNetworkPacketLoss(t *testing.T) { agentState := mock_state.NewMockAgentState(ctrl) metricsFactory := mock_metrics.NewMockEntryFactory(ctrl) + execWrapper := mock_execwrapper.NewMockExec(ctrl) - router := mux.NewRouter() - handler := New(agentState, metricsFactory) if tc.setAgentStateExpectations != nil { tc.setAgentStateExpectations(agentState) } + router := mux.NewRouter() + handler := New(agentState, metricsFactory, execWrapper) router.HandleFunc( NetworkFaultPath(types.PacketLossFaultType), handler.CheckNetworkPacketLoss(), diff --git a/ecs-agent/utils/execwrapper/exec.go b/ecs-agent/utils/execwrapper/exec.go index 776a7dfb57f..d1f94cb82c5 100644 --- a/ecs-agent/utils/execwrapper/exec.go +++ b/ecs-agent/utils/execwrapper/exec.go @@ -25,6 +25,8 @@ import ( // for testing. type Exec interface { CommandContext(ctx context.Context, name string, arg ...string) Cmd + ConvertToExitError(err error) (*exec.ExitError, bool) + GetExitCode(exitErr *exec.ExitError) int } // execWrapper is a placeholder struct which implements the Exec interface. @@ -40,6 +42,17 @@ func (e *execWrapper) CommandContext(ctx context.Context, name string, arg ...st return NewCMDContext(ctx, name, arg...) } +// ConvertToExitError converts an error object to an exec.ExitError +func (e *execWrapper) ConvertToExitError(err error) (*exec.ExitError, bool) { + exitErr, eok := err.(*exec.ExitError) + return exitErr, eok +} + +// GetExitCode gets the exit code of an exec.ExitError object +func (e *execWrapper) GetExitCode(exitErr *exec.ExitError) int { + return exitErr.ExitCode() +} + // Cmd acts as a wrapper to functions exposed by the exec.Cmd object. // Having this interface enables us to create mock objects we can use // for testing. @@ -51,30 +64,42 @@ type Cmd interface { AppendExtraFiles(...*os.File) Args() []string SetIOStreams(io.Reader, io.Writer, io.Writer) + Output() ([]byte, error) + CombinedOutput() ([]byte, error) } type cmdWrapper struct { *exec.Cmd } +// NewCMDContext returns a new cmdWrapper object which will be used to call standard go os exec calls with a context func NewCMDContext(ctx context.Context, name string, arg ...string) Cmd { cmd := exec.CommandContext(ctx, name, arg...) return &cmdWrapper{Cmd: cmd} } +// NewCMDContext returns a new cmdWrapper object which will be used to call standard go os exec calls func NewCMD(name string, arg ...string) Cmd { cmd := exec.Command(name, arg...) return &cmdWrapper{Cmd: cmd} } +// Run is a wrapper to existing Run() method of the os/exec go library. +// Run starts the specified command and waits for it to complete. +// Returns nil if the command runs and exits with a zero code. +// Otherwise returns an error func (c *cmdWrapper) Run() error { return c.Cmd.Run() } +// Start is a wrapper to the existing Start() method of the os/exec go library +// Start starts the specified command but does not wait for it to complete. func (c *cmdWrapper) Start() error { return c.Cmd.Start() } +// Wait is a wrapper to the existing Wait() method of the os/exec go library +// Wait waits for the command started by a Start() call to exit and waits for any copying to stdin or copying from stdout or stderr to complete. func (c *cmdWrapper) Wait() error { return c.Cmd.Wait() } @@ -102,3 +127,15 @@ func (c *cmdWrapper) SetIOStreams(stdin io.Reader, stdout io.Writer, stderr io.W c.Stderr = stderr } } + +// Output is a wrapper to the existing Output() method of the os/exec go library. +// Output runs the command and returns its standard output as well as the standard error output. +func (c *cmdWrapper) Output() ([]byte, error) { + return c.Cmd.Output() +} + +// CombinedOutput is a wrapper to the existing CombinedOutput() method of the os/exec go library. +// CombinedOutput runs the command and returns its combined standard output and standard error. +func (c *cmdWrapper) CombinedOutput() ([]byte, error) { + return c.Cmd.CombinedOutput() +} diff --git a/ecs-agent/utils/execwrapper/mocks/execwrapper_mocks.go b/ecs-agent/utils/execwrapper/mocks/execwrapper_mocks.go index 962077bd141..25bff9f5241 100644 --- a/ecs-agent/utils/execwrapper/mocks/execwrapper_mocks.go +++ b/ecs-agent/utils/execwrapper/mocks/execwrapper_mocks.go @@ -22,6 +22,7 @@ import ( context "context" io "io" os "os" + exec "os/exec" reflect "reflect" execwrapper "github.com/aws/amazon-ecs-agent/ecs-agent/utils/execwrapper" @@ -81,6 +82,21 @@ func (mr *MockCmdMockRecorder) Args() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Args", reflect.TypeOf((*MockCmd)(nil).Args)) } +// CombinedOutput mocks base method. +func (m *MockCmd) CombinedOutput() ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CombinedOutput") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CombinedOutput indicates an expected call of CombinedOutput. +func (mr *MockCmdMockRecorder) CombinedOutput() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CombinedOutput", reflect.TypeOf((*MockCmd)(nil).CombinedOutput)) +} + // KillProcess mocks base method. func (m *MockCmd) KillProcess() error { m.ctrl.T.Helper() @@ -95,6 +111,21 @@ func (mr *MockCmdMockRecorder) KillProcess() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "KillProcess", reflect.TypeOf((*MockCmd)(nil).KillProcess)) } +// Output mocks base method. +func (m *MockCmd) Output() ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Output") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Output indicates an expected call of Output. +func (mr *MockCmdMockRecorder) Output() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Output", reflect.TypeOf((*MockCmd)(nil).Output)) +} + // Run mocks base method. func (m *MockCmd) Run() error { m.ctrl.T.Helper() @@ -190,3 +221,32 @@ func (mr *MockExecMockRecorder) CommandContext(arg0, arg1 interface{}, arg2 ...i varargs := append([]interface{}{arg0, arg1}, arg2...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CommandContext", reflect.TypeOf((*MockExec)(nil).CommandContext), varargs...) } + +// ConvertToExitError mocks base method. +func (m *MockExec) ConvertToExitError(arg0 error) (*exec.ExitError, bool) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ConvertToExitError", arg0) + ret0, _ := ret[0].(*exec.ExitError) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +// ConvertToExitError indicates an expected call of ConvertToExitError. +func (mr *MockExecMockRecorder) ConvertToExitError(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ConvertToExitError", reflect.TypeOf((*MockExec)(nil).ConvertToExitError), arg0) +} + +// GetExitCode mocks base method. +func (m *MockExec) GetExitCode(arg0 *exec.ExitError) int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetExitCode", arg0) + ret0, _ := ret[0].(int) + return ret0 +} + +// GetExitCode indicates an expected call of GetExitCode. +func (mr *MockExecMockRecorder) GetExitCode(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExitCode", reflect.TypeOf((*MockExec)(nil).GetExitCode), arg0) +}