diff --git a/cmd/fuzz.go b/cmd/fuzz.go index 89e1702f..7ff660d4 100644 --- a/cmd/fuzz.go +++ b/cmd/fuzz.go @@ -161,7 +161,7 @@ func cmdRunFuzz(cmd *cobra.Command, args []string) error { signal.Notify(c, os.Interrupt) go func() { <-c - fuzzer.Stop() + fuzzer.Terminate() }() // Start the fuzzing process with our cancellable context. diff --git a/fuzzing/fuzzer.go b/fuzzing/fuzzer.go index 9039105e..0438bca4 100644 --- a/fuzzing/fuzzer.go +++ b/fuzzing/fuzzer.go @@ -43,11 +43,18 @@ import ( // Fuzzer represents an Ethereum smart contract fuzzing provider. type Fuzzer struct { - // ctx describes the context for the fuzzing run, used to cancel running operations. + // ctx is the main context used by the fuzzer. ctx context.Context - // ctxCancelFunc describes a function which can be used to cancel the fuzzing operations ctx tracks. + // mainCtxCancelFunc describes a function which can be used to cancel the fuzzing operations the main ctx tracks. + // Cancelling ctx does _not_ guarantee that all operations will terminate. ctxCancelFunc context.CancelFunc + // emergencyCtx is the context that is used by the fuzzer to react to OS-level interrupts (e.g. SIGINT) or errors. + emergencyCtx context.Context + // emergencyCtxCancelFunc describes a function which can be used to cancel the fuzzing operations due to an OS-level + // interrupt or an error. Cancelling emergencyCtx will guarantee that all operations will terminate. + emergencyCtxCancelFunc context.CancelFunc + // config describes the project configuration which the fuzzing is targeting. config config.ProjectConfig // senders describes a set of account addresses used to send state changing calls in fuzzing campaigns. @@ -722,11 +729,9 @@ func (f *Fuzzer) spawnWorkersLoop(baseTestChain *chain.TestChain) error { }(workerSlotInfo) } - // Explicitly call cancel on our context to ensure all threads exit if we encountered an error. - if err != nil && f.ctxCancelFunc != nil { - // We need to update our context with an error so that anyone subscribing to the cancellation of the fuzzer is aware that an error occured - f.ctx = context.WithValue(f.ctx, fuzzerErrKey, err) - f.ctxCancelFunc() + // Explicitly call cancel on our emergency context to ensure all threads exit if we encountered an error. + if err != nil { + f.Terminate() } // Wait for every worker to be freed, so we don't have a race condition when reporting the order @@ -757,8 +762,9 @@ func (f *Fuzzer) Start() error { // While we're fuzzing, we'll want to have an initialized random provider. f.randomProvider = rand.New(rand.NewSource(time.Now().UnixNano())) - // Create our running context (allows us to cancel across threads) + // Create our main and secondary running context (allows us to cancel across threads) f.ctx, f.ctxCancelFunc = context.WithCancel(context.Background()) + f.emergencyCtx, f.emergencyCtxCancelFunc = context.WithCancel(context.Background()) // If we set a timeout, create the timeout context now, as we're about to begin fuzzing. if f.config.Fuzzing.Timeout > 0 { @@ -833,9 +839,6 @@ func (f *Fuzzer) Start() error { // Start our printing loop now that we're about to begin fuzzing. go f.printMetricsLoop() - // Start our goroutine that monitors when the fuzzing campaign is cancelled - go f.monitorContextCancellation() - // Publish a fuzzer starting event. err = f.Events.FuzzerStarting.Publish(FuzzerStartingEvent{Fuzzer: f}) if err != nil { @@ -871,6 +874,13 @@ func (f *Fuzzer) Start() error { } } + // Publish a fuzzer stopping event. + fuzzerStoppingErr := f.Events.FuzzerStopping.Publish(FuzzerStoppingEvent{Fuzzer: f, err: err}) + if err == nil && fuzzerStoppingErr != nil { + err = fuzzerStoppingErr + f.logger.Error("FuzzerStopping event subscriber returned an error", err) + } + // Print our results on exit. f.printExitingResults() @@ -909,31 +919,26 @@ func (f *Fuzzer) Start() error { return err } -// monitorContextCancellation monitors when the fuzzing campaign is cancelled and emits the fuzzer stopping event once the campaign is over. -func (f *Fuzzer) monitorContextCancellation() { - // Keep checking if the fuzzing campaign is cancelled every 100 milliseconds - for !utils.CheckContextDone(f.ctx) { - time.Sleep(time.Millisecond * 100) - } - - // We are exiting so we need to emit the fuzzer stopping event - err := f.Events.FuzzerStopping.Publish(FuzzerStoppingEvent{Fuzzer: f, err: f.ctx.Value(fuzzerErrKey).(error)}) - // We will log the error but continue to exit gracefully - // TODO: Do we really need to log this error? - if err != nil { - f.logger.Error("FuzzerStopping event subscriber returned an error", err) - } -} - -// Stop stops a running operation invoked by the Start method. This method may return before complete operation teardown -// occurs. +// Stop attempts to stop all running operations invoked by the Start method. Note that Stop is not guaranteed to fully +// terminate the operations across all threads. For example, the optimization testing provider may request a thread to +// shrink some call sequences before the thread is torn down. Stop will not prevent those shrink requests from +// executing. An OS-level interrupt must be used to guarantee the stopping of _all_ operations (see Terminate). func (f *Fuzzer) Stop() { - // Call the cancel function on our running context to stop all working goroutines + // Call the cancel function on our main running context to try stop all working goroutines if f.ctxCancelFunc != nil { f.ctxCancelFunc() } } +// Terminate is called to react to an OS-level interrupt (e.g. SIGINT) or an error. This will stop all operations. +// Note that this function will return before all operations are complete. +func (f *Fuzzer) Terminate() { + // Call the emergency context cancel function on our running context to stop all working goroutines + if f.emergencyCtxCancelFunc != nil { + f.emergencyCtxCancelFunc() + } +} + // printMetricsLoop prints metrics to the console in a loop until ctx signals a stopped operation. func (f *Fuzzer) printMetricsLoop() { // Define our start time diff --git a/fuzzing/fuzzer_worker.go b/fuzzing/fuzzer_worker.go index 57207a40..0538e044 100644 --- a/fuzzing/fuzzer_worker.go +++ b/fuzzing/fuzzer_worker.go @@ -334,8 +334,8 @@ func (fw *FuzzerWorker) testNextCallSequence() ([]ShrinkCallSequenceRequest, err lastCallSequenceElement := currentlyExecutedSequence[len(currentlyExecutedSequence)-1] fw.workerMetrics().gasUsed.Add(fw.workerMetrics().gasUsed, new(big.Int).SetUint64(lastCallSequenceElement.ChainReference.Block.MessageResults[lastCallSequenceElement.ChainReference.TransactionIndex].Receipt.GasUsed)) - // If our fuzzer context is done, exit out immediately without results. - if utils.CheckContextDone(fw.fuzzer.ctx) { + // If our fuzzer context or the emergency context is cancelled, exit out immediately without results. + if utils.CheckContextDone(fw.fuzzer.ctx) || utils.CheckContextDone(fw.fuzzer.emergencyCtx) { return true, nil } @@ -353,8 +353,7 @@ func (fw *FuzzerWorker) testNextCallSequence() ([]ShrinkCallSequenceRequest, err } // If our fuzzer context is done, exit out immediately without results. - // TODO: Probably need to check something here - if utils.CheckContextDone(fw.fuzzer.ctx) { + if utils.CheckContextDone(fw.fuzzer.ctx) || utils.CheckContextDone(fw.fuzzer.emergencyCtx) { return nil, nil } @@ -404,7 +403,7 @@ func (fw *FuzzerWorker) testShrunkenCallSequence(possibleShrunkSequence calls.Ca } // If our fuzzer context is done, exit out immediately without results. - if utils.CheckContextDone(fw.fuzzer.ctx) { + if utils.CheckContextDone(fw.fuzzer.emergencyCtx) { return true, nil } @@ -418,8 +417,7 @@ func (fw *FuzzerWorker) testShrunkenCallSequence(possibleShrunkSequence calls.Ca } // If our fuzzer context is done, exit out immediately without results. - // TODO: Probably need to check something here - if utils.CheckContextDone(fw.fuzzer.ctx) { + if utils.CheckContextDone(fw.fuzzer.emergencyCtx) { return false, nil } @@ -451,7 +449,7 @@ func (fw *FuzzerWorker) shrinkCallSequence(shrinkRequest ShrinkCallSequenceReque shrinkIteration := uint64(0) shrinkLimit := fw.fuzzer.config.Fuzzing.ShrinkLimit shrinkingEnded := func() bool { - return shrinkIteration >= shrinkLimit || utils.CheckContextDone(fw.fuzzer.ctx) + return shrinkIteration >= shrinkLimit || utils.CheckContextDone(fw.fuzzer.emergencyCtx) } if shrinkLimit > 0 { // The first pass of shrinking is greedy towards trying to remove any unnecessary calls. @@ -563,8 +561,9 @@ func (fw *FuzzerWorker) shrinkCallSequence(shrinkRequest ShrinkCallSequenceReque } // run takes a base Chain in a setup state ready for testing, clones it, and begins executing fuzzed transaction calls -// and asserting properties are upheld. This runs until Fuzzer.ctx cancels the operation. -// Returns a boolean indicating whether Fuzzer.ctx has indicated we cancel the operation, and an error if one occurred. +// and asserting properties are upheld. This runs until Fuzzer.ctx or Fuzzer.emergencyCtx cancels the operation. +// Returns a boolean indicating whether Fuzzer.ctx or Fuzzer.emergencyCtx has indicated we cancel the operation, and an +// error if one occurred. func (fw *FuzzerWorker) run(baseTestChain *chain.TestChain) (bool, error) { // Clone our chain, attaching our necessary components for fuzzing post-genesis, prior to all blocks being copied. // This means any tracers added or events subscribed to within this inner function are done so prior to chain @@ -600,7 +599,7 @@ func (fw *FuzzerWorker) run(baseTestChain *chain.TestChain) (bool, error) { // Defer the closing of the test chain object defer fw.chain.Close() - // Emit an event indicating the worker has setup its chain. + // Emit an event indicating the worker has set up its chain. err = fw.Events.FuzzerWorkerChainSetup.Publish(FuzzerWorkerChainSetupEvent{ Worker: fw, Chain: fw.chain, @@ -624,11 +623,16 @@ func (fw *FuzzerWorker) run(baseTestChain *chain.TestChain) (bool, error) { sequencesTested := 0 fuzzingCancelled := false for sequencesTested <= fw.fuzzer.config.Fuzzing.WorkerResetLimit { - // If our context signaled to close the operation, we will complete shrinking any outstanding shrink requests - // and then return immediately + // Immediately exit if the emergency context is triggered + if utils.CheckContextDone(fw.fuzzer.emergencyCtx) { + return true, nil + } + + // If our main context signaled to close the operation, we will emit an event notifying any subscribers that + // this fuzzer worker is going to be shut down. This allows any subscriber (e.g. the optimization provider) + // one last opportunity to shrink a call sequence if necessary. if utils.CheckContextDone(fw.fuzzer.ctx) { fuzzingCancelled = true - // TODO: First figure out if there is error within the context err := fw.Events.TestingComplete.Publish(FuzzerWorkerTestingCompleteEvent{ Worker: fw, }) @@ -646,16 +650,14 @@ func (fw *FuzzerWorker) run(baseTestChain *chain.TestChain) (bool, error) { } } - // If we have cancelled fuzzing, return immediately + // Clean up the shrink requests + fw.shrinkCallSequenceRequests = nil + + // If we have cancelled fuzzing, return now if fuzzingCancelled { return true, nil } - // Need to reset the shrink call sequence request array if it's a non-zero length - if len(fw.shrinkCallSequenceRequests) > 0 { - fw.shrinkCallSequenceRequests = make([]ShrinkCallSequenceRequest, 0) - } - // Emit an event indicating the worker is about to test a new call sequence. err := fw.Events.CallSequenceTesting.Publish(FuzzerWorkerCallSequenceTestingEvent{ Worker: fw, diff --git a/fuzzing/test_case_optimization_provider.go b/fuzzing/test_case_optimization_provider.go index 431169b4..c0bc3142 100644 --- a/fuzzing/test_case_optimization_provider.go +++ b/fuzzing/test_case_optimization_provider.go @@ -162,11 +162,6 @@ func (t *OptimizationTestCaseProvider) onFuzzerStarting(event FuzzerStartingEven // have been destroyed. It clears state tracked for each FuzzerWorker and sets test cases in "running" states to // "passed". func (t *OptimizationTestCaseProvider) onFuzzerStopping(event FuzzerStoppingEvent) error { - // If there is no error, we request a shrink request - if event.err == nil { - // TODO: Request a shrink request - } - // Clear our optimization test methods t.workerStates = nil @@ -197,19 +192,19 @@ func (t *OptimizationTestCaseProvider) onWorkerCreated(event FuzzerWorkerCreated } // onWorkerTestingComplete is the event handler triggered when a FuzzerWorker has completed testing of call sequences -// and is about to exit the fuzzing loop. We use this event to attach a shrink request to the worker, if a request exists. +// and is about to exit the fuzzing loop. We use this event to attach shrink requests to the worker. // This way we are only shrinking once throughout the entire fuzzing campaign in optimization mode. func (t *OptimizationTestCaseProvider) onWorkerTestingComplete(event FuzzerWorkerTestingCompleteEvent) error { - if t.shrinkCallSequenceRequest != nil { - shrunkenSequence, err := event.Worker.shrinkCallSequence(*t.shrinkCallSequenceRequest) - if err != nil { - return err + // Iterate across each test case to see if there is a shrink request for it + t.testCasesLock.Lock() + for _, testCase := range t.testCases { + // We have a shrink request, let's send it to the fuzzer worker + if testCase.shrinkCallSequenceRequest != nil { + event.Worker.shrinkCallSequenceRequests = append(event.Worker.shrinkCallSequenceRequests, *testCase.shrinkCallSequenceRequest) + testCase.shrinkCallSequenceRequest = nil } - - // Reset it to nil so that only the first worker that is trying to exit is requested to handle the shrink request - // The other workers can exit gracefully - t.shrinkCallSequenceRequest = nil } + t.testCasesLock.Unlock() return nil } @@ -289,10 +284,6 @@ func (t *OptimizationTestCaseProvider) onWorkerDeployedContractDeleted(event Fuz // and any underlying FuzzerWorker. It is called after every call made in a call sequence. It checks whether any // optimization test's value has increased. func (t *OptimizationTestCaseProvider) callSequencePostCallTest(worker *FuzzerWorker, callSequence calls.CallSequence) ([]ShrinkCallSequenceRequest, error) { - // Create a list of shrink call sequence verifiers, which we populate for each maximized optimization test we want a call - // sequence shrunk for. - shrinkRequests := make([]ShrinkCallSequenceRequest, 0) - // Obtain the test provider state for this worker workerState := &t.workerStates[worker.WorkerIndex()] @@ -312,9 +303,6 @@ func (t *OptimizationTestCaseProvider) callSequencePostCallTest(worker *FuzzerWo // If we updated the test case's maximum value, we update our state immediately. We provide a shrink verifier which will update // the call sequence for each shrunken sequence provided that still it maintains the maximum value. - // TODO: This is very inefficient since this runs every time a new max value is found. It would be ideal if we - // could perform a one-time shrink request. This code should be refactored when we introduce the high-level - // testing API. if newValue.Cmp(testCase.value) == 1 { // Create a request to shrink this call sequence. shrinkRequest := ShrinkCallSequenceRequest{ @@ -372,11 +360,9 @@ func (t *OptimizationTestCaseProvider) callSequencePostCallTest(worker *FuzzerWo }, RecordResultInCorpus: true, } - - // Add our shrink request to our list. - shrinkRequests = append(shrinkRequests, shrinkRequest) + testCase.shrinkCallSequenceRequest = &shrinkRequest } } - return shrinkRequests, nil + return nil, nil }