add emergency context

cmd/fuzz.go

@@ -161,7 +161,7 @@ func cmdRunFuzz(cmd *cobra.Command, args []string) error {
 	signal.Notify(c, os.Interrupt)
 	go func() {
 		<-c
-		fuzzer.Stop()
+		fuzzer.Terminate()
 	}()
 
 	// Start the fuzzing process with our cancellable context.

fuzzing/fuzzer.go

@@ -43,11 +43,18 @@ import (
 
 // Fuzzer represents an Ethereum smart contract fuzzing provider.
 type Fuzzer struct {
-	// ctx describes the context for the fuzzing run, used to cancel running operations.
+	// ctx is the main context used by the fuzzer.
 	ctx context.Context
-	// ctxCancelFunc describes a function which can be used to cancel the fuzzing operations ctx tracks.
+	// mainCtxCancelFunc describes a function which can be used to cancel the fuzzing operations the main ctx tracks.
+	// Cancelling ctx does _not_ guarantee that all operations will terminate.
 	ctxCancelFunc context.CancelFunc
 
+	// emergencyCtx is the context that is used by the fuzzer to react to OS-level interrupts (e.g. SIGINT) or errors.
+	emergencyCtx context.Context
+	// emergencyCtxCancelFunc describes a function which can be used to cancel the fuzzing operations due to an OS-level
+	// interrupt or an error. Cancelling emergencyCtx will guarantee that all operations will terminate.
+	emergencyCtxCancelFunc context.CancelFunc
+
 	// config describes the project configuration which the fuzzing is targeting.
 	config config.ProjectConfig
 	// senders describes a set of account addresses used to send state changing calls in fuzzing campaigns.
@@ -722,11 +729,9 @@ func (f *Fuzzer) spawnWorkersLoop(baseTestChain *chain.TestChain) error {
 		}(workerSlotInfo)
 	}
 
-	// Explicitly call cancel on our context to ensure all threads exit if we encountered an error.
-	if err != nil && f.ctxCancelFunc != nil {
-		// We need to update our context with an error so that anyone subscribing to the cancellation of the fuzzer is aware that an error occured
-		f.ctx = context.WithValue(f.ctx, fuzzerErrKey, err)
-		f.ctxCancelFunc()
+	// Explicitly call cancel on our emergency context to ensure all threads exit if we encountered an error.
+	if err != nil {
+		f.Terminate()
 	}
 
 	// Wait for every worker to be freed, so we don't have a race condition when reporting the order
@@ -757,8 +762,9 @@ func (f *Fuzzer) Start() error {
 	// While we're fuzzing, we'll want to have an initialized random provider.
 	f.randomProvider = rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	// Create our running context (allows us to cancel across threads)
+	// Create our main and secondary running context (allows us to cancel across threads)
 	f.ctx, f.ctxCancelFunc = context.WithCancel(context.Background())
+	f.emergencyCtx, f.emergencyCtxCancelFunc = context.WithCancel(context.Background())
 
 	// If we set a timeout, create the timeout context now, as we're about to begin fuzzing.
 	if f.config.Fuzzing.Timeout > 0 {
@@ -833,9 +839,6 @@ func (f *Fuzzer) Start() error {
 	// Start our printing loop now that we're about to begin fuzzing.
 	go f.printMetricsLoop()
 
-	// Start our goroutine that monitors when the fuzzing campaign is cancelled
-	go f.monitorContextCancellation()
-
 	// Publish a fuzzer starting event.
 	err = f.Events.FuzzerStarting.Publish(FuzzerStartingEvent{Fuzzer: f})
 	if err != nil {
@@ -871,6 +874,13 @@ func (f *Fuzzer) Start() error {
 		}
 	}
 
+	// Publish a fuzzer stopping event.
+	fuzzerStoppingErr := f.Events.FuzzerStopping.Publish(FuzzerStoppingEvent{Fuzzer: f, err: err})
+	if err == nil && fuzzerStoppingErr != nil {
+		err = fuzzerStoppingErr
+		f.logger.Error("FuzzerStopping event subscriber returned an error", err)
+	}
+
 	// Print our results on exit.
 	f.printExitingResults()
 
@@ -909,31 +919,26 @@ func (f *Fuzzer) Start() error {
 	return err
 }
 
-// monitorContextCancellation monitors when the fuzzing campaign is cancelled and emits the fuzzer stopping event once the campaign is over.
-func (f *Fuzzer) monitorContextCancellation() {
-	// Keep checking if the fuzzing campaign is cancelled every 100 milliseconds
-	for !utils.CheckContextDone(f.ctx) {
-		time.Sleep(time.Millisecond * 100)
-	}
-
-	// We are exiting so we need to emit the fuzzer stopping event
-	err := f.Events.FuzzerStopping.Publish(FuzzerStoppingEvent{Fuzzer: f, err: f.ctx.Value(fuzzerErrKey).(error)})
-	// We will log the error but continue to exit gracefully
-	// TODO: Do we really need to log this error?
-	if err != nil {
-		f.logger.Error("FuzzerStopping event subscriber returned an error", err)
-	}
-}
-
-// Stop stops a running operation invoked by the Start method. This method may return before complete operation teardown
-// occurs.
+// Stop attempts to stop all running operations invoked by the Start method. Note that Stop is not guaranteed to fully
+// terminate the operations across all threads. For example, the optimization testing provider may request a thread to
+// shrink some call sequences before the thread is torn down. Stop will not prevent those shrink requests from
+// executing. An OS-level interrupt must be used to guarantee the stopping of _all_ operations (see Terminate).
 func (f *Fuzzer) Stop() {
-	// Call the cancel function on our running context to stop all working goroutines
+	// Call the cancel function on our main running context to try stop all working goroutines
 	if f.ctxCancelFunc != nil {
 		f.ctxCancelFunc()
 	}
 }
 
+// Terminate is called to react to an OS-level interrupt (e.g. SIGINT) or an error. This will stop all operations.
+// Note that this function will return before all operations are complete.
+func (f *Fuzzer) Terminate() {
+	// Call the emergency context cancel function on our running context to stop all working goroutines
+	if f.emergencyCtxCancelFunc != nil {
+		f.emergencyCtxCancelFunc()
+	}
+}
+
 // printMetricsLoop prints metrics to the console in a loop until ctx signals a stopped operation.
 func (f *Fuzzer) printMetricsLoop() {
 	// Define our start time

fuzzing/fuzzer_worker.go

@@ -334,8 +334,8 @@ func (fw *FuzzerWorker) testNextCallSequence() ([]ShrinkCallSequenceRequest, err
 		lastCallSequenceElement := currentlyExecutedSequence[len(currentlyExecutedSequence)-1]
 		fw.workerMetrics().gasUsed.Add(fw.workerMetrics().gasUsed, new(big.Int).SetUint64(lastCallSequenceElement.ChainReference.Block.MessageResults[lastCallSequenceElement.ChainReference.TransactionIndex].Receipt.GasUsed))
 
-		// If our fuzzer context is done, exit out immediately without results.
-		if utils.CheckContextDone(fw.fuzzer.ctx) {
+		// If our fuzzer context or the emergency context is cancelled, exit out immediately without results.
+		if utils.CheckContextDone(fw.fuzzer.ctx) || utils.CheckContextDone(fw.fuzzer.emergencyCtx) {
 			return true, nil
 		}
 
@@ -353,8 +353,7 @@ func (fw *FuzzerWorker) testNextCallSequence() ([]ShrinkCallSequenceRequest, err
 	}
 
 	// If our fuzzer context is done, exit out immediately without results.
-	// TODO: Probably need to check something here
-	if utils.CheckContextDone(fw.fuzzer.ctx) {
+	if utils.CheckContextDone(fw.fuzzer.ctx) || utils.CheckContextDone(fw.fuzzer.emergencyCtx) {
 		return nil, nil
 	}
 
@@ -404,7 +403,7 @@ func (fw *FuzzerWorker) testShrunkenCallSequence(possibleShrunkSequence calls.Ca
 		}
 
 		// If our fuzzer context is done, exit out immediately without results.
-		if utils.CheckContextDone(fw.fuzzer.ctx) {
+		if utils.CheckContextDone(fw.fuzzer.emergencyCtx) {
 			return true, nil
 		}
 
@@ -418,8 +417,7 @@ func (fw *FuzzerWorker) testShrunkenCallSequence(possibleShrunkSequence calls.Ca
 	}
 
 	// If our fuzzer context is done, exit out immediately without results.
-	// TODO: Probably need to check something here
-	if utils.CheckContextDone(fw.fuzzer.ctx) {
+	if utils.CheckContextDone(fw.fuzzer.emergencyCtx) {
 		return false, nil
 	}
 
@@ -451,7 +449,7 @@ func (fw *FuzzerWorker) shrinkCallSequence(shrinkRequest ShrinkCallSequenceReque
 	shrinkIteration := uint64(0)
 	shrinkLimit := fw.fuzzer.config.Fuzzing.ShrinkLimit
 	shrinkingEnded := func() bool {
-		return shrinkIteration >= shrinkLimit || utils.CheckContextDone(fw.fuzzer.ctx)
+		return shrinkIteration >= shrinkLimit || utils.CheckContextDone(fw.fuzzer.emergencyCtx)
 	}
 	if shrinkLimit > 0 {
 		// The first pass of shrinking is greedy towards trying to remove any unnecessary calls.
@@ -563,8 +561,9 @@ func (fw *FuzzerWorker) shrinkCallSequence(shrinkRequest ShrinkCallSequenceReque
 }
 
 // run takes a base Chain in a setup state ready for testing, clones it, and begins executing fuzzed transaction calls
-// and asserting properties are upheld. This runs until Fuzzer.ctx cancels the operation.
-// Returns a boolean indicating whether Fuzzer.ctx has indicated we cancel the operation, and an error if one occurred.
+// and asserting properties are upheld. This runs until Fuzzer.ctx or Fuzzer.emergencyCtx cancels the operation.
+// Returns a boolean indicating whether Fuzzer.ctx or Fuzzer.emergencyCtx has indicated we cancel the operation, and an
+// error if one occurred.
 func (fw *FuzzerWorker) run(baseTestChain *chain.TestChain) (bool, error) {
 	// Clone our chain, attaching our necessary components for fuzzing post-genesis, prior to all blocks being copied.
 	// This means any tracers added or events subscribed to within this inner function are done so prior to chain
@@ -600,7 +599,7 @@ func (fw *FuzzerWorker) run(baseTestChain *chain.TestChain) (bool, error) {
 	// Defer the closing of the test chain object
 	defer fw.chain.Close()
 
-	// Emit an event indicating the worker has setup its chain.
+	// Emit an event indicating the worker has set up its chain.
 	err = fw.Events.FuzzerWorkerChainSetup.Publish(FuzzerWorkerChainSetupEvent{
 		Worker: fw,
 		Chain:  fw.chain,
@@ -624,11 +623,16 @@ func (fw *FuzzerWorker) run(baseTestChain *chain.TestChain) (bool, error) {
 	sequencesTested := 0
 	fuzzingCancelled := false
 	for sequencesTested <= fw.fuzzer.config.Fuzzing.WorkerResetLimit {
-		// If our context signaled to close the operation, we will complete shrinking any outstanding shrink requests
-		// and then return immediately
+		// Immediately exit if the emergency context is triggered
+		if utils.CheckContextDone(fw.fuzzer.emergencyCtx) {
+			return true, nil
+		}
+
+		// If our main context signaled to close the operation, we will emit an event notifying any subscribers that
+		// this fuzzer worker is going to be shut down. This allows any subscriber (e.g. the optimization provider)
+		// one last opportunity to shrink a call sequence if necessary.
 		if utils.CheckContextDone(fw.fuzzer.ctx) {
 			fuzzingCancelled = true
-			// TODO: First figure out if there is error within the context
 			err := fw.Events.TestingComplete.Publish(FuzzerWorkerTestingCompleteEvent{
 				Worker: fw,
 			})
@@ -646,16 +650,14 @@ func (fw *FuzzerWorker) run(baseTestChain *chain.TestChain) (bool, error) {
 			}
 		}
 
-		// If we have cancelled fuzzing, return immediately
+		// Clean up the shrink requests
+		fw.shrinkCallSequenceRequests = nil
+
+		// If we have cancelled fuzzing, return now
 		if fuzzingCancelled {
 			return true, nil
 		}
 
-		// Need to reset the shrink call sequence request array if it's a non-zero length
-		if len(fw.shrinkCallSequenceRequests) > 0 {
-			fw.shrinkCallSequenceRequests = make([]ShrinkCallSequenceRequest, 0)
-		}
-
 		// Emit an event indicating the worker is about to test a new call sequence.
 		err := fw.Events.CallSequenceTesting.Publish(FuzzerWorkerCallSequenceTestingEvent{
 			Worker: fw,

fuzzing/test_case_optimization_provider.go

@@ -162,11 +162,6 @@ func (t *OptimizationTestCaseProvider) onFuzzerStarting(event FuzzerStartingEven
 // have been destroyed. It clears state tracked for each FuzzerWorker and sets test cases in "running" states to
 // "passed".
 func (t *OptimizationTestCaseProvider) onFuzzerStopping(event FuzzerStoppingEvent) error {
-	// If there is no error, we request a shrink request
-	if event.err == nil {
-		// TODO: Request a shrink request
-	}
-
 	// Clear our optimization test methods
 	t.workerStates = nil
 
@@ -197,19 +192,19 @@ func (t *OptimizationTestCaseProvider) onWorkerCreated(event FuzzerWorkerCreated
 }
 
 // onWorkerTestingComplete is the event handler triggered when a FuzzerWorker has completed testing of call sequences
-// and is about to exit the fuzzing loop. We use this event to attach a shrink request to the worker, if a request exists.
+// and is about to exit the fuzzing loop. We use this event to attach shrink requests to the worker.
 // This way we are only shrinking once throughout the entire fuzzing campaign in optimization mode.
 func (t *OptimizationTestCaseProvider) onWorkerTestingComplete(event FuzzerWorkerTestingCompleteEvent) error {
-	if t.shrinkCallSequenceRequest != nil {
-		shrunkenSequence, err := event.Worker.shrinkCallSequence(*t.shrinkCallSequenceRequest)
-		if err != nil {
-			return err
+	// Iterate across each test case to see if there is a shrink request for it
+	t.testCasesLock.Lock()
+	for _, testCase := range t.testCases {
+		// We have a shrink request, let's send it to the fuzzer worker
+		if testCase.shrinkCallSequenceRequest != nil {
+			event.Worker.shrinkCallSequenceRequests = append(event.Worker.shrinkCallSequenceRequests, *testCase.shrinkCallSequenceRequest)
+			testCase.shrinkCallSequenceRequest = nil
 		}
-
-		// Reset it to nil so that only the first worker that is trying to exit is requested to handle the shrink request
-		// The other workers can exit gracefully
-		t.shrinkCallSequenceRequest = nil
 	}
+	t.testCasesLock.Unlock()
 	return nil
 }
 
@@ -289,10 +284,6 @@ func (t *OptimizationTestCaseProvider) onWorkerDeployedContractDeleted(event Fuz
 // and any underlying FuzzerWorker. It is called after every call made in a call sequence. It checks whether any
 // optimization test's value has increased.
 func (t *OptimizationTestCaseProvider) callSequencePostCallTest(worker *FuzzerWorker, callSequence calls.CallSequence) ([]ShrinkCallSequenceRequest, error) {
-	// Create a list of shrink call sequence verifiers, which we populate for each maximized optimization test we want a call
-	// sequence shrunk for.
-	shrinkRequests := make([]ShrinkCallSequenceRequest, 0)
-
 	// Obtain the test provider state for this worker
 	workerState := &t.workerStates[worker.WorkerIndex()]
 
@@ -312,9 +303,6 @@ func (t *OptimizationTestCaseProvider) callSequencePostCallTest(worker *FuzzerWo
 
 		// If we updated the test case's maximum value, we update our state immediately. We provide a shrink verifier which will update
 		// the call sequence for each shrunken sequence provided that still it maintains the maximum value.
-		// TODO: This is very inefficient since this runs every time a new max value is found. It would be ideal if we
-		//  could perform a one-time shrink request. This code should be refactored when we introduce the high-level
-		//  testing API.
 		if newValue.Cmp(testCase.value) == 1 {
 			// Create a request to shrink this call sequence.
 			shrinkRequest := ShrinkCallSequenceRequest{
@@ -372,11 +360,9 @@ func (t *OptimizationTestCaseProvider) callSequencePostCallTest(worker *FuzzerWo
 				},
 				RecordResultInCorpus: true,
 			}
-
-			// Add our shrink request to our list.
-			shrinkRequests = append(shrinkRequests, shrinkRequest)
+			testCase.shrinkCallSequenceRequest = &shrinkRequest
 		}
 	}
 
-	return shrinkRequests, nil
+	return nil, nil
 }