Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add timeouts to fuzz testing #9394

Merged
merged 11 commits into from
Nov 29, 2024
5 changes: 5 additions & 0 deletions crates/config/src/fuzz.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ pub struct FuzzConfig {
pub failure_persist_file: Option<String>,
/// show `console.log` in fuzz test, defaults to `false`
pub show_logs: bool,
/// Optional timeout (in seconds) for each property test
pub timeout: Option<u32>,
DaniPopes marked this conversation as resolved.
Show resolved Hide resolved
}

impl Default for FuzzConfig {
Expand All @@ -45,6 +47,7 @@ impl Default for FuzzConfig {
failure_persist_dir: None,
failure_persist_file: None,
show_logs: false,
timeout: None,
}
}
}
Expand All @@ -61,6 +64,7 @@ impl FuzzConfig {
failure_persist_dir: Some(cache_dir),
failure_persist_file: Some("failures".to_string()),
show_logs: false,
timeout: None,
}
}
}
Expand Down Expand Up @@ -90,6 +94,7 @@ impl InlineConfigParser for FuzzConfig {
}
"failure-persist-file" => conf_clone.failure_persist_file = Some(value),
"show-logs" => conf_clone.show_logs = parse_config_bool(key, value)?,
"timeout" => conf_clone.timeout = Some(parse_config_u32(key, value)?),
_ => Err(InlineConfigParserError::InvalidConfigProperty(key))?,
}
}
Expand Down
5 changes: 5 additions & 0 deletions crates/config/src/invariant.rs
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ pub struct InvariantConfig {
pub failure_persist_dir: Option<PathBuf>,
/// Whether to collect and display fuzzed selectors metrics.
pub show_metrics: bool,
/// Optional timeout (in seconds) for each invariant test.
pub timeout: Option<u32>,
}

impl Default for InvariantConfig {
Expand All @@ -51,6 +53,7 @@ impl Default for InvariantConfig {
gas_report_samples: 256,
failure_persist_dir: None,
show_metrics: false,
timeout: None,
}
}
}
Expand All @@ -69,6 +72,7 @@ impl InvariantConfig {
gas_report_samples: 256,
failure_persist_dir: Some(cache_dir),
show_metrics: false,
timeout: None,
}
}

Expand Down Expand Up @@ -108,6 +112,7 @@ impl InlineConfigParser for InvariantConfig {
}
"shrink-run-limit" => conf_clone.shrink_run_limit = parse_config_u32(key, value)?,
"show-metrics" => conf_clone.show_metrics = parse_config_bool(key, value)?,
"timeout" => conf_clone.timeout = Some(parse_config_u32(key, value)?),
_ => Err(InlineConfigParserError::InvalidConfigProperty(key.to_string()))?,
}
}
Expand Down
33 changes: 24 additions & 9 deletions crates/evm/evm/src/executors/fuzz/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,19 @@ impl FuzzedExecutor {
let max_traces_to_collect = std::cmp::max(1, self.config.gas_report_samples) as usize;
let show_logs = self.config.show_logs;

// Start a timer if timeout is set.
let start_time = self.config.timeout.map(|timeout| {
(std::time::Instant::now(), std::time::Duration::from_secs(timeout.into()))
});

let run_result = self.runner.clone().run(&strategy, |calldata| {
// Check if the timeout has been reached.
if let Some((start_time, timeout)) = start_time {
if start_time.elapsed() > timeout {
DaniPopes marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess since this is opt-in, checking via elapsed for now is fine, and we can track improvements separately, but I'd suggest to wrap this if let Some into a helper type like Timeout(Option<Instant>) and an is_timed_out then this becomes easier to change

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mattsse I added FuzzTestTimer with da87095 please check

return Err(TestCaseError::fail("Timeout reached"));
}
}

let fuzz_res = self.single_fuzz(address, should_fail, calldata)?;

// If running with progress then increment current run.
Expand Down Expand Up @@ -193,17 +205,20 @@ impl FuzzedExecutor {
}
Err(TestError::Fail(reason, _)) => {
let reason = reason.to_string();
result.reason = (!reason.is_empty()).then_some(reason);

let args = if let Some(data) = calldata.get(4..) {
func.abi_decode_input(data, false).unwrap_or_default()
if reason == "Timeout reached" {
smartcontracts marked this conversation as resolved.
Show resolved Hide resolved
// If the reason is a timeout, we consider the fuzz test successful.
result.success = true;
} else {
vec![]
};
result.reason = (!reason.is_empty()).then_some(reason);
let args = calldata
.get(..4)
.and_then(|data| func.abi_decode_input(data, false).ok())
.unwrap_or_default();

result.counterexample = Some(CounterExample::Single(
BaseCounterExample::from_fuzz_call(calldata, args, call.traces),
));
result.counterexample = Some(CounterExample::Single(
BaseCounterExample::from_fuzz_call(calldata, args, call.traces),
));
}
}
}

Expand Down
15 changes: 15 additions & 0 deletions crates/evm/evm/src/executors/invariant/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,22 @@ impl<'a> InvariantExecutor<'a> {
let (invariant_test, invariant_strategy) =
self.prepare_test(&invariant_contract, fuzz_fixtures)?;

// Start a timer if timeout is set.
let start_time = self.config.timeout.map(|timeout| {
(std::time::Instant::now(), std::time::Duration::from_secs(timeout.into()))
});

let _ = self.runner.run(&invariant_strategy, |first_input| {
// Check if the timeout has been reached.
if let Some((start_time, timeout)) = start_time {
grandizzy marked this conversation as resolved.
Show resolved Hide resolved
if start_time.elapsed() > timeout {
// Since we never record a revert here the test is still considered successful
// even though it timed out. We *want* this behavior for now, so that's ok, but
// future developers should be aware of this.
return Err(TestCaseError::fail("Timeout reached"));
}
}

// Create current invariant run data.
let mut current_run = InvariantTestRun::new(
first_input,
Expand Down
7 changes: 7 additions & 0 deletions crates/forge/bin/cmd/test/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,10 @@ pub struct TestArgs {
#[arg(long, env = "FOUNDRY_FUZZ_RUNS", value_name = "RUNS")]
pub fuzz_runs: Option<u64>,

/// Timeout for each fuzz run in seconds.
#[arg(long, env = "FOUNDRY_FUZZ_TIMEOUT", value_name = "TIMEOUT")]
pub fuzz_timeout: Option<u64>,

/// File to rerun fuzz failures from.
#[arg(long)]
pub fuzz_input_file: Option<String>,
Expand Down Expand Up @@ -870,6 +874,9 @@ impl Provider for TestArgs {
if let Some(fuzz_runs) = self.fuzz_runs {
fuzz_dict.insert("runs".to_string(), fuzz_runs.into());
}
if let Some(fuzz_timeout) = self.fuzz_timeout {
fuzz_dict.insert("timeout".to_string(), fuzz_timeout.into());
}
if let Some(fuzz_input_file) = self.fuzz_input_file.clone() {
fuzz_dict.insert("failure_persist_file".to_string(), fuzz_input_file.into());
}
Expand Down
Loading