From d349318bd412627c7b41245a7a7434794dbf5406 Mon Sep 17 00:00:00 2001 From: Joe Caulfield Date: Mon, 13 Jan 2025 15:53:32 +0800 Subject: [PATCH 1/6] make fixture methods mut self --- harness/src/lib.rs | 42 +++++++++++++++++++++++++++++++- harness/tests/process_fixture.rs | 12 ++++----- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/harness/src/lib.rs b/harness/src/lib.rs index 27fd81b3..7badd9bc 100644 --- a/harness/src/lib.rs +++ b/harness/src/lib.rs @@ -384,7 +384,18 @@ impl Mollusk { /// Fixtures provide an API to `decode` a raw blob, as well as read /// fixtures from files. Those fixtures can then be provided to this /// function to process them and get a Mollusk result. - pub fn process_fixture(fixture: &mollusk_svm_fuzz_fixture::Fixture) -> InstructionResult { + /// + /// Note: This is a mutable method on `Mollusk`, since loading a fixture + /// into the test environment will alter `Mollusk` values, such as compute + /// budget and sysvars. However, the program cache remains unchanged. + /// + /// Therefore, developers can provision a `Mollusk` instance, set up their + /// desired program cache, and then run a series of fixtures against that + /// `Mollusk` instance (and cache). + pub fn process_fixture( + &mut self, + fixture: &mollusk_svm_fuzz_fixture::Fixture, + ) -> InstructionResult { let (mollusk, instruction, accounts, _) = fuzz::mollusk::load_fixture(fixture); mollusk.process_instruction(&instruction, &accounts) } @@ -396,7 +407,17 @@ impl Mollusk { /// Fixtures provide an API to `decode` a raw blob, as well as read /// fixtures from files. Those fixtures can then be provided to this /// function to process them and get a Mollusk result. + /// + /// + /// Note: This is a mutable method on `Mollusk`, since loading a fixture + /// into the test environment will alter `Mollusk` values, such as compute + /// budget and sysvars. However, the program cache remains unchanged. + /// + /// Therefore, developers can provision a `Mollusk` instance, set up their + /// desired program cache, and then run a series of fixtures against that + /// `Mollusk` instance (and cache). pub fn process_and_validate_fixture( + &mut self, fixture: &mollusk_svm_fuzz_fixture::Fixture, ) -> InstructionResult { let (mollusk, instruction, accounts, result) = fuzz::mollusk::load_fixture(fixture); @@ -412,7 +433,16 @@ impl Mollusk { /// Fixtures provide an API to `decode` a raw blob, as well as read /// fixtures from files. Those fixtures can then be provided to this /// function to process them and get a Mollusk result. + /// + /// Note: This is a mutable method on `Mollusk`, since loading a fixture + /// into the test environment will alter `Mollusk` values, such as compute + /// budget and sysvars. However, the program cache remains unchanged. + /// + /// Therefore, developers can provision a `Mollusk` instance, set up their + /// desired program cache, and then run a series of fixtures against that + /// `Mollusk` instance (and cache). pub fn process_firedancer_fixture( + &mut self, fixture: &mollusk_svm_fuzz_fixture_firedancer::Fixture, ) -> InstructionResult { let (mollusk, instruction, accounts, _) = @@ -428,7 +458,17 @@ impl Mollusk { /// Fixtures provide an API to `decode` a raw blob, as well as read /// fixtures from files. Those fixtures can then be provided to this /// function to process them and get a Mollusk result. + /// + /// + /// Note: This is a mutable method on `Mollusk`, since loading a fixture + /// into the test environment will alter `Mollusk` values, such as compute + /// budget and sysvars. However, the program cache remains unchanged. + /// + /// Therefore, developers can provision a `Mollusk` instance, set up their + /// desired program cache, and then run a series of fixtures against that + /// `Mollusk` instance (and cache). pub fn process_and_validate_firedancer_fixture( + &mut self, fixture: &mollusk_svm_fuzz_fixture_firedancer::Fixture, ) -> InstructionResult { let (mollusk, instruction, accounts, result) = diff --git a/harness/tests/process_fixture.rs b/harness/tests/process_fixture.rs index 5cb7bb3c..88f8e557 100644 --- a/harness/tests/process_fixture.rs +++ b/harness/tests/process_fixture.rs @@ -13,7 +13,7 @@ fn test_process_mollusk() { let ok_transfer_amount = 42_000; let too_much = BASE_LAMPORTS + 1; - let mollusk = Mollusk::default(); + let mut mollusk = Mollusk::default(); let sender = Pubkey::new_unique(); let recipient = Pubkey::new_unique(); @@ -41,7 +41,7 @@ fn test_process_mollusk() { &[], ); - Mollusk::process_and_validate_fixture(&fixture); + mollusk.process_and_validate_fixture(&fixture); // Now the error case. let instruction = system_instruction::transfer(&sender, &recipient, too_much); @@ -55,7 +55,7 @@ fn test_process_mollusk() { &[], ); - Mollusk::process_and_validate_fixture(&fixture); + mollusk.process_and_validate_fixture(&fixture); } #[cfg(feature = "fuzz-fd")] @@ -64,7 +64,7 @@ fn test_process_firedancer() { let ok_transfer_amount = 42_000; let too_much = BASE_LAMPORTS + 1; - let mollusk = Mollusk::default(); + let mut mollusk = Mollusk::default(); let sender = Pubkey::new_unique(); let recipient = Pubkey::new_unique(); @@ -92,7 +92,7 @@ fn test_process_firedancer() { &[], ); - Mollusk::process_and_validate_firedancer_fixture(&fixture); + mollusk.process_and_validate_firedancer_fixture(&fixture); // Now the error case. let instruction = system_instruction::transfer(&sender, &recipient, too_much); @@ -106,5 +106,5 @@ fn test_process_firedancer() { &[], ); - Mollusk::process_and_validate_firedancer_fixture(&fixture); + mollusk.process_and_validate_firedancer_fixture(&fixture); } From 46e5b409b91237f7d1af85e660e13a1d10911f4b Mon Sep 17 00:00:00 2001 From: Joe Caulfield Date: Mon, 13 Jan 2025 16:35:23 +0800 Subject: [PATCH 2/6] refactor fixture APIs --- harness/src/fuzz/firedancer.rs | 75 +++++++++++++++++--------------- harness/src/fuzz/mollusk.rs | 64 ++++++++++++++------------- harness/src/lib.rs | 59 +++++++++++++++++++------ harness/tests/fd_test_vectors.rs | 25 +++++++++-- 4 files changed, 141 insertions(+), 82 deletions(-) diff --git a/harness/src/fuzz/firedancer.rs b/harness/src/fuzz/firedancer.rs index 705978c8..c56ff261 100644 --- a/harness/src/fuzz/firedancer.rs +++ b/harness/src/fuzz/firedancer.rs @@ -22,6 +22,7 @@ use { solana_compute_budget::compute_budget::ComputeBudget, solana_sdk::{ account::Account, + feature_set::FeatureSet, instruction::{AccountMeta, Instruction, InstructionError}, pubkey::Pubkey, }, @@ -66,17 +67,12 @@ fn num_to_instr_err(num: i32, custom_code: u32) -> InstructionError { } fn build_fixture_context( - mollusk: &Mollusk, - instruction: &Instruction, accounts: &[(Pubkey, Account)], + compute_budget: &ComputeBudget, + feature_set: &FeatureSet, + instruction: &Instruction, + slot: u64, ) -> FuzzContext { - let Mollusk { - compute_budget, - feature_set, - slot, // FD-Fuzz feature only. - .. - } = mollusk; - let loader_key = if BUILTIN_PROGRAM_IDS.contains(&instruction.program_id) { solana_sdk::native_loader::id() } else { @@ -100,14 +96,22 @@ fn build_fixture_context( instruction_accounts, instruction_data: instruction.data.clone(), compute_units_available: compute_budget.compute_unit_limit, - slot_context: FuzzSlotContext { slot: *slot }, + slot_context: FuzzSlotContext { slot }, epoch_context: FuzzEpochContext { feature_set: feature_set.clone(), }, } } -fn parse_fixture_context(context: &FuzzContext) -> (Mollusk, Instruction, Vec<(Pubkey, Account)>) { +pub struct ParsedFixtureContext { + pub accounts: Vec<(Pubkey, Account)>, + pub compute_budget: ComputeBudget, + pub feature_set: FeatureSet, + pub instruction: Instruction, + pub slot: u64, +} + +pub(crate) fn parse_fixture_context(context: &FuzzContext) -> ParsedFixtureContext { let FuzzContext { program_id, accounts, @@ -128,13 +132,6 @@ fn parse_fixture_context(context: &FuzzContext) -> (Mollusk, Instruction, Vec<(P .map(|(key, acct, _)| (*key, acct.clone())) .collect::>(); - let mollusk = Mollusk { - compute_budget, - feature_set: epoch_context.feature_set.clone(), - slot: slot_context.slot, - ..Default::default() - }; - let metas = instruction_accounts .iter() .map(|ia| { @@ -152,7 +149,13 @@ fn parse_fixture_context(context: &FuzzContext) -> (Mollusk, Instruction, Vec<(P let instruction = Instruction::new_with_bytes(*program_id, instruction_data, metas); - (mollusk, instruction, accounts) + ParsedFixtureContext { + accounts, + compute_budget, + feature_set: epoch_context.feature_set.clone(), + instruction, + slot: slot_context.slot, + } } fn build_fixture_effects(context: &FuzzContext, result: &InstructionResult) -> FuzzEffects { @@ -195,9 +198,9 @@ fn build_fixture_effects(context: &FuzzContext, result: &InstructionResult) -> F } } -fn parse_fixture_effects( - mollusk: &Mollusk, +pub(crate) fn parse_fixture_effects( accounts: &[(Pubkey, Account)], + compute_unit_limit: u64, effects: &FuzzEffects, ) -> InstructionResult { let raw_result = if effects.program_result == 0 { @@ -229,10 +232,7 @@ fn parse_fixture_effects( program_result, raw_result, execution_time: 0, // TODO: Omitted for now. - compute_units_consumed: mollusk - .compute_budget - .compute_unit_limit - .saturating_sub(effects.compute_units_available), + compute_units_consumed: compute_unit_limit.saturating_sub(effects.compute_units_available), return_data, resulting_accounts, } @@ -252,7 +252,13 @@ pub fn build_fixture_from_mollusk_test( result: &InstructionResult, _checks: &[Check], ) -> FuzzFixture { - let input = build_fixture_context(mollusk, instruction, accounts); + let input = build_fixture_context( + accounts, + &mollusk.compute_budget, + &mollusk.feature_set, + instruction, + mollusk.slot, // FD-fuzz feature only. + ); // This should probably be built from the checks, but there's currently no // mechanism to enforce full check coverage on a result. let output = build_fixture_effects(&input, result); @@ -265,15 +271,14 @@ pub fn build_fixture_from_mollusk_test( pub fn load_firedancer_fixture( fixture: &mollusk_svm_fuzz_fixture_firedancer::Fixture, -) -> ( - Mollusk, - Instruction, - Vec<(Pubkey, Account)>, - InstructionResult, -) { - let (mollusk, instruction, accounts) = parse_fixture_context(&fixture.input); - let result = parse_fixture_effects(&mollusk, &accounts, &fixture.output); - (mollusk, instruction, accounts, result) +) -> (ParsedFixtureContext, InstructionResult) { + let parsed = parse_fixture_context(&fixture.input); + let result = parse_fixture_effects( + &parsed.accounts, + parsed.compute_budget.compute_unit_limit, + &fixture.output, + ); + (parsed, result) } #[test] diff --git a/harness/src/fuzz/mollusk.rs b/harness/src/fuzz/mollusk.rs index d13560d7..f77802d7 100644 --- a/harness/src/fuzz/mollusk.rs +++ b/harness/src/fuzz/mollusk.rs @@ -14,8 +14,10 @@ use { context::Context as FuzzContext, effects::Effects as FuzzEffects, sysvars::Sysvars as FuzzSysvars, Fixture as FuzzFixture, }, + solana_compute_budget::compute_budget::ComputeBudget, solana_sdk::{ account::Account, + feature_set::FeatureSet, instruction::{Instruction, InstructionError}, pubkey::Pubkey, slot_hashes::SlotHashes, @@ -103,18 +105,21 @@ impl From<&FuzzEffects> for InstructionResult { } } +pub struct ParsedFixtureContext { + pub accounts: Vec<(Pubkey, Account)>, + pub compute_budget: ComputeBudget, + pub feature_set: FeatureSet, + pub instruction: Instruction, + pub sysvars: Sysvars, +} + fn build_fixture_context( - mollusk: &Mollusk, - instruction: &Instruction, accounts: &[(Pubkey, Account)], + compute_budget: &ComputeBudget, + feature_set: &FeatureSet, + instruction: &Instruction, + sysvars: &Sysvars, ) -> FuzzContext { - let Mollusk { - compute_budget, - feature_set, - sysvars, - .. - } = mollusk; - let instruction_accounts = instruction.accounts.clone(); let instruction_data = instruction.data.clone(); let accounts = accounts.to_vec(); @@ -130,7 +135,7 @@ fn build_fixture_context( } } -fn parse_fixture_context(context: &FuzzContext) -> (Mollusk, Instruction, Vec<(Pubkey, Account)>) { +pub(crate) fn parse_fixture_context(context: &FuzzContext) -> ParsedFixtureContext { let FuzzContext { compute_budget, feature_set, @@ -141,19 +146,16 @@ fn parse_fixture_context(context: &FuzzContext) -> (Mollusk, Instruction, Vec<(P accounts, } = context; - let mollusk = Mollusk { - compute_budget: *compute_budget, - feature_set: feature_set.clone(), - sysvars: sysvars.into(), - ..Default::default() - }; - let instruction = Instruction::new_with_bytes(*program_id, instruction_data, instruction_accounts.clone()); - let accounts = accounts.clone(); - - (mollusk, instruction, accounts) + ParsedFixtureContext { + accounts: accounts.clone(), + compute_budget: *compute_budget, + feature_set: feature_set.clone(), + instruction, + sysvars: sysvars.into(), + } } pub fn build_fixture_from_mollusk_test( @@ -163,7 +165,13 @@ pub fn build_fixture_from_mollusk_test( result: &InstructionResult, _checks: &[Check], ) -> FuzzFixture { - let input = build_fixture_context(mollusk, instruction, accounts); + let input = build_fixture_context( + accounts, + &mollusk.compute_budget, + &mollusk.feature_set, + instruction, + &mollusk.sysvars, + ); // This should probably be built from the checks, but there's currently no // mechanism to enforce full check coverage on a result. let output = FuzzEffects::from(result); @@ -172,13 +180,9 @@ pub fn build_fixture_from_mollusk_test( pub fn load_fixture( fixture: &mollusk_svm_fuzz_fixture::Fixture, -) -> ( - Mollusk, - Instruction, - Vec<(Pubkey, Account)>, - InstructionResult, -) { - let (mollusk, instruction, accounts) = parse_fixture_context(&fixture.input); - let result = InstructionResult::from(&fixture.output); - (mollusk, instruction, accounts, result) +) -> (ParsedFixtureContext, InstructionResult) { + ( + parse_fixture_context(&fixture.input), + InstructionResult::from(&fixture.output), + ) } diff --git a/harness/src/lib.rs b/harness/src/lib.rs index 7badd9bc..c630a34f 100644 --- a/harness/src/lib.rs +++ b/harness/src/lib.rs @@ -396,8 +396,17 @@ impl Mollusk { &mut self, fixture: &mollusk_svm_fuzz_fixture::Fixture, ) -> InstructionResult { - let (mollusk, instruction, accounts, _) = fuzz::mollusk::load_fixture(fixture); - mollusk.process_instruction(&instruction, &accounts) + let fuzz::mollusk::ParsedFixtureContext { + accounts, + compute_budget, + feature_set, + instruction, + sysvars, + } = fuzz::mollusk::parse_fixture_context(&fixture.input); + self.compute_budget = compute_budget; + self.feature_set = feature_set; + self.sysvars = sysvars; + self.process_instruction(&instruction, &accounts) } #[cfg(feature = "fuzz")] @@ -420,10 +429,9 @@ impl Mollusk { &mut self, fixture: &mollusk_svm_fuzz_fixture::Fixture, ) -> InstructionResult { - let (mollusk, instruction, accounts, result) = fuzz::mollusk::load_fixture(fixture); - let this_result = mollusk.process_instruction(&instruction, &accounts); - result.compare(&this_result); - this_result + let result = self.process_fixture(fixture); + InstructionResult::from(&fixture.output).compare(&result); + result } #[cfg(feature = "fuzz-fd")] @@ -445,9 +453,17 @@ impl Mollusk { &mut self, fixture: &mollusk_svm_fuzz_fixture_firedancer::Fixture, ) -> InstructionResult { - let (mollusk, instruction, accounts, _) = - fuzz::firedancer::load_firedancer_fixture(fixture); - mollusk.process_instruction(&instruction, &accounts) + let fuzz::firedancer::ParsedFixtureContext { + accounts, + compute_budget, + feature_set, + instruction, + slot, + } = fuzz::firedancer::parse_fixture_context(&fixture.input); + self.compute_budget = compute_budget; + self.feature_set = feature_set; + self.slot = slot; + self.process_instruction(&instruction, &accounts) } #[cfg(feature = "fuzz-fd")] @@ -471,10 +487,25 @@ impl Mollusk { &mut self, fixture: &mollusk_svm_fuzz_fixture_firedancer::Fixture, ) -> InstructionResult { - let (mollusk, instruction, accounts, result) = - fuzz::firedancer::load_firedancer_fixture(fixture); - let this_result = mollusk.process_instruction(&instruction, &accounts); - result.compare(&this_result); - this_result + let fuzz::firedancer::ParsedFixtureContext { + accounts, + compute_budget, + feature_set, + instruction, + slot, + } = fuzz::firedancer::parse_fixture_context(&fixture.input); + self.compute_budget = compute_budget; + self.feature_set = feature_set; + self.slot = slot; + + let result = self.process_instruction(&instruction, &accounts); + let expected_result = fuzz::firedancer::parse_fixture_effects( + &accounts, + self.compute_budget.compute_unit_limit, + &fixture.output, + ); + + expected_result.compare(&result); + result } } diff --git a/harness/tests/fd_test_vectors.rs b/harness/tests/fd_test_vectors.rs index a702a151..c6830fb9 100644 --- a/harness/tests/fd_test_vectors.rs +++ b/harness/tests/fd_test_vectors.rs @@ -1,7 +1,12 @@ #![cfg(feature = "fuzz-fd")] use { - mollusk_svm::fuzz::firedancer::{build_fixture_from_mollusk_test, load_firedancer_fixture}, + mollusk_svm::{ + fuzz::firedancer::{ + build_fixture_from_mollusk_test, load_firedancer_fixture, ParsedFixtureContext, + }, + Mollusk, + }, mollusk_svm_fuzz_fixture_firedancer::{account::SeedAddress, Fixture}, rayon::prelude::*, solana_sdk::{ @@ -47,8 +52,22 @@ fn test_load_firedancer_fixtures() { let path = entry.unwrap().path(); if path.is_file() && path.extension().is_some_and(|ext| ext == "fix") { let loaded_fixture = Fixture::load_from_blob_file(path.to_str().unwrap()); - let (mollusk, instruction, accounts, result) = - load_firedancer_fixture(&loaded_fixture); + let ( + ParsedFixtureContext { + accounts, + compute_budget, + feature_set, + instruction, + slot, + }, + result, + ) = load_firedancer_fixture(&loaded_fixture); + let mollusk = Mollusk { + compute_budget, + feature_set, + slot, + ..Default::default() + }; let generated_fixture = build_fixture_from_mollusk_test( &mollusk, &instruction, From e88fceee7e420d6aabb6bdbec5efdeff6f409c2e Mon Sep 17 00:00:00 2001 From: Joe Caulfield Date: Mon, 13 Jan 2025 16:43:33 +0800 Subject: [PATCH 3/6] make run_checks pub --- harness/src/result.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/harness/src/result.rs b/harness/src/result.rs index 320523e8..9d42f619 100644 --- a/harness/src/result.rs +++ b/harness/src/result.rs @@ -84,7 +84,7 @@ impl InstructionResult { } /// Perform checks on the instruction result, panicking if any checks fail. - pub(crate) fn run_checks(&self, checks: &[Check]) { + pub fn run_checks(&self, checks: &[Check]) { for check in checks { match &check.check { CheckType::ComputeUnitsConsumed(units) => { From a264bb7b2e28902069337729fc23f3b7635de4df Mon Sep 17 00:00:00 2001 From: Joe Caulfield Date: Mon, 13 Jan 2025 16:59:38 +0800 Subject: [PATCH 4/6] remove unused checks arg --- harness/src/fuzz/firedancer.rs | 3 +-- harness/src/fuzz/mod.rs | 24 +++++------------------- harness/src/fuzz/mollusk.rs | 3 +-- harness/src/lib.rs | 2 +- harness/tests/fd_test_vectors.rs | 9 ++------- harness/tests/process_fixture.rs | 4 ---- 6 files changed, 10 insertions(+), 35 deletions(-) diff --git a/harness/src/fuzz/firedancer.rs b/harness/src/fuzz/firedancer.rs index c56ff261..77497592 100644 --- a/harness/src/fuzz/firedancer.rs +++ b/harness/src/fuzz/firedancer.rs @@ -7,7 +7,7 @@ use { crate::{ accounts::{compile_accounts, CompiledAccounts}, - result::{Check, InstructionResult}, + result::InstructionResult, Mollusk, DEFAULT_LOADER_KEY, }, mollusk_svm_fuzz_fixture_firedancer::{ @@ -250,7 +250,6 @@ pub fn build_fixture_from_mollusk_test( instruction: &Instruction, accounts: &[(Pubkey, Account)], result: &InstructionResult, - _checks: &[Check], ) -> FuzzFixture { let input = build_fixture_context( accounts, diff --git a/harness/src/fuzz/mod.rs b/harness/src/fuzz/mod.rs index 20ff8889..4c32c47c 100644 --- a/harness/src/fuzz/mod.rs +++ b/harness/src/fuzz/mod.rs @@ -4,10 +4,7 @@ pub mod firedancer; pub mod mollusk; use { - crate::{ - result::{Check, InstructionResult}, - Mollusk, - }, + crate::{result::InstructionResult, Mollusk}, mollusk_svm_fuzz_fs::FsHandler, solana_sdk::{account::Account, instruction::Instruction, pubkey::Pubkey}, }; @@ -17,20 +14,14 @@ pub fn generate_fixtures_from_mollusk_test( instruction: &Instruction, accounts: &[(Pubkey, Account)], result: &InstructionResult, - checks: &[Check], ) { #[cfg(feature = "fuzz")] { if std::env::var("EJECT_FUZZ_FIXTURES").is_ok() || std::env::var("EJECT_FUZZ_FIXTURES_JSON").is_ok() { - let fixture = mollusk::build_fixture_from_mollusk_test( - mollusk, - instruction, - accounts, - result, - checks, - ); + let fixture = + mollusk::build_fixture_from_mollusk_test(mollusk, instruction, accounts, result); let handler = FsHandler::new(fixture); if let Ok(blob_dir) = std::env::var("EJECT_FUZZ_FIXTURES") { handler.dump_to_blob_file(&blob_dir); @@ -46,13 +37,8 @@ pub fn generate_fixtures_from_mollusk_test( if std::env::var("EJECT_FUZZ_FIXTURES_FD").is_ok() || std::env::var("EJECT_FUZZ_FIXTURES_JSON_FD").is_ok() { - let fixture = firedancer::build_fixture_from_mollusk_test( - mollusk, - instruction, - accounts, - result, - checks, - ); + let fixture = + firedancer::build_fixture_from_mollusk_test(mollusk, instruction, accounts, result); let handler = FsHandler::new(fixture); if let Ok(blob_dir) = std::env::var("EJECT_FUZZ_FIXTURES_FD") { handler.dump_to_blob_file(&blob_dir); diff --git a/harness/src/fuzz/mollusk.rs b/harness/src/fuzz/mollusk.rs index f77802d7..fc8b047d 100644 --- a/harness/src/fuzz/mollusk.rs +++ b/harness/src/fuzz/mollusk.rs @@ -6,7 +6,7 @@ use { crate::{ - result::{Check, InstructionResult, ProgramResult}, + result::{InstructionResult, ProgramResult}, sysvar::Sysvars, Mollusk, }, @@ -163,7 +163,6 @@ pub fn build_fixture_from_mollusk_test( instruction: &Instruction, accounts: &[(Pubkey, Account)], result: &InstructionResult, - _checks: &[Check], ) -> FuzzFixture { let input = build_fixture_context( accounts, diff --git a/harness/src/lib.rs b/harness/src/lib.rs index c630a34f..82859991 100644 --- a/harness/src/lib.rs +++ b/harness/src/lib.rs @@ -320,7 +320,7 @@ impl Mollusk { let result = self.process_instruction(instruction, accounts); #[cfg(any(feature = "fuzz", feature = "fuzz-fd"))] - fuzz::generate_fixtures_from_mollusk_test(self, instruction, accounts, &result, checks); + fuzz::generate_fixtures_from_mollusk_test(self, instruction, accounts, &result); result.run_checks(checks); result diff --git a/harness/tests/fd_test_vectors.rs b/harness/tests/fd_test_vectors.rs index c6830fb9..d1f1ea25 100644 --- a/harness/tests/fd_test_vectors.rs +++ b/harness/tests/fd_test_vectors.rs @@ -68,13 +68,8 @@ fn test_load_firedancer_fixtures() { slot, ..Default::default() }; - let generated_fixture = build_fixture_from_mollusk_test( - &mollusk, - &instruction, - &accounts, - &result, - /* checks */ &[], - ); + let generated_fixture = + build_fixture_from_mollusk_test(&mollusk, &instruction, &accounts, &result); assert_eq!(loaded_fixture.metadata, generated_fixture.metadata); assert_eq!( diff --git a/harness/tests/process_fixture.rs b/harness/tests/process_fixture.rs index 88f8e557..4cc2c0b6 100644 --- a/harness/tests/process_fixture.rs +++ b/harness/tests/process_fixture.rs @@ -38,7 +38,6 @@ fn test_process_mollusk() { &instruction, &accounts, &result, - &[], ); mollusk.process_and_validate_fixture(&fixture); @@ -52,7 +51,6 @@ fn test_process_mollusk() { &instruction, &accounts, &result, - &[], ); mollusk.process_and_validate_fixture(&fixture); @@ -89,7 +87,6 @@ fn test_process_firedancer() { &instruction, &accounts, &result, - &[], ); mollusk.process_and_validate_firedancer_fixture(&fixture); @@ -103,7 +100,6 @@ fn test_process_firedancer() { &instruction, &accounts, &result, - &[], ); mollusk.process_and_validate_firedancer_fixture(&fixture); From cb51af4fe977562d593f6533e3e4af321a8b4ae4 Mon Sep 17 00:00:00 2001 From: Joe Caulfield Date: Mon, 13 Jan 2025 17:52:30 +0800 Subject: [PATCH 5/6] add fixture partial validation API --- harness/src/fuzz/check.rs | 173 ++++++++++++++++++++++++++++++++++++++ harness/src/fuzz/mod.rs | 2 + harness/src/lib.rs | 94 +++++++++++++++++++++ harness/src/result.rs | 11 ++- 4 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 harness/src/fuzz/check.rs diff --git a/harness/src/fuzz/check.rs b/harness/src/fuzz/check.rs new file mode 100644 index 00000000..e590ee5b --- /dev/null +++ b/harness/src/fuzz/check.rs @@ -0,0 +1,173 @@ +//! Checks to run against a fixture when validating. + +use { + crate::result::{Check, InstructionResult}, + solana_sdk::{ + account::{Account, ReadableAccount}, + pubkey::Pubkey, + }, +}; + +/// Checks to run against a fixture when validating. +/// +/// Similar to Mollusk's `result::Check`, this allows a developer to dictate +/// the type of checks to run on the fixture's effects. +/// +/// Keep in mind that validation of fixtures works slightly differently than +/// typical Mollusk unit tests. In a Mollusk test, you can provide the value to +/// compare a portion of the result against (ie. compute units). However, when +/// comparing the result of a Mollusk invocation against a fixture, the value +/// from the fixture itself is used. +/// +/// For that reason, these are unary variants, and do not offer the developer a +/// way to provide values to check against. +pub enum FixtureCheck { + /// Validate compute units consumed. + ComputeUnits, + /// Validate the program result. + ProgramResult, + /// Validate the return data. + ReturnData, + /// Validate all resulting accounts. + AllResultingAccounts { + /// Whether or not to validate each account's data. + data: bool, + /// Whether or not to validate each account's lamports. + lamports: bool, + /// Whether or not to validate each account's owner. + owner: bool, + /// Whether or not to validate each account's space. + space: bool, + }, + /// Validate the resulting accounts at certain addresses. + OnlyResultingAccounts { + /// The addresses on which to apply the validation. + addresses: Vec, + /// Whether or not to validate each account's data. + data: bool, + /// Whether or not to validate each account's lamports. + lamports: bool, + /// Whether or not to validate each account's owner. + owner: bool, + /// Whether or not to validate each account's space. + space: bool, + }, + /// Validate all of the resulting accounts _except_ the provided addresses. + AllResultingAccountsExcept { + /// The addresses on which to _not_ apply the validation. + ignore_addresses: Vec, + /// On non-ignored accounts, whether or not to validate each account's + /// data. + data: bool, + /// On non-ignored accounts, whether or not to validate each account's + /// lamports. + lamports: bool, + /// On non-ignored accounts, whether or not to validate each account's + /// owner. + owner: bool, + /// On non-ignored accounts, whether or not to validate each account's + /// space. + space: bool, + }, +} + +fn add_account_checks<'a>( + checks: &mut Vec>, + accounts: impl Iterator, + data: bool, + lamports: bool, + owner: bool, + space: bool, +) { + for (pubkey, account) in accounts { + let mut builder = Check::account(pubkey); + if data { + builder = builder.data(account.data()); + } + if lamports { + builder = builder.lamports(account.lamports()); + } + if owner { + builder = builder.owner(account.owner()); + } + if space { + builder = builder.space(account.data().len()); + } + checks.push(builder.build()); + } +} + +pub(crate) fn evaluate_results_with_fixture_checks( + expected: &InstructionResult, + result: &InstructionResult, + fixture_checks: &[FixtureCheck], +) { + let mut checks = vec![]; + + for fixture_check in fixture_checks { + match fixture_check { + FixtureCheck::ComputeUnits => { + checks.push(Check::compute_units(expected.compute_units_consumed)) + } + FixtureCheck::ProgramResult => { + checks.push(Check::program_result(expected.program_result.clone())) + } + FixtureCheck::ReturnData => checks.push(Check::return_data(&expected.return_data)), + FixtureCheck::AllResultingAccounts { + data, + lamports, + owner, + space, + } => { + add_account_checks( + &mut checks, + expected.resulting_accounts.iter(), + *data, + *lamports, + *owner, + *space, + ); + } + FixtureCheck::OnlyResultingAccounts { + addresses, + data, + lamports, + owner, + space, + } => { + add_account_checks( + &mut checks, + expected + .resulting_accounts + .iter() + .filter(|(pubkey, _)| addresses.contains(pubkey)), + *data, + *lamports, + *owner, + *space, + ); + } + FixtureCheck::AllResultingAccountsExcept { + ignore_addresses, + data, + lamports, + owner, + space, + } => { + add_account_checks( + &mut checks, + expected + .resulting_accounts + .iter() + .filter(|(pubkey, _)| !ignore_addresses.contains(pubkey)), + *data, + *lamports, + *owner, + *space, + ); + } + } + } + + result.run_checks(&checks); +} diff --git a/harness/src/fuzz/mod.rs b/harness/src/fuzz/mod.rs index 4c32c47c..528cdc53 100644 --- a/harness/src/fuzz/mod.rs +++ b/harness/src/fuzz/mod.rs @@ -1,3 +1,5 @@ +#[cfg(any(feature = "fuzz", feature = "fuzz-fd"))] +pub mod check; #[cfg(feature = "fuzz-fd")] pub mod firedancer; #[cfg(feature = "fuzz")] diff --git a/harness/src/lib.rs b/harness/src/lib.rs index 82859991..3d88ae91 100644 --- a/harness/src/lib.rs +++ b/harness/src/lib.rs @@ -425,6 +425,9 @@ impl Mollusk { /// Therefore, developers can provision a `Mollusk` instance, set up their /// desired program cache, and then run a series of fixtures against that /// `Mollusk` instance (and cache). + /// + /// Note: To compare the result against the entire fixture effects, pass + /// `&[FixtureCheck::All]` for `checks`. pub fn process_and_validate_fixture( &mut self, fixture: &mollusk_svm_fuzz_fixture::Fixture, @@ -434,6 +437,44 @@ impl Mollusk { result } + #[cfg(feature = "fuzz")] + /// a specific set of checks. + /// + /// This is useful for when you may not want to compare the entire effects, + /// such as omitting comparisons of compute units consumed. + /// Process a fuzz fixture using the minified Solana Virtual Machine (SVM) + /// environment and compare the result against the fixture's effects using + /// a specific set of checks. + /// + /// This is useful for when you may not want to compare the entire effects, + /// such as omitting comparisons of compute units consumed. + /// + /// Fixtures provide an API to `decode` a raw blob, as well as read + /// fixtures from files. Those fixtures can then be provided to this + /// function to process them and get a Mollusk result. + /// + /// + /// Note: This is a mutable method on `Mollusk`, since loading a fixture + /// into the test environment will alter `Mollusk` values, such as compute + /// budget and sysvars. However, the program cache remains unchanged. + /// + /// Therefore, developers can provision a `Mollusk` instance, set up their + /// desired program cache, and then run a series of fixtures against that + /// `Mollusk` instance (and cache). + /// + /// Note: To compare the result against the entire fixture effects, pass + /// `&[FixtureCheck::All]` for `checks`. + pub fn process_and_partially_validate_fixture( + &mut self, + fixture: &mollusk_svm_fuzz_fixture::Fixture, + checks: &[fuzz::check::FixtureCheck], + ) -> InstructionResult { + let result = self.process_fixture(fixture); + let expected = InstructionResult::from(&fixture.output); + fuzz::check::evaluate_results_with_fixture_checks(&expected, &result, checks); + result + } + #[cfg(feature = "fuzz-fd")] /// Process a Firedancer fuzz fixture using the minified Solana Virtual /// Machine (SVM) environment. @@ -483,6 +524,9 @@ impl Mollusk { /// Therefore, developers can provision a `Mollusk` instance, set up their /// desired program cache, and then run a series of fixtures against that /// `Mollusk` instance (and cache). + /// + /// Note: To compare the result against the entire fixture effects, pass + /// `&[FixtureCheck::All]` for `checks`. pub fn process_and_validate_firedancer_fixture( &mut self, fixture: &mollusk_svm_fuzz_fixture_firedancer::Fixture, @@ -508,4 +552,54 @@ impl Mollusk { expected_result.compare(&result); result } + + #[cfg(feature = "fuzz-fd")] + /// Process a Firedancer fuzz fixture using the minified Solana Virtual + /// Machine (SVM) environment and compare the result against the + /// fixture's effects using a specific set of checks. + /// + /// This is useful for when you may not want to compare the entire effects, + /// such as omitting comparisons of compute units consumed. + /// + /// Fixtures provide an API to `decode` a raw blob, as well as read + /// fixtures from files. Those fixtures can then be provided to this + /// function to process them and get a Mollusk result. + /// + /// + /// Note: This is a mutable method on `Mollusk`, since loading a fixture + /// into the test environment will alter `Mollusk` values, such as compute + /// budget and sysvars. However, the program cache remains unchanged. + /// + /// Therefore, developers can provision a `Mollusk` instance, set up their + /// desired program cache, and then run a series of fixtures against that + /// `Mollusk` instance (and cache). + /// + /// Note: To compare the result against the entire fixture effects, pass + /// `&[FixtureCheck::All]` for `checks`. + pub fn process_and_partially_validate_firedancer_fixture( + &mut self, + fixture: &mollusk_svm_fuzz_fixture_firedancer::Fixture, + checks: &[fuzz::check::FixtureCheck], + ) -> InstructionResult { + let fuzz::firedancer::ParsedFixtureContext { + accounts, + compute_budget, + feature_set, + instruction, + slot, + } = fuzz::firedancer::parse_fixture_context(&fixture.input); + self.compute_budget = compute_budget; + self.feature_set = feature_set; + self.slot = slot; + + let result = self.process_instruction(&instruction, &accounts); + let expected = fuzz::firedancer::parse_fixture_effects( + &accounts, + self.compute_budget.compute_unit_limit, + &fixture.output, + ); + + fuzz::check::evaluate_results_with_fixture_checks(&expected, &result, checks); + result + } } diff --git a/harness/src/result.rs b/harness/src/result.rs index 9d42f619..261fe687 100644 --- a/harness/src/result.rs +++ b/harness/src/result.rs @@ -8,7 +8,7 @@ use solana_sdk::{ }; /// The result code of the program's execution. -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub enum ProgramResult { /// The program executed successfully. Success, @@ -258,7 +258,7 @@ enum CheckType<'a> { /// Check the result code of the program's execution. ProgramResult(ProgramResult), /// Check the return data produced by executing the instruction. - ReturnData(Vec), + ReturnData(&'a [u8]), /// Check a resulting account after executing the instruction. ResultingAccount(AccountCheck<'a>), } @@ -297,8 +297,13 @@ impl<'a> Check<'a> { Check::new(CheckType::ProgramResult(ProgramResult::UnknownError(error))) } + /// Assert that the instruction returned the provided result. + pub fn program_result(result: ProgramResult) -> Self { + Check::new(CheckType::ProgramResult(result)) + } + /// Check the return data produced by executing the instruction. - pub fn return_data(return_data: Vec) -> Self { + pub fn return_data(return_data: &'a [u8]) -> Self { Check::new(CheckType::ReturnData(return_data)) } From 4b5991049615c0bb38306da04e6421ef31201a84 Mon Sep 17 00:00:00 2001 From: Joe Caulfield Date: Wed, 15 Jan 2025 10:36:20 +0800 Subject: [PATCH 6/6] add helpers --- harness/src/fuzz/check.rs | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/harness/src/fuzz/check.rs b/harness/src/fuzz/check.rs index e590ee5b..a663e772 100644 --- a/harness/src/fuzz/check.rs +++ b/harness/src/fuzz/check.rs @@ -71,6 +71,51 @@ pub enum FixtureCheck { }, } +impl FixtureCheck { + /// Validate all possible checks for all resulting accounts. + /// + /// Note: To omit certain checks, use the variant directly, ie. + /// `FixtureCheck::AllResultingAccounts { data: false, .. }`. + pub fn all_resulting_accounts() -> Self { + Self::AllResultingAccounts { + data: true, + lamports: true, + owner: true, + space: true, + } + } + + /// Validate all possible checks for only the resulting accounts at certain + /// addresses. + /// + /// Note: To omit certain checks, use the variant directly, ie. + /// `FixtureCheck::OnlyResultingAccounts { data: false, .. }`. + pub fn only_resulting_accounts(addresses: &[Pubkey]) -> Self { + Self::OnlyResultingAccounts { + addresses: addresses.to_vec(), + data: true, + lamports: true, + owner: true, + space: true, + } + } + + /// Validate all possible checks for all of the resulting accounts _except_ + /// the provided addresses. + /// + /// Note: To omit certain checks, use the variant directly, ie. + /// `FixtureCheck::AllResultingAccountsExcept { data: false, .. }`. + pub fn all_resulting_accounts_except(ignore_addresses: &[Pubkey]) -> Self { + Self::AllResultingAccountsExcept { + ignore_addresses: ignore_addresses.to_vec(), + data: true, + lamports: true, + owner: true, + space: true, + } + } +} + fn add_account_checks<'a>( checks: &mut Vec>, accounts: impl Iterator,