From ba9d1329bab9b15e30120e8a916970b45b3054b4 Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Wed, 2 Oct 2024 12:15:56 +0200 Subject: [PATCH 1/3] memory stats --- .gitignore | 2 ++ Cargo.lock | 19 +++++++++++--- Cargo.toml | 2 +- rpc-state-reader/Cargo.toml | 3 ++- .../src/blockifier_state_reader.rs | 26 +++++++++++++++++++ 5 files changed, 46 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 271bc168..3f630d8b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ target/ .envrc compiled_programs/ +traces/ +memory.json diff --git a/Cargo.lock b/Cargo.lock index 80d12302..1820227b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1129,8 +1129,7 @@ dependencies = [ [[package]] name = "cairo-native" -version = "0.2.0" -source = "git+https://github.com/lambdaclass//cairo_native.git?rev=6fb1fecc71c6c604943717d6a72f4810090017a6#6fb1fecc71c6c604943717d6a72f4810090017a6" +version = "0.2.0-alpha.1" dependencies = [ "anyhow", "aquamarine", @@ -1181,12 +1180,13 @@ dependencies = [ [[package]] name = "cairo-native-runtime" -version = "0.2.0" -source = "git+https://github.com/lambdaclass//cairo_native.git?rev=6fb1fecc71c6c604943717d6a72f4810090017a6#6fb1fecc71c6c604943717d6a72f4810090017a6" +version = "0.2.0-alpha.1" dependencies = [ "cairo-lang-sierra-gas", + "itertools 0.13.0", "lazy_static", "libc", + "num-traits 0.2.19", "rand", "starknet-curve 0.5.1", "starknet-types-core", @@ -2780,6 +2780,16 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memory-stats" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c73f5c649995a115e1a0220b35e4df0a1294500477f97a91d0660fb5abeb574a" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "mime" version = "0.3.17" @@ -3585,6 +3595,7 @@ dependencies = [ "cairo-vm", "dotenv", "flate2", + "memory-stats", "pretty_assertions_sorted", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 3d912a32..300e858b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,4 +14,4 @@ cairo-native = { git = "https://github.com/lambdaclass/cairo_native" } tracing = "0.1" [patch.'https://github.com/lambdaclass/cairo_native'] -cairo-native = { git = 'https://github.com/lambdaclass//cairo_native.git', rev = "6fb1fecc71c6c604943717d6a72f4810090017a6" } +cairo-native = { path = "/data2/edgar/work/cairo_native"} diff --git a/rpc-state-reader/Cargo.toml b/rpc-state-reader/Cargo.toml index a40454b5..8b4250ed 100644 --- a/rpc-state-reader/Cargo.toml +++ b/rpc-state-reader/Cargo.toml @@ -21,13 +21,14 @@ cairo-lang-sierra = "2.8.2" cairo-lang-starknet-classes = "2.8.2" cairo-lang-utils = "2.8.2" cairo-native = { workspace = true } -starknet = "0.7.0" +starknet = "0.7.0" thiserror = { workspace = true } flate2 = "1.0.25" dotenv = "0.15.0" cairo-vm = "1.0.0-rc5" blockifier = { workspace = true } tracing = { workspace = true } +memory-stats = "1.2.0" [dev-dependencies] pretty_assertions_sorted = "1.2.3" diff --git a/rpc-state-reader/src/blockifier_state_reader.rs b/rpc-state-reader/src/blockifier_state_reader.rs index 92fec65a..ec4e57f6 100644 --- a/rpc-state-reader/src/blockifier_state_reader.rs +++ b/rpc-state-reader/src/blockifier_state_reader.rs @@ -22,6 +22,7 @@ use blockifier::{ }; use cairo_vm::types::program::Program; +use serde::{Deserialize, Serialize}; use starknet::core::types::ContractClass as SNContractClass; use starknet_api::{ block::BlockNumber, @@ -270,6 +271,13 @@ fn calculate_class_info_for_testing(contract_class: ContractClass) -> ClassInfo ClassInfo::new(&contract_class, sierra_program_length, 100).unwrap() } +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +struct MemoryData { + physical_mem: usize, + virtual_mem: usize, + tx_hash: TransactionHash, +} + pub fn execute_tx_configurable_with_state( tx_hash: &TransactionHash, tx: SNTransaction, @@ -278,6 +286,24 @@ pub fn execute_tx_configurable_with_state( _skip_nonce_check: bool, state: &mut CachedState, ) -> TransactionExecutionResult { + if let Some(usage) = memory_stats::memory_stats() { + tracing::info!("Current physical memory usage: {}", usage.physical_mem); + tracing::info!("Current virtual memory usage: {}", usage.virtual_mem); + let data = if let Ok(data) = std::fs::read_to_string("memory.json") { + data + } else { + "[]".to_string() + }; + + let mut data: Vec = serde_json::from_str(&data).unwrap(); + data.push(MemoryData { + physical_mem: usage.physical_mem, + virtual_mem: usage.virtual_mem, + tx_hash: *tx_hash, + }); + std::fs::write("memory.json", serde_json::to_string_pretty(&data).unwrap()).unwrap(); + } + let fee_token_address = FeeTokenAddresses { strk_fee_token_address: ContractAddress( felt!("0x04718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d") From 26bb838cc32ce68052c42e2a805b93852d9d4ab7 Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Wed, 2 Oct 2024 14:36:45 +0200 Subject: [PATCH 2/3] plot script --- .gitignore | 3 ++ memory.gnuplot | 38 +++++++++++++++++++ plot.sh | 2 + .../src/blockifier_state_reader.rs | 24 ++++++++---- 4 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 memory.gnuplot create mode 100755 plot.sh diff --git a/.gitignore b/.gitignore index 3f630d8b..075d42f9 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ target/ compiled_programs/ traces/ memory.json +*.csv +*.png +*.svg diff --git a/memory.gnuplot b/memory.gnuplot new file mode 100644 index 00000000..56f5c205 --- /dev/null +++ b/memory.gnuplot @@ -0,0 +1,38 @@ +set encoding utf8 +set style line 1 lc rgb '#E41A1C' pt 1 ps 1 lt 1 lw 2 # red +set style line 2 lc rgb '#377EB8' pt 6 ps 1 lt 1 lw 2 # blue +set style line 3 lc rgb '#4DAF4A' pt 2 ps 1 lt 1 lw 2 # green +set style line 4 lc rgb '#984EA3' pt 3 ps 1 lt 1 lw 2 # purple +set style line 5 lc rgb '#FF7F00' pt 4 ps 1 lt 1 lw 2 # orange +set style line 6 lc rgb '#FFFF33' pt 5 ps 1 lt 1 lw 2 # yellow +set style line 7 lc rgb '#A65628' pt 7 ps 1 lt 1 lw 2 # brown +set style line 8 lc rgb '#F781BF' pt 8 ps 1 lt 1 lw 2 # pink +# Palette +set palette maxcolors 8 +set palette defined ( 0 '#E41A1C', 1 '#377EB8', 2 '#4DAF4A', 3 '#984EA3',\ +4 '#FF7F00', 5 '#FFFF33', 6 '#A65628', 7 '#F781BF' ) + +# Standard border +set style line 11 lc rgb '#808080' lt 1 lw 3 +set border 0 back ls 11 +set tics out nomirror + +# Standard grid +set style line 12 lc rgb '#808080' lt 0 lw 1 +set grid back ls 12 +unset grid + +set title 'Memory per tx' +set key autotitle columnhead +set ylabel 'Memory (MB)' +set xlabel 'Tx Count' +set grid +set grid xtics +set grid ytics +set grid mxtics +set grid mytics +set yrange [0:] +set term svg size 1024, 768 dynamic rounded +set output 'out.svg' +set datafile separator ',' +plot 'memory.csv' using 0:1 with lines, '' using 0:4 with lines diff --git a/plot.sh b/plot.sh new file mode 100755 index 00000000..3f955f13 --- /dev/null +++ b/plot.sh @@ -0,0 +1,2 @@ +cat memory.json | jq -r '(map(keys) | add | unique) as $cols | map(. as $row | $cols | map($row[.])) as $rows | $cols, $rows[] | @csv' > memory.csv +gnuplot -p memory.gnuplot diff --git a/rpc-state-reader/src/blockifier_state_reader.rs b/rpc-state-reader/src/blockifier_state_reader.rs index ec4e57f6..3fe5779f 100644 --- a/rpc-state-reader/src/blockifier_state_reader.rs +++ b/rpc-state-reader/src/blockifier_state_reader.rs @@ -32,7 +32,7 @@ use starknet_api::{ state::StorageKey, transaction::{Transaction as SNTransaction, TransactionHash}, }; -use std::sync::Arc; +use std::sync::{atomic::AtomicU64, Arc}; use crate::{ rpc_state::{RpcBlockInfo, RpcChain, RpcState, RpcTransactionReceipt, TransactionTrace}, @@ -273,11 +273,14 @@ fn calculate_class_info_for_testing(contract_class: ContractClass) -> ClassInfo #[derive(Debug, Serialize, Deserialize, Clone, Copy)] struct MemoryData { - physical_mem: usize, - virtual_mem: usize, + physical: usize, + r#virtual: usize, tx_hash: TransactionHash, + tx_count: u64, } +static TX_COUNT: AtomicU64 = AtomicU64::new(1); + pub fn execute_tx_configurable_with_state( tx_hash: &TransactionHash, tx: SNTransaction, @@ -287,8 +290,14 @@ pub fn execute_tx_configurable_with_state( state: &mut CachedState, ) -> TransactionExecutionResult { if let Some(usage) = memory_stats::memory_stats() { - tracing::info!("Current physical memory usage: {}", usage.physical_mem); - tracing::info!("Current virtual memory usage: {}", usage.virtual_mem); + tracing::info!( + "Current physical memory usage: {}", + usage.physical_mem / 1024 / 1024 + ); + tracing::info!( + "Current virtual memory usage: {}", + usage.virtual_mem / 1024 / 1024 + ); let data = if let Ok(data) = std::fs::read_to_string("memory.json") { data } else { @@ -297,9 +306,10 @@ pub fn execute_tx_configurable_with_state( let mut data: Vec = serde_json::from_str(&data).unwrap(); data.push(MemoryData { - physical_mem: usage.physical_mem, - virtual_mem: usage.virtual_mem, + physical: usage.physical_mem / 1024 / 1024, + r#virtual: usage.virtual_mem / 1024 / 1024, tx_hash: *tx_hash, + tx_count: TX_COUNT.fetch_add(1, std::sync::atomic::Ordering::Relaxed), }); std::fs::write("memory.json", serde_json::to_string_pretty(&data).unwrap()).unwrap(); } From 31e66655a374cd2a4e6f043138c6fa829eb1a0bd Mon Sep 17 00:00:00 2001 From: Edgar Luque Date: Wed, 2 Oct 2024 16:51:50 +0200 Subject: [PATCH 3/3] fix --- Cargo.lock | 2 ++ Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 1820227b..a85dd8de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1130,6 +1130,7 @@ dependencies = [ [[package]] name = "cairo-native" version = "0.2.0-alpha.1" +source = "git+https://github.com/lambdaclass//cairo_native.git?branch=main#c866fbf8417cfcf522fd809348a483c5daef2e45" dependencies = [ "anyhow", "aquamarine", @@ -1181,6 +1182,7 @@ dependencies = [ [[package]] name = "cairo-native-runtime" version = "0.2.0-alpha.1" +source = "git+https://github.com/lambdaclass//cairo_native.git?branch=main#c866fbf8417cfcf522fd809348a483c5daef2e45" dependencies = [ "cairo-lang-sierra-gas", "itertools 0.13.0", diff --git a/Cargo.toml b/Cargo.toml index 300e858b..3b5228b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,4 +14,4 @@ cairo-native = { git = "https://github.com/lambdaclass/cairo_native" } tracing = "0.1" [patch.'https://github.com/lambdaclass/cairo_native'] -cairo-native = { path = "/data2/edgar/work/cairo_native"} +cairo-native = { git = 'https://github.com/lambdaclass//cairo_native.git', branch = "main"}