Merge branch 'ci/levm/compare_ef' of github.com:lambdaclass/ethrex in…

…to ci/levm/compare_ef

.dockerignore

@@ -8,7 +8,7 @@
 **/*.rs.bk
 
 cmd/ef_tests/blockchain/vectors
-cmd/ef_tests/levm/vectors
+cmd/ef_tests/state/vectors
 
 **/.DS_Store
 **/.vscode
@@ -24,4 +24,4 @@ crates/levm_mlir/ethtests
 crates/levm_mlir/*.tar.gz
 
 # used by Flamegraph and Samply
-cmd/ef_tests/levm/levm_perfgraphs
+cmd/ef_tests/state/levm_perfgraphs

.github/scripts/publish_levm_ef_tests.sh

@@ -1,3 +1,3 @@
 curl -X POST $1 \
 -H 'Content-Type: application/json; charset=utf-8' \
---data "$(cat cmd/ef_tests/levm/levm_ef_tests_summary_slack.txt)"
+--data "$(cat cmd/ef_tests/state/levm_ef_tests_summary_slack.txt)"

.github/workflows/ci_l1.yaml

@@ -197,6 +197,7 @@ jobs:
         run: |
           docker load --input /tmp/ethrex_image.tar
 
+      # By default ethrex uses revm as evm backend.
       - name: Run Hive Simulation
         run: chmod +x hive && ./hive --client ethrex --sim ${{ matrix.simulation }} --sim.limit "${{ matrix.test_pattern }}" --sim.parallelism 4
 

.github/workflows/ci_levm.yaml

@@ -204,7 +204,10 @@ jobs:
           name: results_levm_trigger.md
 
       - name: Rename result (1)
-        run: cp results.md results_levm.md
+        run: |
+          cp results.md results_levm.md
+          echo "RESULTS:"
+          cat results_levm.md
 
       - name: Download results (main)
         uses: actions/download-artifact@v4
@@ -214,12 +217,17 @@ jobs:
         continue-on-error: true
 
       - name: Rename result (2)
-        run: cp results.md results_levm_main.md
+        run: |
+          cp results.md results_levm_main.md
+          echo "RESULTS:"
+          cat results_levm_main.md
 
       - name: Create diff message
         run: |
           bash .github/scripts/hive_levm_revm_diff.sh results_levm_main.md results_levm.md >> diff.md
           cat diff.md >> $GITHUB_STEP_SUMMARY
+          echo "SUMMARY:"
+          cat diff.md
 
       - name: Check Regression
         run: |

.github/workflows/common_hive_reports.yaml

@@ -38,7 +38,6 @@ jobs:
 
     steps:
       - name: Pull image
-        if: ${{ inputs.evm == 'revm' }}
         run: |
           docker pull ghcr.io/lambdaclass/ethrex:latest
           docker tag ghcr.io/lambdaclass/ethrex:latest ethrex:latest
@@ -53,18 +52,14 @@ jobs:
         with:
           ref: main
 
-      - name: Build Image with LEVM
-        if: ${{ inputs.evm == 'levm' }}
-        run: cd crates/vm/levm && make build-image-levm
-
       - name: Setup Go
         uses: actions/setup-go@v5
 
       - name: Setup Hive
         run: make setup-hive
 
       - name: Run Hive Simulation
-        run: cd hive && ./hive --client ethrex --sim ${{ matrix.test.simulation }} --sim.parallelism 16
+        run: cd hive && ./hive --client ethrex --sim ${{ matrix.test.simulation }} --ethrex.flags "--evm ${{ inputs.evm }}" --sim.parallelism 16
         continue-on-error: true
 
       - name: Upload results

.github/workflows/flamegraph_reporter.yaml

@@ -40,7 +40,7 @@ jobs:
             ${{ env.HOME }}/.cargo/bin/addr2line
             ${{ env.HOME }}/.cargo/bin/flamegraph
             ${{ env.HOME }}/.cargo/bin/inferno-*
-            ${{ env.HOME }}/ethrex/ethrex/cmd/ef_tests/levm/vectors
+            ${{ env.HOME }}/ethrex/ethrex/cmd/ef_tests/state/vectors
           key: ${{ runner.os }}-${{ env.RUST_VERSION }}-extra-binaries
 
       - name: Change perf settings
@@ -91,7 +91,7 @@ jobs:
 
       - name: Download EF Tests
         run: |
-          if [ ! -f "$HOME/ethrex/ethrex/cmd/ef_tests/levm/vectors" ]; then
+          if [ ! -f "$HOME/ethrex/ethrex/cmd/ef_tests/state/vectors" ]; then
             cd crates/vm/levm
             make download-evm-ef-tests
           fi
@@ -101,13 +101,13 @@ jobs:
           start_time=$(date +%s)
 
           if [ "${{ matrix.name }}" == "levm" ]; then
-            rm -rf target/debug/ef_tests_levm
+            rm -rf target/debug/ef_tests_state
             CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph -c "record -o perf.data -F997 --call-graph dwarf,16384 -g" \
-            -p ef_tests-levm -- --tests stSolidityTest,stCallCodes
+            -p ef_tests-state --test all -- --tests stSolidityTest,stCallCodes
           elif [ "${{ matrix.name }}" == "revm" ]; then
-            rm -rf target/debug/ef_tests_levm
+            rm -rf target/debug/ef_tests_state
             CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph -c "record -o perf.data -F997 --call-graph dwarf,16384 -g" \
-            -p ef_tests-levm -- --revm --tests stSolidityTest,stCallCodes
+            -p ef_tests-state --test all -- --revm --tests stSolidityTest,stCallCodes
           fi
 
           end_time=$(date +%s)
@@ -252,6 +252,7 @@ jobs:
           ethrex_l2 config create default --default
           ethrex_l2 config set default
 
+      # By default ethrex uses revm as evm backend.
       - id: generate-flamegraph-ethrex
         name: Generate Flamegraph data for Ethrex
         shell: bash

.gitignore

@@ -11,7 +11,7 @@ debug/
 
 cmd/ef_tests/blockchain/vectors
 
-cmd/ef_tests/levm/vectors
+cmd/ef_tests/state/vectors
 
 # Repos checked out by make target
 hive/
@@ -59,7 +59,7 @@ loc_report.json
 ethrex.redb
 
 # used by Flamegraph and Samply
-cmd/ef_tests/levm/levm_perfgraphs
+cmd/ef_tests/state/levm_perfgraphs
 *.svg
 prof_*.json
 prof.json

Cargo.toml

@@ -11,7 +11,7 @@ members = [
     "crates/common/rlp",
     "cmd/ethrex",
     "cmd/ef_tests/blockchain",
-    "cmd/ef_tests/levm",
+    "cmd/ef_tests/state",
     "cmd/ethrex_l2",
     "cmd/hive_report",
     "crates/vm/levm",

Makefile

@@ -13,7 +13,7 @@ lint: ## 🧹 Linter check
 
 CRATE ?= *
 test: ## 🧪 Run each crate's tests
-	cargo test -p '$(CRATE)' --workspace --exclude ethrex-prover --exclude ethrex-levm --exclude ef_tests-blockchain --exclude ef_tests-levm --exclude ethrex-l2 -- --skip test_contract_compilation
+	cargo test -p '$(CRATE)' --workspace --exclude ethrex-prover --exclude ethrex-levm --exclude ef_tests-blockchain --exclude ef_tests-state --exclude ethrex-l2 -- --skip test_contract_compilation
 	$(MAKE) -C cmd/ef_tests/blockchain test
 
 clean: clean-vectors ## 🧹 Remove build artifacts
@@ -66,12 +66,13 @@ stop-localnet-silent:
 	@kurtosis enclave stop $(ENCLAVE) >/dev/null 2>&1 || true
 	@kurtosis enclave rm $(ENCLAVE) --force >/dev/null 2>&1 || true
 
-HIVE_REVISION := b0b0f98bd24676239722e3aa7885e29ef856d804
+HIVE_REVISION := feb4333db7fe9f6dc161326ebb11957d4306d2f9
 # Shallow clones can't specify a single revision, but at least we avoid working
 # the whole history by making it shallow since a given date (one day before our
 # target revision).
 HIVE_SHALLOW_SINCE := 2024-09-02
 QUIET ?= false
+
 hive:
 	if [ "$(QUIET)" = "true" ]; then \
 		git clone --quiet --single-branch --branch master --shallow-since=$(HIVE_SHALLOW_SINCE) https://github.com/lambdaclass/hive && \
@@ -106,6 +107,9 @@ SIM_LOG_LEVEL ?= 4
 run-hive: build-image setup-hive ## 🧪 Run Hive testing suite
 	cd hive && ./hive --client ethrex --sim $(SIMULATION) --sim.limit "$(TEST_PATTERN)"
 
+run-hive-levm: build-image setup-hive ## 🧪 Run Hive testing suite with LEVM
+	cd hive && ./hive --client ethrex --ethrex.flags "--evm levm" --sim $(SIMULATION) --sim.limit "$(TEST_PATTERN)"
+
 run-hive-all: build-image setup-hive ## 🧪 Run all Hive testing suites
 	cd hive && ./hive --client ethrex --sim ".*" --sim.parallelism 4
 
@@ -115,6 +119,16 @@ run-hive-debug: build-image setup-hive ## 🐞 Run Hive testing suite in debug m
 clean-hive-logs: ## 🧹 Clean Hive logs
 	rm -rf ./hive/workspace/logs
 
+SIM_PARALLELISM := 48
+EVM_BACKEND := revm
+# `make run-hive-report SIM_PARALLELISM=24 EVM_BACKEND="levm"`
+run-hive-report: build-image setup-hive clean-hive-logs ## 🐝 Run Hive and Build report
+	cd hive && ./hive --ethrex.flags "--evm $(EVM_BACKEND)" --sim ethereum/rpc-compat --client ethrex --sim.limit "$(TEST_PATTERN)" --sim.parallelism $(SIM_PARALLELISM) || exit 0
+	cd hive && ./hive --ethrex.flags "--evm $(EVM_BACKEND)" --sim devp2p --client ethrex --sim.limit "$(TEST_PATTERN)" --sim.parallelism $(SIM_PARALLELISM) || exit 0
+	cd hive && ./hive --ethrex.flags "--evm $(EVM_BACKEND)" --sim ethereum/engine --client ethrex --sim.limit "$(TEST_PATTERN)" --sim.parallelism $(SIM_PARALLELISM) || exit 0
+	cd hive && ./hive --ethrex.flags "--evm $(EVM_BACKEND)" --sim ethereum/sync --client ethrex --sim.limit "$(TEST_PATTERN)" --sim.parallelism $(SIM_PARALLELISM) || exit 0
+	cargo run --release -p hive_report
+
 loc:
 	cargo run -p loc
 
@@ -152,17 +166,18 @@ install-cli: ## 🛠️ Installs the ethrex-l2 cli
 
 start-node-with-flamegraph: rm-test-db ## 🚀🔥 Starts an ethrex client used for testing
 	@if [ -z "$$L" ]; then \
-		LEVM=""; \
+		LEVM="revm"; \
 		echo "Running the test-node without the LEVM feature"; \
 		echo "If you want to use levm, run the target with an L at the end: make <target> L=1"; \
 	else \
-		LEVM=",levm"; \
+		LEVM="levm"; \
 		echo "Running the test-node with the LEVM feature"; \
 	fi; \
-	sudo CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph \
+	sudo -E CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph \
 	--bin ethrex \
-	--features "dev$$LEVM" \
+	--features "dev" \
 	--  \
+	--evm $$LEVM \
 	--network test_data/genesis-l2.json \
 	--http.port 1729 \
 	--datadir test_ethrex
@@ -176,14 +191,10 @@ load-node: install-cli ## 🚧 Runs a load-test. Run make start-node-with-flameg
 		CONTRACT_INTERACTION="-c"; \
 		echo "Running the load-test with contract interaction"; \
 	fi; \
-	ethrex_l2 test load --path test_data/private_keys.txt -i 100 -v  --value 1 $$CONTRACT_INTERACTION
+	ethrex_l2 test load --path test_data/private_keys.txt -i 1000 -v  --value 100000 $$CONTRACT_INTERACTION
 
 rm-test-db:  ## 🛑 Removes the DB used by the ethrex client used for testing
 	sudo cargo run --release --bin ethrex -- removedb --datadir test_ethrex
 
-flamegraph:
-	sudo -E CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph --bin ethrex --features dev  --  --network test_data/genesis-l2.json --http.port 1729 >/dev/null &
+flamegraph: ## 🚧 Runs a load-test. Run make start-node-with-flamegraph and in a new terminal make flamegraph
 	bash scripts/flamegraph.sh
-
-test-load:
-	ethrex_l2 test load --path ./test_data/private_keys.txt -i 1000 -v  --value 10000000 --to 0xFCbaC0713ACf16708aB6BC977227041FA1BC618D

README.md

@@ -282,6 +282,7 @@ ethrex supports the following command line arguments:
 - `--bootnodes <BOOTNODE_LIST>`: Comma separated enode URLs for P2P discovery bootstrap.
 - `--log.level <LOG_LEVEL>`: The verbosity level used for logs. Default value: info. possible values: info, debug, trace, warn, error
 - `--syncmode <SYNC_MODE>`: The way in which the node will sync its state. Can be either "full" or "snap" with "snap" as default value.
+- `--evm <EVM_BACKEND>`: Has to be `levm` or `revm`. Default value: `revm`.
 
 # ethrex L2
 

cmd/ef_tests/levm/Cargo.toml → cmd/ef_tests/state/Cargo.toml

@@ -1,14 +1,14 @@
 [package]
-name = "ef_tests-levm"
+name = "ef_tests-state"
 version.workspace = true
 edition.workspace = true
 
 [dependencies]
-ethrex-blockchain = { workspace = true, features = ["levm"] }
+ethrex-blockchain = { workspace = true }
 ethrex-core.workspace = true
 ethrex-storage.workspace = true
 ethrex-rlp.workspace = true
-ethrex-vm = { workspace = true, features = ["levm"] }
+ethrex-vm = { workspace = true }
 ethrex-levm = { path = "../../../crates/vm/levm" }
 serde.workspace = true
 serde_json.workspace = true
@@ -33,12 +33,8 @@ revm = { version = "18.0.0", features = [
 hex = "0.4.3"
 
 [lib]
-path = "./ef_tests.rs"
-
-[[bin]]
-name = "ef_tests_levm"
-path = "tests/ef_tests_levm.rs"
+path = "./lib.rs"
 
 [[test]]
-name = "ef_tests_levm"
+name = "all"
 harness = false

cmd/ef_tests/state/README.md

@@ -0,0 +1,9 @@
+# State Tests
+
+https://ethereum.github.io/execution-spec-tests/main/consuming_tests/state_test/
+
+## Running the tests
+
+```bash
+make run-evm-ef-tests
+```

cmd/ef_tests/levm/runner/levm_runner.rs → cmd/ef_tests/state/runner/levm_runner.rs

@@ -15,7 +15,7 @@ use ethrex_levm::{
     Environment,
 };
 use ethrex_storage::AccountUpdate;
-use ethrex_vm::{db::StoreWrapper, EvmState};
+use ethrex_vm::db::{EvmState, StoreWrapper};
 use keccak_hash::keccak;
 use std::{collections::HashMap, sync::Arc};
 

cmd/ef_tests/levm/runner/revm_runner.rs → cmd/ef_tests/state/runner/revm_runner.rs

@@ -17,7 +17,10 @@ use ethrex_levm::{
     Account, StorageSlot,
 };
 use ethrex_storage::{error::StoreError, AccountUpdate};
-use ethrex_vm::{db::StoreWrapper, fork_to_spec_id, EvmState, RevmAddress, RevmU256};
+use ethrex_vm::{
+    db::{EvmState, StoreWrapper},
+    fork_to_spec_id, RevmAddress, RevmU256,
+};
 use revm::{
     db::State,
     inspectors::TracerEip3155 as RevmTracerEip3155,

cmd/ef_tests/levm/tests/ef_tests_levm.rs → cmd/ef_tests/state/tests/all.rs

@@ -1,5 +1,5 @@
 use clap::Parser;
-use ef_tests_levm::{
+use ef_tests_state::{
     parser,
     runner::{self, EFTestRunnerOptions},
 };

cmd/ef_tests/levm/utils.rs → cmd/ef_tests/state/utils.rs

@@ -4,7 +4,7 @@ use crate::{
 };
 use ethrex_core::{types::Genesis, H256, U256};
 use ethrex_storage::{EngineType, Store};
-use ethrex_vm::{evm_state, EvmState};
+use ethrex_vm::db::{evm_state, EvmState};
 use spinoff::Spinner;
 
 pub fn load_initial_state(test: &EFTest) -> (EvmState, H256) {

cmd/ethrex/Cargo.toml

@@ -48,4 +48,3 @@ metrics = ["ethrex-blockchain/metrics", "ethrex-l2/metrics"]
 libmdbx = ["dep:libmdbx", "ethrex-storage/libmdbx"]
 redb = ["dep:redb", "ethrex-storage/redb"]
 l2 = ["dep:ethrex-l2", "ethrex-vm/l2"]
-levm = ["default", "ethrex-vm/levm", "ethrex-blockchain/levm"]

cmd/ethrex/cli.rs

@@ -1,5 +1,6 @@
 use clap::{Arg, ArgAction, Command};
 use ethrex_p2p::types::Node;
+use ethrex_vm::backends::EVM;
 use tracing::Level;
 
 pub fn cli() -> Command {
@@ -128,6 +129,7 @@ pub fn cli() -> Command {
                 .required(false)
                 .default_value("revm")
                 .value_name("EVM_BACKEND")
+                .value_parser(clap::value_parser!(EVM))
                 .help("Has to be `levm` or `revm`"),
         )
         .subcommand(