diff --git a/parse.py b/parse.py new file mode 100644 index 0000000000..27c054ecea --- /dev/null +++ b/parse.py @@ -0,0 +1,29 @@ +import csv +import re +import sys + +pattern = re.compile('.*PrecompileId::(\w+), (\w+)[^/]*(/(\d+))?_(mean|stddev)') + +data = [] + +with open(sys.argv[1], newline='') as csvfile: + reader = csv.reader(csvfile) + for row in reader: + name = row[0] + r = re.match(pattern, name) + if r: + id = r.group(1) + impl = r.group(2) + count = r.group(4) if r.group(4) else '1' + type = r.group(5) + if type == 'mean': + data.append([id, impl, count, row[3]]) + elif type == 'stddev': + data[-1].append(row[3]) + # print(count, row[3]) + # print(r.group(0), r.group(1), r.group(2), r.group(3), r.group(4), r.group(5)) + # if "_mean" in name: + # print(', '.join(row)) + +for d in data: + print(",".join(d)) diff --git a/test/precompiles_bench/CMakeLists.txt b/test/precompiles_bench/CMakeLists.txt index 28237cf48b..1fda588a41 100644 --- a/test/precompiles_bench/CMakeLists.txt +++ b/test/precompiles_bench/CMakeLists.txt @@ -8,6 +8,8 @@ target_include_directories(evmone-precompiles-bench PRIVATE ..) target_link_libraries(evmone-precompiles-bench PRIVATE evmone::state benchmark::benchmark) target_sources( evmone-precompiles-bench PRIVATE + bls_bench_input.hpp + bls_bench_input.cpp precompiles_bench.cpp ) diff --git a/test/precompiles_bench/bls_bench_input.cpp b/test/precompiles_bench/bls_bench_input.cpp new file mode 100644 index 0000000000..73ce3b681e --- /dev/null +++ b/test/precompiles_bench/bls_bench_input.cpp @@ -0,0 +1,173 @@ +#include "bls_bench_input.hpp" +#include "intx/intx.hpp" +#include "utils/utils.hpp" + +#include + +namespace evmone::test +{ +namespace +{ + +using PointsArrayType = std::array, 10>; + +[[nodiscard]] std::array generate_mul_input( + const intx::uint256& scalar, const PointsArrayType& test_points) noexcept +{ + std::array result; + + const auto scalar_be = bytes(intx::be::store(scalar).bytes, 32); + + for (size_t i = 0; i < 5; ++i) + { + const auto array_idx = i % test_points.size(); + result[i] = test_points[array_idx].first + test_points[array_idx].second + scalar_be; + } + + return result; +} + +[[nodiscard]] evmc::bytes generate_msm_input( + size_t num, const intx::uint256& scalar, const PointsArrayType& test_points) noexcept +{ + evmc::bytes result; + + const auto scalar_be = bytes(intx::be::store(scalar).bytes, 32); + + for (size_t i = 0; i < num; ++i) + { + const auto array_idx = i % test_points.size(); + result += test_points[array_idx].first + test_points[array_idx].second + scalar_be; + } + + return result; +} + +using evmone::test::operator""_hex; + +const PointsArrayType g1_test_points{ + {{"00000000000000000000000000000000076d5eda8e65c56c1195d2095892454f00da8760b4a42065869bee08c127872540f116e6dd447ed08f61e87655b7d802"_hex, + "0000000000000000000000000000000012e9a031b228aa9f4293db75008d6eb48142854deed8273a24f7f4e9ead2e73659e47f3dcb9d80f33fd66686bdb380bd"_hex}, + {"0000000000000000000000000000000015e68a7b3257373d0d7d661a863a95ee722e3576d2208283553b602c405a0b9bf4ef03f50754b28352d1eee37f98afc5"_hex, + "00000000000000000000000000000000123438727f2e017f2ce0acd494cd7ea7394467fa6c16e87c8ef48027136bb345be8c5a2f129db2eaee60f3e45053f95f"_hex}, + { + "0000000000000000000000000000000014e14fe19cd378a3351b764afc8c13b63d1bda36f3f261d53ae3bc189677d4f12ef388b195e9ae4edf630ee43fd5ad4d"_hex, + "000000000000000000000000000000000de02bdcac42f674df16b61208bf8fec06abb73f725a0742dbf699b681bfa05168a865688bd6a7a3c0eda7ccd57f5202"_hex, + }, + { + "0000000000000000000000000000000006a217646dd7ce72eb0ecf0fe42b26d19261010afb633030a0ecd65388d6e76d96df34bd9a2182be81a7ac674d61158f"_hex, + "00000000000000000000000000000000178f6a309845d6ec5b57dcc8874294d3ba18bc04af364f038597d680c18ae070e5ec056b83a762b407b4232e37459a16"_hex, + }, + { + "000000000000000000000000000000000ddd34ecdb7c52c453a9f33d691f38680b8c7d8539b1c123808b6fcd42d032d7cb49828b85d904dc9980c292d8ce06b0"_hex, + "0000000000000000000000000000000011aabaa0a78e641433e5bcef78e88b4a68dd8a513ebe339d87d85fd6860b57c6efbd0a90c44851a18f87455d926b4861"_hex, + }, + { + "0000000000000000000000000000000004a815eb8606282028f5f7558f910948e905b447727f5d88c7c0b51c2510357a7b8012adb826fd8cb7bb9e384ca5a68a"_hex, + "000000000000000000000000000000000cf73659de07444291a03aa69936bb9549acef9cb193d2ac7aed79db3517ef61b848c4623a971e2ddeb7a548aa34b9f6"_hex, + }, + { + "000000000000000000000000000000000a1f21645a31d991a4509cee86763dcd655af7fb1a6d04a9493e0244fa04eca099e63af149f71caf642472bc1761199f"_hex, + "000000000000000000000000000000001790e866be5d24247231e49ebb6eeaccb791af09948c3928403121dbe6338af66bf330de78c6b56b92a1681721c8607f"_hex, + }, + { + "00000000000000000000000000000000094f6f4bb3a1a35036ddfe8d3bde859afe22066bb038079c6b67c5b4ba31bd6e472db7bb84b16adc90661afd5dba75e0"_hex, + "000000000000000000000000000000000aab57295627e79773a94c81cd6b8fa9be5b7ef5533ed2d3b5edf693c4b2ca269f9ed8431112b01419ef02f119dd94f9"_hex, + }, + { + "0000000000000000000000000000000004f174170152492f64695d020f0ece14ee99d4a306eb8d30cd7357640e66f3e22153e98c7208e234a874c889b127e44f"_hex, + "0000000000000000000000000000000002077eb3d39567e26934b9457f4ca1f761991992d85c26302cefa543424c5497b472cac33e94f6bdb03c99916013aee9"_hex, + }, + { + "0000000000000000000000000000000014e14fe19cd378a3351b764afc8c13b63d1bda36f3f261d53ae3bc189677d4f12ef388b195e9ae4edf630ee43fd5ad4d"_hex, + "000000000000000000000000000000000de02bdcac42f674df16b61208bf8fec06abb73f725a0742dbf699b681bfa05168a865688bd6a7a3c0eda7ccd57f5202"_hex, + }}}; + + +const PointsArrayType g2_test_points{{{"00000000000000000000000000000000113b300891bb9007cb40a8bb9e0" + "5629b6e3bfa1eb5e67750e75dce2b8a67" + "3572187705ae16e62daa55b224853d1d623b" + "000000000000000000000000000000000331bd0b51374d6e580da7fd40f61af6b0e6458cbe07d9d2e74914a486dec84b544a874a264e93c4f320841dce129db8"_hex, + "0000000000000000000000000000000007598231ae9c7943e6c03292" + "e1aa8d61ff6110d28d0e80407de986059" + "7be123c3b5343adca1e6b5ea96f9dfc41f6d772" + "000000000000000000000000000000001954c40957a2a8d34cbe9c644691bde055af975ce327e4abd97da45dba983276d88395d5e0ae0806d65aacdefafe4749"_hex}, + {"0000000000000000000000000000000008b59a299a01cd2ef1fd625753b7c97e530a85938eb3171cd9c44615f" + "42841f11684b69c483e03ea8972a85d348a146f" + "0000000000000000000000000000000014db525c9e44adc16b99b5b78ac0fe7ba3af20358aeec9d783486a4fd4fa8a5efcdf929037ebbe1308e309e5d6b50dfe"_hex, + "0000000000000000000000000000000010b467dbccf8b4f6ebd644967641122cd213501b435c2bc62d473e" + "892e7fa99b8c1af77a8617f26f2e08df17a82f8cb8" + "000000000000000000000000000000000b77070c4b9cc67975b26a368f0f14f2411ea81ba777034b09a524a5bc17479e436c890db4443e2e28a17879d202925e"_hex}, + {"0000000000000000000000000000000003e90e6c5e60ac33bd061282a5040ff6d3d4e1f4239c8e5c8d500e395" + "3cd5f619e65e13ce9d9ca6f1f0cc4888234992c" + "0000000000000000000000000000000006bad719a05061f7a54a346dd58edaaad5b4613db7f52b805cfc1586dcca354d61f4abee0e20b0955f521dc798cd7e00"_hex, + "000000000000000000000000000000000f1802768a1bf9f8ebc188f732ce0ae7d23c629366df60a3d08e9e" + "6d4c11b9d6d81b09876667d463eeb16810c70e7a1c" + "0000000000000000000000000000000010c83fe489c77ae21ee823eb4d6a2ca5e7f0080ed186ec472e6253b69c6f9019650f2991eb9e05e590f7fb225a550573"_hex}, + {"000000000000000000000000000000000a7d777912b54412c493999d646f69309023b1c8a69012519c40b22af" + "3debae2e046726263e999a76b6acacb0408a5a3" + "0000000000000000000000000000000010ec46701a0843ce4435f0d991e9477e294aee25d7f2caf5db44b1ca21683909eb421aa8e07eeb0ab689d53e08bcaf69"_hex, + "0000000000000000000000000000000009a2d76f1adf9042a6b02b6f3ff17f52e3897248e45996da3af0fb" + "8b0945f7c2767229aa1ebb388e775f47b2f6f9ec7a" + "0000000000000000000000000000000004f214fe1504929b04f5baf87ac95e4ef75275fd97916919c691130f8b723eba3084ffc11b3604552f40eaec0a8b0bdd"_hex}, + {"0000000000000000000000000000000019fc237c473e25469def1f455f1c939791037893b7debdc3d93315012" + "1c8e499c70c361bdc6b34d1ab77b21ba6953341" + "0000000000000000000000000000000001275d6ac87a6b3c0da206fa8871b431dafb7e75511951f05039eed30554b3b5647fe41429104431704e01ae05899844"_hex, + "00000000000000000000000000000000103468609158d387794c08bd64b43d22be5c6d99841e674108b6d3" + "84198e97bb37df40bd28036d9ac129c46af63d74cc" + "00000000000000000000000000000000065420291f27a4da3c781f13bef9a8901fb6676afc0a993ad0d2527826d0134223291eba0d2674ad5bc99433b93414b2"_hex}, + {"000000000000000000000000000000000d5c2bec503b0615709a671ac54266ebbe9f75b6f8bbf0698349f0e21" + "fde07747ee66d1d4841365fb72761803eee3f18" + "0000000000000000000000000000000013124e27787729d0869c1386ec9671656d3894691eb6ca6a2193255a0f405edb63a2ec07bf8237f8999874a03b539222"_hex, + "000000000000000000000000000000000368556ca1d34334be1d2a8f8063b5d6d9737207ff0e4f2aa722ea" + "f0ce12ffdf7755fd8751617e08d05debaa64aa8995" + "000000000000000000000000000000000fba5ca2cb230d450b5a091b4a3b2a4d8a29776697d0cb84609cb705bdcedff01043d0c2a4607f0fe8cf468dea631ba5"_hex}, + {"000000000000000000000000000000000c50453a39f252ce2e32973ebe69dde4fb3e2a9ac24de1376cffc8289" + "a6a215800a99aaf2aecf97f134cb8ea6002dea9" + "00000000000000000000000000000000018485f14e8e2987c9298f72cc5ce53d3529f789438453c0302218f4b77ffdc1185eb9a8724bb8e0d550f2ef4b5d1370"_hex, + "000000000000000000000000000000001953407348ec087c91fc5745696282146a154c944f027121e66632" + "97986bf7597825644145b481c7a53d0a624ad33a7d" + "0000000000000000000000000000000011472eaad741eae3da87f0a4f7ec4dde77c199a9d8b549bf19f7c25244e211dbeb291889445bcbdf2b36d3f753eec282"_hex}, + {"0000000000000000000000000000000019fc237c473e25469def1f455f1c939791037893b7debdc3d93315012" + "1c8e499c70c361bdc6b34d1ab77b21ba6953341" + "0000000000000000000000000000000001275d6ac87a6b3c0da206fa8871b431dafb7e75511951f05039eed30554b3b5647fe41429104431704e01ae05899844"_hex, + "00000000000000000000000000000000103468609158d387794c08bd64b43d22be5c6d99841e674108b6d3" + "84198e97bb37df40bd28036d9ac129c46af63d74cc" + "00000000000000000000000000000000065420291f27a4da3c781f13bef9a8901fb6676afc0a993ad0d2527826d0134223291eba0d2674ad5bc99433b93414b2"_hex}, + {"0000000000000000000000000000000008cf4f93e2806ef2630423d6bf5c9e0b9ed60bef6fe0ff0776b66fed0" + "3f1e58822bad241e93725a5790803beb2830237" + "0000000000000000000000000000000008ed0b1f59b0d3ecfda4778e038a16db110200054928fcfaa3fc93c7ea82cf75e294b61671afc41ab5d9313773d863c5"_hex, + "0000000000000000000000000000000004770867fd6be78bb114bf41c0a8c5e9a52a8a969e7a23dc609705" + "c3e9262167f7471cf1078c8708c9e50aae0750ec34" + "000000000000000000000000000000000df0ec257b5ed7c8c344c8e18bf1b2b19c9deed84d2cd7d189c6bae6c4bec78945bc7c9f88101b5146e23fc8e96d5120"_hex}, + {"0000000000000000000000000000000008b0a8d57c2950ec115f3332361a5938af35ac01061ce7f9ad112d7ee" + "e3a858e647ebef9392625d2886d114bb8b02465" + "00000000000000000000000000000000031fcd4f4ce56194f9ac39b82c4133a0111772895769e8f8b8f5de4c61a88538928a9cfcf7ca2da51031cbd09ca4a90c"_hex, + "00000000000000000000000000000000137a807ecef1af91a6d85bf4c85a705a77ece3b8354fd3ff50d46a" + "db99e5299e6b6e8864e279311e05fd9ceed524e842" + "0000000000000000000000000000000018e7d81a098e5b4c94f8b2cf644c8e9b3b99fe2ed99e25f9b797b34c0c497c56ce3283c2064311fe554d154939fac68d"_hex}}}; + +} // namespace + + +[[nodiscard]] std::array generate_g1_mul_input(const intx::uint256& scalar) noexcept +{ + return generate_mul_input(scalar, g1_test_points); +} + +[[nodiscard]] std::array generate_g2_mul_input(const intx::uint256& scalar) noexcept +{ + return generate_mul_input(scalar, g2_test_points); +} + +[[nodiscard]] bytes generate_g1_msm_input(size_t num, const intx::uint256& scalar) noexcept +{ + return generate_msm_input(num, scalar, g1_test_points); +} + +[[nodiscard]] bytes generate_g2_msm_input(size_t num, const intx::uint256& scalar) noexcept +{ + return generate_msm_input(num, scalar, g2_test_points); +} + +} // namespace evmone::test diff --git a/test/precompiles_bench/bls_bench_input.hpp b/test/precompiles_bench/bls_bench_input.hpp new file mode 100644 index 0000000000..0654b2d276 --- /dev/null +++ b/test/precompiles_bench/bls_bench_input.hpp @@ -0,0 +1,19 @@ +// evmone: Fast Ethereum Virtual Machine implementation +// Copyright 2019-2020 The evmone Authors. +// SPDX-License-Identifier: Apache-2.0 +#pragma once + +#include "evmc/evmc.hpp" +#include "intx/intx.hpp" + +namespace evmone::test +{ +[[nodiscard]] std::array generate_g1_mul_input( + const intx::uint256& scalar = std::numeric_limits::max()) noexcept; +[[nodiscard]] std::array generate_g2_mul_input( + const intx::uint256& scalar = std::numeric_limits::max()) noexcept; +[[nodiscard]] evmc::bytes generate_g1_msm_input( + size_t num, const intx::uint256& scalar = std::numeric_limits::max()) noexcept; +[[nodiscard]] evmc::bytes generate_g2_msm_input( + size_t num, const intx::uint256& scalar = std::numeric_limits::max()) noexcept; +} // namespace evmone::test diff --git a/test/precompiles_bench/precompiles_bench.cpp b/test/precompiles_bench/precompiles_bench.cpp index 0209feb3d2..f4dc3e08ed 100644 --- a/test/precompiles_bench/precompiles_bench.cpp +++ b/test/precompiles_bench/precompiles_bench.cpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #include "../utils/utils.hpp" +#include "bls_bench_input.hpp" #include #include #include @@ -35,6 +36,14 @@ template <> [[maybe_unused]] constexpr auto analyze = ecpairing_analyze; template <> constexpr auto analyze = point_evaluation_analyze; +template <> +constexpr auto analyze = bls12_g1mul_analyze; +template <> +constexpr auto analyze = bls12_g1msm_analyze; +template <> +constexpr auto analyze = bls12_g2mul_analyze; +template <> +constexpr auto analyze = bls12_g2msm_analyze; template const inline std::array inputs{0}; @@ -138,6 +147,15 @@ const inline std::array inputs{ "01ff83fbe4488061b695d9e4632e9ab23c5b9dfb046241fac921659c2e3b6a4e5e88a0c20ef27ae5dea6e83a748115df152b207362c022a43c8ebbfbec73ea2a2b38376a1de4ed9abd35175ac422af7df2c6ddf1503979964b1d217747c108978969cc0fcbd47dba8b1e3cbba16920f3fc694fadbf203b55aa66aaf138a38ace946a7ccddc20f45d796cd60268df8a2487d06c11cdc50247c580c5dd25f6fb486053b73868038cb92c2ae6d429a7bde850177f31c9fde00ca824c8c59cd4ef49"_hex, }; +template <> +const inline std::array inputs{generate_g1_mul_input()}; +template <> +const inline std::array inputs{generate_g2_mul_input()}; +template <> +const inline std::array inputs{generate_g1_msm_input(256)}; +template <> +const inline std::array inputs{generate_g2_msm_input(256)}; + template void precompile(benchmark::State& state) { @@ -145,7 +163,12 @@ void precompile(benchmark::State& state) size_t max_output_size = 0; for (const auto& input : inputs) { - const auto r = analyze(input, EVMC_LATEST_STABLE_REVISION); + size_t input_size = input.size(); // NOLINT(misc-const-correctness) + if constexpr (Id == PrecompileId::bls12_g1msm) + input_size = 160 * static_cast(state.range(0)); + else if constexpr (Id == PrecompileId::bls12_g2msm) + input_size = 288 * static_cast(state.range(0)); + const auto r = analyze(input.substr(0, input_size), EVMC_LATEST_STABLE_REVISION); batch_gas_cost += r.gas_cost; max_output_size = std::max(max_output_size, r.max_output_size); } @@ -157,7 +180,13 @@ void precompile(benchmark::State& state) { for (const auto& input : inputs) { - const auto [status, _] = Fn(input.data(), input.size(), output.get(), max_output_size); + size_t input_size = input.size(); // NOLINT(misc-const-correctness) + if constexpr (Id == PrecompileId::bls12_g1msm) + input_size = 160 * static_cast(state.range(0)); + else if constexpr (Id == PrecompileId::bls12_g2msm) + input_size = 288 * static_cast(state.range(0)); + + const auto [status, _] = Fn(input.data(), input_size, output.get(), max_output_size); if (status != EVMC_SUCCESS) [[unlikely]] { state.SkipWithError("invalid result"); @@ -170,6 +199,7 @@ void precompile(benchmark::State& state) using benchmark::Counter; state.counters["gas_used"] = Counter(static_cast(batch_gas_cost)); state.counters["gas_rate"] = Counter(static_cast(total_gas_used), Counter::kIsRate); + // state.SetComplexityN(state.range(0)); } BENCHMARK_TEMPLATE(precompile, PrecompileId::identity, identity_execute); @@ -177,10 +207,12 @@ BENCHMARK_TEMPLATE(precompile, PrecompileId::identity, identity_execute); namespace bench_ecrecovery { constexpr auto evmmax_cpp = ecrecover_execute; -BENCHMARK_TEMPLATE(precompile, PrecompileId::ecrecover, evmmax_cpp); +BENCHMARK_TEMPLATE(precompile, PrecompileId::ecrecover, evmmax_cpp) + ->Unit(benchmark::TimeUnit::kMicrosecond); #ifdef EVMONE_PRECOMPILES_SILKPRE constexpr auto libsecp256k1 = silkpre_ecrecover_execute; -BENCHMARK_TEMPLATE(precompile, PrecompileId::ecrecover, libsecp256k1); +BENCHMARK_TEMPLATE(precompile, PrecompileId::ecrecover, libsecp256k1) + ->Unit(benchmark::TimeUnit::kMicrosecond); #endif } // namespace bench_ecrecovery @@ -218,6 +250,39 @@ constexpr auto evmone_blst = point_evaluation_execute; BENCHMARK_TEMPLATE(precompile, PrecompileId::point_evaluation, evmone_blst); } // namespace bench_kzg +namespace bench_mul_g1 +{ +constexpr auto evmone_blst = bls12_g1mul_execute; +BENCHMARK_TEMPLATE(precompile, PrecompileId::bls12_g1mul, evmone_blst) + ->Unit(benchmark::TimeUnit::kMicrosecond); +} // namespace bench_mul_g1 + +namespace bench_msm_g1 +{ +constexpr auto evmone_blst = bls12_g1msm_execute; +BENCHMARK_TEMPLATE(precompile, PrecompileId::bls12_g1msm, evmone_blst) + ->DenseRange(1, 256) + // ->Complexity(benchmark::oN) + ->Unit(benchmark::TimeUnit::kMicrosecond); +} // namespace bench_msm_g1 + +namespace bench_mul_g2 +{ +constexpr auto evmone_blst = bls12_g2mul_execute; +BENCHMARK_TEMPLATE(precompile, PrecompileId::bls12_g2mul, evmone_blst) + // ->Complexity(benchmark::oN) + ->Unit(benchmark::TimeUnit::kMicrosecond); +} // namespace bench_mul_g2 + +namespace bench_msm_g2 +{ +constexpr auto evmone_blst = bls12_g2msm_execute; +BENCHMARK_TEMPLATE(precompile, PrecompileId::bls12_g2msm, evmone_blst) + ->DenseRange(1, 256) + // ->Complexity(benchmark::oN) + ->Unit(benchmark::TimeUnit::kMicrosecond); +} // namespace bench_msm_g2 + } // namespace BENCHMARK_MAIN();