Merge branch 'hotstuff_integration' into GH-1523-hotstuff-activate

.github/actions/parallel-ctest-containers/action.yml

@@ -12,5 +12,5 @@ inputs:
   test-timeout:
     required: true
 runs:
-  using: 'node16'
+  using: 'node20'
   main: 'dist/index.mjs'

.github/workflows/build.yaml

@@ -89,7 +89,7 @@ jobs:
     runs-on: ubuntu-latest
     container: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Download builddir
@@ -138,9 +138,9 @@ jobs:
     runs-on: ["self-hosted", "enf-x86-hightier"]
     container:
       image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.cfg.base].image}}
-      options: --security-opt seccomp=unconfined
+      options: --security-opt seccomp=unconfined --mount type=bind,source=/var/lib/systemd/coredump,target=/cores
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
@@ -152,6 +152,13 @@ jobs:
           zstdcat build.tar.zst | tar x
           cd build
           ctest --output-on-failure -j $(nproc) -LE "(nonparallelizable_tests|long_running_tests)" --timeout 420
+      - name: Upload core files from failed tests
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: ${{matrix.cfg.name}}-tests-logs
+          if-no-files-found: ignore
+          path: /cores
       - name: Check CPU Features
         run: awk 'BEGIN {err = 1} /bmi2/ && /adx/ {err = 0} END {exit err}' /proc/cpuinfo
 
@@ -168,7 +175,7 @@ jobs:
           - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-midtier"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
@@ -181,12 +188,17 @@ jobs:
           log-tarball-prefix: ${{matrix.cfg.name}}
           tests-label: nonparallelizable_tests
           test-timeout: 420
+      - name: Export core dumps
+        run: docker run --mount type=bind,source=/var/lib/systemd/coredump,target=/cores alpine sh -c 'tar -C /cores/ -c .' | tar x
+        if: failure()
       - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: ${{matrix.cfg.name}}-np-logs
-          path: '*-logs.tar.gz'
+          path: |
+            *-logs.tar.gz
+            core*.zst
 
   lr-tests:
     name: LR Tests (${{matrix.cfg.name}})
@@ -201,7 +213,7 @@ jobs:
           - cfg: {name: 'ubuntu22repro', base: 'ubuntu22', builddir: 'reproducible'}
     runs-on: ["self-hosted", "enf-x86-lowtier"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:
@@ -214,12 +226,17 @@ jobs:
           log-tarball-prefix: ${{matrix.cfg.name}}
           tests-label: long_running_tests
           test-timeout: 1800
+      - name: Export core dumps
+        run: docker run --mount type=bind,source=/var/lib/systemd/coredump,target=/cores alpine sh -c 'tar -C /cores/ -c .' | tar x
+        if: failure()
       - name: Upload logs from failed tests
         uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: ${{matrix.cfg.name}}-lr-logs
-          path: '*-logs.tar.gz'
+          path: |
+            *-logs.tar.gz
+            core*.zst
 
   libtester-tests:
     name: libtester tests
@@ -243,7 +260,7 @@ jobs:
       # LEAP
       - if: ${{ matrix.test != 'deb-install' }}
         name: Clone leap
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
       - if: ${{ matrix.test != 'deb-install' }}
@@ -296,7 +313,7 @@ jobs:
 
       # Reference Contracts
       - name: checkout eos-system-contracts
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: eosnetworkfoundation/eos-system-contracts
           path: eos-system-contracts

.github/workflows/build_base.yaml

@@ -30,7 +30,7 @@ jobs:
     runs-on: ["self-hosted", "enf-x86-beefy"]
     container: ${{fromJSON(inputs.platforms)[matrix.platform].image}}
     steps:
-        - uses: actions/checkout@v3
+        - uses: actions/checkout@v4
           with:
             submodules: recursive
         - name: Build

.github/workflows/ph_backward_compatibility.yaml

@@ -46,7 +46,7 @@ jobs:
       image: ${{fromJSON(needs.platform-cache.outputs.platforms)[matrix.platform].image}}
       options: --security-opt seccomp=unconfined
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:

.github/workflows/release.yaml

@@ -21,7 +21,7 @@ jobs:
           file: 'leap-dev.*amd64.deb'
           target: ${{github.sha}}
           artifact-name: leap-dev-ubuntu20-amd64
-          wait-for-exact-target-workflow: true
+          wait-for-exact-target: true
       - name: Get ubuntu22 leap-dev.deb
         uses: AntelopeIO/asset-artifact-download-action@v3
         with:
@@ -30,7 +30,7 @@ jobs:
           file: 'leap-dev.*amd64.deb'
           target: ${{github.sha}}
           artifact-name: leap-dev-ubuntu22-amd64
-          wait-for-exact-target-workflow: true
+          wait-for-exact-target: true
       - name: Create Dockerfile
         run: |
           cat <<EOF > Dockerfile

CMakeLists.txt

@@ -175,6 +175,10 @@ endif()
 # do not attempt to use an external OpenSSL in any manner
 set(CMAKE_DISABLE_FIND_PACKAGE_OpenSSL On)
 
+# many tests require handling of signals themselves and even when they don't we'd like for them to generate a core dump, this
+# is effectively --catch_system_errors=no broadly across all tests
+add_compile_definitions(BOOST_TEST_DEFAULTS_TO_CORE_DUMP)
+
 add_subdirectory( libraries )
 add_subdirectory( plugins )
 add_subdirectory( programs )

docs/01_nodeos/03_plugins/producer_plugin/index.md

@@ -111,8 +111,6 @@ Config Options for eosio::producer_plugin:
   --disable-subjective-api-billing arg (=1)
                                         Disable subjective CPU billing for API
                                         transactions
-  --producer-threads arg (=2)           Number of worker threads in producer
-                                        thread pool
   --snapshots-dir arg (="snapshots")    the location of the snapshots directory
                                         (absolute path or relative to
                                         application data dir)

libraries/chain/include/eosio/chain/database_utils.hpp

@@ -182,8 +182,8 @@ namespace fc {
 
    inline
    void from_variant( const variant& v, eosio::chain::shared_blob& b ) {
-      std::string _s = base64_decode(v.as_string());
-      b = eosio::chain::shared_blob(_s.begin(), _s.end(), b.get_allocator());
+      std::vector<char> b64 = base64_decode(v.as_string());
+      b = eosio::chain::shared_blob(b64.begin(), b64.end(), b.get_allocator());
    }
 
    template<typename T>

libraries/chain/include/eosio/chain/transaction_metadata.hpp

@@ -83,6 +83,12 @@ class transaction_metadata {
       start_recover_keys( packed_transaction_ptr trx, boost::asio::io_context& thread_pool,
                           const chain_id_type& chain_id, fc::microseconds time_limit,
                           trx_type t, uint32_t max_variable_sig_size = UINT32_MAX );
+      /// Thread safe.
+      /// @returns transaction_metadata_ptr or throws
+      static transaction_metadata_ptr
+      recover_keys( packed_transaction_ptr trx,
+                    const chain_id_type& chain_id, fc::microseconds time_limit,
+                    trx_type t, uint32_t max_variable_sig_size = UINT32_MAX );
 
       /// @returns constructed transaction_metadata with no key recovery (sig_cpu_usage=0, recovered_pub_keys=empty)
       static transaction_metadata_ptr

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp

@@ -69,6 +69,7 @@ class code_cache_base {
       code_cache_index _cache_index;
 
       const chainbase::database& _db;
+      eosvmoc::config            _eosvmoc_config;
 
       std::filesystem::path _cache_file_path;
       int                   _cache_fd;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/config.hpp

@@ -7,11 +7,24 @@
 
 #include <fc/reflect/reflect.hpp>
 
+#include <sys/resource.h>
+
 namespace eosio { namespace chain { namespace eosvmoc {
 
 struct config {
    uint64_t cache_size = 1024u*1024u*1024u;
    uint64_t threads    = 1u;
+
+   // subjective limits for OC compilation.
+   // nodeos enforces the limits by the default values.
+   // libtester disables the limits in all tests, except enforces the limits
+   // in the tests in unittests/eosvmoc_limits_tests.cpp.
+   std::optional<rlim_t>   cpu_limit {20u};
+   std::optional<rlim_t>   vm_limit  {512u*1024u*1024u};
+   std::optional<uint64_t> stack_size_limit {16u*1024u};
+   std::optional<size_t>   generated_code_size_limit {16u*1024u*1024u};
 };
 
 }}}
+
+FC_REFLECT(eosio::chain::eosvmoc::config, (cache_size)(threads)(cpu_limit)(vm_limit)(stack_size_limit)(generated_code_size_limit))

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/ipc_protocol.hpp

@@ -22,6 +22,7 @@ struct code_tuple {
 
 struct compile_wasm_message {
    code_tuple code;
+   eosvmoc::config eosvmoc_config;
    //Two sent fd: 1) communication socket for result, 2) the wasm to compile
 };
 
@@ -62,7 +63,7 @@ using eosvmoc_message = std::variant<initialize_message,
 FC_REFLECT(eosio::chain::eosvmoc::initialize_message, )
 FC_REFLECT(eosio::chain::eosvmoc::initalize_response_message, (error_message))
 FC_REFLECT(eosio::chain::eosvmoc::code_tuple, (code_id)(vm_version))
-FC_REFLECT(eosio::chain::eosvmoc::compile_wasm_message, (code))
+FC_REFLECT(eosio::chain::eosvmoc::compile_wasm_message, (code)(eosvmoc_config))
 FC_REFLECT(eosio::chain::eosvmoc::evict_wasms_message, (codes))
 FC_REFLECT(eosio::chain::eosvmoc::code_compilation_result_message, (start)(apply_offset)(starting_memory_pages)(initdata_prologue_size))
 FC_REFLECT(eosio::chain::eosvmoc::compilation_result_unknownfailure, )

libraries/chain/transaction_metadata.cpp

@@ -12,15 +12,23 @@ recover_keys_future transaction_metadata::start_recover_keys( packed_transaction
                                                               uint32_t max_variable_sig_size )
 {
    return post_async_task( thread_pool, [trx{std::move(trx)}, chain_id, time_limit, t, max_variable_sig_size]() mutable {
-         fc::time_point deadline = time_limit == fc::microseconds::maximum() ?
-                                   fc::time_point::maximum() : fc::time_point::now() + time_limit;
-         check_variable_sig_size( trx, max_variable_sig_size );
-         const signed_transaction& trn = trx->get_signed_transaction();
-         flat_set<public_key_type> recovered_pub_keys;
-         fc::microseconds cpu_usage = trn.get_signature_keys( chain_id, deadline, recovered_pub_keys );
-         return std::make_shared<transaction_metadata>( private_type(), std::move( trx ), cpu_usage, std::move( recovered_pub_keys ), t );
-      }
-   );
+      return recover_keys( std::move(trx), chain_id, time_limit, t, max_variable_sig_size );
+   });
+}
+
+transaction_metadata_ptr transaction_metadata::recover_keys( packed_transaction_ptr trx,
+                                                              const chain_id_type& chain_id,
+                                                              fc::microseconds time_limit,
+                                                              trx_type t,
+                                                              uint32_t max_variable_sig_size )
+{
+   fc::time_point deadline = time_limit == fc::microseconds::maximum() ?
+                             fc::time_point::maximum() : fc::time_point::now() + time_limit;
+   check_variable_sig_size( trx, max_variable_sig_size );
+   const signed_transaction& trn = trx->get_signed_transaction();
+   flat_set<public_key_type> recovered_pub_keys;
+   fc::microseconds cpu_usage = trn.get_signature_keys( chain_id, deadline, recovered_pub_keys );
+   return std::make_shared<transaction_metadata>( private_type(), std::move( trx ), cpu_usage, std::move( recovered_pub_keys ), t );
 }
 
 size_t transaction_metadata::get_estimated_size() const {

libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMJIT.cpp

@@ -274,7 +274,7 @@ namespace LLVMJIT
 		final_pic_code = std::move(*unitmemorymanager->code);
 	}
 
-	instantiated_code instantiateModule(const IR::Module& module)
+	instantiated_code instantiateModule(const IR::Module& module, uint64_t stack_size_limit, size_t generated_code_size_limit)
 	{
 		static bool inited;
 		if(!inited) {
@@ -315,13 +315,13 @@ namespace LLVMJIT
 				WAVM_ASSERT_THROW(!!c);
 
 				++num_functions_stack_size_found;
-				if(stack_size > 16u*1024u)
+				if(stack_size > stack_size_limit)
 					_exit(1);
 			}
 		}
 		if(num_functions_stack_size_found != module.functions.defs.size())
 			_exit(1);
-		if(jitModule->final_pic_code.size() >= 16u*1024u*1024u)
+		if(jitModule->final_pic_code.size() >= generated_code_size_limit)
 			_exit(1);
 
 		instantiated_code ret;

libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMJIT.h

@@ -15,6 +15,6 @@ struct instantiated_code {
 };
 
 namespace LLVMJIT {
-   instantiated_code instantiateModule(const IR::Module& module);
+   instantiated_code instantiateModule(const IR::Module& module, uint64_t stack_size_limit, size_t generated_code_size_limit);
 }
 }}}

libraries/chain/webassembly/runtimes/eos-vm-oc/code_cache.cpp

@@ -126,7 +126,7 @@ const code_descriptor* const code_cache_async::get_descriptor_for_code(bool high
             _outstanding_compiles_and_poison.emplace(*nextup, false);
             std::vector<wrapped_fd> fds_to_pass;
             fds_to_pass.emplace_back(memfd_for_bytearray(codeobject->code));
-            FC_ASSERT(write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ *nextup }, fds_to_pass), "EOS VM failed to communicate to OOP manager");
+            FC_ASSERT(write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ *nextup, _eosvmoc_config }, fds_to_pass), "EOS VM failed to communicate to OOP manager");
             --count_processed;
          }
          _queued_compiles.erase(nextup);
@@ -179,7 +179,7 @@ const code_descriptor* const code_cache_async::get_descriptor_for_code(bool high
    _outstanding_compiles_and_poison.emplace(ct, false);
    std::vector<wrapped_fd> fds_to_pass;
    fds_to_pass.emplace_back(memfd_for_bytearray(codeobject->code));
-   write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ ct }, fds_to_pass);
+   write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ ct, _eosvmoc_config }, fds_to_pass);
    failure = get_cd_failure::temporary; // Compile might not be done yet
    return nullptr;
 }
@@ -211,7 +211,7 @@ const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const
    std::vector<wrapped_fd> fds_to_pass;
    fds_to_pass.emplace_back(memfd_for_bytearray(codeobject->code));
 
-   write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ {code_id, vm_version} }, fds_to_pass);
+   write_message_with_fds(_compile_monitor_write_socket, compile_wasm_message{ {code_id, vm_version}, _eosvmoc_config }, fds_to_pass);
    auto [success, message, fds] = read_message_with_fds(_compile_monitor_read_socket);
    EOS_ASSERT(success, wasm_execution_error, "failed to read response from monitor process");
    EOS_ASSERT(std::holds_alternative<wasm_compilation_result_message>(message), wasm_execution_error, "unexpected response from monitor process");
@@ -226,6 +226,7 @@ const code_descriptor* const code_cache_sync::get_descriptor_for_code_sync(const
 
 code_cache_base::code_cache_base(const std::filesystem::path& data_dir, const eosvmoc::config& eosvmoc_config, const chainbase::database& db) :
    _db(db),
+   _eosvmoc_config(eosvmoc_config),
    _cache_file_path(data_dir/"code_cache.bin") {
    static_assert(sizeof(allocator_t) <= header_offset, "header offset intersects with allocator");
 

libraries/chain/webassembly/runtimes/eos-vm-oc/compile_monitor.cpp

@@ -71,7 +71,7 @@ struct compile_monitor_session {
                   connection_dead_signal();
                   return;
                }
-               kick_compile_off(compile.code, std::move(fds[0]));
+               kick_compile_off(compile.code, compile.eosvmoc_config, std::move(fds[0]));
             },
             [&](const evict_wasms_message& evict) {
                for(const code_descriptor& cd : evict.codes) {
@@ -90,7 +90,7 @@ struct compile_monitor_session {
       });
    }
 
-   void kick_compile_off(const code_tuple& code_id, wrapped_fd&& wasm_code) {
+   void kick_compile_off(const code_tuple& code_id, const eosvmoc::config& eosvmoc_config, wrapped_fd&& wasm_code) {
       //prepare a requst to go out to the trampoline
       int socks[2];
       socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, socks);
@@ -100,7 +100,7 @@ struct compile_monitor_session {
       fds_pass_to_trampoline.emplace_back(socks[1]);
       fds_pass_to_trampoline.emplace_back(std::move(wasm_code));
 
-      eosvmoc_message trampoline_compile_request = compile_wasm_message{code_id};
+      eosvmoc_message trampoline_compile_request = compile_wasm_message{code_id, eosvmoc_config};
       if(write_message_with_fds(_trampoline_socket, trampoline_compile_request, fds_pass_to_trampoline) == false) {
          wasm_compilation_result_message reply{code_id, compilation_result_unknownfailure{}, _allocator->get_free_memory()};
          write_message_with_fds(_nodeos_instance_socket, reply);