diff --git a/.cirrus.yml b/.cirrus.yml index 393237af66b74..5e2f9eea9bd58 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,6 +4,7 @@ env: # Global defaults MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling processes and setting this variable avoids killing the CI script itself on error + RUN_FUNCTIONAL_TESTS: false # Template Provider doesn't have its own functional tests # A self-hosted machine(s) can be used via Cirrus CI. It can be configured with # multiple users to run tasks in parallel. No sudo permission is required. @@ -187,13 +188,13 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: 'multiprocess, i686, DEBUG' + name: 'no multiprocess, i686, DEBUG' << : *GLOBAL_TASK_TEMPLATE persistent_worker: labels: type: medium env: - FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh" + FILE_ENV: "./ci/test/00_setup_env_i686_no_multiprocess.sh" task: name: 'no wallet, libbitcoinkernel' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1da02a86b07b4..86e82648dd861 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,12 +67,12 @@ jobs: echo "TEST_BASE=$(git rev-list -n$((${{ env.MAX_COUNT }} + 1)) --reverse HEAD $EXCLUDE_MERGE_BASE_ANCESTORS | head -1)" >> "$GITHUB_ENV" - run: | sudo apt-get update - sudo apt-get install clang ccache build-essential cmake pkgconf python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev -y + sudo apt-get install clang ccache build-essential cmake pkgconf python3-zmq libevent-dev libboost-dev libsqlite3-dev libdb++-dev systemtap-sdt-dev libzmq3-dev qtbase5-dev qttools5-dev qttools5-dev-tools qtwayland5 libqrencode-dev libcapnp-dev capnproto -y - name: Compile and run tests run: | # Run tests on commits after the last merge commit and before the PR head commit # Use clang++, because it is a bit faster and uses less memory than g++ - git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWERROR=ON -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DWITH_BDB=ON -DWITH_USDT=ON -DCMAKE_CXX_FLAGS='-Wno-error=unused-member-function' && cmake --build build -j $(nproc) && ctest --output-on-failure --stop-on-failure --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 )) --combinedlogslen=99999999" ${{ env.TEST_BASE }} + git rebase --exec "echo Running test-one-commit on \$( git log -1 ) && CC=clang CXX=clang++ cmake -B build -DWERROR=ON -DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON -DCMAKE_CXX_FLAGS='-Wno-error=unused-member-function' && cmake --build build -j $(nproc) && ctest --output-on-failure --stop-on-failure --test-dir build -j $(nproc) && ./build/test/functional/test_runner.py -j $(( $(nproc) * 2 )) --combinedlogslen=99999999" ${{ env.TEST_BASE }} macos-native-arm64: name: ${{ matrix.job-name }} @@ -124,7 +124,7 @@ jobs: run: | # A workaround for "The `brew link` step did not complete successfully" error. brew install --quiet python@3 || brew link --overwrite python@3 - brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent zeromq qt@5 qrencode + brew install --quiet coreutils ninja pkgconf gnu-getopt ccache boost libevent zeromq qt@5 qrencode capnp - name: Set Ccache directory run: echo "CCACHE_DIR=${RUNNER_TEMP}/ccache_dir" >> "$GITHUB_ENV" diff --git a/CMakeLists.txt b/CMakeLists.txt index 5fa1ae668c1c8..1154ecbf76e6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -141,15 +141,16 @@ endif() cmake_dependent_option(WITH_DBUS "Enable DBus support." ON "CMAKE_SYSTEM_NAME STREQUAL \"Linux\" AND BUILD_GUI" OFF) -option(WITH_MULTIPROCESS "Build multiprocess bitcoin-node and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables. Requires libmultiprocess library. Experimental." OFF) -if(WITH_MULTIPROCESS) +cmake_dependent_option(ENABLE_IPC "Build multiprocess bitcoin-node and bitcoin-gui executables in addition to monolithic bitcoind and bitcoin-qt executables. Requires libmultiprocess library. Experimental." ON "NOT WIN32" OFF) +option(WITH_EXTERNAL_LIBMULTIPROCESS "Build with external libmultiprocess library instead of with local git subtree when ENABLE_IPC is enabled. This is not normally recommended, but can be useful when cross-compiling or making changes to the upstream project." OFF) +if(ENABLE_IPC AND WITH_EXTERNAL_LIBMULTIPROCESS) find_package(Libmultiprocess REQUIRED COMPONENTS Lib) find_package(LibmultiprocessNative REQUIRED COMPONENTS Bin NAMES Libmultiprocess ) endif() -option(WITH_SV2 "Enable Stratum v2 functionality." ON) +cmake_dependent_option(WITH_SV2 "Enable Stratum v2 functionality." ON "NOT WIN32" OFF) cmake_dependent_option(BUILD_GUI_TESTS "Build test_bitcoin-qt executable." ON "BUILD_GUI;BUILD_TESTS" OFF) if(BUILD_GUI) @@ -611,9 +612,9 @@ message("Configure summary") message("=================") message("Executables:") message(" bitcoind ............................ ${BUILD_DAEMON}") -message(" bitcoin-node (multiprocess) ......... ${WITH_MULTIPROCESS}") +message(" bitcoin-node (multiprocess) ......... ${ENABLE_IPC}") message(" bitcoin-qt (GUI) .................... ${BUILD_GUI}") -if(BUILD_GUI AND WITH_MULTIPROCESS) +if(BUILD_GUI AND ENABLE_IPC) set(bitcoin_gui_status ON) else() set(bitcoin_gui_status OFF) diff --git a/CMakePresets.json b/CMakePresets.json index da838f2b0e3f5..dcbac45dfc1e4 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -81,7 +81,7 @@ "ENABLE_WALLET": "ON", "WARN_INCOMPATIBLE_BDB": "OFF", "WITH_BDB": "ON", - "WITH_MULTIPROCESS": "ON", + "ENABLE_IPC": "ON", "WITH_QRENCODE": "ON", "WITH_SQLITE": "ON", "WITH_USDT": "ON", diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 67e43aca0d1cc..5995dab66c114 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -16,6 +16,7 @@ export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false export GOAL="install" +export DEP_OPTS="MULTIPROCESS=1" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized'" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_no_multiprocess.sh similarity index 82% rename from ci/test/00_setup_env_i686_multiprocess.sh rename to ci/test/00_setup_env_i686_no_multiprocess.sh index 1aa143a7f08f9..f1b8af0b8268d 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_no_multiprocess.sh @@ -7,11 +7,11 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu -export CONTAINER_NAME=ci_i686_multiprocess +export CONTAINER_NAME=ci_i686_no_multiprocess export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CI_IMAGE_PLATFORM="linux/amd64" export PACKAGES="llvm clang g++-multilib" -export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" +export DEP_OPTS="DEBUG=1 NO_MULTIPROCESS=1" export GOAL="install" export TEST_RUNNER_EXTRA="--v2transport" export BITCOIN_CONFIG="\ @@ -20,4 +20,3 @@ export BITCOIN_CONFIG="\ -DCMAKE_CXX_COMPILER='clang++;-m32' \ -DAPPEND_CPPFLAGS='-DBOOST_MULTI_INDEX_ENABLE_SAFE_MODE' \ " -export BITCOIND=bitcoin-node # Used in functional tests diff --git a/ci/test/00_setup_env_mac_cross.sh b/ci/test/00_setup_env_mac_cross.sh index 6a1e116a547b7..4fc2006da8c9a 100755 --- a/ci/test/00_setup_env_mac_cross.sh +++ b/ci/test/00_setup_env_mac_cross.sh @@ -17,4 +17,4 @@ export XCODE_BUILD_ID=15A240d export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="-DBUILD_GUI=ON -DREDUCE_EXPORTS=ON" +export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON" diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh index e01a56895bfb6..86b34aeb8e756 100755 --- a/ci/test/00_setup_env_mac_native.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -11,7 +11,11 @@ export LC_ALL=C.UTF-8 export PIP_PACKAGES="--break-system-packages zmq" export GOAL="install" export CMAKE_GENERATOR="Ninja" -export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DREDUCE_EXPORTS=ON" +export BITCOIN_CONFIG="\ +-DENABLE_IPC=ON \ +-DREDUCE_EXPORTS=ON \ +" export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" +export BITCOIND=bitcoin-node # Used in functional tests diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index c59a506a71989..ab2826f6dacd1 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -20,11 +20,11 @@ fi export CONTAINER_NAME=ci_native_asan export APT_LLVM_V="20" -export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" +export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev python3-zmq qtbase5-dev qttools5-dev qttools5-dev-tools libevent-dev libboost-dev libdb5.3++-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE} libcapnp-dev capnproto" export NO_DEPENDS=1 export GOAL="install" export BITCOIN_CONFIG="\ - -DWITH_USDT=ON -DWITH_ZMQ=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF -DBUILD_GUI=ON \ + -DENABLE_IPC=ON \ -DSANITIZERS=address,float-divide-by-zero,integer,undefined \ -DCMAKE_C_COMPILER=clang-${APT_LLVM_V} \ -DCMAKE_CXX_COMPILER=clang++-${APT_LLVM_V} \ diff --git a/ci/test/00_setup_env_native_centos.sh b/ci/test/00_setup_env_native_centos.sh index c423d788eb846..1c691827a4de3 100755 --- a/ci/test/00_setup_env_native_centos.sh +++ b/ci/test/00_setup_env_native_centos.sh @@ -10,6 +10,7 @@ export CONTAINER_NAME=ci_native_centos export CI_IMAGE_NAME_TAG="quay.io/centos/centos:stream10" export CI_BASE_PACKAGES="gcc-c++ glibc-devel libstdc++-devel ccache make git python3 python3-pip which patch xz procps-ng ksh rsync coreutils bison e2fsprogs cmake" export PIP_PACKAGES="pyzmq" -export DEP_OPTS="DEBUG=1" # Temporarily enable a DEBUG=1 build to check for GCC-bug-117966 regressions. This can be removed once the minimum GCC version is bumped to 12 in the previous releases task, see https://github.com/bitcoin/bitcoin/issues/31436#issuecomment-2530717875 +export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" # Temporarily enable a DEBUG=1 build to check for GCC-bug-117966 regressions. This can be removed once the minimum GCC version is bumped to 12 in the previous releases task, see https://github.com/bitcoin/bitcoin/issues/31436#issuecomment-2530717875 export GOAL="install" export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DBUILD_GUI=ON -DREDUCE_EXPORTS=ON -DCMAKE_BUILD_TYPE=Debug" +export BITCOIND=bitcoin-node # Used in functional tests diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index 1b5a27bb6cd15..ca1939a022208 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_fuzz export APT_LLVM_V="20" -export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libevent-dev libboost-dev libsqlite3-dev" +export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libevent-dev libboost-dev libsqlite3-dev libcapnp-dev capnproto" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -17,6 +17,7 @@ export RUN_FUZZ_TESTS=true export GOAL="install" export CI_CONTAINER_CAP="--cap-add SYS_PTRACE" # If run with (ASan + LSan), the container needs access to ptrace (https://github.com/google/sanitizers/issues/764) export BITCOIN_CONFIG="\ + -DENABLE_IPC=ON \ -DBUILD_FOR_FUZZING=ON \ -DSANITIZERS=fuzzer,address,undefined,float-divide-by-zero,integer \ -DCMAKE_C_COMPILER=clang-${APT_LLVM_V} \ diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index cfdbc8c0142a7..562e3f24bd4b2 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -15,7 +15,7 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_fuzz_msan" export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 MULTIPROCESS=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" # Setting CMAKE_{C,CXX}_FLAGS_DEBUG flags to an empty string ensures that the flags set in MSAN_FLAGS remain unaltered. # _FORTIFY_SOURCE is not compatible with MSAN. diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index c65c05bff9ea0..6706886d75336 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libevent-dev libboost-dev libsqlite3-dev valgrind" +export PACKAGES="clang-16 llvm-16 libclang-rt-16-dev libevent-dev libboost-dev libsqlite3-dev valgrind libcapnp-dev capnproto" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -16,6 +16,7 @@ export RUN_FUZZ_TESTS=true export FUZZ_TESTS_CONFIG="--valgrind" export GOAL="install" export BITCOIN_CONFIG="\ + -DENABLE_IPC=ON \ -DBUILD_FOR_FUZZING=ON \ -DSANITIZERS=fuzzer \ -DCMAKE_C_COMPILER=clang-16 \ diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index cd4ac99942890..8558cae2f6aaf 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -15,11 +15,12 @@ export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" export CONTAINER_NAME="ci_native_msan" export PACKAGES="ninja-build" # BDB generates false-positives and will be removed in future -export DEP_OPTS="DEBUG=1 NO_BDB=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="DEBUG=1 NO_WALLET=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" # Setting CMAKE_{C,CXX}_FLAGS_DEBUG flags to an empty string ensures that the flags set in MSAN_FLAGS remain unaltered. # _FORTIFY_SOURCE is not compatible with MSAN. export BITCOIN_CONFIG="\ + -DENABLE_WALLET=OFF \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_C_FLAGS_DEBUG='' \ -DCMAKE_CXX_FLAGS_DEBUG='' \ diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 3d5d1b7745634..0b9406de39db7 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -10,6 +10,6 @@ export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel export CI_IMAGE_NAME_TAG="docker.io/debian:bookworm" # Use minimum supported python3.10 (or best-effort 3.11) and clang-16, see doc/dependencies.md export PACKAGES="python3-zmq clang-16 llvm-16 libc++abi-16-dev libc++-16-dev" -export DEP_OPTS="NO_WALLET=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'" +export DEP_OPTS="NO_WALLET=1 MULTIPROCESS=1 CC=clang-16 CXX='clang++-16 -stdlib=libc++'" export GOAL="install" export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DBUILD_UTIL_CHAINSTATE=ON -DBUILD_KERNEL_LIB=ON -DBUILD_SHARED_LIBS=ON" diff --git a/ci/test/00_setup_env_native_previous_releases.sh b/ci/test/00_setup_env_native_previous_releases.sh index 19a33f14dd858..4b490c3c0e40f 100755 --- a/ci/test/00_setup_env_native_previous_releases.sh +++ b/ci/test/00_setup_env_native_previous_releases.sh @@ -10,14 +10,14 @@ export CONTAINER_NAME=ci_native_previous_releases export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04" # Use minimum supported python3.10 and gcc-11, see doc/dependencies.md export PACKAGES="gcc-11 g++-11 python3-zmq" -export DEP_OPTS="DEBUG=1 CC=gcc-11 CXX=g++-11" +export DEP_OPTS="DEBUG=1 MULTIPROCESS=1 CC=gcc-11 CXX=g++-11" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" export DOWNLOAD_PREVIOUS_RELEASES="true" export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=ON -DREDUCE_EXPORTS=ON \ + -DWITH_ZMQ=OFF -DBUILD_GUI=OFF -DREDUCE_EXPORTS=ON \ -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_C_FLAGS='-funsigned-char' \ -DCMAKE_C_FLAGS_DEBUG='-g0 -O2' \ diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index b85fae859c270..f325dd26272ee 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -10,7 +10,7 @@ export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_tidy export TIDY_LLVM_V="19" export APT_LLVM_V="${TIDY_LLVM_V}" -export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" +export PACKAGES="clang-${TIDY_LLVM_V} libclang-${TIDY_LLVM_V}-dev llvm-${TIDY_LLVM_V}-dev libomp-${TIDY_LLVM_V}-dev clang-tidy-${TIDY_LLVM_V} jq libevent-dev libboost-dev libzmq3-dev systemtap-sdt-dev qtbase5-dev qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev libcapnp-dev capnproto" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false @@ -18,8 +18,10 @@ export RUN_FUZZ_TESTS=false export RUN_CHECK_DEPS=true export RUN_TIDY=true export GOAL="install" +# Wallet support is kept, because check-deps assumes it export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DBUILD_GUI=ON -DBUILD_BENCH=ON -DWITH_USDT=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF \ + -DENABLE_IPC=ON \ + -DBUILD_BENCH=ON \ -DENABLE_HARDENING=OFF \ -DCMAKE_C_COMPILER=clang-${TIDY_LLVM_V} \ -DCMAKE_CXX_COMPILER=clang++-${TIDY_LLVM_V} \ diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 988107e7fb49d..2c1e4df369f36 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -10,7 +10,10 @@ export CONTAINER_NAME=ci_native_tsan export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export APT_LLVM_V="20" export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libc++abi-${APT_LLVM_V}-dev libc++-${APT_LLVM_V}-dev python3-zmq" -export DEP_OPTS="CC=clang-${APT_LLVM_V} CXX='clang++-${APT_LLVM_V} -stdlib=libc++'" +export DEP_OPTS="CC=clang-${APT_LLVM_V} CXX='clang++-${APT_LLVM_V} -stdlib=libc++' MULTIPROCESS=1" export GOAL="install" -export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DSANITIZERS=thread \ --DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES'" +export BITCOIN_CONFIG="\ +-DENABLE_IPC=ON \ +-DSANITIZERS=thread \ +-DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES'\ +" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 3acb5842a41d6..f401b376620d4 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -8,14 +8,14 @@ export LC_ALL=C.UTF-8 export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04" export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libzmq3-dev libsqlite3-dev" +export PACKAGES="valgrind clang-16 llvm-16 libclang-rt-16-dev python3-zmq libevent-dev libboost-dev libdb5.3++-dev libzmq3-dev libsqlite3-dev libcapnp-dev capnproto" export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # feature_init excluded for now, see https://github.com/bitcoin/bitcoin/issues/30011 ; bind tests excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export GOAL="install" -# TODO enable GUI export BITCOIN_CONFIG="\ - -DWITH_ZMQ=ON -DWITH_BDB=ON -DWARN_INCOMPATIBLE_BDB=OFF -DBUILD_GUI=OFF \ + -DENABLE_IPC=ON \ + -DBUILD_GUI=OFF \ -DCMAKE_C_COMPILER=clang-16 \ -DCMAKE_CXX_COMPILER=clang++-16 \ " diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 0e0ddf3382532..6da6f22f861b8 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -14,4 +14,5 @@ export CI_IMAGE_PLATFORM="linux/s390x" export TEST_RUNNER_EXTRA="--exclude rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true export GOAL="install" +export DEP_OPTS="MULTIPROCESS=1" export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON" diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index bd5c86bfbe701..b392591224d34 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -118,7 +118,10 @@ BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build-$HOST} mkdir -p "${BASE_BUILD_DIR}" cd "${BASE_BUILD_DIR}" -BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DENABLE_EXTERNAL_SIGNER=ON -DCMAKE_INSTALL_PREFIX=$BASE_OUTDIR" +# Disable things the Template Provider binary doesn't use +# TODO: +# -DENABLE_WALLET=OFF : needs handling in check-deps.sh (called by tidy job) +BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_INSTALL_PREFIX=$BASE_OUTDIR" if [[ "${RUN_TIDY}" == "true" ]]; then BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_EXPORT_COMPILE_COMMANDS=ON" diff --git a/cmake/module/GenerateSetupNsi.cmake b/cmake/module/GenerateSetupNsi.cmake index 97a53b071dbb8..00d6fcf9687aa 100644 --- a/cmake/module/GenerateSetupNsi.cmake +++ b/cmake/module/GenerateSetupNsi.cmake @@ -7,11 +7,8 @@ function(generate_setup_nsi) set(abs_top_builddir ${PROJECT_BINARY_DIR}) set(CLIENT_URL ${PROJECT_HOMEPAGE_URL}) set(CLIENT_TARNAME "bitcoin") - set(BITCOIN_GUI_NAME "bitcoin-qt") - set(BITCOIN_DAEMON_NAME "bitcoind") + # TODO: add bitcoin-node and bitcoin-miner set(BITCOIN_CLI_NAME "bitcoin-cli") - set(BITCOIN_TX_NAME "bitcoin-tx") - set(BITCOIN_WALLET_TOOL_NAME "bitcoin-wallet") set(BITCOIN_TEST_NAME "test_bitcoin") set(EXEEXT ${CMAKE_EXECUTABLE_SUFFIX}) configure_file(${PROJECT_SOURCE_DIR}/share/setup.nsi.in ${PROJECT_BINARY_DIR}/bitcoin-win64-setup.nsi USE_SOURCE_PERMISSIONS @ONLY) diff --git a/cmake/module/Maintenance.cmake b/cmake/module/Maintenance.cmake index 61251d24397b3..c554340cda60d 100644 --- a/cmake/module/Maintenance.cmake +++ b/cmake/module/Maintenance.cmake @@ -42,7 +42,7 @@ function(add_maintenance_targets) VERBATIM ) - foreach(target IN ITEMS bitcoind bitcoin-qt bitcoin-cli bitcoin-tx bitcoin-util bitcoin-wallet test_bitcoin bench_bitcoin) + foreach(target IN ITEMS bitcoind bitcoin-node bitcoin-qt bitcoin-gui bitcoin-cli bitcoin-tx bitcoin-util bitcoin-wallet bitcoin-mine test_bitcoin bench_bitcoin) if(TARGET ${target}) list(APPEND executables $) endif() @@ -66,7 +66,8 @@ function(add_maintenance_targets) endfunction() function(add_windows_deploy_target) - if(MINGW AND TARGET bitcoin-qt AND TARGET bitcoind AND TARGET bitcoin-cli AND TARGET bitcoin-tx AND TARGET bitcoin-wallet AND TARGET bitcoin-util AND TARGET test_bitcoin) + # TODO: add TARGET bitcoin-node TARGET bitcoin-mine + if(MINGW AND TARGET bitcoin-cli AND TARGET test_bitcoin) # TODO: Consider replacing this code with the CPack NSIS Generator. # See https://cmake.org/cmake/help/latest/cpack_gen/nsis.html include(GenerateSetupNsi) @@ -74,12 +75,7 @@ function(add_windows_deploy_target) add_custom_command( OUTPUT ${PROJECT_BINARY_DIR}/bitcoin-win64-setup.exe COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/release - COMMAND ${CMAKE_STRIP} $ -o ${PROJECT_BINARY_DIR}/release/$ - COMMAND ${CMAKE_STRIP} $ -o ${PROJECT_BINARY_DIR}/release/$ COMMAND ${CMAKE_STRIP} $ -o ${PROJECT_BINARY_DIR}/release/$ - COMMAND ${CMAKE_STRIP} $ -o ${PROJECT_BINARY_DIR}/release/$ - COMMAND ${CMAKE_STRIP} $ -o ${PROJECT_BINARY_DIR}/release/$ - COMMAND ${CMAKE_STRIP} $ -o ${PROJECT_BINARY_DIR}/release/$ COMMAND ${CMAKE_STRIP} $ -o ${PROJECT_BINARY_DIR}/release/$ COMMAND makensis -V2 ${PROJECT_BINARY_DIR}/bitcoin-win64-setup.nsi VERBATIM diff --git a/contrib/devtools/check-deps.sh b/contrib/devtools/check-deps.sh index cdfc4e7533d04..529eeca7cb559 100755 --- a/contrib/devtools/check-deps.sh +++ b/contrib/devtools/check-deps.sh @@ -44,12 +44,6 @@ declare -A SUPPRESS # init.cpp file currently calls Berkeley DB sanity check function on startup, so # there is an undocumented dependency of the node library on the wallet library. SUPPRESS["init.cpp.o bdb.cpp.o _ZN6wallet27BerkeleyDatabaseSanityCheckEv"]=1 -# init/common.cpp file calls InitError and InitWarning from interface_ui which -# is currently part of the node library. interface_ui should just be part of the -# common library instead, and is moved in -# https://github.com/bitcoin/bitcoin/issues/10102 -SUPPRESS["common.cpp.o interface_ui.cpp.o _Z11InitWarningRK13bilingual_str"]=1 -SUPPRESS["common.cpp.o interface_ui.cpp.o _Z9InitErrorRK13bilingual_str"]=1 # rpc/external_signer.cpp adds defines node RPC methods but is built as part of the # common library. It should be moved to the node library instead. SUPPRESS["external_signer.cpp.o server.cpp.o _ZN9CRPCTable13appendCommandERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPK11CRPCCommand"]=1 diff --git a/contrib/devtools/gen-manpages.py b/contrib/devtools/gen-manpages.py index 14c8c408e83af..b94054df1a6f6 100755 --- a/contrib/devtools/gen-manpages.py +++ b/contrib/devtools/gen-manpages.py @@ -11,6 +11,7 @@ BINARIES = [ 'src/bitcoind', 'src/bitcoin-cli', +'src/bitcoin-mine', 'src/bitcoin-tx', 'src/bitcoin-wallet', 'src/bitcoin-util', diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 6c252e787022a..78f8d2b2b2ae6 100755 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -176,7 +176,11 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \ x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \ x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \ x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \ - x86_64_linux_STRIP=x86_64-linux-gnu-strip + x86_64_linux_STRIP=x86_64-linux-gnu-strip \ + NO_QT=1 \ + NO_WALLET=1 \ + NO_UPNP=1 \ + NO_ZMQ=1 case "$HOST" in *darwin*) @@ -205,8 +209,10 @@ mkdir -p "$OUTDIR" # Binary Tarball Building # ########################### -# CONFIGFLAGS -CONFIGFLAGS="-DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF" +# TODO: once bitcoin-node with Mining interface is in a release: +# -DBUILD_CLI=OFF +# - don't build bitcoin-node and bitcoin-gui +CONFIGFLAGS="-DBUILD_DAEMON=OFF -DBUILD_BUILD_TESTS=OFF -DBUILD_UTIL=off -DENABLE_WALLET=OFF -DENABLE_EXTERNAL_SIGNER=OFF -DWITH_NATPMP=OFF -DWITH_MINIUPNPC=OFF -DWITH_USDT=OFF -DWITH_ZMQ=OFF -DREDUCE_EXPORTS=ON -DBUILD_BENCH=OFF -DBUILD_GUI_TESTS=OFF -DBUILD_FUZZ_BINARY=OFF" # CFLAGS HOST_CFLAGS="-O2 -g" @@ -285,21 +291,7 @@ mkdir -p "$DISTSRC" case "$HOST" in *darwin*) - cmake --build build --target deploy ${V:+--verbose} - mv build/dist/Bitcoin-Core.zip "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.zip" - mkdir -p "unsigned-app-${HOST}" - cp --target-directory="unsigned-app-${HOST}" \ - contrib/macdeploy/detached-sig-create.sh - mv --target-directory="unsigned-app-${HOST}" build/dist - ( - cd "unsigned-app-${HOST}" - find . -print0 \ - | sort --zero-terminated \ - | tar --create --no-recursion --mode='u+rw,go+r-w,a+X' --null --files-from=- \ - | gzip -9n > "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" \ - || ( rm -f "${OUTDIR}/${DISTNAME}-${HOST}-unsigned.tar.gz" && exit 1 ) - ) - ;; + # Don't create desktop zip esac ( cd installed diff --git a/depends/Makefile b/depends/Makefile index 7eb0988d635aa..3dc21061a320f 100644 --- a/depends/Makefile +++ b/depends/Makefile @@ -41,7 +41,10 @@ NO_SQLITE ?= NO_WALLET ?= NO_ZMQ ?= NO_USDT ?= -MULTIPROCESS ?= +# Default NO_MULTIPROCESS value is 1 on unsupported host platforms: +# - Windows +# - OpenBSD: https://github.com/capnproto/capnproto/pull/1907 +NO_MULTIPROCESS ?= $(if $(findstring mingw32,$(HOST))$(findstring openbsd,$(HOST)),1,) LTO ?= NO_HARDEN ?= FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources @@ -165,7 +168,7 @@ sqlite_packages_$(NO_SQLITE) = $(sqlite_packages) wallet_packages_$(NO_WALLET) = $(bdb_packages_) $(sqlite_packages_) zmq_packages_$(NO_ZMQ) = $(zmq_packages) -multiprocess_packages_$(MULTIPROCESS) = $(multiprocess_packages) +multiprocess_packages_$(NO_MULTIPROCESS) = $(multiprocess_packages) usdt_packages_$(NO_USDT) = $(usdt_$(host_os)_packages) packages += $($(host_arch)_$(host_os)_packages) $($(host_os)_packages) $(boost_packages_) $(libevent_packages_) $(qt_packages_) $(wallet_packages_) $(usdt_packages_) @@ -175,7 +178,7 @@ ifneq ($(zmq_packages_),) packages += $(zmq_packages) endif -ifeq ($(multiprocess_packages_),) +ifneq ($(multiprocess_packages_),) packages += $(multiprocess_packages) native_packages += $(multiprocess_native_packages) endif @@ -237,7 +240,7 @@ $(host_prefix)/toolchain.cmake : toolchain.cmake.in $(host_prefix)/.stamp_$(fina -e 's|@sqlite_packages@|$(sqlite_packages_)|' \ -e 's|@usdt_packages@|$(usdt_packages_)|' \ -e 's|@no_harden@|$(NO_HARDEN)|' \ - -e 's|@multiprocess@|$(MULTIPROCESS)|' \ + -e 's|@multiprocess_packages@|$(multiprocess_packages_)|' \ $< > $@ touch $@ diff --git a/depends/README.md b/depends/README.md index 848137f03faf7..d2fb423db4651 100644 --- a/depends/README.md +++ b/depends/README.md @@ -119,7 +119,7 @@ The following can be set when running make: `make FOO=bar` - `NO_BDB`: Don't download/build/cache BerkeleyDB - `NO_SQLITE`: Don't download/build/cache SQLite - `NO_USDT`: Don't download/build/cache packages needed for enabling USDT tracepoints -- `MULTIPROCESS`: Build libmultiprocess (experimental) +- `NO_MULTIPROCESS`: Don't build experimental libmultiprocess (default on Windows and OpenBSD) - `DEBUG`: Disable some optimizations and enable more runtime checking - `HOST_ID_SALT`: Optional salt to use when generating host package ids - `BUILD_ID_SALT`: Optional salt to use when generating build package ids diff --git a/depends/funcs.mk b/depends/funcs.mk index b07432adec960..1246d6f2fcaac 100644 --- a/depends/funcs.mk +++ b/depends/funcs.mk @@ -39,8 +39,32 @@ define fetch_file $(call fetch_file_inner,$(1),$(FALLBACK_DOWNLOAD_PATH),$(3),$(4),$(5)))) endef +# Shell script to create a source tarball in $(1)_source from local directory +# $(1)_local_dir instead of downloading remote sources. Tarball is recreated if +# any paths in the local directory have a newer mtime, and checksum of the +# tarball is saved to $(1)_fetched and returned as output. +define fetch_local_dir_sha256 + if ! [ -f $($(1)_source) ] || [ -n "$$(find $($(1)_local_dir) -newer $($(1)_source) | head -n1)" ]; then \ + mkdir -p $(dir $($(1)_source)) && \ + $(build_TAR) -c -f $($(1)_source) -C $($(1)_local_dir) . && \ + rm -f $($(1)_fetched); \ + fi && \ + if ! [ -f $($(1)_fetched) ] || [ -n "$$(find $($(1)_source) -newer $($(1)_fetched))" ]; then \ + mkdir -p $(dir $($(1)_fetched)) && \ + cd $($(1)_source_dir) && \ + $(build_SHA256SUM) $($(1)_all_sources) > $($(1)_fetched); \ + fi && \ + cut -d" " -f1 $($(1)_fetched) +endef + define int_get_build_recipe_hash $(eval $(1)_all_file_checksums:=$(shell $(build_SHA256SUM) $(meta_depends) packages/$(1).mk $(addprefix $(PATCHES_PATH)/$(1)/,$($(1)_patches)) | cut -d" " -f1)) +# If $(1)_local_dir is set, create a tarball of the local directory contents to +# use as the source of the package, and include a hash of the tarball in the +# package id, so if directory contents change, the package and packages +# depending on it will be rebuilt. +$(if $($(1)_local_dir),$(eval $(1)_sha256_hash:=$(shell $(call fetch_local_dir_sha256,$(1))))) +$(if $($(1)_local_dir),$(eval $(1)_all_file_checksums+=$($(1)_sha256_hash))) $(eval $(1)_recipe_hash:=$(shell echo -n "$($(1)_all_file_checksums)" | $(build_SHA256SUM) | cut -d" " -f1)) endef @@ -53,20 +77,32 @@ $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SU final_build_id_long+=$($(package)_build_id_long) #compute package-specific paths -$(1)_build_subdir?=. -$(1)_download_file?=$($(1)_file_name) -$(1)_source_dir:=$(SOURCES_PATH) -$(1)_source:=$$($(1)_source_dir)/$($(1)_file_name) $(1)_staging_dir=$(base_staging_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id) $(1)_staging_prefix_dir:=$$($(1)_staging_dir)$($($(1)_type)_prefix) $(1)_extract_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id) -$(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version) $(1)_build_dir:=$$($(1)_extract_dir)/$$($(1)_build_subdir) $(1)_cached_checksum:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz.hash $(1)_patch_dir:=$(base_build_dir)/$(host)/$(1)/$($(1)_version)-$($(1)_build_id)/.patches-$($(1)_build_id) -$(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_cached:=$(BASE_CACHE)/$(host)/$(1)/$(1)-$($(1)_version)-$($(1)_build_id).tar.gz $(1)_build_log:=$(BASEDIR)/$(1)-$($(1)_version)-$($(1)_build_id).log +endef + +# Convert a string to a human-readable filename, replacing dot, slash, and space +# characters that could cause problems with dashes and collapsing them. +define int_friendly_file_name +$(subst $(null) $(null),-,$(strip $(subst ., ,$(subst /, ,$(1))))) +endef + +define int_get_build_properties +$(1)_build_subdir?=. +$(1)_download_file?=$($(1)_file_name) +$(1)_source_dir:=$(SOURCES_PATH) +# If $(1)_file_name is empty and $(1)_local_dir is nonempty, set file name to a +# .tar file with a friendly filename named after the directory path. +$(if $($(1)_file_name),,$(if $($(1)_local_dir),$(eval $(1)_file_name:=$(call int_friendly_file_name,$($(1)_local_dir)).tar))) +$(1)_source:=$$($(1)_source_dir)/$($(1)_file_name) +$(1)_download_dir:=$(base_download_dir)/$(1)-$($(1)_version) +$(1)_prefixbin:=$($($(1)_type)_prefix)/bin/ $(1)_all_sources=$($(1)_file_name) $($(1)_extra_sources) #stamps @@ -287,6 +323,9 @@ $(foreach package,$(all_packages),$(eval $(call int_vars,$(package)))) $(foreach native_package,$(native_packages),$(eval include packages/$(native_package).mk)) $(foreach package,$(packages),$(eval include packages/$(package).mk)) +#set build properties for included package files +$(foreach package,$(all_packages),$(eval $(call int_get_build_properties,$(package)))) + #compute a hash of all files that comprise this package's build recipe $(foreach package,$(all_packages),$(eval $(call int_get_build_recipe_hash,$(package)))) diff --git a/depends/packages.md b/depends/packages.md index 6b458f22dd434..f304f7e028ab0 100644 --- a/depends/packages.md +++ b/depends/packages.md @@ -11,7 +11,8 @@ General tips: [below](#secondary-dependencies) for more details. ## Identifiers -Each package is required to define at least these variables: +If package does not define a `$(package)_local_dir` variable, it is required to +define these variables: $(package)_version: Version of the upstream library or program. If there is no version, a @@ -28,6 +29,9 @@ Each package is required to define at least these variables: $(package)_sha256_hash: The sha256 hash of the upstream file +If a package does define a `$(package)_local_dir` variable, the above variables +are not required and will be ignored. + These variables are optional: $(package)_build_subdir: @@ -48,6 +52,18 @@ These variables are optional: Any extra files that will be fetched via $(package)_fetch_cmds. These are specified so that they can be fetched and verified via 'make download'. +## Local packages + +If a package defines a `$(package)_local_dir` variable, the specified directory +will be treated as a download source, and a tarball of its contents will be +saved to `sources/`. A hash of the tarball will also become part of the package +build id, so if the directory contents change, the package and everything +depending on it will be rebuilt. For efficiency, the tarball is cached once it +has been created, but if the local directory is touched, it will be rebuilt. + +Local packages can be useful for using git submodules or subtrees to manage +package sources, or for testing local changes that are not available to +download from an external source. ## Build Variables: After defining the main identifiers, build variables may be added or customized diff --git a/depends/packages/libmultiprocess.mk b/depends/packages/libmultiprocess.mk deleted file mode 100644 index afbd315e38806..0000000000000 --- a/depends/packages/libmultiprocess.mk +++ /dev/null @@ -1,29 +0,0 @@ -package=libmultiprocess -$(package)_version=$(native_$(package)_version) -$(package)_download_path=$(native_$(package)_download_path) -$(package)_file_name=$(native_$(package)_file_name) -$(package)_sha256_hash=$(native_$(package)_sha256_hash) -$(package)_dependencies=native_$(package) capnp -ifneq ($(host),$(build)) -$(package)_dependencies += native_capnp -endif - -define $(package)_set_vars := -ifneq ($(host),$(build)) -$(package)_config_opts := -DCAPNP_EXECUTABLE="$$(native_capnp_prefixbin)/capnp" -$(package)_config_opts += -DCAPNPC_CXX_EXECUTABLE="$$(native_capnp_prefixbin)/capnpc-c++" -endif -$(package)_cxxflags += -fdebug-prefix-map=$($(package)_extract_dir)=/usr -fmacro-prefix-map=$($(package)_extract_dir)=/usr -endef - -define $(package)_config_cmds - $($(package)_cmake) . -endef - -define $(package)_build_cmds - $(MAKE) multiprocess -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install-lib -endef diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk index 74d22207a07d8..f494f6e8a5808 100644 --- a/depends/packages/native_libmultiprocess.mk +++ b/depends/packages/native_libmultiprocess.mk @@ -1,8 +1,5 @@ package=native_libmultiprocess -$(package)_version=07c917f7ca910d66abc6d3873162fc9061704074 -$(package)_download_path=https://github.com/chaincodelabs/libmultiprocess/archive -$(package)_file_name=$($(package)_version).tar.gz -$(package)_sha256_hash=ac9db311e3b22aac3c7b7b7b3f6b7fee5cf3043ebb3c3bf412049e8b17166de8 +$(package)_local_dir=../src/ipc/libmultiprocess $(package)_dependencies=native_capnp define $(package)_config_cmds diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 61cf66230c7d9..9118cae46baa9 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -17,7 +17,7 @@ sqlite_packages=sqlite zmq_packages=zeromq -multiprocess_packages = libmultiprocess capnp +multiprocess_packages = capnp multiprocess_native_packages = native_libmultiprocess native_capnp usdt_linux_packages=systemtap diff --git a/depends/toolchain.cmake.in b/depends/toolchain.cmake.in index 89a6e369690fd..a3f02a7764f2e 100644 --- a/depends/toolchain.cmake.in +++ b/depends/toolchain.cmake.in @@ -152,10 +152,13 @@ else() set(ENABLE_HARDENING ON CACHE BOOL "") endif() -if("@multiprocess@" STREQUAL "1") - set(WITH_MULTIPROCESS ON CACHE BOOL "") - set(Libmultiprocess_ROOT "${CMAKE_CURRENT_LIST_DIR}" CACHE PATH "") - set(LibmultiprocessNative_ROOT "${CMAKE_CURRENT_LIST_DIR}/native" CACHE PATH "") +set(multiprocess_packages @multiprocess_packages@) +if("${multiprocess_packages}" STREQUAL "") + set(ENABLE_IPC OFF CACHE BOOL "") + set(WITH_SV2 OFF CACHE BOOL "") else() - set(WITH_MULTIPROCESS OFF CACHE BOOL "") + set(ENABLE_IPC ON CACHE BOOL "") + set(MPGEN_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/native/bin/mpgen" CACHE FILEPATH "") + set(CAPNP_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/native/bin/capnp" CACHE FILEPATH "") + set(CAPNPC_CXX_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/native/bin/capnpc-c++" CACHE FILEPATH "") endif() diff --git a/doc/design/libraries.md b/doc/design/libraries.md index 8448fb7011d69..64ef199b4f9fd 100644 --- a/doc/design/libraries.md +++ b/doc/design/libraries.md @@ -8,7 +8,7 @@ | *libbitcoin_crypto* | Hardware-optimized functions for data encryption, hashing, message authentication, and key derivation. | | *libbitcoin_kernel* | Consensus engine and support library used for validation by *libbitcoin_node*. | | *libbitcoinqt* | GUI functionality used by *bitcoin-qt* and *bitcoin-gui* executables. | -| *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`-DWITH_MULTIPROCESS=ON`](multiprocess.md) is used. | +| *libbitcoin_ipc* | IPC functionality used by *bitcoin-node*, *bitcoin-wallet*, *bitcoin-gui* executables to communicate when [`-DENABLE_IPC=ON`](multiprocess.md) is used. | | *libbitcoin_node* | P2P and RPC server functionality used by *bitcoind* and *bitcoin-qt* executables. | | *libbitcoin_util* | Home for common functionality shared by different executables and libraries. Similar to *libbitcoin_common*, but lower-level (see [Dependencies](#dependencies)). | | *libbitcoin_wallet* | Wallet functionality used by *bitcoind* and *bitcoin-wallet* executables. | diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 37e594e762861..9402f4848427d 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -432,8 +432,8 @@ RPC that, when enabled, logs the location and duration of each lock contention to the `debug.log` file. The `-DCMAKE_BUILD_TYPE=Debug` build option adds `-DDEBUG_LOCKCONTENTION` to the -compiler flags. You may also enable it manually by building with `-DDEBUG_LOCKCONTENTION` added to your CPPFLAGS, -i.e. `CPPFLAGS="-DDEBUG_LOCKCONTENTION"`, then build and run bitcoind. +compiler flags. You may also enable it manually by building with `-DDEBUG_LOCKCONTENTION` +added to your CPPFLAGS, i.e. `-DAPPEND_CPPFLAGS="-DDEBUG_LOCKCONTENTION"`. You can then use the `-debug=lock` configuration option at bitcoind startup or `bitcoin-cli logging '["lock"]'` at runtime to turn on lock contention logging. @@ -1232,6 +1232,9 @@ Current subtrees include: - src/minisketch - Upstream at https://github.com/sipa/minisketch ; maintained by Core contributors. +- src/ipc/libmultiprocess + - Upstream at https://github.com/chaincodelabs/libmultiprocess ; maintained by Core contributors. + Upgrading LevelDB --------------------- diff --git a/doc/multiprocess.md b/doc/multiprocess.md index 1757296eed6b7..c9ad34f54210c 100644 --- a/doc/multiprocess.md +++ b/doc/multiprocess.md @@ -4,7 +4,7 @@ _This document describes usage of the multiprocess feature. For design informati ## Build Option -On Unix systems, the `-DWITH_MULTIPROCESS=ON` build option can be passed to build the supplemental `bitcoin-node` and `bitcoin-gui` multiprocess executables. +On Unix systems, the `-DENABLE_IPC=ON` build option can be passed to build the supplemental `bitcoin-node` and `bitcoin-gui` multiprocess executables. ## Debugging @@ -12,11 +12,15 @@ The `-debug=ipc` command line option can be used to see requests and responses b ## Installation -The multiprocess feature requires [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) as dependencies. A simple way to get started using it without installing these dependencies manually is to use the [depends system](../depends) with the `MULTIPROCESS=1` [dependency option](../depends#dependency-options) passed to make: +Specifying `-DENABLE_IPC=ON` requires [Cap'n Proto](https://capnproto.org/) to be installed. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies. + +### Depends installation + +Alternatively the [depends system](../depends) can be used to avoid needing to local install dependencies: ``` cd -make -C depends NO_QT=1 MULTIPROCESS=1 +make -C depends NO_QT=1 # Set host platform to output of gcc -dumpmachine or clang -dumpmachine or check the depends/ directory for the generated subdirectory name HOST_PLATFORM="x86_64-pc-linux-gnu" cmake -B build --toolchain=depends/$HOST_PLATFORM/toolchain.cmake @@ -25,9 +29,11 @@ build/src/bitcoin-node -regtest -printtoconsole -debug=ipc BITCOIND=$(pwd)/build/src/bitcoin-node build/test/functional/test_runner.py ``` -The `cmake` build will pick up settings and library locations from the depends directory, so there is no need to pass `-DWITH_MULTIPROCESS=ON` as a separate flag when using the depends system (it's controlled by the `MULTIPROCESS=1` option). +The `cmake` build will pick up settings and library locations from the depends directory, so there is no need to pass `-DENABLE_IPC=ON` as a separate flag when using the depends system (it's controlled by the `NO_MULTIPROCESS=1` option). + +### Cross-compiling -Alternately, you can install [Cap'n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) packages on your system, and just run `cmake -B build -DWITH_MULTIPROCESS=ON` without using the depends system. The `cmake` build will be able to locate the installed packages via [pkg-config](https://www.freedesktop.org/wiki/Software/pkg-config/). See [Installation](https://github.com/chaincodelabs/libmultiprocess/blob/master/doc/install.md) section of the libmultiprocess readme for install steps. See [build-unix.md](build-unix.md) and [build-osx.md](build-osx.md) for information about installing dependencies in general. +When cross-compiling and not using depends, a native `mpgen` tool from [libmultiprocess](https://github.com/chaincodelabs/libmultiprocess) is required. It can be set manually by passing `-DMPGEN_EXECUTABLE=/path/to/mpgen` ## Usage diff --git a/share/setup.nsi.in b/share/setup.nsi.in index d1a85cdb7f350..5ad42faf70100 100644 --- a/share/setup.nsi.in +++ b/share/setup.nsi.in @@ -72,17 +72,14 @@ ShowUninstDetails show Section -Main SEC0000 SetOutPath $INSTDIR SetOverwrite on - File @abs_top_builddir@/release/@BITCOIN_GUI_NAME@@EXEEXT@ File /oname=COPYING.txt @abs_top_srcdir@/COPYING File /oname=readme.txt @abs_top_srcdir@/doc/README_windows.txt File @abs_top_srcdir@/share/examples/bitcoin.conf SetOutPath $INSTDIR\share\rpcauth File @abs_top_srcdir@/share/rpcauth/*.* SetOutPath $INSTDIR\daemon - File @abs_top_builddir@/release/@BITCOIN_DAEMON_NAME@@EXEEXT@ + # TODO: add bitcoin-node and bitcoin-miner File @abs_top_builddir@/release/@BITCOIN_CLI_NAME@@EXEEXT@ - File @abs_top_builddir@/release/@BITCOIN_TX_NAME@@EXEEXT@ - File @abs_top_builddir@/release/@BITCOIN_WALLET_TOOL_NAME@@EXEEXT@ File @abs_top_builddir@/release/@BITCOIN_TEST_NAME@@EXEEXT@ SetOutPath $INSTDIR WriteRegStr HKCU "${REGKEY}\Components" Main 1 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bed9864960a5a..95f7d6ed45d1f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -26,11 +26,27 @@ add_dependencies(bitcoin_clientversion generate_build_info) add_subdirectory(crypto) add_subdirectory(univalue) add_subdirectory(util) -if(WITH_MULTIPROCESS) + +set(installable_targets) + +if(ENABLE_IPC) add_subdirectory(ipc) endif() if(WITH_SV2) add_subdirectory(sv2) + + # TOOD: move into sv2 directory + add_executable(bitcoin-mine + bitcoin-mine.cpp + init/bitcoin-mine.cpp + ) + target_link_libraries(bitcoin-mine + core_interface + bitcoin_common + bitcoin_sv2 + bitcoin_ipc + ) +list(APPEND installable_targets bitcoin-mine) endif() #============================= @@ -145,6 +161,7 @@ add_library(bitcoin_common STATIC EXCLUDE_FROM_ALL net_types.cpp netaddress.cpp netbase.cpp + node/interface_ui.cpp outputtype.cpp policy/feerate.cpp policy/policy.cpp @@ -176,7 +193,6 @@ target_link_libraries(bitcoin_common ) -set(installable_targets) if(ENABLE_WALLET) add_subdirectory(wallet) @@ -242,7 +258,6 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL node/context.cpp node/database_args.cpp node/eviction.cpp - node/interface_ui.cpp node/interfaces.cpp node/kernel_notifications.cpp node/mempool_args.cpp @@ -324,7 +339,7 @@ if(BUILD_DAEMON) ) list(APPEND installable_targets bitcoind) endif() -if(WITH_MULTIPROCESS) +if(ENABLE_IPC) add_executable(bitcoin-node bitcoind.cpp init/bitcoin-node.cpp diff --git a/src/bitcoin-mine.cpp b/src/bitcoin-mine.cpp new file mode 100644 index 0000000000000..39c9fd4a8cd2b --- /dev/null +++ b/src/bitcoin-mine.cpp @@ -0,0 +1,209 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include // IWYU pragma: keep + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef WIN32 +// #include +#include +// #include +#endif + +static const char* const HELP_USAGE{R"( +bitcoin-mine is a test program for interacting with bitcoin-node via IPC. + +Usage: + bitcoin-mine [options] +)"}; + +static const char* HELP_EXAMPLES{R"( +Examples: + # Start separate bitcoin-node that bitcoin-mine can connect to. + bitcoin-node -regtest -ipcbind=unix + + # Connect to bitcoin-node and print tip block hash. + bitcoin-mine -regtest + + # Run with debug output. + bitcoin-mine -regtest -debug=sv2 -loglevel=sv2:trace +)"}; + +const std::function G_TRANSLATION_FUN = nullptr; + +static void AddArgs(ArgsManager& args) +{ + SetupHelpOptions(args); + SetupChainParamsBaseOptions(args); + + const auto defaultBaseParams = CreateBaseChainParams(ChainType::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(ChainType::TESTNET); + const auto testnet4BaseParams = CreateBaseChainParams(ChainType::TESTNET4); + const auto signetBaseParams = CreateBaseChainParams(ChainType::SIGNET); + const auto regtestBaseParams = CreateBaseChainParams(ChainType::REGTEST); + + args.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + args.AddArg("-datadir=", "Specify data directory", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + args.AddArg("-ipcconnect=
", "Connect to bitcoin-node process in the background to perform online operations. Valid
values are 'unix' to connect to the default socket, 'unix:' to connect to a socket at a nonstandard path. Default value: unix", ArgsManager::ALLOW_ANY, OptionsCategory::IPC); + args.AddArg("-sv2bind=[:]", strprintf("Bind to given address and always listen on it (default: 127.0.0.1). Use [host]:port notation for IPv6."), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + args.AddArg("-sv2port=", strprintf("Listen for Stratum v2 connections on (default: %u, testnet3: %u, testnet4: %u, signet: %u, regtest: %u).", defaultBaseParams->Sv2Port(), testnetBaseParams->Sv2Port(), testnet4BaseParams->Sv2Port(), signetBaseParams->Sv2Port(), regtestBaseParams->Sv2Port()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); + args.AddArg("-sv2interval", strprintf("Template Provider block template update interval (default: %d seconds)", Sv2TemplateProviderOptions().fee_check_interval.count()), ArgsManager::ALLOW_ANY, OptionsCategory::BLOCK_CREATION); + args.AddArg("-sv2feedelta", strprintf("Minimum fee delta for Template Provider to send update upstream (default: %d sat)", uint64_t(Sv2TemplateProviderOptions().fee_delta)), ArgsManager::ALLOW_ANY, OptionsCategory::BLOCK_CREATION); + init::AddLoggingArgs(args); +} + +static bool g_interrupt{false}; + +#ifndef WIN32 +static void registerSignalHandler(int signal, void(*handler)(int)) +{ + struct sigaction sa; + sa.sa_handler = handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(signal, &sa, nullptr); +} +static void HandleSIGTERM(int) +{ + g_interrupt = true; +} + +#endif + +MAIN_FUNCTION +{ + ArgsManager& args = gArgs; + AddArgs(args); + std::string error_message; + if (!args.ParseParameters(argc, argv, error_message)) { + tfm::format(std::cerr, "Error parsing command line arguments: %s\n", error_message); + return EXIT_FAILURE; + } + if (!args.ReadConfigFiles(error_message, true)) { + tfm::format(std::cerr, "Error reading config files: %s\n", error_message); + return EXIT_FAILURE; + } + if (HelpRequested(args) || args.IsArgSet("-version")) { + std::string output{strprintf("%s bitcoin-mine version", CLIENT_NAME) + " " + FormatFullVersion() + "\n"}; + if (args.IsArgSet("-version")) { + output += FormatParagraph(LicenseInfo()); + } else { + output += HELP_USAGE; + output += args.GetHelpMessage(); + output += HELP_EXAMPLES; + } + tfm::format(std::cout, "%s", output); + return EXIT_SUCCESS; + } + if (!CheckDataDirOption(args)) { + tfm::format(std::cerr, "Error: Specified data directory \"%s\" does not exist.\n", args.GetArg("-datadir", "")); + return EXIT_FAILURE; + } + SelectParams(args.GetChainType()); + + // Set logging options but override -printtoconsole default to depend on -debug rather than -daemon + init::SetLoggingOptions(args); + if (auto result{init::SetLoggingCategories(args)}; !result) { + tfm::format(std::cerr, "Error: %s\n", util::ErrorString(result).original); + return EXIT_FAILURE; + } + if (auto result{init::SetLoggingLevel(args)}; !result) { + tfm::format(std::cerr, "Error: %s\n", util::ErrorString(result).original); + return EXIT_FAILURE; + } + LogInstance().m_print_to_console = args.GetBoolArg("-printtoconsole", LogInstance().GetCategoryMask()); + if (!init::StartLogging(args)) { + tfm::format(std::cerr, "Error: StartLogging failed\n"); + return EXIT_FAILURE; + } + + ECC_Context ecc_context{}; + + // Parse -sv2... params + Sv2TemplateProviderOptions options{}; + + const std::string sv2_port_arg = args.GetArg("-sv2port", ""); + + if (sv2_port_arg.empty()) { + options.port = BaseParams().Sv2Port(); + } else { + if (!ParseUInt16(sv2_port_arg, &options.port) || options.port == 0) { + tfm::format(std::cerr, "Invalid port %s\n", sv2_port_arg); + return EXIT_FAILURE; + } + } + + if (args.IsArgSet("-sv2bind")) { // Specific bind address + std::optional sv2_bind{args.GetArg("-sv2bind")}; + if (sv2_bind) { + if (!SplitHostPort(sv2_bind.value(), options.port, options.host)) { + tfm::format(std::cerr, "Invalid port %d\n", options.port); + return EXIT_FAILURE; + } + } + } + + options.fee_delta = args.GetIntArg("-sv2feedelta", Sv2TemplateProviderOptions().fee_delta); + + if (args.IsArgSet("-sv2interval")) { + if (args.GetIntArg("-sv2interval", 0) < 1) { + tfm::format(std::cerr, "-sv2interval must be at least one second\n"); + return EXIT_FAILURE; + } + options.fee_check_interval = std::chrono::seconds(args.GetIntArg("-sv2interval", 0)); + } + + // Connect to existing bitcoin-node process or spawn new one. + std::unique_ptr mine_init{interfaces::MakeMineInit(argc, argv)}; + assert(mine_init); + std::unique_ptr node_init; + try { + std::string address{args.GetArg("-ipcconnect", "unix")}; + node_init = mine_init->ipc()->connectAddress(address); + } catch (const std::exception& exception) { + tfm::format(std::cerr, "Error: %s\n", exception.what()); + tfm::format(std::cerr, "Probably bitcoin-node is not running or not listening on a unix socket. Can be started with:\n\n"); + tfm::format(std::cerr, " bitcoin-node -chain=%s -ipcbind=unix\n", args.GetChainTypeString()); + return EXIT_FAILURE; + } + assert(node_init); + tfm::format(std::cout, "Connected to bitcoin-node\n"); + std::unique_ptr mining{node_init->makeMining()}; + assert(mining); + + auto tp = std::make_unique(*mining); + + if (!tp->Start(options)) { + tfm::format(std::cerr, "Unable to start Stratum v2 Template Provider"); + return EXIT_FAILURE; + } + +#ifndef WIN32 + registerSignalHandler(SIGTERM, HandleSIGTERM); + registerSignalHandler(SIGINT, HandleSIGTERM); +#endif + + while(!g_interrupt) { + UninterruptibleSleep(100ms); + } + + tp->Interrupt(); + tp->StopThreads(); + tp.reset(); + + return EXIT_SUCCESS; +} diff --git a/src/init/bitcoin-mine.cpp b/src/init/bitcoin-mine.cpp new file mode 100644 index 0000000000000..ad2d4a0f14a60 --- /dev/null +++ b/src/init/bitcoin-mine.cpp @@ -0,0 +1,29 @@ +// Copyright (c) 2024 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include + +namespace init { +namespace { +const char* EXE_NAME = "bitcoin-mine"; + +class BitcoinMineInit : public interfaces::Init +{ +public: + BitcoinMineInit(const char* arg0) : m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this)) + { + } + interfaces::Ipc* ipc() override { return m_ipc.get(); } + std::unique_ptr m_ipc; +}; +} // namespace +} // namespace init + +namespace interfaces { +std::unique_ptr MakeMineInit(int argc, char* argv[]) +{ + return std::make_unique(argc > 0 ? argv[0] : ""); +} +} // namespace interfaces diff --git a/src/interfaces/init.h b/src/interfaces/init.h index b496ada05f4ab..3c7525d7cda49 100644 --- a/src/interfaces/init.h +++ b/src/interfaces/init.h @@ -53,6 +53,9 @@ std::unique_ptr MakeWalletInit(int argc, char* argv[], int& exit_status); //! Return implementation of Init interface for the gui process. std::unique_ptr MakeGuiInit(int argc, char* argv[]); + +//! Return implementation of Init interface for the bitcoin-mine process. +std::unique_ptr MakeMineInit(int argc, char* argv[]); } // namespace interfaces #endif // BITCOIN_INTERFACES_INIT_H diff --git a/src/ipc/CMakeLists.txt b/src/ipc/CMakeLists.txt index 904d72f56e130..c1b743b586602 100644 --- a/src/ipc/CMakeLists.txt +++ b/src/ipc/CMakeLists.txt @@ -2,6 +2,49 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or https://opensource.org/license/mit/. +# Build local libmultiprocess/ directory, unless an external libmultiprocess +# package is being used instead. +if (NOT WITH_EXTERNAL_LIBMULTIPROCESS) + # Unconditionally set BUILD_TESTING to ON. BUILD_TESTING is a standard cmake + # variable that controls whether CTest is used. In the bitcoin build, CTest is + # always used and we do not want an option to turn it off. But in + # libmultiprocess, CTest is optional and BUILD_TESTING needs to be set in + # order to add libmultiprocess unit tests to the CTest configuration. + set(BUILD_TESTING ON) + add_subdirectory(libmultiprocess EXCLUDE_FROM_ALL) + # Apply core_interface compile options to libmultiprocess runtime library. + target_link_libraries(multiprocess PUBLIC $) + target_link_libraries(mputil PUBLIC $) + target_link_libraries(mpgen PUBLIC $) + # Mark capproto options as advanced to hide by default from cmake UI + mark_as_advanced(CapnProto_DIR) + mark_as_advanced(CapnProto_capnpc_IMPORTED_LOCATION) + mark_as_advanced(CapnProto_capnp_IMPORTED_LOCATION) + mark_as_advanced(CapnProto_capnp-json_IMPORTED_LOCATION) + mark_as_advanced(CapnProto_capnp-rpc_IMPORTED_LOCATION) + mark_as_advanced(CapnProto_capnp-websocket_IMPORTED_LOCATION) + mark_as_advanced(CapnProto_kj-async_IMPORTED_LOCATION) + mark_as_advanced(CapnProto_kj-gzip_IMPORTED_LOCATION) + mark_as_advanced(CapnProto_kj-http_IMPORTED_LOCATION) + mark_as_advanced(CapnProto_kj_IMPORTED_LOCATION) + mark_as_advanced(CapnProto_kj-test_IMPORTED_LOCATION) + mark_as_advanced(CapnProto_kj-tls_IMPORTED_LOCATION) + mark_as_advanced(CAPNP_EXECUTABLE) + mark_as_advanced(CAPNPC_CXX_EXECUTABLE) + # Share MP_INCLUDE_DIR variable with parent scope so target_capnp_sources + # build rules can find the libmultiprocess include directory and avoid + # "error: Import failed: /mp/proxy.capnp" errors from capnp. + set(MP_INCLUDE_DIR "${MP_INCLUDE_DIR}" PARENT_SCOPE) + # Add tests to "all" target so ctest can run them + set_target_properties(mptests PROPERTIES EXCLUDE_FROM_ALL OFF) + # Exclude examples from compilation database, because the examples are not + # built by default, and they contain generated c++ code. Without this + # exclusion, tools like clang-tidy and IWYU that make use of compilation + # database would complain that the generated c++ source files do not exist. An + # alternate fix could build "mpexamples" by default like "mptests" above. + set_target_properties(mpcalculator mpprinter mpexample PROPERTIES EXPORT_COMPILE_COMMANDS OFF) +endif() + add_library(bitcoin_ipc STATIC EXCLUDE_FROM_ALL capnp/mining.cpp capnp/protocol.cpp diff --git a/src/ipc/libmultiprocess/.clang-tidy b/src/ipc/libmultiprocess/.clang-tidy new file mode 100644 index 0000000000000..2d29f120ae4ff --- /dev/null +++ b/src/ipc/libmultiprocess/.clang-tidy @@ -0,0 +1,41 @@ +Checks: ' +-*, +bugprone-*, +-bugprone-easily-swappable-parameters, +-bugprone-exception-escape, +-bugprone-move-forwarding-reference, +-bugprone-narrowing-conversions, +-bugprone-reserved-identifier, +misc-*, +-misc-non-private-member-variables-in-classes, +-misc-no-recursion, +-misc-unconventional-assign-operator, +-misc-unused-parameters, +-misc-use-anonymous-namespace, +modernize-*, +-modernize-avoid-c-arrays, +-modernize-concat-nested-namespaces, +-modernize-deprecated-headers, +-modernize-use-nodiscard, +-modernize-use-trailing-return-type, +-modernize-use-using, +performance-*, +-performance-avoid-endl, +-performance-noexcept-move-constructor, +readability-*, +-readability-braces-around-statements, +-readability-convert-member-functions-to-static, +-readability-else-after-return, +-readability-function-cognitive-complexity, +-readability-identifier-length, +-readability-implicit-bool-conversion, +-readability-inconsistent-declaration-parameter-name, +-readability-magic-numbers, +-readability-named-parameter, +-readability-uppercase-literal-suffix, +-readability-use-anyofallof, +' +CheckOptions: + - key: modernize-use-override.IgnoreDestructors + value: true +HeaderFilterRegex: 'example/calculator.h|example/init.h|example/printer.h|include/mp/proxy-io.h|include/mp/proxy-types.h|include/mp/proxy.h|include/mp/util.h|test/mp/test/foo-types.h|test/mp/test/foo.h' diff --git a/src/ipc/libmultiprocess/CMakeLists.txt b/src/ipc/libmultiprocess/CMakeLists.txt new file mode 100644 index 0000000000000..13cc5ee0ab8ca --- /dev/null +++ b/src/ipc/libmultiprocess/CMakeLists.txt @@ -0,0 +1,178 @@ +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +cmake_minimum_required(VERSION 3.8) + +project("Libmultiprocess" CXX) +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(CMAKE_CXX_STANDARD 20) + set(CMAKE_CXX_STANDARD_REQUIRED YES) +endif() + +include("cmake/compat_find.cmake") + +find_package(CapnProto REQUIRED) +find_package(Threads REQUIRED) + +option(Libmultiprocess_ENABLE_CLANG_TIDY "Run clang-tidy with the compiler." OFF) +if(Libmultiprocess_ENABLE_CLANG_TIDY) + find_program(CLANG_TIDY_EXECUTABLE NAMES clang-tidy) + if(NOT CLANG_TIDY_EXECUTABLE) + message(FATAL_ERROR "Libmultiprocess_ENABLE_CLANG_TIDY is ON but clang-tidy is not found.") + endif() + set(CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_EXECUTABLE}") +endif() + +set(MPGEN_EXECUTABLE "" CACHE FILEPATH "If specified, should be full path to an external mpgen binary to use rather than the one built internally.") + +include("cmake/compat_config.cmake") +include("cmake/pthread_checks.cmake") +include(GNUInstallDirs) + +# Set convenience variables for subdirectories. +set(MP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(MP_STANDALONE TRUE) + include(CTest) +else() + # Set MP_INCLUDE_DIR for parent directories too, so target_capnp_sources calls + # in parent directories can use it and not need to specify include directories + # manually or see capnproto error "error: Import failed: /mp/proxy.capnp" + set(MP_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include" PARENT_SCOPE) + set(MP_STANDALONE FALSE) +endif() + +# Prevent include directories from parent project from leaking into this one. +set_property(DIRECTORY PROPERTY INCLUDE_DIRECTORIES "") + +# Generated C++ preprocessor defines +configure_file(include/mp/config.h.in "${CMAKE_CURRENT_BINARY_DIR}/include/mp/config.h") + +# Generated C++ Capn'Proto schema files +capnp_generate_cpp(MP_PROXY_SRCS MP_PROXY_HDRS include/mp/proxy.capnp) + +# util library +add_library(mputil OBJECT src/mp/util.cpp) +target_include_directories(mputil PRIVATE + $ + $) +target_link_libraries(mputil PUBLIC CapnProto::kj) + +# libmultiprocess.a runtime library +set(MP_PUBLIC_HEADERS + ${MP_PROXY_HDRS} + include/mp/proxy-io.h + include/mp/proxy-types.h + include/mp/proxy.h + include/mp/type-char.h + include/mp/type-chrono.h + include/mp/type-context.h + include/mp/type-data.h + include/mp/type-decay.h + include/mp/type-exception.h + include/mp/type-function.h + include/mp/type-interface.h + include/mp/type-map.h + include/mp/type-message.h + include/mp/type-number.h + include/mp/type-optional.h + include/mp/type-pair.h + include/mp/type-pointer.h + include/mp/type-set.h + include/mp/type-string.h + include/mp/type-struct.h + include/mp/type-threadmap.h + include/mp/type-tuple.h + include/mp/type-vector.h + include/mp/type-void.h + include/mp/util.h) +add_library(multiprocess STATIC + ${MP_PROXY_SRCS} + ${MP_PUBLIC_HEADERS} + src/mp/proxy.cpp + $) +add_library(Libmultiprocess::multiprocess ALIAS multiprocess) +target_include_directories(multiprocess PUBLIC + $ + $ + $) +target_link_libraries(multiprocess PUBLIC CapnProto::capnp) +target_link_libraries(multiprocess PUBLIC CapnProto::capnp-rpc) +target_link_libraries(multiprocess PUBLIC CapnProto::kj) +target_link_libraries(multiprocess PUBLIC CapnProto::kj-async) +set_target_properties(multiprocess PROPERTIES + PUBLIC_HEADER "${MP_PUBLIC_HEADERS}") +install(TARGETS multiprocess EXPORT LibTargets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT lib + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/mp COMPONENT lib) + +# mpgen code generator +add_executable(mpgen src/mp/gen.cpp $) +add_executable(Libmultiprocess::mpgen ALIAS mpgen) +target_include_directories(mpgen PRIVATE $) +target_include_directories(mpgen PUBLIC $ $) +target_link_libraries(mpgen PRIVATE CapnProto::capnp) +target_link_libraries(mpgen PRIVATE CapnProto::capnp-rpc) +target_link_libraries(mpgen PRIVATE CapnProto::capnpc) +target_link_libraries(mpgen PRIVATE CapnProto::kj) +target_link_libraries(mpgen PRIVATE Threads::Threads) +set_target_properties(mpgen PROPERTIES + INSTALL_RPATH_USE_LINK_PATH TRUE) +set_target_properties(mpgen PROPERTIES + PUBLIC_HEADER include/mp/proxy.capnp) +install(TARGETS mpgen EXPORT BinTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT bin + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/mp COMPONENT bin) + +# makefile include to invoke mpgen code generator, for downstream Make projects +install(FILES "include/mpgen.mk" + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT bin) + +# pkg-config module to build against libmultiprocess library, for downstream autoconf projects +configure_file(pkgconfig/libmultiprocess.pc.in "${CMAKE_CURRENT_BINARY_DIR}/pkgconfig/libmultiprocess.pc" @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pkgconfig/libmultiprocess.pc" + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig COMPONENT lib) + +# cmake include to invoke mpgen code generator, for downstream CMake projects +install( + FILES + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/TargetCapnpSources.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Libmultiprocess COMPONENT bin) + +# CMake target import files, for downstream CMake projects +install(EXPORT BinTargets + NAMESPACE Libmultiprocess:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Libmultiprocess COMPONENT bin) +install(EXPORT LibTargets + NAMESPACE Libmultiprocess:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Libmultiprocess COMPONENT lib) + +# CMake find_package config file, for downstream CMake projects +include(CMakePackageConfigHelpers) +configure_package_config_file( + ${PROJECT_SOURCE_DIR}/cmake/Config.cmake.in + LibmultiprocessConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Libmultiprocess + NO_SET_AND_CHECK_MACRO) +install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/LibmultiprocessConfig.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/Libmultiprocess + COMPONENT common) + +# Makefile targets to support "make install-bin" "make install-lib" +add_custom_target(install-bin + COMMAND ${CMAKE_COMMAND} -DCOMPONENT=bin -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_install.cmake + COMMAND ${CMAKE_COMMAND} -DCOMPONENT=common -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_install.cmake + VERBATIM) +add_dependencies(install-bin mpgen) +add_custom_target(install-lib + COMMAND ${CMAKE_COMMAND} -DCOMPONENT=lib -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_install.cmake + COMMAND ${CMAKE_COMMAND} -DCOMPONENT=common -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_install.cmake + VERBATIM) +add_dependencies(install-lib multiprocess) + +# Example and test subdirectories +add_subdirectory(example EXCLUDE_FROM_ALL) +add_subdirectory(test EXCLUDE_FROM_ALL) diff --git a/src/ipc/libmultiprocess/COPYING b/src/ipc/libmultiprocess/COPYING new file mode 100644 index 0000000000000..9d54ecbde1229 --- /dev/null +++ b/src/ipc/libmultiprocess/COPYING @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2009-2019 The Bitcoin Core developers +Copyright (c) 2009-2019 Bitcoin Developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/src/ipc/libmultiprocess/README.md b/src/ipc/libmultiprocess/README.md new file mode 100644 index 0000000000000..5d55f85c03cfe --- /dev/null +++ b/src/ipc/libmultiprocess/README.md @@ -0,0 +1,8 @@ +# libmultiprocess + +`libmultiprocess` is a C++ library and code generator making it easy to call functions and reference objects in different processes. + +For more information see the [usage instructions](doc/usage.md), [installation instructions](doc/install.md), or [design documentation](doc/design.md). + +If you have any questions, comments, or feedback, please submit an [issue](https://github.com/chaincodelabs/libmultiprocess/issues/new). +Duplicate issues are perfectly fine and all discussion about the project is welcome, since there isn't another discussion forum currently. diff --git a/src/ipc/libmultiprocess/cmake/Config.cmake.in b/src/ipc/libmultiprocess/cmake/Config.cmake.in new file mode 100644 index 0000000000000..edff7d143be71 --- /dev/null +++ b/src/ipc/libmultiprocess/cmake/Config.cmake.in @@ -0,0 +1,33 @@ +@PACKAGE_INIT@ + +# CMake find_package compatible package file, for downstream CMake projects +# +# Based on https://cmake.org/cmake/help/latest/guide/importing-exporting/index.html#adding-components + +set(_Libmultiprocess_supported_components Bin Lib) + +# If no components specified, include all components. +list(LENGTH ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS_len) +if(${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS_len EQUAL 0) + set(${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS ${_Libmultiprocess_supported_components}) +endif() + +if ("Bin" IN_LIST ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + include("${CMAKE_CURRENT_LIST_DIR}/TargetCapnpSources.cmake") +endif() + +if ("Lib" IN_LIST ${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS) + # Setting FOUND_LIBATOMIC is needed on debian & ubuntu systems to work around bug in + # their capnproto packages. See compat_find.cmake for a more complete explanation. + set(FOUND_LIBATOMIC TRUE) + include(CMakeFindDependencyMacro) + find_dependency(CapnProto) +endif() + +foreach(_comp ${${CMAKE_FIND_PACKAGE_NAME}_FIND_COMPONENTS}) + if (NOT _comp IN_LIST _Libmultiprocess_supported_components) + set(${CMAKE_FIND_PACKAGE_NAME}_FOUND False) + set(${CMAKE_FIND_PACKAGE_NAME}_NOT_FOUND_MESSAGE "Unsupported component: ${_comp}") + endif() + include("${CMAKE_CURRENT_LIST_DIR}/${_comp}Targets.cmake") +endforeach() diff --git a/src/ipc/libmultiprocess/cmake/TargetCapnpSources.cmake b/src/ipc/libmultiprocess/cmake/TargetCapnpSources.cmake new file mode 100644 index 0000000000000..963d67d025cf8 --- /dev/null +++ b/src/ipc/libmultiprocess/cmake/TargetCapnpSources.cmake @@ -0,0 +1,112 @@ +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +#[=[ + +target_capnp_sources +-------------------- + +This function adds build steps to generate C++ files from Cap'n Proto files +and build them as part of a specified target. + +Arguments: + + target: The name of the CMake target (e.g., a library or executable) to + which the generated source files will be added. This target must already + be defined elsewhere in the CMake scripts. + + include_prefix: Absolute path indicating what portion of capnp source paths + should be used in relative #include statements in the generated C++ + files. For example, if the .capnp path is /home/src/lib/schema.capnp + and include_prefix is /home/src, generated includes look like: + + #include + + And if include_prefix is /home/src/lib, generated includes look like: + + #include + + The specified include_prefix should be ${CMAKE_SOURCE_DIR} or a + subdirectory of it to include files relative to the project root. It can + be ${CMAKE_CURRENT_SOURCE_DIR} to include files relative to the current + source directory. + +Additional Unnamed Arguments: + + After `target` and `include_prefix`, all unnamed arguments are treated as + paths to `.capnp` schema files. These should be paths relative to + ${CMAKE_CURRENT_SOURCE_DIR}. + +Optional Keyword Arguments: + + IMPORT_PATHS: Specifies additional directories to search for imported + `.capnp` files. + +Example: + # Assuming `my_library` is a target and `lib/` contains `.capnp` schema + # files with imports from `include/`. + target_capnp_sources(my_library "${CMAKE_SOURCE_DIR}" + lib/schema1.capnp lib/schema2.capnp + IMPORT_PATHS ${CMAKE_SOURCE_DIR}/include) + +#]=] + +function(target_capnp_sources target include_prefix) + cmake_parse_arguments(PARSE_ARGV 2 + "TCS" # prefix + "" # options + "" # one_value_keywords + "IMPORT_PATHS" # multi_value_keywords + ) + + set(MPGEN_BINARY "") + if(MPGEN_EXECUTABLE) + set(MPGEN_BINARY "${MPGEN_EXECUTABLE}") + if(NOT EXISTS "${MPGEN_BINARY}") + message(FATAL_ERROR "MPGEN_EXECUTABLE: \"${MPGEN_BINARY}\" does not exist.") + endif() + elseif(TARGET Libmultiprocess::multiprocess) + set(MPGEN_BINARY Libmultiprocess::mpgen) + else() + message(FATAL_ERROR "No usable mpgen. Set MPGEN_EXECUTABLE or enable the internal target.") + endif() + + set(generated_headers "") + foreach(capnp_file IN LISTS TCS_UNPARSED_ARGUMENTS) + add_custom_command( + OUTPUT ${capnp_file}.c++ ${capnp_file}.h ${capnp_file}.proxy-client.c++ ${capnp_file}.proxy-types.h ${capnp_file}.proxy-server.c++ ${capnp_file}.proxy-types.c++ ${capnp_file}.proxy.h + COMMAND ${MPGEN_BINARY} ${CMAKE_CURRENT_SOURCE_DIR} ${include_prefix} ${CMAKE_CURRENT_SOURCE_DIR}/${capnp_file} ${TCS_IMPORT_PATHS} ${MP_INCLUDE_DIR} + DEPENDS ${capnp_file} + VERBATIM + ) + target_sources(${target} PRIVATE + ${CMAKE_CURRENT_BINARY_DIR}/${capnp_file}.c++ + ${CMAKE_CURRENT_BINARY_DIR}/${capnp_file}.proxy-client.c++ + ${CMAKE_CURRENT_BINARY_DIR}/${capnp_file}.proxy-server.c++ + ${CMAKE_CURRENT_BINARY_DIR}/${capnp_file}.proxy-types.c++ + ) + + list(APPEND generated_headers ${capnp_file}.h) + endforeach() + + # Translate include_prefix from a source path to a binary path and add it as a + # target include directory. + set(build_include_prefix ${CMAKE_BINARY_DIR}) + file(RELATIVE_PATH relative_path ${CMAKE_SOURCE_DIR} ${include_prefix}) + if(relative_path) + string(APPEND build_include_prefix "/" "${relative_path}") + endif() + target_include_directories(${target} PUBLIC $ ${MP_INCLUDE_DIR}) + + if(TARGET Libmultiprocess::multiprocess) + target_link_libraries(${target} PRIVATE Libmultiprocess::multiprocess) + endif() + + # Add a custom target that can be specified as a dependency of c++ targets + # that include generated headers. It can be necessary to specify these + # dependencies explicitly because while cmake detect dependencies of non + # generated files on generated headers, it does not reliably detect + # dependencies of generated headers on other generated headers. + add_custom_target("${target}_headers" DEPENDS ${generated_headers}) +endfunction() diff --git a/src/ipc/libmultiprocess/cmake/compat_config.cmake b/src/ipc/libmultiprocess/cmake/compat_config.cmake new file mode 100644 index 0000000000000..736d420894fde --- /dev/null +++ b/src/ipc/libmultiprocess/cmake/compat_config.cmake @@ -0,0 +1,59 @@ +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# compat_config.cmake -- compatibility workarounds meant to be included after +# cmake find_package() calls are made, before configuring the ebuild + +# Define capnp_PREFIX if not defined to avoid issue on macos +# https://github.com/chaincodelabs/libmultiprocess/issues/26 + +if (NOT DEFINED capnp_PREFIX AND DEFINED CAPNP_INCLUDE_DIRS) + get_filename_component(capnp_PREFIX "${CAPNP_INCLUDE_DIRS}" DIRECTORY) +endif() + +if (NOT DEFINED CAPNPC_OUTPUT_DIR) + set(CAPNPC_OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}") +endif() + +# CMake target definitions for backwards compatibility with Ubuntu bionic +# capnproto 0.6.1 package (https://packages.ubuntu.com/bionic/libcapnp-dev) +# https://github.com/chaincodelabs/libmultiprocess/issues/27 + +if (NOT DEFINED CAPNP_LIB_CAPNPC AND DEFINED CAPNP_LIB_CAPNP-RPC) + string(REPLACE "-rpc" "c" CAPNP_LIB_CAPNPC "${CAPNP_LIB_CAPNP-RPC}") +endif() + +if (NOT DEFINED CapnProto_capnpc_IMPORTED_LOCATION AND DEFINED CapnProto_capnp-rpc_IMPORTED_LOCATION) + string(REPLACE "-rpc" "c" CapnProto_capnpc_IMPORTED_LOCATION "${CapnProto_capnp-rpc_IMPORTED_LOCATION}") +endif() + +if (NOT TARGET CapnProto::capnp AND DEFINED CAPNP_LIB_CAPNP) + add_library(CapnProto::capnp SHARED IMPORTED) + set_target_properties(CapnProto::capnp PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNP}") +endif() + +if (NOT TARGET CapnProto::capnpc AND DEFINED CAPNP_LIB_CAPNPC) + add_library(CapnProto::capnpc SHARED IMPORTED) + set_target_properties(CapnProto::capnpc PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNPC}") +endif() + +if (NOT TARGET CapnProto::capnpc AND DEFINED CapnProto_capnpc_IMPORTED_LOCATION) + add_library(CapnProto::capnpc SHARED IMPORTED) + set_target_properties(CapnProto::capnpc PROPERTIES IMPORTED_LOCATION "${CapnProto_capnpc_IMPORTED_LOCATION}") +endif() + +if (NOT TARGET CapnProto::capnp-rpc AND DEFINED CAPNP_LIB_CAPNP-RPC) + add_library(CapnProto::capnp-rpc SHARED IMPORTED) + set_target_properties(CapnProto::capnp-rpc PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_CAPNP-RPC}") +endif() + +if (NOT TARGET CapnProto::kj AND DEFINED CAPNP_LIB_KJ) + add_library(CapnProto::kj SHARED IMPORTED) + set_target_properties(CapnProto::kj PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_KJ}") +endif() + +if (NOT TARGET CapnProto::kj-async AND DEFINED CAPNP_LIB_KJ-ASYNC) + add_library(CapnProto::kj-async SHARED IMPORTED) + set_target_properties(CapnProto::kj-async PROPERTIES IMPORTED_LOCATION "${CAPNP_LIB_KJ-ASYNC}") +endif() diff --git a/src/ipc/libmultiprocess/cmake/compat_find.cmake b/src/ipc/libmultiprocess/cmake/compat_find.cmake new file mode 100644 index 0000000000000..f838e323c8a6e --- /dev/null +++ b/src/ipc/libmultiprocess/cmake/compat_find.cmake @@ -0,0 +1,20 @@ +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# compat_find.cmake -- compatibility workarounds meant to be included before +# cmake find_package() calls are made + +# Set FOUND_LIBATOMIC to work around bug in debian capnproto package that is +# debian-specific and does not happpen upstream. Debian includes a patch +# https://sources.debian.org/patches/capnproto/1.0.1-4/07_libatomic.patch/ which +# uses check_library_exists(atomic __atomic_load_8 ...) and it fails because the +# symbol name conflicts with a compiler instrinsic as described +# https://github.com/chaincodelabs/libmultiprocess/issues/68#issuecomment-1135150171. +# This could be fixed by improving the check_library_exists function as +# described in the github comment, or by changing the debian patch to check for +# the symbol a different way, but simplest thing to do is work around the +# problem by setting FOUND_LIBATOMIC. This problem has probably not +# been noticed upstream because it only affects CMake packages depending on +# capnproto, not autoconf packages. +set(FOUND_LIBATOMIC TRUE) diff --git a/src/ipc/libmultiprocess/cmake/pthread_checks.cmake b/src/ipc/libmultiprocess/cmake/pthread_checks.cmake new file mode 100644 index 0000000000000..b54c0b45b8d4e --- /dev/null +++ b/src/ipc/libmultiprocess/cmake/pthread_checks.cmake @@ -0,0 +1,41 @@ +# Copyright (c) 2024 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# Define HAVE_PTHREAD_* variables depending on what pthread functions are +# available. + +include(CMakePushCheckState) +include(CheckCXXSourceCompiles) + +cmake_push_check_state() +set(CMAKE_REQUIRED_LIBRARIES Threads::Threads) +check_cxx_source_compiles(" + #include + int main(int argc, char** argv) + { + char thread_name[16]; + return pthread_getname_np(pthread_self(), thread_name, sizeof(thread_name)); + }" + HAVE_PTHREAD_GETNAME_NP) + +check_cxx_source_compiles(" + #include + #include + int main(int argc, char** argv) + { + uint64_t tid; + pthread_threadid_np(NULL, &tid); + return 0; + }" + HAVE_PTHREAD_THREADID_NP) + +check_cxx_source_compiles(" + #include + #include + int main(int argc, char** argv) + { + return pthread_getthreadid_np(); + }" + HAVE_PTHREAD_GETTHREADID_NP) +cmake_pop_check_state() diff --git a/src/ipc/libmultiprocess/doc/design.md b/src/ipc/libmultiprocess/doc/design.md new file mode 100644 index 0000000000000..ff4b30d063cac --- /dev/null +++ b/src/ipc/libmultiprocess/doc/design.md @@ -0,0 +1,44 @@ +# libmultiprocess Design + +Given an interface description of an object with one or more methods, libmultiprocess generates: + +* A C++ `ProxyClient` class with an implementation of each interface method that sends a request over a socket, waits for a response, and returns the result. +* A C++ `ProxyServer` class that listens for requests over a socket and calls a wrapped C++ object implementing the same interface to actually execute the requests. + +The function call ⇆ request translation supports input and output arguments, standard types like `unique_ptr`, `vector`, `map`, and `optional`, and bidirectional calls between processes through interface pointer and `std::function` arguments. + +If the wrapped C++ object inherits from an abstract base class declaring virtual methods, the generated `ProxyClient` objects can inherit from the same class, allowing interprocess calls to replace local calls without changes to existing code. + +There is also optional support for thread mapping, so each thread making interprocess calls can have a dedicated thread processing requests from it, and callbacks from processing threads are executed on corresponding request threads (so recursive mutexes and thread names function as expected in callbacks). + +Libmultiprocess acts as a pure wrapper or layer over the underlying protocol. Clients and servers written in other languages, but using a shared capnproto schema can communicate with interprocess counterparties using libmultiprocess without having to use libmultiprocess themselves or having to know about the implementation details of libmultiprocess. + +### Internals + +The `ProxyClient` and `ProxyServer` generated classes are not directly exposed to the user, as described in [usage.md](usage.md). Instead, they wrap c++ interfaces and appear to the user as pointers to an interface. They are first instantiated when calling `ConnectStream` and `ServeStream` respectively for creating the `InitInterface`. These methods establish connections through sockets, internally creating `Connection` objects wrapping a `capnp::RpcSystem` configured for client and server mode respectively. + +The `InitInterface` interface will typically have methods which return other interfaces, giving the connecting process the ability to call other functions in the serving process. Interfaces can also have methods accepting other interfaces as parameters, giving serving processes the ability to call back and invoke functions in connecting processes. Creating new interfaces does not create new connections, and typically many interface objects will share the same connection. + +Both `ConnectStream` and `ServeStream` also require an instantiation of the `EventLoop`. The `EventLoop` owns pending requests, notifies on request dispatch, allows clients from multiple threads to make synchronous calls, and handles some cleanup routines on exit. It must be run in a separate thread so it is always active and can process incoming requests from local clients and remote connections. + +When a generated method on the `ProxyClient` is called, it calls `clientInvoke` with the capnp-translated types. `clientInvoke` creates a self-executing promise (`kj::TaskSet`) that drives the execution of the request and gives ownership of it to the `EventLoop`. `clientInvoke` blocks until a response is received, or until there is a call from the server that needs to run on the same client thread, using a `Waiter` object. + +On the server side, the `capnp::RpcSystem` receives the capnp request and invokes the corresponding c++ method through the corresponding `ProxyServer` and the heavily templated `serverInvoke` triggering a `ServerCall`. Its return values from the actual c++ methods are copied into capnp responses by `ServerRet` and exceptions are caught and copied by `ServerExcept`. The two are connected through `ServerField`. The main method driving execution of a request is `PassField`, which is invoked through `ServerField`. Instantiated interfaces, or capabilities in capnp speak, are tracked and owned by the server's `capnp::RpcSystem`. + +## Interface descriptions + +As explained in the [usage](usage.md) document, interface descriptions need to be consumed both by the _libmultiprocess_ code generator, and by C++ code that calls and implements the interfaces. The C++ code only needs to know about C++ arguments and return types, while the code generator only needs to know about capnp arguments and return types, but both need to know class and method names, so the corresponding `.h` and `.capnp` source files contain some of the same information, and have to be kept in sync manually when methods or parameters change. Despite the redundancy, reconciling the interface definitions is designed to be _straightforward_ and _safe_. _Straightforward_ because there is no need to write manual serialization code or use awkward intermediate types like [`UniValue`](https://github.com/bitcoin/bitcoin/blob/master/src/univalue/include/univalue.h) instead of native types. _Safe_ because if there are any inconsistencies between API and data definitions (even minor ones like using a narrow int data type for a wider int API input), there are errors at build time instead of errors or bugs at runtime. + +In the future, it would be possible to combine API and data definitions together using [C++ attributes](https://en.cppreference.com/w/cpp/language/attributes). To do this we would add attributes to the API definition files, and then generate the data definitions from the API definitions and attributes. I didn't take this approach mostly because it would be extra work, but also because until c++ standardizes reflection, this would require either hooking into compiler APIs like https://github.com/RosettaCommons/binder, or parsing c++ code manually like http://www.swig.org/. + +## What is `kj`? + +KJ is a concurrency framework [bundled with +capnproto](https://capnproto.org/cxxrpc.html#kj-concurrency-framework); it is used as a +basis in this library to construct the event-loop necessary to service IPC requests. + +## Future directions + +_libmultiprocess_ uses the [Cap'n Proto](https://capnproto.org) interface description language and protocol, but it could be extended or changed to use a different IDL/protocol like [gRPC](https://grpc.io). The nice thing about _Cap'n Proto_ compared to _gRPC_ and most other lower level protocols is that it allows interface pointers (_Services_ in gRPC parlance) to be passed as method arguments and return values, so object references and bidirectional requests work out of the box. Supporting a lower-level protocol would require writing adding maps and tracking code to proxy objects. + +_libmultiprocess_ is currently compatible with sandboxing but could add platform-specific sandboxing support or integration with a sandboxing library like [SAPI](https://github.com/google/sandboxed-api). diff --git a/src/ipc/libmultiprocess/doc/install.md b/src/ipc/libmultiprocess/doc/install.md new file mode 100644 index 0000000000000..3fe7c7b29a450 --- /dev/null +++ b/src/ipc/libmultiprocess/doc/install.md @@ -0,0 +1,52 @@ +# libmultiprocess Installation + +Installation currently requires Cap'n Proto: + +```sh +apt install libcapnp-dev capnproto +brew install capnp cmake +dnf install capnproto +``` + +Installation steps are: + +```sh +mkdir build +cd build +cmake .. +make +make check # Optionally build and run tests +make install +``` + +To build with libmultiprocess in a CMake project can specify: + +```cmake +find_package(Libmultiprocess) +target_capnp_sources(mytarget ${CMAKE_CURRENT_SOURCE_DIR} myschema.capnp) +``` + +Which will locate the libmultiprocess cmake package, and call the +`target_capnp_sources` function to generate C++ files and link them into a +library or executable target. See `example/CMakeLists.txt` for a complete +example. + +To build with libmultiprocess in a non-CMake project can use installed +`/include/mpgen.mk` Makefile rule to generate C++ files, and +`/lib/pkgconfig/libmultiprocess.pc` pkg-config definition to link +against the runtime library. + +For cross-compilation, it may be useful to build the runtime library and code +generation binaries separately, which can be done with: + +```sh +make install-bin # install bin/mpgen and related files +make install-lib # install lib/libmultiprocess.a and related files +``` + +It is also possible to import CMake targets separately with: + +```cmake +find_package(Libmultiprocess COMPONENTS Bin) +find_package(Libmultiprocess COMPONENTS Lib) +``` diff --git a/src/ipc/libmultiprocess/doc/usage.md b/src/ipc/libmultiprocess/doc/usage.md new file mode 100644 index 0000000000000..0ef029f159cee --- /dev/null +++ b/src/ipc/libmultiprocess/doc/usage.md @@ -0,0 +1,24 @@ +# libmultiprocess Usage + +## Overview + +_libmultiprocess_ is a library and code generator that allows calling C++ class interfaces across different processes. For an interface to be available from other processes, it needs two definitions: + +- An **API definition** declaring how the interface is called. Included examples: [calculator.h](https://github.com/chaincodelabs/libmultiprocess/blob/master/example/calculator.h), [printer.h](https://github.com/chaincodelabs/libmultiprocess/blob/master/example/printer.h), [init.h](https://github.com/chaincodelabs/libmultiprocess/blob/master/example/init.h). Bitcoin examples: [node.h](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/interfaces/node.h), [wallet.h](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/interfaces/wallet.h), [echo.h](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/interfaces/echo.h), [init.h](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/interfaces/init.h). + +- A **data definition** declaring how interface calls get sent across the wire. Included examples: [calculator.capnp](https://github.com/chaincodelabs/libmultiprocess/blob/master/example/calculator.capnp), [printer.capnp](https://github.com/chaincodelabs/libmultiprocess/blob/master/example/printer.capnp), [init.capnp](https://github.com/chaincodelabs/libmultiprocess/blob/master/example/init.capnp). Bitcoin examples: [node.capnp](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/ipc/capnp/node.capnp), [wallet.capnp](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/ipc/capnp/wallet.capnp), [echo.capnp](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/ipc/capnp/echo.capnp), [init.capnp](https://github.com/ryanofsky/bitcoin/blob/ipc-export/src/ipc/capnp/init.capnp). + +The `*.capnp` data definition files are consumed by the _libmultiprocess_ code generator and each `X.capnp` file generates `X.capnp.c++`, `X.capnp.h`, `X.capnp.proxy-client.c++`, `X.capnp.proxy-server.c++`, `X.capnp.proxy-types.c++`, `X.capnp.proxy-types.h`, and `X.capnp.proxy.h` output files. The generated files include `mp::ProxyClient` and `mp::ProxyServer` class specializations for all the interfaces in the `.capnp` files. These allow methods on C++ objects in one process to be called from other processes over IPC sockets. + +The `ProxyServer` objects help translate IPC requests from a socket to method calls on a local object. The `ProxyServer` objects are just used internally by the `mp::ServeStream(loop, socket, wrapped_object)` and `mp::ListenConnections(loop, socket, wrapped_object)` functions, and aren't exposed externally. The `ProxyClient` classes are exposed, and returned from the `mp::ConnectStream(loop, socket)` function and meant to be used directly. The classes implement methods described in `.capnp` definitions, and whenever any method is called, a request with the method arguments is sent over the associated IPC connection, and the corresponding `wrapped_object` method on the other end of the connection is called, with the `ProxyClient` method blocking until it returns and forwarding back any return value to the `ProxyClient` method caller. + +## Example + +A simple interface description can be found at [test/mp/test/foo.capnp](../test/mp/test/foo.capnp), implementation in [test/mp/test/foo.h](../test/mp/test/foo.h), and usage in [test/mp/test/test.cpp](../test/mp/test/test.cpp). + +A more complete example can be found in [example](../example/) and run with: + +```sh +make -C build example +build/example/mpexample +``` diff --git a/src/ipc/libmultiprocess/example/CMakeLists.txt b/src/ipc/libmultiprocess/example/CMakeLists.txt new file mode 100644 index 0000000000000..333462b8249c4 --- /dev/null +++ b/src/ipc/libmultiprocess/example/CMakeLists.txt @@ -0,0 +1,29 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +include(${PROJECT_SOURCE_DIR}/cmake/TargetCapnpSources.cmake) + +add_executable(mpcalculator + calculator.cpp +) +target_capnp_sources(mpcalculator ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) +target_include_directories(mpcalculator PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(mpcalculator PRIVATE Threads::Threads) + +add_executable(mpprinter + printer.cpp +) +target_capnp_sources(mpprinter ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) +target_include_directories(mpprinter PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(mpprinter PRIVATE Threads::Threads) + +add_executable(mpexample + example.cpp +) +target_capnp_sources(mpexample ${CMAKE_CURRENT_SOURCE_DIR} init.capnp calculator.capnp printer.capnp) +target_include_directories(mpexample PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_link_libraries(mpexample PRIVATE Threads::Threads) +target_link_libraries(mpexample PRIVATE stdc++fs) + +add_custom_target(mpexamples DEPENDS mpexample mpcalculator mpprinter) diff --git a/src/ipc/libmultiprocess/example/calculator.capnp b/src/ipc/libmultiprocess/example/calculator.capnp new file mode 100644 index 0000000000000..8f546552f7057 --- /dev/null +++ b/src/ipc/libmultiprocess/example/calculator.capnp @@ -0,0 +1,16 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xb67dbf34061180a9; + +using Cxx = import "/capnp/c++.capnp"; +using Proxy = import "/mp/proxy.capnp"; + +$Proxy.include("calculator.h"); +$Proxy.includeTypes("types.h"); + +interface CalculatorInterface $Proxy.wrap("Calculator") { + destroy @0 (context :Proxy.Context) -> (); + solveEquation @1 (context :Proxy.Context, eqn: Text) -> (); +} diff --git a/src/ipc/libmultiprocess/example/calculator.cpp b/src/ipc/libmultiprocess/example/calculator.cpp new file mode 100644 index 0000000000000..ae69ce8a626c5 --- /dev/null +++ b/src/ipc/libmultiprocess/example/calculator.cpp @@ -0,0 +1,58 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include // NOLINT(misc-include-cleaner) +#include +#include +#include +#include +#include +#include +#include +#include + +class CalculatorImpl : public Calculator +{ +public: + CalculatorImpl(std::unique_ptr printer) : m_printer(std::move(printer)) {} + void solveEquation(const std::string& eqn) override { m_printer->print("Wow " + eqn + ", that's a tough one.\n"); } + std::unique_ptr m_printer; +}; + +class InitImpl : public Init +{ +public: + std::unique_ptr makeCalculator(std::unique_ptr printer) override + { + return std::make_unique(std::move(printer)); + } +}; + +static void LogPrint(bool raise, const std::string& message) +{ + if (raise) throw std::runtime_error(message); + std::ofstream("debug.log", std::ios_base::app) << message << std::endl; +} + +int main(int argc, char** argv) +{ + if (argc != 2) { + std::cout << "Usage: mpcalculator \n"; + return 1; + } + int fd; + if (std::from_chars(argv[1], argv[1] + strlen(argv[1]), fd).ec != std::errc{}) { + std::cerr << argv[1] << " is not a number or is larger than an int\n"; + return 1; + } + mp::EventLoop loop("mpcalculator", LogPrint); + std::unique_ptr init = std::make_unique(); + mp::ServeStream(loop, fd, *init); + loop.loop(); + return 0; +} diff --git a/src/ipc/libmultiprocess/example/calculator.h b/src/ipc/libmultiprocess/example/calculator.h new file mode 100644 index 0000000000000..749e435547dec --- /dev/null +++ b/src/ipc/libmultiprocess/example/calculator.h @@ -0,0 +1,17 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef EXAMPLE_CALCULATOR_H +#define EXAMPLE_CALCULATOR_H + +#include + +class Calculator +{ +public: + virtual ~Calculator() = default; + virtual void solveEquation(const std::string& eqn) = 0; +}; + +#endif // EXAMPLE_CALCULATOR_H diff --git a/src/ipc/libmultiprocess/example/example.cpp b/src/ipc/libmultiprocess/example/example.cpp new file mode 100644 index 0000000000000..a4f84c55a7587 --- /dev/null +++ b/src/ipc/libmultiprocess/example/example.cpp @@ -0,0 +1,72 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +static auto Spawn(mp::EventLoop& loop, const std::string& process_argv0, const std::string& new_exe_name) +{ + int pid; + const int fd = mp::SpawnProcess(pid, [&](int fd) -> std::vector { + fs::path path = process_argv0; + path.remove_filename(); + path.append(new_exe_name); + return {path.string(), std::to_string(fd)}; + }); + return std::make_tuple(mp::ConnectStream(loop, fd), pid); +} + +static void LogPrint(bool raise, const std::string& message) +{ + if (raise) throw std::runtime_error(message); + std::ofstream("debug.log", std::ios_base::app) << message << std::endl; +} + +int main(int argc, char** argv) +{ + if (argc != 1) { + std::cout << "Usage: mpexample\n"; + return 1; + } + + std::promise promise; + std::thread loop_thread([&] { + mp::EventLoop loop("mpexample", LogPrint); + promise.set_value(&loop); + loop.loop(); + }); + mp::EventLoop* loop = promise.get_future().get(); + + auto [printer_init, printer_pid] = Spawn(*loop, argv[0], "mpprinter"); + auto [calc_init, calc_pid] = Spawn(*loop, argv[0], "mpcalculator"); + auto calc = calc_init->makeCalculator(printer_init->makePrinter()); + while (true) { + std::string eqn; + std::cout << "Enter the equation, or \"exit\" to quit: "; + std::getline(std::cin, eqn); + if (eqn == "exit") break; + calc->solveEquation(eqn); + } + calc.reset(); + calc_init.reset(); + mp::WaitProcess(calc_pid); + printer_init.reset(); + mp::WaitProcess(printer_pid); + loop_thread.join(); + std::cout << "Bye!" << std::endl; + return 0; +} diff --git a/src/ipc/libmultiprocess/example/init.capnp b/src/ipc/libmultiprocess/example/init.capnp new file mode 100644 index 0000000000000..2b0b5113972c3 --- /dev/null +++ b/src/ipc/libmultiprocess/example/init.capnp @@ -0,0 +1,22 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xba5a7448664901b1; + +using Cxx = import "/capnp/c++.capnp"; +using Proxy = import "/mp/proxy.capnp"; +using Calculator = import "calculator.capnp"; +using Printer = import "printer.capnp"; + +$Proxy.include("calculator.h"); +$Proxy.include("init.h"); +$Proxy.include("printer.h"); +$Proxy.includeTypes("calculator.capnp.proxy-types.h"); +$Proxy.includeTypes("printer.capnp.proxy-types.h"); + +interface InitInterface $Proxy.wrap("Init") { + construct @0 (threadMap: Proxy.ThreadMap) -> (threadMap :Proxy.ThreadMap); + makeCalculator @1 (context :Proxy.Context, print :Printer.PrinterInterface) -> (result :Calculator.CalculatorInterface); + makePrinter @2 (context :Proxy.Context) -> (result :Printer.PrinterInterface); +} diff --git a/src/ipc/libmultiprocess/example/init.h b/src/ipc/libmultiprocess/example/init.h new file mode 100644 index 0000000000000..314d5d7f23805 --- /dev/null +++ b/src/ipc/libmultiprocess/example/init.h @@ -0,0 +1,20 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef EXAMPLE_INIT_H +#define EXAMPLE_INIT_H + +#include +#include +#include + +class Init +{ +public: + virtual ~Init() = default; + virtual std::unique_ptr makePrinter() { return nullptr; } + virtual std::unique_ptr makeCalculator(std::unique_ptr printer) { return nullptr; } +}; + +#endif // EXAMPLE_INIT_H diff --git a/src/ipc/libmultiprocess/example/printer.capnp b/src/ipc/libmultiprocess/example/printer.capnp new file mode 100644 index 0000000000000..e27ce41204855 --- /dev/null +++ b/src/ipc/libmultiprocess/example/printer.capnp @@ -0,0 +1,16 @@ +# Copyright (c) 2021 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0x893db95f456ed0e3; + +using Cxx = import "/capnp/c++.capnp"; +using Proxy = import "/mp/proxy.capnp"; + +$Proxy.include("printer.h"); +$Proxy.includeTypes("types.h"); + +interface PrinterInterface $Proxy.wrap("Printer") { + destroy @0 (context :Proxy.Context) -> (); + print @1 (context :Proxy.Context, text: Text) -> (); +} diff --git a/src/ipc/libmultiprocess/example/printer.cpp b/src/ipc/libmultiprocess/example/printer.cpp new file mode 100644 index 0000000000000..9f85d450b66bf --- /dev/null +++ b/src/ipc/libmultiprocess/example/printer.cpp @@ -0,0 +1,51 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include +#include // NOLINT(misc-include-cleaner) +#include +#include +#include +#include +#include +#include +#include + +class PrinterImpl : public Printer +{ +public: + void print(const std::string& message) override { std::cout << "mpprinter: " << message << std::endl; } +}; + +class InitImpl : public Init +{ +public: + std::unique_ptr makePrinter() override { return std::make_unique(); } +}; + +static void LogPrint(bool raise, const std::string& message) +{ + if (raise) throw std::runtime_error(message); + std::ofstream("debug.log", std::ios_base::app) << message << std::endl; +} + +int main(int argc, char** argv) +{ + if (argc != 2) { + std::cout << "Usage: mpprinter \n"; + return 1; + } + int fd; + if (std::from_chars(argv[1], argv[1] + strlen(argv[1]), fd).ec != std::errc{}) { + std::cerr << argv[1] << " is not a number or is larger than an int\n"; + return 1; + } + mp::EventLoop loop("mpprinter", LogPrint); + std::unique_ptr init = std::make_unique(); + mp::ServeStream(loop, fd, *init); + loop.loop(); + return 0; +} diff --git a/src/ipc/libmultiprocess/example/printer.h b/src/ipc/libmultiprocess/example/printer.h new file mode 100644 index 0000000000000..066facf1e2620 --- /dev/null +++ b/src/ipc/libmultiprocess/example/printer.h @@ -0,0 +1,17 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef EXAMPLE_PRINTER_H +#define EXAMPLE_PRINTER_H + +#include + +class Printer +{ +public: + virtual ~Printer() = default; + virtual void print(const std::string& message) = 0; +}; + +#endif // EXAMPLE_PRINTER_H diff --git a/src/ipc/libmultiprocess/example/types.h b/src/ipc/libmultiprocess/example/types.h new file mode 100644 index 0000000000000..0c0bd9342b9c5 --- /dev/null +++ b/src/ipc/libmultiprocess/example/types.h @@ -0,0 +1,14 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef EXAMPLE_TYPES_H +#define EXAMPLE_TYPES_H + +#include +#include +#include +#include +#include + +#endif // EXAMPLE_TYPES_H diff --git a/src/ipc/libmultiprocess/include/mp/config.h.in b/src/ipc/libmultiprocess/include/mp/config.h.in new file mode 100644 index 0000000000000..79ebc4790b880 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/config.h.in @@ -0,0 +1,16 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_CONFIG_H +#define MP_CONFIG_H + +#cmakedefine CMAKE_INSTALL_PREFIX "@CMAKE_INSTALL_PREFIX@" +#cmakedefine capnp_PREFIX "@capnp_PREFIX@" +#cmakedefine HAVE_KJ_FILESYSTEM + +#cmakedefine HAVE_PTHREAD_GETNAME_NP @HAVE_PTHREAD_GETNAME_NP@ +#cmakedefine HAVE_PTHREAD_THREADID_NP @HAVE_PTHREAD_THREADID_NP@ +#cmakedefine HAVE_PTHREAD_GETTHREADID_NP @HAVE_PTHREAD_GETTHREADID_NP@ + +#endif // MP_CONFIG_H diff --git a/src/ipc/libmultiprocess/include/mp/proxy-io.h b/src/ipc/libmultiprocess/include/mp/proxy-io.h new file mode 100644 index 0000000000000..4eb27fae7c423 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/proxy-io.h @@ -0,0 +1,641 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_IO_H +#define MP_PROXY_IO_H + +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace mp { +struct ThreadContext; + +struct InvokeContext +{ + Connection& connection; +}; + +struct ClientInvokeContext : InvokeContext +{ + ThreadContext& thread_context; + ClientInvokeContext(Connection& conn, ThreadContext& thread_context) + : InvokeContext{conn}, thread_context{thread_context} + { + } +}; + +template +struct ServerInvokeContext : InvokeContext +{ + using CallContext = CallContext_; + + ProxyServer& proxy_server; + CallContext& call_context; + int req; + + ServerInvokeContext(ProxyServer& proxy_server, CallContext& call_context, int req) + : InvokeContext{*proxy_server.m_context.connection}, proxy_server{proxy_server}, call_context{call_context}, req{req} + { + } +}; + +template +using ServerContext = ServerInvokeContext, ::capnp::CallContext>; + +template <> +struct ProxyClient : public ProxyClientBase +{ + using ProxyClientBase::ProxyClientBase; + // https://stackoverflow.com/questions/22357887/comparing-two-mapiterators-why-does-it-need-the-copy-constructor-of-stdpair + ProxyClient(const ProxyClient&) = delete; + ~ProxyClient(); + + void setCleanup(const std::function& fn); + + //! Cleanup function to run when the connection is closed. If the Connection + //! gets destroyed before this ProxyClient object, this cleanup + //! callback lets it destroy this object and remove its entry in the + //! thread's request_threads or callback_threads map (after resetting + //! m_cleanup_it so the destructor does not try to access it). But if this + //! object gets destroyed before the Connection, there's no need to run the + //! cleanup function and the destructor will unregister it. + std::optional m_cleanup_it; +}; + +template <> +struct ProxyServer final : public Thread::Server +{ +public: + ProxyServer(ThreadContext& thread_context, std::thread&& thread); + ~ProxyServer(); + kj::Promise getName(GetNameContext context) override; + ThreadContext& m_thread_context; + std::thread m_thread; +}; + +//! Handler for kj::TaskSet failed task events. +class LoggingErrorHandler : public kj::TaskSet::ErrorHandler +{ +public: + LoggingErrorHandler(EventLoop& loop) : m_loop(loop) {} + void taskFailed(kj::Exception&& exception) override; + EventLoop& m_loop; +}; + +using LogFn = std::function; + +class Logger +{ +public: + Logger(bool raise, LogFn& fn) : m_raise(raise), m_fn(fn) {} + Logger(Logger&& logger) : m_raise(logger.m_raise), m_fn(logger.m_fn), m_buffer(std::move(logger.m_buffer)) {} + ~Logger() noexcept(false) + { + if (m_fn) m_fn(m_raise, m_buffer.str()); + } + + template + friend Logger& operator<<(Logger& logger, T&& value) + { + if (logger.m_fn) logger.m_buffer << std::forward(value); + return logger; + } + + template + friend Logger& operator<<(Logger&& logger, T&& value) + { + return logger << std::forward(value); + } + + bool m_raise; + LogFn& m_fn; + std::ostringstream m_buffer; +}; + +std::string LongThreadName(const char* exe_name); + +//! Event loop implementation. +//! +//! Based on https://groups.google.com/d/msg/capnproto/TuQFF1eH2-M/g81sHaTAAQAJ +class EventLoop +{ +public: + //! Construct event loop object. + EventLoop(const char* exe_name, LogFn log_fn, void* context = nullptr); + ~EventLoop(); + + //! Run event loop. Does not return until shutdown. This should only be + //! called once from the m_thread_id thread. This will block until + //! the m_num_clients reference count is 0. + void loop(); + + //! Run function on event loop thread. Does not return until function completes. + //! Must be called while the loop() function is active. + void post(const std::function& fn); + + //! Wrapper around EventLoop::post that takes advantage of the + //! fact that callable will not go out of scope to avoid requirement that it + //! be copyable. + template + void sync(Callable&& callable) + { + post(std::ref(callable)); + } + + //! Start asynchronous worker thread if necessary. This is only done if + //! there are ProxyServerBase::m_impl objects that need to be destroyed + //! asynchronously, without tying up the event loop thread. This can happen + //! when an interface does not declare a destroy() method that would allow + //! the client to wait for the destructor to finish and run it on a + //! dedicated thread. It can also happen whenever this is a broken + //! connection and the client is no longer around to call the destructors + //! and the server objects need to be garbage collected. In both cases, it + //! is important that ProxyServer::m_impl destructors do not run on the + //! eventloop thread because they may need it to do I/O if they perform + //! other IPC calls. + void startAsyncThread(std::unique_lock& lock); + + //! Add/remove remote client reference counts. + void addClient(std::unique_lock& lock); + bool removeClient(std::unique_lock& lock); + //! Check if loop should exit. + bool done(std::unique_lock& lock); + + Logger log() + { + Logger logger(false, m_log_fn); + logger << "{" << LongThreadName(m_exe_name) << "} "; + return logger; + } + Logger logPlain() { return {false, m_log_fn}; } + Logger raise() { return {true, m_log_fn}; } + + //! Process name included in thread names so combined debug output from + //! multiple processes is easier to understand. + const char* m_exe_name; + + //! ID of the event loop thread + std::thread::id m_thread_id = std::this_thread::get_id(); + + //! Handle of an async worker thread. Joined on destruction. Unset if async + //! method has not been called. + std::thread m_async_thread; + + //! Callback function to run on event loop thread during post() or sync() call. + const std::function* m_post_fn = nullptr; + + //! Callback functions to run on async thread. + CleanupList m_async_fns; + + //! Pipe read handle used to wake up the event loop thread. + int m_wait_fd = -1; + + //! Pipe write handle used to wake up the event loop thread. + int m_post_fd = -1; + + //! Number of clients holding references to ProxyServerBase objects that + //! reference this event loop. + int m_num_clients = 0; + + //! Mutex and condition variable used to post tasks to event loop and async + //! thread. + std::mutex m_mutex; + std::condition_variable m_cv; + + //! Capnp IO context. + kj::AsyncIoContext m_io_context; + + //! Capnp error handler. Needs to outlive m_task_set. + LoggingErrorHandler m_error_handler{*this}; + + //! Capnp list of pending promises. + std::unique_ptr m_task_set; + + //! List of connections. + std::list m_incoming_connections; + + //! External logging callback. + LogFn m_log_fn; + + //! External context pointer. + void* m_context; +}; + +//! Single element task queue used to handle recursive capnp calls. (If server +//! makes an callback into the client in the middle of a request, while client +//! thread is blocked waiting for server response, this is what allows the +//! client to run the request in the same thread, the same way code would run in +//! single process, with the callback sharing same thread stack as the original +//! call. +struct Waiter +{ + Waiter() = default; + + template + void post(Fn&& fn) + { + const std::unique_lock lock(m_mutex); + assert(!m_fn); + m_fn = std::move(fn); + m_cv.notify_all(); + } + + template + void wait(std::unique_lock& lock, Predicate pred) + { + m_cv.wait(lock, [&] { + // Important for this to be "while (m_fn)", not "if (m_fn)" to avoid + // a lost-wakeup bug. A new m_fn and m_cv notification might be sent + // after the fn() call and before the lock.lock() call in this loop + // in the case where a capnp response is sent and a brand new + // request is immediately received. + while (m_fn) { + auto fn = std::move(m_fn); + m_fn = nullptr; + lock.unlock(); + fn(); + lock.lock(); + } + const bool done = pred(); + return done; + }); + } + + std::mutex m_mutex; + std::condition_variable m_cv; + std::function m_fn; +}; + +//! Object holding network & rpc state associated with either an incoming server +//! connection, or an outgoing client connection. It must be created and destroyed +//! on the event loop thread. +//! In addition to Cap'n Proto state, it also holds lists of callbacks to run +//! when the connection is closed. +class Connection +{ +public: + Connection(EventLoop& loop, kj::Own&& stream_) + : m_loop(loop), m_stream(kj::mv(stream_)), + m_network(*m_stream, ::capnp::rpc::twoparty::Side::CLIENT, ::capnp::ReaderOptions()), + m_rpc_system(::capnp::makeRpcClient(m_network)) + { + std::unique_lock lock(m_loop.m_mutex); + m_loop.addClient(lock); + } + Connection(EventLoop& loop, + kj::Own&& stream_, + const std::function<::capnp::Capability::Client(Connection&)>& make_client) + : m_loop(loop), m_stream(kj::mv(stream_)), + m_network(*m_stream, ::capnp::rpc::twoparty::Side::SERVER, ::capnp::ReaderOptions()), + m_rpc_system(::capnp::makeRpcServer(m_network, make_client(*this))) + { + std::unique_lock lock(m_loop.m_mutex); + m_loop.addClient(lock); + } + + //! Run cleanup functions. Must be called from the event loop thread. First + //! calls synchronous cleanup functions while blocked (to free capnp + //! Capability::Client handles owned by ProxyClient objects), then schedules + //! asynchronous cleanup functions to run in a worker thread (to run + //! destructors of m_impl instances owned by ProxyServer objects). + ~Connection(); + + //! Register synchronous cleanup function to run on event loop thread (with + //! access to capnp thread local variables) when disconnect() is called. + //! any new i/o. + CleanupIt addSyncCleanup(std::function fn); + void removeSyncCleanup(CleanupIt it); + + //! Register asynchronous cleanup function to run on worker thread when + //! disconnect() is called. + void addAsyncCleanup(std::function fn); + + //! Add disconnect handler. + template + void onDisconnect(F&& f) + { + // Add disconnect handler to local TaskSet to ensure it is cancelled and + // will never run after connection object is destroyed. But when disconnect + // handler fires, do not call the function f right away, instead add it + // to the EventLoop TaskSet to avoid "Promise callback destroyed itself" + // error in cases where f deletes this Connection object. + m_on_disconnect.add(m_network.onDisconnect().then( + [f = std::move(f), this]() mutable { m_loop.m_task_set->add(kj::evalLater(kj::mv(f))); })); + } + + EventLoop& m_loop; + kj::Own m_stream; + LoggingErrorHandler m_error_handler{m_loop}; + kj::TaskSet m_on_disconnect{m_error_handler}; + ::capnp::TwoPartyVatNetwork m_network; + std::optional<::capnp::RpcSystem<::capnp::rpc::twoparty::VatId>> m_rpc_system; + + // ThreadMap interface client, used to create a remote server thread when an + // client IPC call is being made for the first time from a new thread. + ThreadMap::Client m_thread_map{nullptr}; + + //! Collection of server-side IPC worker threads (ProxyServer objects previously returned by + //! ThreadMap.makeThread) used to service requests to clients. + ::capnp::CapabilityServerSet m_threads; + + //! Cleanup functions to run if connection is broken unexpectedly. + //! Lists will be empty if all ProxyClient and ProxyServer objects are + //! destroyed cleanly before the connection is destroyed. + CleanupList m_sync_cleanup_fns; + CleanupList m_async_cleanup_fns; +}; + +//! Vat id for server side of connection. Required argument to RpcSystem::bootStrap() +//! +//! "Vat" is Cap'n Proto nomenclature for a host of various objects that facilitates +//! bidirectional communication with other vats; it is often but not always 1-1 with +//! processes. Cap'n Proto doesn't reference clients or servers per se; instead everything +//! is just a vat. +//! +//! See also: https://github.com/capnproto/capnproto/blob/9021f0c722b36cb11e3690b0860939255ebad39c/c%2B%2B/src/capnp/rpc.capnp#L42-L56 +struct ServerVatId +{ + ::capnp::word scratch[4]{}; + ::capnp::MallocMessageBuilder message{scratch}; + ::capnp::rpc::twoparty::VatId::Builder vat_id{message.getRoot<::capnp::rpc::twoparty::VatId>()}; + ServerVatId() { vat_id.setSide(::capnp::rpc::twoparty::Side::SERVER); } +}; + +template +ProxyClientBase::ProxyClientBase(typename Interface::Client client, + Connection* connection, + bool destroy_connection) + : m_client(std::move(client)), m_context(connection) + +{ + { + std::unique_lock lock(m_context.connection->m_loop.m_mutex); + m_context.connection->m_loop.addClient(lock); + } + + // Handler for the connection getting destroyed before this client object. + auto cleanup_it = m_context.connection->addSyncCleanup([this]() { + // Release client capability by move-assigning to temporary. + { + typename Interface::Client(std::move(m_client)); + } + { + std::unique_lock lock(m_context.connection->m_loop.m_mutex); + m_context.connection->m_loop.removeClient(lock); + } + m_context.connection = nullptr; + }); + + // Two shutdown sequences are supported: + // + // - A normal sequence where client proxy objects are deleted by external + // code that no longer needs them + // + // - A garbage collection sequence where the connection or event loop shuts + // down while external code is still holding client references. + // + // The first case is handled here when m_context.connection is not null. The + // second case is handled by the cleanup function, which sets m_context.connection to + // null so nothing happens here. + m_context.cleanup_fns.emplace_front([this, destroy_connection, cleanup_it]{ + if (m_context.connection) { + // Remove cleanup callback so it doesn't run and try to access + // this object after it's already destroyed. + m_context.connection->removeSyncCleanup(cleanup_it); + + // If the capnp interface defines a destroy method, call it to destroy + // the remote object, waiting for it to be deleted server side. If the + // capnp interface does not define a destroy method, this will just call + // an empty stub defined in the ProxyClientBase class and do nothing. + Sub::destroy(*this); + + // FIXME: Could just invoke removed addCleanup fn here instead of duplicating code + m_context.connection->m_loop.sync([&]() { + // Release client capability by move-assigning to temporary. + { + typename Interface::Client(std::move(m_client)); + } + { + std::unique_lock lock(m_context.connection->m_loop.m_mutex); + m_context.connection->m_loop.removeClient(lock); + } + + if (destroy_connection) { + delete m_context.connection; + m_context.connection = nullptr; + } + }); + } + }); + Sub::construct(*this); +} + +template +ProxyClientBase::~ProxyClientBase() noexcept +{ + CleanupRun(m_context.cleanup_fns); +} + +template +ProxyServerBase::ProxyServerBase(std::shared_ptr impl, Connection& connection) + : m_impl(std::move(impl)), m_context(&connection) +{ + assert(m_impl); + std::unique_lock lock(m_context.connection->m_loop.m_mutex); + m_context.connection->m_loop.addClient(lock); +} + +//! ProxyServer destructor, called from the EventLoop thread by Cap'n Proto +//! garbage collection code after there are no more references to this object. +template +ProxyServerBase::~ProxyServerBase() +{ + if (m_impl) { + // If impl is non-null at this point, it means no client is waiting for + // the m_impl server object to be destroyed synchronously. This can + // happen either if the interface did not define a "destroy" method (see + // invokeDestroy method below), or if a destroy method was defined, but + // the connection was broken before it could be called. + // + // In either case, be conservative and run the cleanup on an + // asynchronous thread, to avoid destructors or cleanup functions + // blocking or deadlocking the current EventLoop thread, since they + // could be making IPC calls. + // + // Technically this is a little too conservative since if the interface + // defines a "destroy" method, but the destroy method does not accept a + // Context parameter specifying a worker thread, the cleanup method + // would run on the EventLoop thread normally (when connection is + // unbroken), but will not run on the EventLoop thread now (when + // connection is broken). Probably some refactoring of the destructor + // and invokeDestroy function is possible to make this cleaner and more + // consistent. + m_context.connection->addAsyncCleanup([impl=std::move(m_impl), fns=std::move(m_context.cleanup_fns)]() mutable { + impl.reset(); + CleanupRun(fns); + }); + } + assert(m_context.cleanup_fns.empty()); + std::unique_lock lock(m_context.connection->m_loop.m_mutex); + m_context.connection->m_loop.removeClient(lock); +} + +//! If the capnp interface defined a special "destroy" method, as described the +//! ProxyClientBase class, this method will be called and synchronously destroy +//! m_impl before returning to the client. +//! +//! If the capnp interface does not define a "destroy" method, this will never +//! be called and the ~ProxyServerBase destructor will be responsible for +//! deleting m_impl asynchronously, whenever the ProxyServer object gets garbage +//! collected by Cap'n Proto. +//! +//! This method is called in the same way other proxy server methods are called, +//! via the serverInvoke function. Basically serverInvoke just calls this as a +//! substitute for a non-existent m_impl->destroy() method. If the destroy +//! method has any parameters or return values they will be handled in the +//! normal way by PassField/ReadField/BuildField functions. Particularly if a +//! Context.thread parameter was passed, this method will run on the worker +//! thread specified by the client. Otherwise it will run on the EventLoop +//! thread, like other server methods without an assigned thread. +template +void ProxyServerBase::invokeDestroy() +{ + m_impl.reset(); + CleanupRun(m_context.cleanup_fns); +} + +using ConnThreads = std::map>; +using ConnThread = ConnThreads::iterator; + +// Retrieve ProxyClient object associated with this connection from a +// map, or create a new one and insert it into the map. Return map iterator and +// inserted bool. +std::tuple SetThread(ConnThreads& threads, std::mutex& mutex, Connection* connection, const std::function& make_thread); + +struct ThreadContext +{ + //! Identifying string for debug. + std::string thread_name; + + //! Waiter object used to allow client threads blocked waiting for a server + //! response to execute callbacks made from the client's corresponding + //! server thread. + std::unique_ptr waiter = nullptr; + + //! When client is making a request to a server, this is the + //! `callbackThread` argument it passes in the request, used by the server + //! in case it needs to make callbacks into the client that need to execute + //! while the client is waiting. This will be set to a local thread object. + ConnThreads callback_threads; + + //! When client is making a request to a server, this is the `thread` + //! argument it passes in the request, used to control which thread on + //! server will be responsible for executing it. If client call is being + //! made from a local thread, this will be a remote thread object returned + //! by makeThread. If a client call is being made from a thread currently + //! handling a server request, this will be set to the `callbackThread` + //! request thread argument passed in that request. + ConnThreads request_threads; + + //! Whether this thread is a capnp event loop thread. Not really used except + //! to assert false if there's an attempt to execute a blocking operation + //! which could deadlock the thread. + bool loop_thread = false; +}; + +//! Given stream file descriptor, make a new ProxyClient object to send requests +//! over the stream. Also create a new Connection object embedded in the +//! client that is freed when the client is closed. +template +std::unique_ptr> ConnectStream(EventLoop& loop, int fd) +{ + typename InitInterface::Client init_client(nullptr); + std::unique_ptr connection; + loop.sync([&] { + auto stream = + loop.m_io_context.lowLevelProvider->wrapSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP); + connection = std::make_unique(loop, kj::mv(stream)); + init_client = connection->m_rpc_system->bootstrap(ServerVatId().vat_id).castAs(); + Connection* connection_ptr = connection.get(); + connection->onDisconnect([&loop, connection_ptr] { + loop.log() << "IPC client: unexpected network disconnect."; + delete connection_ptr; + }); + }); + return std::make_unique>( + kj::mv(init_client), connection.release(), /* destroy_connection= */ true); +} + +//! Given stream and init objects, construct a new ProxyServer object that +//! handles requests from the stream by calling the init object. Embed the +//! ProxyServer in a Connection object that is stored and erased if +//! disconnected. This should be called from the event loop thread. +template +void _Serve(EventLoop& loop, kj::Own&& stream, InitImpl& init) +{ + loop.m_incoming_connections.emplace_front(loop, kj::mv(stream), [&](Connection& connection) { + // Disable deleter so proxy server object doesn't attempt to delete the + // init implementation when the proxy client is destroyed or + // disconnected. + return kj::heap>(std::shared_ptr(&init, [](InitImpl*){}), connection); + }); + auto it = loop.m_incoming_connections.begin(); + it->onDisconnect([&loop, it] { + loop.log() << "IPC server: socket disconnected."; + loop.m_incoming_connections.erase(it); + }); +} + +//! Given connection receiver and an init object, handle incoming connections by +//! calling _Serve, to create ProxyServer objects and forward requests to the +//! init object. +template +void _Listen(EventLoop& loop, kj::Own&& listener, InitImpl& init) +{ + auto* ptr = listener.get(); + loop.m_task_set->add(ptr->accept().then( + [&loop, &init, listener = kj::mv(listener)](kj::Own&& stream) mutable { + _Serve(loop, kj::mv(stream), init); + _Listen(loop, kj::mv(listener), init); + })); +} + +//! Given stream file descriptor and an init object, handle requests on the +//! stream by calling methods on the Init object. +template +void ServeStream(EventLoop& loop, int fd, InitImpl& init) +{ + _Serve( + loop, loop.m_io_context.lowLevelProvider->wrapSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP), init); +} + +//! Given listening socket file descriptor and an init object, handle incoming +//! connections and requests by calling methods on the Init object. +template +void ListenConnections(EventLoop& loop, int fd, InitImpl& init) +{ + loop.sync([&]() { + _Listen(loop, + loop.m_io_context.lowLevelProvider->wrapListenSocketFd(fd, kj::LowLevelAsyncIoProvider::TAKE_OWNERSHIP), + init); + }); +} + +extern thread_local ThreadContext g_thread_context; + +} // namespace mp + +#endif // MP_PROXY_IO_H diff --git a/src/ipc/libmultiprocess/include/mp/proxy-types.h b/src/ipc/libmultiprocess/include/mp/proxy-types.h new file mode 100644 index 0000000000000..1a519efde02b3 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/proxy-types.h @@ -0,0 +1,727 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPES_H +#define MP_PROXY_TYPES_H + +#include + +#include +#include +#include +#include +#include + +namespace mp { + +template +class ValueField +{ +public: + ValueField(Value& value) : m_value(value) {} + ValueField(Value&& value) : m_value(value) {} + Value& m_value; + + Value& get() { return m_value; } + Value& init() { return m_value; } + bool has() { return true; } +}; + +template +struct StructField +{ + template + StructField(S& struct_) : m_struct(struct_) + { + } + Struct& m_struct; + + // clang-format off + template auto get() const -> decltype(A::get(this->m_struct)) { return A::get(this->m_struct); } + template auto has() const -> std::enable_if_t { return A::getHas(m_struct); } + template auto has() const -> std::enable_if_t { return A::has(m_struct); } + template auto has() const -> std::enable_if_t { return true; } + template auto want() const -> std::enable_if_t { return A::getWant(m_struct); } + template auto want() const -> std::enable_if_t { return true; } + template decltype(auto) set(Args&&... args) const { return A::set(this->m_struct, std::forward(args)...); } + template decltype(auto) init(Args&&... args) const { return A::init(this->m_struct, std::forward(args)...); } + template auto setHas() const -> std::enable_if_t { return A::setHas(m_struct); } + template auto setHas() const -> std::enable_if_t { } + template auto setWant() const -> std::enable_if_t { return A::setWant(m_struct); } + template auto setWant() const -> std::enable_if_t { } + // clang-format on +}; + + + +// Destination parameter type that can be passed to ReadField function as an +// alternative to ReadDestUpdate. It allows the ReadField implementation to call +// the provided emplace_fn function with constructor arguments, so it only needs +// to determine the arguments, and can let the emplace function decide how to +// actually construct the read destination object. For example, if a std::string +// is being read, the ReadField call will call the custom emplace_fn with char* +// and size_t arguments, and the emplace function can decide whether to call the +// constructor via the operator or make_shared or emplace or just return a +// temporary string that is moved from. +template +struct ReadDestEmplace +{ + ReadDestEmplace(TypeList, EmplaceFn&& emplace_fn) : m_emplace_fn(emplace_fn) {} + + //! Simple case. If ReadField impementation calls this construct() method + //! with constructor arguments, just pass them on to the emplace function. + template + decltype(auto) construct(Args&&... args) + { + return m_emplace_fn(std::forward(args)...); + } + + //! More complicated case. If ReadField implementation works by calling this + //! update() method, adapt it call construct() instead. This requires + //! LocalType to have a default constructor to create new object that can be + //! passed to update() + template + decltype(auto) update(UpdateFn&& update_fn) + { + if constexpr (std::is_const_v>>) { + // If destination type is const, default construct temporary + // to pass to update, then call move constructor via construct() to + // move from that temporary. + std::remove_cv_t temp; + update_fn(temp); + return construct(std::move(temp)); + } else { + // Default construct object and pass it to update_fn. + decltype(auto) temp = construct(); + update_fn(temp); + return temp; + } + } + EmplaceFn& m_emplace_fn; +}; + +//! Helper function to create a ReadDestEmplace object that constructs a +//! temporary, ReadField can return. +template +auto ReadDestTemp() +{ + return ReadDestEmplace{TypeList(), [&](auto&&... args) -> decltype(auto) { + return LocalType{std::forward(args)...}; + }}; +} + +//! Destination parameter type that can be passed to ReadField function as an +//! alternative to ReadDestEmplace. Instead of requiring an emplace callback to +//! construct a new value, it just takes a reference to an existing value and +//! assigns a new value to it. +template +struct ReadDestUpdate +{ + ReadDestUpdate(Value& value) : m_value(value) {} + + //! Simple case. If ReadField works by calling update() just forward arguments to update_fn. + template + Value& update(UpdateFn&& update_fn) + { + update_fn(m_value); + return m_value; + } + + //! More complicated case. If ReadField works by calling construct(), need + //! to reconstruct m_value in place. + template + Value& construct(Args&&... args) + { + m_value.~Value(); + new (&m_value) Value(std::forward(args)...); + return m_value; + } + + Value& m_value; +}; + +template +decltype(auto) ReadField(TypeList, Args&&... args) +{ + return CustomReadField(TypeList...>(), Priority<2>(), std::forward(args)...); +} + +template +void ThrowField(TypeList, InvokeContext& invoke_context, Input&& input) +{ + ReadField( + TypeList(), invoke_context, input, ReadDestEmplace(TypeList(), + [](auto&& ...args) -> const LocalType& { throw LocalType{std::forward(args)...}; })); +} + +//! Special case for generic std::exception. It's an abstract type so it can't +//! be created directly. Rethrow as std::runtime_error so callers expecting it +//! will still catch it. +template +void ThrowField(TypeList, InvokeContext& invoke_context, Input&& input) +{ + auto data = input.get(); + throw std::runtime_error(std::string(CharCast(data.begin()), data.size())); +} + +template +bool CustomHasValue(InvokeContext& invoke_context, Values&&... value) +{ + return true; +} + +template +void BuildField(TypeList, Context& context, Output&& output, Values&&... values) +{ + if (CustomHasValue(context, std::forward(values)...)) { + CustomBuildField(TypeList(), Priority<3>(), context, std::forward(values)..., + std::forward(output)); + } +} + +// Adapter to let BuildField overloads methods work set & init list elements as +// if they were fields of a struct. If BuildField is changed to use some kind of +// accessor class instead of calling method pointers, then then maybe this could +// go away or be simplified, because would no longer be a need to return +// ListOutput method pointers emulating capnp struct method pointers.. +template +struct ListOutput; + +template +struct ListOutput<::capnp::List> +{ + using Builder = typename ::capnp::List::Builder; + + ListOutput(Builder& builder, size_t index) : m_builder(builder), m_index(index) {} + Builder& m_builder; + size_t m_index; + + // clang-format off + decltype(auto) get() const { return this->m_builder[this->m_index]; } + decltype(auto) init() const { return this->m_builder[this->m_index]; } + template decltype(auto) set(Arg&& arg) const { return static_cast(this->m_builder).set(m_index, std::forward(arg)); } + template decltype(auto) init(Arg&& arg) const { return static_cast(this->m_builder).init(m_index, std::forward(arg)); } + // clang-format on +}; + +template +void CustomBuildField(TypeList, Priority<0>, InvokeContext& invoke_context, Value&& value, Output&& output) +{ + output.set(BuildPrimitive(invoke_context, std::forward(value), TypeList())); +} + +//! PassField override for callable interface reference arguments. +template +auto PassField(Priority<1>, TypeList, ServerContext& server_context, Fn&& fn, Args&&... args) + -> Require +{ + // Just create a temporary ProxyClient if argument is a reference to an + // interface client. If argument needs to have a longer lifetime and not be + // destroyed after this call, a CustomPassField overload can be implemented + // to bypass this code, and a custom ProxyServerMethodTraits overload can be + // implemented in order to read the capability pointer out of params and + // construct a ProxyClient with a longer lifetime. + const auto& params = server_context.call_context.getParams(); + const auto& input = Make(params); + using Interface = typename Decay::Calls; + auto param = std::make_unique>(input.get(), server_context.proxy_server.m_context.connection, false); + fn.invoke(server_context, std::forward(args)..., *param); +} + +template +void MaybeBuildField(std::true_type, Args&&... args) +{ + BuildField(std::forward(args)...); +} +template +void MaybeBuildField(std::false_type, Args&&...) +{ +} +template +void MaybeReadField(std::true_type, Args&&... args) +{ + ReadField(std::forward(args)...); +} +template +void MaybeReadField(std::false_type, Args&&...) +{ +} + +template +void MaybeSetWant(TypeList, Priority<1>, Value&& value, Output&& output) +{ + if (value) { + output.setWant(); + } +} + +template +void MaybeSetWant(LocalTypes, Priority<0>, Args&&...) +{ +} + +//! Default PassField implementation calling MaybeReadField/MaybeBuildField. +template +void PassField(Priority<0>, TypeList, ServerContext& server_context, Fn&& fn, Args&&... args) +{ + InvokeContext& invoke_context = server_context; + using ArgType = RemoveCvRef; + std::optional param; + const auto& params = server_context.call_context.getParams(); + MaybeReadField(std::integral_constant(), TypeList(), invoke_context, + Make(params), ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + param.emplace(std::forward(args)...); + return *param; + })); + if constexpr (Accessor::in) { + assert(param); + } else { + if (!param) param.emplace(); + } + fn.invoke(server_context, std::forward(args)..., static_cast(*param)); + auto&& results = server_context.call_context.getResults(); + MaybeBuildField(std::integral_constant(), TypeList(), invoke_context, + Make(results), *param); +} + +//! Default PassField implementation for count(0) arguments, calling ReadField/BuildField +template +void PassField(Priority<0>, TypeList<>, ServerContext& server_context, const Fn& fn, Args&&... args) +{ + const auto& params = server_context.call_context.getParams(); + const auto& input = Make(params); + ReadField(TypeList<>(), server_context, input); + fn.invoke(server_context, std::forward(args)...); + auto&& results = server_context.call_context.getResults(); + BuildField(TypeList<>(), server_context, Make(results)); +} + +template +struct IterateFieldsHelper +{ + template + void handleChain(Arg1&& arg1, Arg2&& arg2, ParamList, NextFn&& next_fn, NextFnArgs&&... next_fn_args) + { + using S = Split; + handleChain(std::forward(arg1), std::forward(arg2), typename S::First()); + next_fn.handleChain(std::forward(arg1), std::forward(arg2), typename S::Second(), + std::forward(next_fn_args)...); + } + + template + void handleChain(Arg1&& arg1, Arg2&& arg2, ParamList) + { + static_cast(this)->handleField(std::forward(arg1), std::forward(arg2), ParamList()); + } +private: + IterateFieldsHelper() = default; + friend Derived; +}; + +struct IterateFields : IterateFieldsHelper +{ + template + void handleField(Arg1&&, Arg2&&, ParamList) + { + } +}; + +template +struct ClientException +{ + struct BuildParams : IterateFieldsHelper + { + template + void handleField(InvokeContext& invoke_context, Params& params, ParamList) + { + } + + BuildParams(ClientException* client_exception) : m_client_exception(client_exception) {} + ClientException* m_client_exception; + }; + + struct ReadResults : IterateFieldsHelper + { + template + void handleField(InvokeContext& invoke_context, Results& results, ParamList) + { + StructField input(results); + if (input.has()) { + ThrowField(TypeList(), invoke_context, input); + } + } + + ReadResults(ClientException* client_exception) : m_client_exception(client_exception) {} + ClientException* m_client_exception; + }; +}; + +template +struct ClientParam +{ + ClientParam(Types&&... values) : m_values(values...) {} + + struct BuildParams : IterateFieldsHelper + { + template + void handleField(Args&&... args) + { + callBuild<0>(std::forward(args)...); + } + + // TODO Possible optimization to speed up compile time: + // https://stackoverflow.com/a/7858971 Using enable_if below to check + // position when unpacking tuple might be slower than pattern matching + // approach in the stack overflow solution + template + auto callBuild(Args&&... args) -> std::enable_if_t<(I < sizeof...(Types))> + { + callBuild(std::forward(args)..., std::get(m_client_param->m_values)); + } + + template + auto callBuild(ClientInvokeContext& invoke_context, Params& params, ParamList, Values&&... values) -> + std::enable_if_t<(I == sizeof...(Types))> + { + MaybeBuildField(std::integral_constant(), ParamList(), invoke_context, + Make(params), std::forward(values)...); + MaybeSetWant( + ParamList(), Priority<1>(), std::forward(values)..., Make(params)); + } + + BuildParams(ClientParam* client_param) : m_client_param(client_param) {} + ClientParam* m_client_param; + }; + + struct ReadResults : IterateFieldsHelper + { + template + void handleField(Args&&... args) + { + callRead<0>(std::forward(args)...); + } + + template + auto callRead(Args&&... args) -> std::enable_if_t<(I < sizeof...(Types))> + { + callRead(std::forward(args)..., std::get(m_client_param->m_values)); + } + + template + auto callRead(ClientInvokeContext& invoke_context, Results& results, TypeList, Values&&... values) + -> std::enable_if_t + { + MaybeReadField(std::integral_constant(), TypeList...>(), invoke_context, + Make(results), ReadDestUpdate(values)...); + } + + ReadResults(ClientParam* client_param) : m_client_param(client_param) {} + ClientParam* m_client_param; + }; + + std::tuple m_values; +}; + +template +ClientParam MakeClientParam(Types&&... values) +{ + return {std::forward(values)...}; +} + +struct ServerCall +{ + // FIXME: maybe call call_context.releaseParams() + template + decltype(auto) invoke(ServerContext& server_context, TypeList<>, Args&&... args) const + { + return ProxyServerMethodTraits::invoke( + server_context, + std::forward(args)...); + } +}; + +struct ServerDestroy +{ + template + void invoke(ServerContext& server_context, TypeList<>, Args&&... args) const + { + server_context.proxy_server.invokeDestroy(std::forward(args)...); + } +}; + +template +struct ServerRet : Parent +{ + ServerRet(Parent parent) : Parent(parent) {} + + template + void invoke(ServerContext& server_context, TypeList<>, Args&&... args) const + { + auto&& result = Parent::invoke(server_context, TypeList<>(), std::forward(args)...); + auto&& results = server_context.call_context.getResults(); + InvokeContext& invoke_context = server_context; + BuildField(TypeList(), invoke_context, Make(results), + std::forward(result)); + } +}; + +template +struct ServerExcept : Parent +{ + ServerExcept(Parent parent) : Parent(parent) {} + + template + void invoke(ServerContext& server_context, TypeList<>, Args&&... args) const + { + try { + return Parent::invoke(server_context, TypeList<>(), std::forward(args)...); + } catch (const Exception& exception) { + auto&& results = server_context.call_context.getResults(); + BuildField(TypeList(), server_context, Make(results), exception); + } + } +}; + +//! Helper for CustomPassField below. Call Accessor::get method if it has one, +//! otherwise return capnp::Void. +template +decltype(auto) MaybeGet(Message&& message, decltype(Accessor::get(message))* enable = nullptr) +{ + return Accessor::get(message); +} + +template +::capnp::Void MaybeGet(...) +{ + return {}; +} + +template +void CustomPassField(); + +//! PassField override calling CustomPassField function, if it exists. +//! Defining a CustomPassField or CustomPassMessage overload is useful for +//! input/output parameters. If an overload is not defined these parameters will +//! just be deserialized on the server side with ReadField into a temporary +//! variable, then the server method will be called passing the temporary +//! variable as a parameter, then the temporary variable will be serialized and +//! sent back to the client with BuildField. But if a PassField or PassMessage +//! overload is defined, the overload is called with a callback to invoke and +//! pass parameters to the server side function, and run arbitrary code before +//! and after invoking the function. +template +auto PassField(Priority<2>, Args&&... args) -> decltype(CustomPassField(std::forward(args)...)) +{ + return CustomPassField(std::forward(args)...); +}; + +template +struct ServerField : Parent +{ + ServerField(Parent parent) : Parent(parent) {} + + const Parent& parent() const { return *this; } + + template + decltype(auto) invoke(ServerContext& server_context, ArgTypes, Args&&... args) const + { + return PassField(Priority<2>(), + typename Split::First(), + server_context, + this->parent(), + typename Split::Second(), + std::forward(args)...); + } +}; + +template +ServerField MakeServerField(Parent parent) +{ + return {parent}; +} + +template +struct CapRequestTraits; + +template +struct CapRequestTraits<::capnp::Request<_Params, _Results>> +{ + using Params = _Params; + using Results = _Results; +}; + +//! Entry point called by all generated ProxyClient destructors. This only logs +//! the object destruction. The actual cleanup happens in the ProxyClient base +//! destructor. +template +void clientDestroy(Client& client) +{ + if (client.m_context.connection) { + client.m_context.connection->m_loop.log() << "IPC client destroy " << typeid(client).name(); + } else { + KJ_LOG(INFO, "IPC interrupted client destroy", typeid(client).name()); + } +} + +template +void serverDestroy(Server& server) +{ + server.m_context.connection->m_loop.log() << "IPC server destroy " << typeid(server).name(); +} + +//! Entry point called by generated client code that looks like: +//! +//! ProxyClient::M0::Result ProxyClient::methodName(M0::Param<0> arg0, M0::Param<1> arg1) { +//! typename M0::Result result; +//! clientInvoke(*this, &InterfaceName::Client::methodNameRequest, MakeClientParam<...>(arg0), MakeClientParam<...>(arg1), MakeClientParam<...>(result)); +//! return result; +//! } +//! +//! Ellipses above are where generated Accessor<> type declarations are inserted. +template +void clientInvoke(ProxyClient& proxy_client, const GetRequest& get_request, FieldObjs&&... fields) +{ + if (!proxy_client.m_context.connection) { + throw std::logic_error("clientInvoke call made after disconnect"); + } + if (!g_thread_context.waiter) { + assert(g_thread_context.thread_name.empty()); + g_thread_context.thread_name = ThreadName(proxy_client.m_context.connection->m_loop.m_exe_name); + // If next assert triggers, it means clientInvoke is being called from + // the capnp event loop thread. This can happen when a ProxyServer + // method implementation that runs synchronously on the event loop + // thread tries to make a blocking callback to the client. Any server + // method that makes a blocking callback or blocks in general needs to + // run asynchronously off the event loop thread. This is easy to fix by + // just adding a 'context :Proxy.Context' argument to the capnp method + // declaration so the server method runs in a dedicated thread. + assert(!g_thread_context.loop_thread); + g_thread_context.waiter = std::make_unique(); + proxy_client.m_context.connection->m_loop.logPlain() + << "{" << g_thread_context.thread_name + << "} IPC client first request from current thread, constructing waiter"; + } + ClientInvokeContext invoke_context{*proxy_client.m_context.connection, g_thread_context}; + std::exception_ptr exception; + std::string kj_exception; + bool done = false; + proxy_client.m_context.connection->m_loop.sync([&]() { + auto request = (proxy_client.m_client.*get_request)(nullptr); + using Request = CapRequestTraits; + using FieldList = typename ProxyClientMethodTraits::Fields; + IterateFields().handleChain(invoke_context, request, FieldList(), typename FieldObjs::BuildParams{&fields}...); + proxy_client.m_context.connection->m_loop.logPlain() + << "{" << invoke_context.thread_context.thread_name << "} IPC client send " + << TypeName() << " " << LogEscape(request.toString()); + + proxy_client.m_context.connection->m_loop.m_task_set->add(request.send().then( + [&](::capnp::Response&& response) { + proxy_client.m_context.connection->m_loop.logPlain() + << "{" << invoke_context.thread_context.thread_name << "} IPC client recv " + << TypeName() << " " << LogEscape(response.toString()); + try { + IterateFields().handleChain( + invoke_context, response, FieldList(), typename FieldObjs::ReadResults{&fields}...); + } catch (...) { + exception = std::current_exception(); + } + const std::unique_lock lock(invoke_context.thread_context.waiter->m_mutex); + done = true; + invoke_context.thread_context.waiter->m_cv.notify_all(); + }, + [&](const ::kj::Exception& e) { + kj_exception = kj::str("kj::Exception: ", e).cStr(); + proxy_client.m_context.connection->m_loop.logPlain() + << "{" << invoke_context.thread_context.thread_name << "} IPC client exception " << kj_exception; + const std::unique_lock lock(invoke_context.thread_context.waiter->m_mutex); + done = true; + invoke_context.thread_context.waiter->m_cv.notify_all(); + })); + }); + + std::unique_lock lock(invoke_context.thread_context.waiter->m_mutex); + invoke_context.thread_context.waiter->wait(lock, [&done]() { return done; }); + if (exception) std::rethrow_exception(exception); + if (!kj_exception.empty()) proxy_client.m_context.connection->m_loop.raise() << kj_exception; +} + +//! Invoke callable `fn()` that may return void. If it does return void, replace +//! return value with value of `ret()`. This is useful for avoiding code +//! duplication and branching in generic code that forwards calls to functions. +template +auto ReplaceVoid(Fn&& fn, Ret&& ret) -> + std::enable_if_t, decltype(ret())> +{ + fn(); + return ret(); +} + +//! Overload of above for non-void `fn()` case. +template +auto ReplaceVoid(Fn&& fn, Ret&& ret) -> + std::enable_if_t, decltype(fn())> +{ + return fn(); +} + +extern std::atomic server_reqs; + +//! Entry point called by generated server code that looks like: +//! +//! kj::Promise ProxyServer::methodName(CallContext call_context) { +//! return serverInvoke(*this, call_context, MakeServerField<0, ...>(MakeServerField<1, ...>(Make(ServerCall())))); +//! } +//! +//! Ellipses above are where generated Accessor<> type declarations are inserted. +template +kj::Promise serverInvoke(Server& server, CallContext& call_context, Fn fn) +{ + auto params = call_context.getParams(); + using Params = decltype(params); + using Results = typename decltype(call_context.getResults())::Builds; + + int req = ++server_reqs; + server.m_context.connection->m_loop.log() << "IPC server recv request #" << req << " " + << TypeName() << " " << LogEscape(params.toString()); + + try { + using ServerContext = ServerInvokeContext; + using ArgList = typename ProxyClientMethodTraits::Params; + ServerContext server_context{server, call_context, req}; + // ReplaceVoid is used to support fn.invoke implementations that + // execute asynchronously and return promises, as well as + // implementations that execute synchronously and return void. The + // invoke function will be synchronous by default, but asynchronous if + // an mp.Context argument is passed, and the mp.Context PassField + // overload returns a promise executing the request in a worker thread + // and waiting for it to complete. + return ReplaceVoid([&]() { return fn.invoke(server_context, ArgList()); }, + [&]() { return kj::Promise(kj::mv(call_context)); }) + .then([&server, req](CallContext call_context) { + server.m_context.connection->m_loop.log() << "IPC server send response #" << req << " " << TypeName() + << " " << LogEscape(call_context.getResults().toString()); + }); + } catch (const std::exception& e) { + server.m_context.connection->m_loop.log() << "IPC server unhandled exception: " << e.what(); + throw; + } catch (...) { + server.m_context.connection->m_loop.log() << "IPC server unhandled exception"; + throw; + } +} + +//! Map to convert client interface pointers to ProxyContext struct references +//! at runtime using typeids. +struct ProxyTypeRegister { + template + ProxyTypeRegister(TypeList) { + types().emplace(typeid(Interface), [](void* iface) -> ProxyContext& { return static_cast::Client&>(*static_cast(iface)).m_context; }); + } + using Types = std::map; + static Types& types() { static Types types; return types; } +}; + +} // namespace mp + +#endif // MP_PROXY_TYPES_H diff --git a/src/ipc/libmultiprocess/include/mp/proxy.capnp b/src/ipc/libmultiprocess/include/mp/proxy.capnp new file mode 100644 index 0000000000000..abd02e437fc50 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/proxy.capnp @@ -0,0 +1,65 @@ +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +@0xcc316e3f71a040fb; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("mp"); + +annotation include(file): Text; +annotation includeTypes(file): Text; +# Extra include paths to add to generated files. + +annotation wrap(interface, struct): Text; +# Wrap capnp interface generating ProxyClient / ProxyServer C++ classes that +# forward calls to a C++ interface with same methods and parameters. Text +# string should be the name of the C++ interface. +# If applied to struct rather than an interface, this will generate a ProxyType +# struct with get methods for introspection and copying fields between C++ and +# capnp structs. + +annotation count(param, struct, interface): Int32; +# Indicate how many C++ method parameters there are corresponding to one capnp +# parameter (default is 1). If not 1, multiple C++ method arguments will be +# condensed into a single capnp parameter by the client and then expanded by +# the server by CustomReadField/CustomBuildField overloads which need to be +# provided separately. An example would be a capnp Text parameter initialized +# from C++ char* and size arguments. Can be 0 to fill an implicit capnp +# parameter from client or server side context. If annotation is applied to an +# interface or struct type it will apply to all parameters of that type. + +annotation exception(param): Text; +# Indicate that a result parameter corresponds to a C++ exception. Text string +# should be the name of a C++ exception type that the generated server class +# will catch and the client class will rethrow. + +annotation name(field, method): Text; +# Name of the C++ method or field corresponding to a capnp method or field. + +annotation skip(field): Void; +# Synonym for count(0). + +interface ThreadMap $count(0) { + # Interface letting clients control which thread a method call should + # execute on. Clients create and name threads and pass the thread handle as + # a call parameter. + makeThread @0 (name :Text) -> (result :Thread); +} + +interface Thread { + # Thread handle returned by makeThread corresponding to one server thread. + + getName @0 () -> (result: Text); +} + +struct Context $count(0) { + # Execution context passed as a parameter from the client class to the server class. + + thread @0 : Thread; + # Handle of the server thread the current method call should execute on. + + callbackThread @1 : Thread; + # Handle of the client thread that is calling the current method, and that + # any callbacks made by the server thread should be made on. +} diff --git a/src/ipc/libmultiprocess/include/mp/proxy.h b/src/ipc/libmultiprocess/include/mp/proxy.h new file mode 100644 index 0000000000000..76be09921090f --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/proxy.h @@ -0,0 +1,299 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_H +#define MP_PROXY_H + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace mp { +class Connection; +class EventLoop; +//! Mapping from capnp interface type to proxy client implementation (specializations are generated by +//! proxy-codegen.cpp). +template +struct ProxyClient; +//! Mapping from capnp interface type to proxy server implementation (specializations are generated by +//! proxy-codegen.cpp). +template +struct ProxyServer; +//! Mapping from capnp method params type to method traits (specializations are generated by proxy-codegen.cpp). +template +struct ProxyMethod; +//! Mapping from capnp struct type to struct traits (specializations are generated by proxy-codegen.cpp). +template +struct ProxyStruct; +//! Mapping from local c++ type to capnp type and traits (specializations are generated by proxy-codegen.cpp). +template +struct ProxyType; + +using CleanupList = std::list>; +using CleanupIt = typename CleanupList::iterator; + +inline void CleanupRun(CleanupList& fns) { + while (!fns.empty()) { + auto fn = std::move(fns.front()); + fns.pop_front(); + fn(); + } +} + +//! Context data associated with proxy client and server classes. +struct ProxyContext +{ + Connection* connection; + CleanupList cleanup_fns; + + ProxyContext(Connection* connection) : connection(connection) {} +}; + +//! Base class for generated ProxyClient classes that implement a C++ interface +//! and forward calls to a capnp interface. +template +class ProxyClientBase : public Impl_ +{ +public: + using Interface = Interface_; + using Impl = Impl_; + using Sub = ProxyClient; + using Super = ProxyClientBase; + + ProxyClientBase(typename Interface::Client client, Connection* connection, bool destroy_connection); + ~ProxyClientBase() noexcept; + + // construct/destroy methods called during client construction/destruction + // that can optionally be defined in capnp interfaces to invoke code on the + // server when proxy client objects are created and destroyed. + // + // The construct() method is not generally very useful, but can be used to + // run custom code on the server automatically when a ProxyClient client is + // constructed. The only current use is adding a construct method to Init + // interfaces that is called automatically on construction, so client and + // server exchange ThreadMap references and set Connection::m_thread_map + // values as soon as the Init client is created. + // + // construct @0 (threadMap: Proxy.ThreadMap) -> (threadMap: Proxy.ThreadMap); + // + // But construct() is not necessary for this, thread maps could be passed + // through a normal method that is just called explicitly rather than + // implicitly. + // + // The destroy() method is more generally useful than construct(), because + // it ensures that the server object will be destroyed synchronously before + // the client destructor returns, instead of asynchronously at some + // unpredictable time after the client object is already destroyed and + // client code has moved on. If the destroy method accepts a Context + // parameter like: + // + // destroy @0 (context: Proxy.Context) -> (); + // + // then it will also ensure that the destructor runs on the same thread the + // client used to make other RPC calls, instead of running on the server + // EventLoop thread and possibly blocking it. + static void construct(Super&) {} + static void destroy(Super&) {} + + typename Interface::Client m_client; + ProxyContext m_context; +}; + +//! Customizable (through template specialization) base class used in generated ProxyClient implementations from +//! proxy-codegen.cpp. +template +class ProxyClientCustom : public ProxyClientBase +{ + using ProxyClientBase::ProxyClientBase; +}; + +//! Base class for generated ProxyServer classes that implement capnp server +//! methods and forward calls to a wrapped c++ implementation class. +template +struct ProxyServerBase : public virtual Interface_::Server +{ +public: + using Interface = Interface_; + using Impl = Impl_; + + ProxyServerBase(std::shared_ptr impl, Connection& connection); + virtual ~ProxyServerBase(); + void invokeDestroy(); + + /** + * Implementation pointer that may or may not be owned and deleted when this + * capnp server goes out of scope. It is owned for servers created to wrap + * unique_ptr method arguments, but unowned for servers created to + * wrap Impl& method arguments. + * + * In the case of Impl& arguments, custom code is required on other side of + * the connection to delete the capnp client & server objects since native + * code on that side of the connection will just be taking a plain reference + * rather than a pointer, so won't be able to do its own cleanup. Right now + * this is implemented with addCloseHook callbacks to delete clients at + * appropriate times depending on semantics of the particular method being + * wrapped. */ + std::shared_ptr m_impl; + ProxyContext m_context; +}; + +//! Customizable (through template specialization) base class which ProxyServer +//! classes produced by generated code will inherit from. The default +//! specialization of this class just inherits from ProxyServerBase, but custom +//! specializations can be defined to control ProxyServer behavior. +//! +//! Specifically, it can be useful to specialize this class to add additional +//! state to ProxyServer classes, for example to cache state between IPC calls. +//! If this is done, however, care should be taken to ensure that the extra +//! state can be destroyed without blocking, because ProxyServer destructors are +//! called from the EventLoop thread, and if they block, it could deadlock the +//! program. One way to do avoid blocking is to clean up the state by pushing +//! cleanup callbacks to the m_context.cleanup_fns list, which run after the server +//! m_impl object is destroyed on the same thread destroying it (which will +//! either be an IPC worker thread if the ProxyServer is being explicitly +//! destroyed by a client calling a destroy() method with a Context argument and +//! Context.thread value set, or the temporary EventLoop::m_async_thread used to +//! run destructors without blocking the event loop when no-longer used server +//! objects are garbage collected by Cap'n Proto.) Alternately, if cleanup needs +//! to run before m_impl is destroyed, the specialization can override +//! invokeDestroy and destructor methods to do that. +template +struct ProxyServerCustom : public ProxyServerBase +{ + using ProxyServerBase::ProxyServerBase; +}; + +//! Function traits class used to get method parameter and result types, used in +//! generated ProxyClient and ProxyServer classes produced by gen.cpp to get C++ +//! method type information. The generated code accesses these traits via +//! intermediate ProxyClientMethodTraits and ProxyServerMethodTraits classes, +//! which it is possible to specialize to change the way method arguments and +//! return values are handled. +//! +//! Fields of the trait class are: +//! +//! Params - TypeList of C++ ClassName::methodName parameter types +//! Result - Return type of ClassName::method +//! Param - helper to access individual parameters by index number. +//! Fields - helper alias that appends Result type to the Params typelist if +//! it not void. +template +struct FunctionTraits; + +//! Specialization of above extracting result and params types assuming the +//! template argument is a pointer-to-method type, +//! decltype(&ClassName::methodName) +template +struct FunctionTraits<_Result (_Class::*const)(_Params...)> +{ + using Params = TypeList<_Params...>; + using Result = _Result; + template + using Param = typename std::tuple_element>::type; + using Fields = + std::conditional_t, Params, TypeList<_Params..., _Result>>; +}; + +//! Traits class for a proxy method, providing the same +//! Params/Result/Param/Fields described in the FunctionTraits class above, plus +//! an additional invoke() method that calls the C++ method which is being +//! proxied, forwarding any arguments. +//! +//! The template argument should be the InterfaceName::MethodNameParams class +//! (generated by Cap'n Proto) associated with the method. +//! +//! Note: The class definition here is just the fallback definition used when +//! the other specialization below doesn't match. The fallback is only used for +//! capnp methods which do not have corresponding C++ methods, which in practice +//! is just the two special construct() and destroy() methods described in \ref +//! ProxyClientBase. These methods don't have any C++ parameters or return +//! types, so the trait information below reflects that. +template +struct ProxyMethodTraits +{ + using Params = TypeList<>; + using Result = void; + using Fields = Params; + + template + static void invoke(ServerContext&) + { + } +}; + +//! Specialization of above for proxy methods that have a +//! ProxyMethod::impl pointer-to-method +//! constant defined by generated code. This includes all functions defined in +//! the capnp interface except any construct() or destroy() methods, that are +//! assumed not to correspond to real member functions in the C++ class, and +//! will use the fallback traits definition above. The generated code this +//! specialization relies on looks like: +//! +//! struct ProxyMethod +//! { +//! static constexpr auto impl = &ClassName::methodName; +//! }; +template +struct ProxyMethodTraits::impl)>> + : public FunctionTraits::impl)> +{ + template + static decltype(auto) invoke(ServerContext& server_context, Args&&... args) + { + return (server_context.proxy_server.m_impl.get()->*ProxyMethod::impl)(std::forward(args)...); + } +}; + +//! Customizable (through template specialization) traits class used in generated ProxyClient implementations from +//! proxy-codegen.cpp. +template +struct ProxyClientMethodTraits : public ProxyMethodTraits +{ +}; + +//! Customizable (through template specialization) traits class used in generated ProxyServer implementations from +//! proxy-codegen.cpp. +template +struct ProxyServerMethodTraits : public ProxyMethodTraits +{ +}; + +static constexpr int FIELD_IN = 1; +static constexpr int FIELD_OUT = 2; +static constexpr int FIELD_OPTIONAL = 4; +static constexpr int FIELD_REQUESTED = 8; +static constexpr int FIELD_BOXED = 16; + +//! Accessor type holding flags that determine how to access a message field. +template +struct Accessor : public Field +{ + static const bool in = flags & FIELD_IN; + static const bool out = flags & FIELD_OUT; + static const bool optional = flags & FIELD_OPTIONAL; + static const bool requested = flags & FIELD_REQUESTED; + static const bool boxed = flags & FIELD_BOXED; +}; + +//! Wrapper around std::function for passing std::function objects between client and servers. +template +class ProxyCallback; + +//! Specialization of above to separate Result and Arg types. +template +class ProxyCallback> +{ +public: + virtual Result call(Args&&... args) = 0; +}; + +} // namespace mp + +#endif // MP_PROXY_H diff --git a/src/ipc/libmultiprocess/include/mp/type-char.h b/src/ipc/libmultiprocess/include/mp/type-char.h new file mode 100644 index 0000000000000..d1d27b6241442 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-char.h @@ -0,0 +1,36 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_CHAR_H +#define MP_PROXY_TYPE_CHAR_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList, + Priority<3>, + InvokeContext& invoke_context, + const unsigned char (&value)[size], + Output&& output) +{ + auto result = output.init(size); + memcpy(result.begin(), value, size); +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + memcpy(value, data.begin(), size); + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_CHAR_H diff --git a/src/ipc/libmultiprocess/include/mp/type-chrono.h b/src/ipc/libmultiprocess/include/mp/type-chrono.h new file mode 100644 index 0000000000000..a17d9a994bf6f --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-chrono.h @@ -0,0 +1,34 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_CHRONO_H +#define MP_PROXY_TYPE_CHRONO_H + +#include + +#include + +namespace mp { +//! Overload CustomBuildField and CustomReadField to serialize std::chrono +//! parameters and return values as numbers. +template +void CustomBuildField(TypeList>, Priority<1>, InvokeContext& invoke_context, Value&& value, + Output&& output) +{ + static_assert(std::numeric_limits::lowest() <= std::numeric_limits::lowest(), + "capnp type does not have enough range to hold lowest std::chrono::duration value"); + static_assert(std::numeric_limits::max() >= std::numeric_limits::max(), + "capnp type does not have enough range to hold highest std::chrono::duration value"); + output.set(value.count()); +} + +template +decltype(auto) CustomReadField(TypeList>, Priority<1>, InvokeContext& invoke_context, + Input&& input, ReadDest&& read_dest) +{ + return read_dest.construct(input.get()); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_CHRONO_H diff --git a/src/ipc/libmultiprocess/include/mp/type-context.h b/src/ipc/libmultiprocess/include/mp/type-context.h new file mode 100644 index 0000000000000..7c12afe2ff027 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-context.h @@ -0,0 +1,173 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_CONTEXT_H +#define MP_PROXY_TYPE_CONTEXT_H + +#include +#include + +namespace mp { +template +void CustomBuildField(TypeList<>, + Priority<1>, + ClientInvokeContext& invoke_context, + Output&& output, + typename std::enable_if::value>::type* enable = nullptr) +{ + auto& connection = invoke_context.connection; + auto& thread_context = invoke_context.thread_context; + + // Create local Thread::Server object corresponding to the current thread + // and pass a Thread::Client reference to it in the Context.callbackThread + // field so the function being called can make callbacks to this thread. + // Also store the Thread::Client reference in the callback_threads map so + // future calls over this connection can reuse it. + auto [callback_thread, _]{SetThread( + thread_context.callback_threads, thread_context.waiter->m_mutex, &connection, + [&] { return connection.m_threads.add(kj::heap>(thread_context, std::thread{})); })}; + + // Call remote ThreadMap.makeThread function so server will create a + // dedicated worker thread to run function calls from this thread. Store the + // Thread::Client reference it returns in the request_threads map. + auto make_request_thread{[&]{ + // This code will only run if an IPC client call is being made for the + // first time on this thread. After the first call, subsequent calls + // will use the existing request thread. This code will also never run at + // all if the current thread is a request thread created for a different + // IPC client, because in that case PassField code (below) will have set + // request_thread to point to the calling thread. + auto request = connection.m_thread_map.makeThreadRequest(); + request.setName(thread_context.thread_name); + return request.send().getResult(); // Nonblocking due to capnp request pipelining. + }}; + auto [request_thread, _1]{SetThread( + thread_context.request_threads, thread_context.waiter->m_mutex, + &connection, make_request_thread)}; + + auto context = output.init(); + context.setThread(request_thread->second.m_client); + context.setCallbackThread(callback_thread->second.m_client); +} + +//! PassField override for mp.Context arguments. Return asynchronously and call +//! function on other thread found in context. +template +auto PassField(Priority<1>, TypeList<>, ServerContext& server_context, const Fn& fn, Args&&... args) -> + typename std::enable_if< + std::is_same::value, + kj::Promise>::type +{ + const auto& params = server_context.call_context.getParams(); + Context::Reader context_arg = Accessor::get(params); + auto future = kj::newPromiseAndFulfiller(); + auto& server = server_context.proxy_server; + int req = server_context.req; + auto invoke = MakeAsyncCallable( + [fulfiller = kj::mv(future.fulfiller), + call_context = kj::mv(server_context.call_context), &server, req, fn, args...]() mutable { + const auto& params = call_context.getParams(); + Context::Reader context_arg = Accessor::get(params); + ServerContext server_context{server, call_context, req}; + bool disconnected{false}; + { + // Before invoking the function, store a reference to the + // callbackThread provided by the client in the + // thread_local.request_threads map. This way, if this + // server thread needs to execute any RPCs that call back to + // the client, they will happen on the same client thread + // that is waiting for this function, just like what would + // happen if this were a normal function call made on the + // local stack. + // + // If the request_threads map already has an entry for this + // connection, it will be left unchanged, and it indicates + // that the current thread is an RPC client thread which is + // in the middle of an RPC call, and the current RPC call is + // a nested call from the remote thread handling that RPC + // call. In this case, the callbackThread value should point + // to the same thread already in the map, so there is no + // need to update the map. + auto& thread_context = g_thread_context; + auto& request_threads = thread_context.request_threads; + auto [request_thread, inserted]{SetThread( + request_threads, thread_context.waiter->m_mutex, + server.m_context.connection, + [&] { return context_arg.getCallbackThread(); })}; + + // If an entry was inserted into the requests_threads map, + // remove it after calling fn.invoke. If an entry was not + // inserted, one already existed, meaning this must be a + // recursive call (IPC call calling back to the caller which + // makes another IPC call), so avoid modifying the map. + const bool erase_thread{inserted}; + KJ_DEFER({ + std::unique_lock lock(thread_context.waiter->m_mutex); + // Call erase here with a Connection* argument instead + // of an iterator argument, because the `request_thread` + // iterator may be invalid if the connection is closed + // during this function call. More specifically, the + // iterator may be invalid because SetThread adds a + // cleanup callback to the Connection destructor that + // erases the thread from the map, and also because the + // ProxyServer destructor calls + // request_threads.clear(). + if (erase_thread) { + disconnected = !request_threads.erase(server.m_context.connection); + } else { + disconnected = !request_threads.count(server.m_context.connection); + } + }); + fn.invoke(server_context, args...); + } + if (disconnected) { + // If disconnected is true, the Connection object was + // destroyed during the method call. Deal with this by + // returning without ever fulfilling the promise, which will + // cause the ProxyServer object to leak. This is not ideal, + // but fixing the leak will require nontrivial code changes + // because there is a lot of code assuming ProxyServer + // objects are destroyed before Connection objects. + return; + } + KJ_IF_MAYBE(exception, kj::runCatchingExceptions([&]() { + server.m_context.connection->m_loop.sync([&] { + auto fulfiller_dispose = kj::mv(fulfiller); + fulfiller_dispose->fulfill(kj::mv(call_context)); + }); + })) + { + server.m_context.connection->m_loop.sync([&]() { + auto fulfiller_dispose = kj::mv(fulfiller); + fulfiller_dispose->reject(kj::mv(*exception)); + }); + } + }); + + // Lookup Thread object specified by the client. The specified thread should + // be a local Thread::Server object, but it needs to be looked up + // asynchronously with getLocalServer(). + auto thread_client = context_arg.getThread(); + return server.m_context.connection->m_threads.getLocalServer(thread_client) + .then([&server, invoke, req](const kj::Maybe& perhaps) { + // Assuming the thread object is found, pass it a pointer to the + // `invoke` lambda above which will invoke the function on that + // thread. + KJ_IF_MAYBE (thread_server, perhaps) { + const auto& thread = static_cast&>(*thread_server); + server.m_context.connection->m_loop.log() + << "IPC server post request #" << req << " {" << thread.m_thread_context.thread_name << "}"; + thread.m_thread_context.waiter->post(std::move(invoke)); + } else { + server.m_context.connection->m_loop.log() + << "IPC server error request #" << req << ", missing thread to execute request"; + throw std::runtime_error("invalid thread handle"); + } + }) + // Wait for the invocation to finish before returning to the caller. + .then([invoke_wait = kj::mv(future.promise)]() mutable { return kj::mv(invoke_wait); }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_CONTEXT_H diff --git a/src/ipc/libmultiprocess/include/mp/type-data.h b/src/ipc/libmultiprocess/include/mp/type-data.h new file mode 100644 index 0000000000000..46a2b2fc7250b --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-data.h @@ -0,0 +1,46 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_DATA_H +#define MP_PROXY_TYPE_DATA_H + +#include + +namespace mp { +template +concept IsSpanOf = + std::convertible_to> && + std::constructible_from; + +template +concept IsByteSpan = + IsSpanOf || + IsSpanOf || + IsSpanOf || + IsSpanOf; + +//! Generic ::capnp::Data field builder for any C++ type that can be converted +//! to a span of bytes, like std::vector or std::array, or custom +//! blob types like uint256 or PKHash with data() and size() methods pointing to +//! bytes. +template +void CustomBuildField(TypeList, Priority<2>, InvokeContext& invoke_context, Value&& value, Output&& output) +requires (std::is_same_v && IsByteSpan) +{ + auto data = std::span{value}; + auto result = output.init(data.size()); + memcpy(result.begin(), data.data(), data.size()); +} + +template +decltype(auto) CustomReadField(TypeList, Priority<2>, InvokeContext& invoke_context, Input&& input, ReadDest&& read_dest) +requires (std::is_same_v && IsByteSpan) +{ + using ByteType = decltype(std::span{std::declval().begin(), std::declval().end()})::element_type; + const kj::byte *begin{input.get().begin()}, *end{input.get().end()}; + return read_dest.construct(reinterpret_cast(begin), reinterpret_cast(end)); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_DATA_H diff --git a/src/ipc/libmultiprocess/include/mp/type-decay.h b/src/ipc/libmultiprocess/include/mp/type-decay.h new file mode 100644 index 0000000000000..7b203c8628cd1 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-decay.h @@ -0,0 +1,38 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_DECAY_H +#define MP_PROXY_TYPE_DECAY_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList, + Priority<0>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + BuildField(TypeList(), invoke_context, output, std::forward(value)); +} + +template +void CustomBuildField(TypeList, Priority<0>, InvokeContext& invoke_context, Value&& value, Output&& output) +{ + BuildField(TypeList(), invoke_context, output, std::forward(value)); +} + +template +void CustomBuildField(TypeList, + Priority<0>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + BuildField(TypeList(), invoke_context, output, std::forward(value)); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_DECAY_H diff --git a/src/ipc/libmultiprocess/include/mp/type-exception.h b/src/ipc/libmultiprocess/include/mp/type-exception.h new file mode 100644 index 0000000000000..3e2fcac273721 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-exception.h @@ -0,0 +1,22 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_EXCEPTION_H +#define MP_PROXY_TYPE_EXCEPTION_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + const std::exception& value, + Output&& output) +{ + BuildField(TypeList(), invoke_context, output, std::string(value.what())); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_EXCEPTION_H diff --git a/src/ipc/libmultiprocess/include/mp/type-function.h b/src/ipc/libmultiprocess/include/mp/type-function.h new file mode 100644 index 0000000000000..bf00c5811979b --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-function.h @@ -0,0 +1,67 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_FUNCTION_H +#define MP_PROXY_TYPE_FUNCTION_H + +#include + +namespace mp { +//! Adapter to convert ProxyCallback object call to function object call. +template +class ProxyCallbackImpl final : public ProxyCallback> +{ + using Fn = std::function; + Fn m_fn; + +public: + ProxyCallbackImpl(Fn fn) : m_fn(std::move(fn)) {} + Result call(Args&&... args) override { return m_fn(std::forward(args)...); } +}; + +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value& value, + Output&& output) +{ + if (value) { + using Interface = typename decltype(output.get())::Calls; + using Callback = ProxyCallbackImpl; + output.set(kj::heap>( + std::make_shared(std::forward(value)), invoke_context.connection)); + } +} + +// ProxyCallFn class is needed because c++11 doesn't support auto lambda parameters. +// It's equivalent c++14: [invoke_context](auto&& params) { +// invoke_context->call(std::forward(params)...) +template +struct ProxyCallFn +{ + InvokeContext m_proxy; + + template + decltype(auto) operator()(CallParams&&... params) { return this->m_proxy->call(std::forward(params)...); } +}; + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + if (input.has()) { + using Interface = typename Decay::Calls; + auto client = std::make_shared>( + input.get(), &invoke_context.connection, /* destroy_connection= */ false); + return read_dest.construct(ProxyCallFn{std::move(client)}); + } + return read_dest.construct(); +}; +} // namespace mp + +#endif // MP_PROXY_TYPE_FUNCTION_H diff --git a/src/ipc/libmultiprocess/include/mp/type-interface.h b/src/ipc/libmultiprocess/include/mp/type-interface.h new file mode 100644 index 0000000000000..99adf2ab94eb9 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-interface.h @@ -0,0 +1,112 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_INTERFACE_H +#define MP_PROXY_TYPE_INTERFACE_H + +#include + +namespace mp { +template +kj::Own MakeProxyServer(InvokeContext& context, std::shared_ptr impl) +{ + return kj::heap>(std::move(impl), context.connection); +} + +template +kj::Own CustomMakeProxyServer(InvokeContext& context, std::shared_ptr&& impl) +{ + return MakeProxyServer(context, std::move(impl)); +} + +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output, + typename Decay::Calls* enable = nullptr) +{ + if (value) { + using Interface = typename decltype(output.get())::Calls; + output.set(CustomMakeProxyServer(invoke_context, std::shared_ptr(value.release()))); + } +} + +template +void CustomBuildField(TypeList>, + Priority<2>, + InvokeContext& invoke_context, + Value&& value, + Output&& output, + typename Decay::Calls* enable = nullptr) +{ + if (value) { + using Interface = typename decltype(output.get())::Calls; + output.set(CustomMakeProxyServer(invoke_context, std::move(value))); + } +} + +template +void CustomBuildField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Impl& value, + Output&& output, + typename decltype(output.get())::Calls* enable = nullptr) +{ + // Disable deleter so proxy server object doesn't attempt to delete the + // wrapped implementation when the proxy client is destroyed or + // disconnected. + using Interface = typename decltype(output.get())::Calls; + output.set(CustomMakeProxyServer(invoke_context, std::shared_ptr(&value, [](Impl*){}))); +} + +template +std::unique_ptr MakeProxyClient(InvokeContext& context, typename Interface::Client&& client) +{ + return std::make_unique>( + std::move(client), &context.connection, /* destroy_connection= */ false); +} + +template +std::unique_ptr CustomMakeProxyClient(InvokeContext& context, typename Interface::Client&& client) +{ + return MakeProxyClient(context, kj::mv(client)); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename Decay::Calls* enable = nullptr) +{ + using Interface = typename Decay::Calls; + if (input.has()) { + return read_dest.construct( + CustomMakeProxyClient(invoke_context, std::move(input.get()))); + } + return read_dest.construct(); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename Decay::Calls* enable = nullptr) +{ + using Interface = typename Decay::Calls; + if (input.has()) { + return read_dest.construct( + CustomMakeProxyClient(invoke_context, std::move(input.get()))); + } + return read_dest.construct(); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_INTERFACE_H diff --git a/src/ipc/libmultiprocess/include/mp/type-map.h b/src/ipc/libmultiprocess/include/mp/type-map.h new file mode 100644 index 0000000000000..bc1b22769df1f --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-map.h @@ -0,0 +1,52 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_MAP_H +#define MP_PROXY_TYPE_MAP_H + +#include +#include +#include + +namespace mp { +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + // FIXME dededup with vector handler above + auto list = output.init(value.size()); + size_t i = 0; + for (const auto& elem : value) { + BuildField(TypeList>(), invoke_context, + ListOutput(list, i), elem); + ++i; + } +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.clear(); + for (auto item : data) { + ReadField(TypeList>(), invoke_context, + Make(item), + ReadDestEmplace( + TypeList>(), [&](auto&&... args) -> auto& { + return *value.emplace(std::forward(args)...).first; + })); + } + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_MAP_H diff --git a/src/ipc/libmultiprocess/include/mp/type-message.h b/src/ipc/libmultiprocess/include/mp/type-message.h new file mode 100644 index 0000000000000..d80f43c8d3710 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-message.h @@ -0,0 +1,64 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_MESSAGE_H +#define MP_PROXY_TYPE_MESSAGE_H + +#include + +namespace mp { +//! Overload CustomBuildField to serialize objects that have CustomBuildMessage +//! overloads. Defining a CustomBuildMessage overload is simpler than defining a +//! CustomBuildField overload because it only requires defining a normal +//! function, not a template function, but less flexible. +template +void CustomBuildField(TypeList, Priority<2>, InvokeContext& invoke_context, Value&& value, Output&& output, + decltype(CustomBuildMessage(invoke_context, value, std::move(output.get())))* enable = nullptr) +{ + CustomBuildMessage(invoke_context, value, std::move(output.init())); +} + +//! Overload CustomReadField to serialize objects that have CustomReadMessage +//! overloads. Defining a CustomReadMessage overload is simpler than defining a +//! CustomReadField overload because it only requires defining a normal +//! function, not a template function, but less flexible. +template +decltype(auto) CustomReadField(TypeList, Priority<2>, InvokeContext& invoke_context, Reader&& reader, + ReadDest&& read_dest, + decltype(CustomReadMessage(invoke_context, reader.get(), + std::declval()))* enable = nullptr) +{ + return read_dest.update([&](auto& value) { if (reader.has()) CustomReadMessage(invoke_context, reader.get(), value); }); +} + +//! Helper for CustomPassField below. Call Accessor::init method if it has one, +//! otherwise do nothing. +template +decltype(auto) MaybeInit(Message&& message, decltype(Accessor::get(message))* enable = nullptr) +{ + return Accessor::init(message); +} + +template +::capnp::Void MaybeInit(...) +{ + return {}; +} + +//! Overload CustomPassField to serialize objects that have CustomPassMessage +//! overloads. Defining a CustomPassMessage overload is simpler than defining a +//! CustomPassField overload because it only requires defining a normal +//! function, not a template function, but less flexible. +template +auto CustomPassField(TypeList, ServerContext& server_context, Fn&& fn, Args&&... args) + -> decltype(CustomPassMessage(server_context, MaybeGet(server_context.call_context.getParams()), + MaybeGet(server_context.call_context.getResults()), nullptr)) +{ + CustomPassMessage(server_context, MaybeGet(server_context.call_context.getParams()), + MaybeInit(server_context.call_context.getResults()), + [&](LocalTypes... param) { fn.invoke(server_context, std::forward(args)..., param...); }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_MESSAGE_H diff --git a/src/ipc/libmultiprocess/include/mp/type-number.h b/src/ipc/libmultiprocess/include/mp/type-number.h new file mode 100644 index 0000000000000..9d269be60a8fd --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-number.h @@ -0,0 +1,87 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_NUMBER_H +#define MP_PROXY_TYPE_NUMBER_H + +#include + +namespace mp { +template +LocalType BuildPrimitive(InvokeContext& invoke_context, + const Value& value, + TypeList, + typename std::enable_if::value>::type* enable = nullptr) +{ + using E = std::make_unsigned_t>; + using T = std::make_unsigned_t; + static_assert(std::numeric_limits::max() >= std::numeric_limits::max(), "mismatched integral/enum types"); + return static_cast(value); +} + +template +LocalType BuildPrimitive(InvokeContext& invoke_context, + const Value& value, + TypeList, + typename std::enable_if::value, int>::type* enable = nullptr) +{ + static_assert( + std::numeric_limits::lowest() <= std::numeric_limits::lowest(), "mismatched integral types"); + static_assert( + std::numeric_limits::max() >= std::numeric_limits::max(), "mismatched integral types"); + return value; +} + +template +LocalType BuildPrimitive(InvokeContext& invoke_context, + const Value& value, + TypeList, + typename std::enable_if::value>::type* enable = nullptr) +{ + static_assert(std::is_same::value, + "mismatched floating point types. please fix message.capnp type declaration to match wrapped interface"); + return value; +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename std::enable_if::value>::type* enable = 0) +{ + return read_dest.construct(static_cast(input.get())); +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename std::enable_if::value>::type* enable = nullptr) +{ + auto value = input.get(); + if (value < std::numeric_limits::min() || value > std::numeric_limits::max()) { + throw std::range_error("out of bound int received"); + } + return read_dest.construct(static_cast(value)); +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename std::enable_if::value>::type* enable = 0) +{ + auto value = input.get(); + static_assert(std::is_same::value, "floating point type mismatch"); + return read_dest.construct(value); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_NUMBER_H diff --git a/src/ipc/libmultiprocess/include/mp/type-optional.h b/src/ipc/libmultiprocess/include/mp/type-optional.h new file mode 100644 index 0000000000000..822508d5533f6 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-optional.h @@ -0,0 +1,48 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_OPTIONAL_H +#define MP_PROXY_TYPE_OPTIONAL_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + if (value) { + output.setHas(); + // FIXME: should std::move value if destvalue is rref? + BuildField(TypeList(), invoke_context, output, *value); + } +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + if (!input.has()) { + value.reset(); + } else if (value) { + ReadField(TypeList(), invoke_context, input, ReadDestUpdate(*value)); + } else { + ReadField(TypeList(), invoke_context, input, + ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + value.emplace(std::forward(args)...); + return *value; + })); + } + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_OPTIONAL_H diff --git a/src/ipc/libmultiprocess/include/mp/type-pair.h b/src/ipc/libmultiprocess/include/mp/type-pair.h new file mode 100644 index 0000000000000..3af9c9313de97 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-pair.h @@ -0,0 +1,50 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_PAIR_H +#define MP_PROXY_TYPE_PAIR_H + +#include + +namespace mp { +// FIXME: Overload on output type instead of value type and switch to std::get and merge with next overload +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + auto pair = output.init(); + using Accessors = typename ProxyStruct::Accessors; + BuildField(TypeList(), invoke_context, Make>(pair), value.first); + BuildField(TypeList(), invoke_context, Make>(pair), value.second); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + const auto& pair = input.get(); + using Accessors = typename ProxyStruct::Reads>::Accessors; + + ReadField(TypeList(), invoke_context, Make>(pair), + ReadDestEmplace(TypeList(), [&](auto&&... key_args) -> auto& { + KeyLocalType* key = nullptr; + ReadField(TypeList(), invoke_context, Make>(pair), + ReadDestEmplace(TypeList(), [&](auto&&... value_args) -> auto& { + auto& ret = read_dest.construct(std::piecewise_construct, std::forward_as_tuple(key_args...), + std::forward_as_tuple(value_args...)); + key = &ret.first; + return ret.second; + })); + return *key; + })); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_PAIR_H diff --git a/src/ipc/libmultiprocess/include/mp/type-pointer.h b/src/ipc/libmultiprocess/include/mp/type-pointer.h new file mode 100644 index 0000000000000..5c79e8d2e8a75 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-pointer.h @@ -0,0 +1,113 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_POINTER_H +#define MP_PROXY_TYPE_POINTER_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList, Priority<3>, InvokeContext& invoke_context, Value&& value, Output&& output) +{ + if (value) { + BuildField(TypeList(), invoke_context, output, *value); + } +} + +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + if (value) { + BuildField(TypeList(), invoke_context, output, *value); + } +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + if (value) { + ReadField(TypeList(), invoke_context, std::forward(input), ReadDestUpdate(*value)); + } + }); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<0>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + if (!input.has()) { + value.reset(); + } else if (value) { + ReadField(TypeList(), invoke_context, input, ReadDestUpdate(*value)); + } else { + ReadField(TypeList(), invoke_context, input, + ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + value = std::make_shared(std::forward(args)...); + return *value; + })); + } + }); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + if (!input.has()) { + value.reset(); + return; + } + ReadField(TypeList(), invoke_context, std::forward(input), + ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + value = std::make_shared(std::forward(args)...); + return *value; + })); + }); +} + +//! PassField override for C++ pointer arguments. +template +void PassField(Priority<1>, TypeList, ServerContext& server_context, const Fn& fn, Args&&... args) +{ + const auto& params = server_context.call_context.getParams(); + const auto& input = Make(params); + + if (!input.want()) { + fn.invoke(server_context, std::forward(args)..., nullptr); + return; + } + + InvokeContext& invoke_context = server_context; + Decay param; + + MaybeReadField(std::integral_constant(), TypeList(), invoke_context, input, + ReadDestUpdate(param)); + + fn.invoke(server_context, std::forward(args)..., ¶m); + + auto&& results = server_context.call_context.getResults(); + MaybeBuildField(std::integral_constant(), TypeList(), invoke_context, + Make(results), param); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_POINTER_H diff --git a/src/ipc/libmultiprocess/include/mp/type-set.h b/src/ipc/libmultiprocess/include/mp/type-set.h new file mode 100644 index 0000000000000..ea60dc4d1a02d --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-set.h @@ -0,0 +1,48 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_SET_H +#define MP_PROXY_TYPE_SET_H + +#include +#include + +namespace mp { +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + // FIXME dededup with vector handler above + auto list = output.init(value.size()); + size_t i = 0; + for (const auto& elem : value) { + BuildField(TypeList(), invoke_context, ListOutput(list, i), elem); + ++i; + } +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.clear(); + for (auto item : data) { + ReadField(TypeList(), invoke_context, Make(item), + ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + return *value.emplace(std::forward(args)...).first; + })); + } + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_SET_H diff --git a/src/ipc/libmultiprocess/include/mp/type-string.h b/src/ipc/libmultiprocess/include/mp/type-string.h new file mode 100644 index 0000000000000..77d04acb2804e --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-string.h @@ -0,0 +1,34 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_STRING_H +#define MP_PROXY_TYPE_STRING_H + +#include + +namespace mp { +template +void CustomBuildField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + auto result = output.init(value.size()); + memcpy(result.begin(), value.data(), value.size()); +} + +template +decltype(auto) CustomReadField(TypeList, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + auto data = input.get(); + return read_dest.construct(CharCast(data.begin()), data.size()); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_STRING_H diff --git a/src/ipc/libmultiprocess/include/mp/type-struct.h b/src/ipc/libmultiprocess/include/mp/type-struct.h new file mode 100644 index 0000000000000..d282e20e9a4a6 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-struct.h @@ -0,0 +1,85 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_STRUCT_H +#define MP_PROXY_TYPE_STRUCT_H + +#include + +namespace mp { +template +void BuildOne(TypeList param, + InvokeContext& invoke_context, + Output&& output, + Value&& value, + typename std::enable_if < index::fields>::type * enable = nullptr) +{ + using Index = std::integral_constant; + using Struct = typename ProxyType::Struct; + using Accessor = typename std::tuple_element::Accessors>::type; + auto&& field_output = Make(output); + auto&& field_value = value.*ProxyType::get(Index()); + BuildField(TypeList>(), invoke_context, field_output, field_value); + BuildOne(param, invoke_context, output, value); +} + +template +void BuildOne(TypeList param, + InvokeContext& invoke_context, + Output&& output, + Value&& value, + typename std::enable_if::fields>::type* enable = nullptr) +{ +} + +template +void CustomBuildField(TypeList local_type, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output, + typename ProxyType::Struct* enable = nullptr) +{ + BuildOne<0>(local_type, invoke_context, output.init(), value); +} + +template +void ReadOne(TypeList param, + InvokeContext& invoke_context, + Input&& input, + Value&& value, + typename std::enable_if::fields>::type* enable = nullptr) +{ + using Index = std::integral_constant; + using Struct = typename ProxyType::Struct; + using Accessor = typename std::tuple_element::Accessors>::type; + const auto& struc = input.get(); + auto&& field_value = value.*ProxyType::get(Index()); + ReadField(TypeList>(), invoke_context, Make(struc), + ReadDestUpdate(field_value)); + ReadOne(param, invoke_context, input, value); +} + +template +void ReadOne(TypeList param, + InvokeContext& invoke_context, + Input& input, + Value& value, + typename std::enable_if::fields>::type* enable = nullptr) +{ +} + +template +decltype(auto) CustomReadField(TypeList param, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest, + typename ProxyType::Struct* enable = nullptr) +{ + return read_dest.update([&](auto& value) { ReadOne<0>(param, invoke_context, input, value); }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_STRUCT_H diff --git a/src/ipc/libmultiprocess/include/mp/type-threadmap.h b/src/ipc/libmultiprocess/include/mp/type-threadmap.h new file mode 100644 index 0000000000000..683586fbc6e83 --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-threadmap.h @@ -0,0 +1,41 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_THREADMAP_H +#define MP_PROXY_TYPE_THREADMAP_H + +#include + +namespace mp { +template <> +struct ProxyServer final : public virtual ThreadMap::Server +{ +public: + ProxyServer(Connection& connection); + kj::Promise makeThread(MakeThreadContext context) override; + Connection& m_connection; +}; + +template +void CustomBuildField(TypeList<>, + Priority<1>, + InvokeContext& invoke_context, + Output&& output, + typename std::enable_if::value>::type* enable = nullptr) +{ + output.set(kj::heap>(invoke_context.connection)); +} + +template +decltype(auto) CustomReadField(TypeList<>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + typename std::enable_if::value>::type* enable = nullptr) +{ + invoke_context.connection.m_thread_map = input.get(); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_THREADMAP_H diff --git a/src/ipc/libmultiprocess/include/mp/type-tuple.h b/src/ipc/libmultiprocess/include/mp/type-tuple.h new file mode 100644 index 0000000000000..5083887258f0c --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-tuple.h @@ -0,0 +1,45 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_TUPLE_H +#define MP_PROXY_TYPE_TUPLE_H + +#include + +namespace mp { +// TODO: Should generalize this to work with arbitrary length tuples, not just length 2-tuples. +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + auto pair = output.init(); + using Accessors = typename ProxyStruct::Accessors; + BuildField(TypeList(), invoke_context, Make>(pair), std::get<0>(value)); + BuildField(TypeList(), invoke_context, Make>(pair), std::get<1>(value)); +} + +// TODO: Should generalize this to work with arbitrary length tuples, not just length 2-tuples. +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + const auto& pair = input.get(); + using Struct = ProxyStruct::Reads>; + using Accessors = typename Struct::Accessors; + ReadField(TypeList(), invoke_context, Make>(pair), + ReadDestUpdate(std::get<0>(value))); + ReadField(TypeList(), invoke_context, Make>(pair), + ReadDestUpdate(std::get<1>(value))); + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_TUPLE_H diff --git a/src/ipc/libmultiprocess/include/mp/type-vector.h b/src/ipc/libmultiprocess/include/mp/type-vector.h new file mode 100644 index 0000000000000..e4996e930432d --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-vector.h @@ -0,0 +1,71 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_VECTOR_H +#define MP_PROXY_TYPE_VECTOR_H + +#include +#include + +namespace mp { +template +void CustomBuildField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Value&& value, + Output&& output) +{ + // FIXME dedup with set handler below + auto list = output.init(value.size()); + size_t i = 0; + for (auto it = value.begin(); it != value.end(); ++it, ++i) { + BuildField(TypeList(), invoke_context, ListOutput(list, i), *it); + } +} + +inline static bool BuildPrimitive(InvokeContext& invoke_context, std::vector::const_reference value, TypeList) +{ + return value; +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.clear(); + value.reserve(data.size()); + for (auto item : data) { + ReadField(TypeList(), invoke_context, Make(item), + ReadDestEmplace(TypeList(), [&](auto&&... args) -> auto& { + value.emplace_back(std::forward(args)...); + return value.back(); + })); + } + }); +} + +template +decltype(auto) CustomReadField(TypeList>, + Priority<1>, + InvokeContext& invoke_context, + Input&& input, + ReadDest&& read_dest) +{ + return read_dest.update([&](auto& value) { + auto data = input.get(); + value.clear(); + value.reserve(data.size()); + for (auto item : data) { + value.push_back(ReadField(TypeList(), invoke_context, Make(item), ReadDestTemp())); + } + }); +} +} // namespace mp + +#endif // MP_PROXY_TYPE_VECTOR_H diff --git a/src/ipc/libmultiprocess/include/mp/type-void.h b/src/ipc/libmultiprocess/include/mp/type-void.h new file mode 100644 index 0000000000000..0a8876805291b --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/type-void.h @@ -0,0 +1,23 @@ +// Copyright (c) 2025 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_PROXY_TYPE_VOID_H +#define MP_PROXY_TYPE_VOID_H + +#include + +namespace mp { +template +::capnp::Void BuildPrimitive(InvokeContext& invoke_context, Value&&, TypeList<::capnp::Void>) +{ + return {}; +} + +template +void CustomBuildField(TypeList, Priority<1>, InvokeContext& invoke_context, ::capnp::Void, Output&& output) +{ +} +} // namespace mp + +#endif // MP_PROXY_TYPE_VOID_H diff --git a/src/ipc/libmultiprocess/include/mp/util.h b/src/ipc/libmultiprocess/include/mp/util.h new file mode 100644 index 0000000000000..ebfc3b5e7206b --- /dev/null +++ b/src/ipc/libmultiprocess/include/mp/util.h @@ -0,0 +1,220 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef MP_UTIL_H +#define MP_UTIL_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mp { + +//! Generic utility functions used by capnp code. + +//! Type holding a list of types. +//! +//! Example: +//! TypeList +template +struct TypeList +{ + static constexpr size_t size = sizeof...(Types); +}; + +//! Construct a template class value by deducing template arguments from the +//! types of constructor arguments, so they don't need to be specified manually. +//! +//! Uses of this can go away with class template deduction in C++17 +//! (https://en.cppreference.com/w/cpp/language/class_template_argument_deduction) +//! +//! Example: +//! Make(5, true) // Constructs std::pair(5, true); +template