Remove rocksdb usage from fift (ton-blockchain#1064)

* add github action for macOS 14 (arm64, M1) * add github action (portable) for macOS 14 (arm64, M1) * rename macOS arm64 output artifact * Update libsodium on windows * Compile libsodium * Update build-windows.bat * use upgraded libsodium 1.0.20; use compiled static libsodium for Windows instead of precompiled; * revert libsodium 1.0.20; use compiled static libsodium for Windows instead of precompiled; * use upgraded libsodium 1.0.20; use compiled static libsodium for Windows instead of precompiled; * fix libsodium version 1.0.19; use compiled static libsodium for Windows instead of precompiled; * try 1.0.20 libsodium precompiled on github * try 1.0.18 libsodium precompiled on github * try windows build on win server 2019 * and use PlatformToolset=v142 * use cmake -G "Visual Studio 16 2019" * fix path to msvc 2019 on github * separate github windows build on win server 2019 and build on win server 2022 * Update assembly/native/build-windows-2019.bat add retry mechanism Co-authored-by: Dr. Awesome Doge <[email protected]> * add test-emulator; disable test groovy pipeline * trigger all gh actions * fix win build * call test-emulator * remove usage of rocksdb in fift-lib * put back some code for test-db * fix test-emulator * remove usage of db-path parameter in fift * some func adjustments * fix checkout of openssl in fift-func-wasm-build-ubuntu.sh * typo * improve wasm build script for quicker turn around * remove sENVIRONMENT=web,worker for funcfiftlib. will be added later. * remove sENVIRONMENT=web,worker for funcfiftlib. will be added later. * remove sENVIRONMENT=web,worker for funcfiftlib. will be added later. * minor adjustments * remove -d option in fift; optimize fift-lib usage * reduce tondb usage --------- Co-authored-by: neodiX <[email protected]> Co-authored-by: Dr. Awesome Doge <[email protected]> Co-authored-by: ms <[email protected]>
zjg555543 · Jul 26, 2024 · 679e6be · 679e6be
1 parent 28f9a9b
commit 679e6be
Show file tree

Hide file tree

Showing 12 changed files with 123 additions and 329 deletions.
diff --git a/.github/workflows/build-ton-wasm-emscripten.yml b/.github/workflows/build-ton-wasm-emscripten.yml
@@ -19,7 +19,7 @@ jobs:
 
     - name: Build TON WASM artifacts
       run: |
-        cd assembly/wasm
+        cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
         chmod +x fift-func-wasm-build-ubuntu.sh
         ./fift-func-wasm-build-ubuntu.sh -a
 

diff --git a/assembly/cicd/jenkins/test-builds.groovy b/assembly/cicd/jenkins/test-builds.groovy
@@ -219,7 +219,7 @@ pipeline {
                     steps {
                         timeout(time: 180, unit: 'MINUTES') {
                             sh '''
-                                cd assembly/wasm
+                                cp assembly/wasm/fift-func-wasm-build-ubuntu.sh .
                                 chmod +x fift-func-wasm-build-ubuntu.sh
                                 ./fift-func-wasm-build-ubuntu.sh -a
                             '''

diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh
@@ -1,5 +1,3 @@
-# The script builds funcfift compiler to WASM
-
 # Execute these prerequisites first
 # sudo apt update
 # sudo apt install -y build-essential git make cmake ninja-build clang libgflags-dev zlib1g-dev libssl-dev \
@@ -11,10 +9,12 @@
 # sudo ./llvm.sh 16 all
 
 with_artifacts=false
+scratch_new=false
 
-while getopts 'a' flag; do
+while getopts 'af' flag; do
   case "${flag}" in
     a) with_artifacts=true ;;
+    f) scratch_new=true ;;
     *) break
        ;;
   esac
@@ -24,131 +24,155 @@ export CC=$(which clang-16)
 export CXX=$(which clang++-16)
 export CCACHE_DISABLE=1
 
-cd ../..
-rm -rf openssl zlib emsdk secp256k1 libsodium build
 echo `pwd`
+if [ "$scratch_new" = true ]; then
+  echo Compiling openssl zlib lz4 emsdk secp256k1 libsodium emsdk ton
+  rm -rf openssl zlib lz4 emsdk secp256k1 libsodium build
+fi
 
-git clone https://github.com/openssl/openssl.git
-cd openssl
-git checkout checkout openssl-3.1.4
-./config
-make -j16
-OPENSSL_DIR=`pwd`
-cd ..
-
-git clone https://github.com/madler/zlib.git
-cd zlib
-git checkout v1.3.1
-ZLIB_DIR=`pwd`
-cd ..
-
-git clone https://github.com/lz4/lz4.git
-cd lz4
-git checkout v1.9.4
-LZ4_DIR=`pwd`
-cd ..
-
-git clone https://github.com/bitcoin-core/secp256k1.git
-cd secp256k1
-git checkout v0.3.2
-./autogen.sh
-SECP256K1_DIR=`pwd`
-cd ..
-
-git clone https://github.com/jedisct1/libsodium
-cd libsodium
-git checkout 1.0.18-RELEASE
-SODIUM_DIR=`pwd`
-cd ..
-
-mkdir build
-cd build
-cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
--DCMAKE_CXX_STANDARD=17 \
--DOPENSSL_FOUND=1 \
--DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
--DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
--DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \
--DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.so \
--DTON_USE_ABSEIL=OFF ..
-
-test $? -eq 0 || { echo "Can't configure TON build"; exit 1; }
-
-ninja fift smc-envelope
 
-test $? -eq 0 || { echo "Can't compile fift "; exit 1; }
+if [ ! -d "openssl" ]; then
+  git clone https://github.com/openssl/openssl.git
+  cd openssl
+  git checkout openssl-3.1.4
+  ./config
+  make -j16
+  OPENSSL_DIR=`pwd`
+  cd ..
+else
+  OPENSSL_DIR=`pwd`/openssl
+  echo Using compiled openssl at $OPENSSL_DIR
+fi
 
-rm -rf *
+if [ ! -d "build" ]; then
+  mkdir build
+  cd build
+  cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CXX_STANDARD=17 \
+  -DOPENSSL_FOUND=1 \
+  -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
+  -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.so \
+  -DTON_USE_ABSEIL=OFF ..
+
+  test $? -eq 0 || { echo "Can't configure TON build"; exit 1; }
+  ninja fift smc-envelope
+  test $? -eq 0 || { echo "Can't compile fift "; exit 1; }
+  rm -rf *
+  cd ..
+else
+  echo cleaning build...
+  rm -rf build/*
+fi
 
-cd ..
+if [ ! -d "emsdk" ]; then
+  git clone https://github.com/emscripten-core/emsdk.git
+echo
+  echo Using cloned emsdk
+fi
 
-git clone https://github.com/emscripten-core/emsdk.git
 cd emsdk
 ./emsdk install 3.1.19
 ./emsdk activate 3.1.19
 EMSDK_DIR=`pwd`
-ls $EMSDK_DIR
 
 . $EMSDK_DIR/emsdk_env.sh
 export CC=$(which emcc)
 export CXX=$(which em++)
 export CCACHE_DISABLE=1
 
-cd ../openssl
-
-make clean
-emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
-sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
-sed -i 's/-ldl//g' Makefile
-sed -i 's/-O3/-Os/g' Makefile
-emmake make depend
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; }
-
-cd ../zlib
-
-emconfigure ./configure --static
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
+cd ..
 
-cd ../lz4
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }
+if [ ! -f "openssl/openssl_em" ]; then
+  cd openssl
+  make clean
+  emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test
+  sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile
+  sed -i 's/-ldl//g' Makefile
+  sed -i 's/-O3/-Os/g' Makefile
+  emmake make depend
+  emmake make -j16
+  test $? -eq 0 || { echo "Can't compile OpenSSL with emmake "; exit 1; }
+  touch openssl_em
+  cd ..
+else
+  echo Using compiled openssl with emscripten
+fi
 
-cd ../secp256k1
+if [ ! -d "zlib" ]; then
+  git clone https://github.com/madler/zlib.git
+  cd zlib
+  git checkout v1.3.1
+  ZLIB_DIR=`pwd`
+  emconfigure ./configure --static
+  emmake make -j16
+  test $? -eq 0 || { echo "Can't compile zlib with emmake "; exit 1; }
+  cd ..
+else
+  ZLIB_DIR=`pwd`/zlib
+  echo Using compiled zlib with emscripten at $ZLIB_DIR
+fi
 
-emconfigure ./configure --enable-module-recovery
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; }
+if [ ! -d "lz4" ]; then
+  git clone https://github.com/lz4/lz4.git
+  cd lz4
+  git checkout v1.9.4
+  LZ4_DIR=`pwd`
+  emmake make -j16
+  test $? -eq 0 || { echo "Can't compile lz4 with emmake "; exit 1; }
+  cd ..
+else
+  LZ4_DIR=`pwd`/lz4
+  echo Using compiled lz4 with emscripten at $LZ4_DIR
+fi
 
-cd ../libsodium
+if [ ! -d "secp256k1" ]; then
+  git clone https://github.com/bitcoin-core/secp256k1.git
+  cd secp256k1
+  git checkout v0.3.2
+  ./autogen.sh
+  SECP256K1_DIR=`pwd`
+  emconfigure ./configure --enable-module-recovery
+  emmake make -j16
+  test $? -eq 0 || { echo "Can't compile secp256k1 with emmake "; exit 1; }
+  cd ..
+else
+  SECP256K1_DIR=`pwd`/secp256k1
+  echo Using compiled secp256k1 with emscripten at $SECP256K1_DIR
+fi
 
-emconfigure ./configure --disable-ssp
-emmake make -j16
-test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; }
+if [ ! -d "libsodium" ]; then
+  git clone https://github.com/jedisct1/libsodium
+  cd libsodium
+  git checkout 1.0.18-RELEASE
+  SODIUM_DIR=`pwd`
+  emconfigure ./configure --disable-ssp
+  emmake make -j16
+  test $? -eq 0 || { echo "Can't compile libsodium with emmake "; exit 1; }
+  cd ..
+else
+  SODIUM_DIR=`pwd`/libsodium
+  echo Using compiled libsodium with emscripten at $SODIUM_DIR
+fi
 
-cd ../build
+cd build
 
-emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release \
+emcmake cmake -DUSE_EMSCRIPTEN=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
 -DZLIB_FOUND=1 \
 -DZLIB_LIBRARIES=$ZLIB_DIR/libz.a \
 -DZLIB_INCLUDE_DIR=$ZLIB_DIR \
 -DLZ4_FOUND=1 \
 -DLZ4_LIBRARIES=$LZ4_DIR/lib/liblz4.a \
 -DLZ4_INCLUDE_DIRS=$LZ4_DIR/lib \
 -DOPENSSL_FOUND=1 \
--DOPENSSL_ROOT_DIR=$OPENSSL_DIR \
 -DOPENSSL_INCLUDE_DIR=$OPENSSL_DIR/include \
 -DOPENSSL_CRYPTO_LIBRARY=$OPENSSL_DIR/libcrypto.a \
--DOPENSSL_SSL_LIBRARY=$OPENSSL_DIR/libssl.a \
 -DCMAKE_TOOLCHAIN_FILE=$EMSDK_DIR/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake \
 -DCMAKE_CXX_FLAGS="-sUSE_ZLIB=1" \
--DSECP256K1_FOUND=1 \
 -DSECP256K1_INCLUDE_DIR=$SECP256K1_DIR/include \
 -DSECP256K1_LIBRARY=$SECP256K1_DIR/.libs/libsecp256k1.a \
 -DSODIUM_INCLUDE_DIR=$SODIUM_DIR/src/libsodium/include \
 -DSODIUM_LIBRARY_RELEASE=$SODIUM_DIR/src/libsodium/.libs/libsodium.a \
--DSODIUM_USE_STATIC_LIBS=ON ..
+..
 
 test $? -eq 0 || { echo "Can't configure TON with emmake "; exit 1; }
 cp -R ../crypto/smartcont ../crypto/fift/lib crypto

diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt
@@ -358,7 +358,8 @@ target_link_libraries(test-ed25519-crypto PUBLIC ton_crypto)
 
 add_library(fift-lib STATIC ${FIFT_SOURCE})
 target_include_directories(fift-lib PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
-target_link_libraries(fift-lib PUBLIC ton_crypto ton_db tdutils ton_block)
+target_link_libraries(fift-lib PUBLIC ton_crypto tdutils ton_block)
+
 if (USE_EMSCRIPTEN)
   target_link_options(fift-lib PRIVATE -fexceptions)
   target_compile_options(fift-lib PRIVATE -fexceptions)

diff --git a/crypto/fift/Fift.cpp b/crypto/fift/Fift.cpp
@@ -49,7 +49,6 @@ td::Result<int> Fift::interpret_istream(std::istream& stream, std::string curren
 }
 
 td::Result<int> Fift::do_interpret(IntCtx& ctx, bool is_interactive) {
-  ctx.ton_db = &config_.ton_db;
   ctx.source_lookup = &config_.source_lookup;
   ctx.dictionary = ctx.main_dictionary = ctx.context = config_.dictionary;
   ctx.output_stream = config_.output_stream;

diff --git a/crypto/fift/Fift.h b/crypto/fift/Fift.h
@@ -19,7 +19,6 @@
 #pragma once
 
 #include "SourceLookup.h"
-#include "vm/db/TonDb.h"
 #include "Dictionary.h"
 
 #include "td/utils/Status.h"
@@ -31,13 +30,11 @@ struct Fift {
  public:
   struct Config {
     fift::SourceLookup source_lookup;
-    vm::TonDb ton_db;
     fift::Dictionary dictionary;
     std::ostream* output_stream{&std::cout};
     std::ostream* error_stream{&std::cerr};
     bool show_backtrace{true};
   };
-  // Fift must own ton_db and dictionary, no concurrent access is allowed
   explicit Fift(Config config);
 
   td::Result<int> interpret_file(std::string fname, std::string current_dir, bool interactive = false);

diff --git a/crypto/fift/fift-main.cpp b/crypto/fift/fift-main.cpp
@@ -46,8 +46,6 @@
 #include "SourceLookup.h"
 #include "words.h"
 
-#include "vm/db/TonDb.h"
-
 #include "td/utils/logging.h"
 #include "td/utils/misc.h"
 #include "td/utils/Parser.h"
@@ -65,7 +63,6 @@ void usage(const char* progname) {
                "\t-I<source-search-path>\tSets colon-separated (unix) or at-separated (windows) library source include path. If not indicated, "
                "$FIFTPATH is used instead.\n"
                "\t-L<library-fif-file>\tPre-loads a library source file\n"
-               "\t-d<ton-db-path>\tUse a ton database\n"
                "\t-s\tScript mode: use first argument as a fift source file and import remaining arguments as $n)\n"
                "\t-v<verbosity-level>\tSet verbosity level\n"
                "\t-V<version>\tShow fift build information\n";
@@ -94,13 +91,12 @@ int main(int argc, char* const argv[]) {
   bool script_mode = false;
   std::vector<std::string> library_source_files, source_list;
   std::vector<std::string> source_include_path;
-  std::string ton_db_path;
 
   fift::Fift::Config config;
 
   int i;
   int new_verbosity_level = VERBOSITY_NAME(INFO);
-  while (!script_mode && (i = getopt(argc, argv, "hinI:L:d:sv:V")) != -1) {
+  while (!script_mode && (i = getopt(argc, argv, "hinI:L:sv:V")) != -1) {
     switch (i) {
       case 'i':
         interactive = true;
@@ -115,9 +111,6 @@ int main(int argc, char* const argv[]) {
       case 'L':
         library_source_files.emplace_back(optarg);
         break;
-      case 'd':
-        ton_db_path = optarg;
-        break;
       case 's':
         script_mode = true;
         break;
@@ -158,16 +151,6 @@ int main(int argc, char* const argv[]) {
     config.source_lookup.add_include_path(path);
   }
 
-  if (!ton_db_path.empty()) {
-    auto r_ton_db = vm::TonDbImpl::open(ton_db_path);
-    if (r_ton_db.is_error()) {
-      LOG(ERROR) << "Error opening ton database: " << r_ton_db.error().to_string();
-      std::exit(2);
-    }
-    config.ton_db = r_ton_db.move_as_ok();
-    // FIXME //std::atexit([&] { config.ton_db.reset(); });
-  }
-
   fift::init_words_common(config.dictionary);
   fift::init_words_vm(config.dictionary, true);  // enable vm debug
   fift::init_words_ton(config.dictionary);

diff --git a/crypto/fift/utils.cpp b/crypto/fift/utils.cpp
@@ -22,6 +22,8 @@
 #include "td/utils/filesystem.h"
 #include "td/utils/misc.h"
 #include "td/utils/port/path.h"
+#include "vm/boc.h"
+#include <map>
 
 namespace fift {
 namespace {