build: support cmake for benchmark_client, binlog_sender, manifest_ge…

…nerator, rdb_to_pika, txt_to_pika, pika_to_txt, pika_port (OpenAtomFoundation#1451)

* update README in aof_to_pika

* edit compile commands in aof_to_pika README

* support cmake for binlog_sender

* support cmake for manifest_generator

* support cmake for benchmark_client

* support cmake for rdb_to_pika

* deprecate nemo_to_blackwidow tools

* delete makefile in tools that support cmake

* add button to control compilation of pika-tools

* add build.sh parameter to control compilation

* rename pika-tools to tools

* rename pika-tools in CMakeLists.txt

* use pip to install rdb-tools

* resolve name conflict in cmake after renaming pika-tools to tools

* support cmake for txt_to_pika and pika_to_txt

* remove support for pika_port_2

* support cmake for pika_port_3

* remove redundant makefile

* change workflow working directory

* check tools compile in workflow

---------

Co-authored-by: J1senn <[email protected]>

.github/workflows/operator.yml

@@ -29,12 +29,12 @@ jobs:
 
     - name: Build
       run: |
-        cd pika-tools/pika_operator && make 
+        cd tools/pika_operator && make 
 
     - name: Unit Test
       run: |
-        cd pika-tools/pika_operator && make test
+        cd tools/pika_operator && make test
 
     - name: E2E Test
       run: |
-        cd pika-tools/pika_operator && make e2e-test-local
+        cd tools/pika_operator && make e2e-test-local

.github/workflows/pika.yml

@@ -35,7 +35,7 @@ jobs:
       - name: Configure CMake
         # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
         # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
-        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+        run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DUSE_PIKA_TOOLS=ON
 
       - name: Build
         # Build your program with the given configuration

.github/workflows/pika_exporter.yml

@@ -6,7 +6,7 @@ on:
   pull_request:
     branches: [ "unstable" ]
     paths:
-      - 'pika-tools/pika_exporter/**'
+      - 'tools/pika_exporter/**'
 
 jobs:
 
@@ -22,7 +22,7 @@ jobs:
 
     - name: Build
       run: |
-        cd pika-tools/pika_exporter && make -j
+        cd tools/pika_exporter && make -j
     - name: Test
       run: |
-        cd pika-tools/pika_exporter && make -j
+        cd tools/pika_exporter && make -j

.gitignore

@@ -35,7 +35,6 @@ cmake-build-release
 make_config.mk
 log/
 lib/
-tools/
 output/
 
 # DB

CMakeLists.txt

@@ -580,6 +580,65 @@ ExternalProject_Add(rocksdb
   make -j${CPU_CORE}
 )
 
+option(USE_PIKA_TOOLS "compile pika-tools" OFF)
+if (USE_PIKA_TOOLS)
+  ExternalProject_Add(hiredis
+    URL
+    https://github.com/vipshop/hiredis-vip/archive/refs/tags/0.3.0.tar.gz
+    URL_HASH
+    MD5=e4d4a2928188b11f696d35ca1c52e9a4
+    DOWNLOAD_NO_PROGRESS
+    1
+    UPDATE_COMMAND
+    ""
+    LOG_BUILD
+    1
+    LOG_INSTALL
+    1
+    BUILD_IN_SOURCE
+    1
+    SOURCE_SUBDIR
+    ""
+    BUILD_ALWAYS
+    1
+    CONFIGURE_COMMAND
+    ""
+    BUILD_COMMAND
+    make PREFIX=${STAGED_INSTALL_PREFIX} -j${CPU_CORE} all
+    INSTALL_COMMAND
+    make PREFIX=${STAGED_INSTALL_PREFIX} -j${CPU_CORE} install
+  )
+  set(HIREDIS_LIBRARY ${INSTALL_LIBDIR}/libhiredis_vip.a)
+
+  ExternalProject_Add(bz2
+    URL
+    https://sourceware.org/pub/bzip2/bzip2-1.0.8.tar.gz
+    URL_HASH
+    MD5=67e051268d0c475ea773822f7500d0e5
+    DOWNLOAD_NO_PROGRESS
+    1
+    UPDATE_COMMAND
+    ""
+    LOG_BUILD
+    1
+    LOG_INSTALL
+    1
+    BUILD_IN_SOURCE
+    1
+    SOURCE_SUBDIR
+    ""
+    BUILD_ALWAYS
+    1
+    CONFIGURE_COMMAND
+    ""
+    BUILD_COMMAND
+    make PREFIX=${STAGED_INSTALL_PREFIX} -j${CPU_CORE} all
+    INSTALL_COMMAND
+    make PREFIX=${STAGED_INSTALL_PREFIX} -j${CPU_CORE} install
+  )
+  set(BZ2_LIBRARY ${INSTALL_LIBDIR}/libbz2.a)
+endif()
+
 if(${OS_VERSION} MATCHES "CentOS")
   set(ROCKSDB_LIBRARY ${INSTALL_LIBDIR_64}/librocksdb.a)
 else()
@@ -592,7 +651,9 @@ set(ROCKSDB_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/${EP_BASE_SUFFIX}/Source/rock
 add_subdirectory(src/pstd)
 add_subdirectory(src/net)
 add_subdirectory(src/storage)
-add_subdirectory(pika-tools)
+if (USE_PIKA_TOOLS)
+  add_subdirectory(tools)
+endif()
 aux_source_directory(src DIR_SRCS)
 
 # # generate version

build.sh

@@ -92,7 +92,12 @@ fi
 
 cd ${BUILD_DIR}
 
-${CMAKE} .. .
+use_pika_tools=""
+if [ $1 = "tools" ]; then
+  use_pika_tools="-DUSE_PIKA_TOOLS=ON"
+fi
+
+${CMAKE} ${use_pika_tools} .. .
 
 if [ $? -ne 0 ]; then
     echo -e "${C_RED} cmake execution error ${C_END}"