Merge pull request #920 from ouonline/merged

CMakeLists.txt

@@ -28,6 +28,18 @@ option(PPLNN_USE_AARCH64 "" OFF)
 option(PPLNN_USE_ARMV7 "" OFF)
 option(PPLNN_USE_RISCV64 "" OFF)
 option(PPLNN_USE_CUDA "" OFF)
+option(PPLNN_USE_LLM_CUDA "" OFF)
+
+if(PPLNN_USE_LLM_CUDA)
+    if(NOT CMAKE_CXX_STANDARD)
+        set(CMAKE_CXX_STANDARD 17)
+    endif()
+    if(NOT CMAKE_CUDA_STANDARD)
+        set(CMAKE_CUDA_STANDARD 17)
+    endif()
+    set(PPLNN_PROTOBUF_VERSION "v23.4")
+    set(PPLCOMMON_USE_CUDA ON)
+endif()
 
 # deprecated options
 if(HPCC_USE_X86_64)
@@ -95,6 +107,10 @@ if(PPLNN_USE_AARCH64 OR PPLNN_USE_ARMV7)
     include(cmake/arm.cmake)
 endif()
 
+if(PPLNN_USE_LLM_CUDA)
+    include(cmake/llm_cuda.cmake)
+endif()
+
 # pplcommon MUST be placed after engines because engines may set pplcommon options
 hpcc_populate_dep(pplcommon)
 

README.md

@@ -76,6 +76,8 @@ Refer to [Documents](#documents) for more details.
   - ARM
     - [Adding Ops](docs/en/arm-doc/add_op.md)（[中文版](docs/cn/arm-doc/add_op.md)）
     - [Benchmark](docs/en/arm-doc/benchmark_tool.md)（[中文版](docs/cn/arm-doc/benchmark_tool.md)）
+  - LLM-CUDA
+    - [Overview](docs/en/llm-cuda-overview.md)
 * Models
   - [Converting ONNX Opset](docs/en/onnx-model-opset-convert-guide.md)
   - [Generating ONNX models from OpenMMLab](docs/en/model-convert-guide.md)

build.sh

@@ -2,12 +2,9 @@
 
 workdir=`pwd`
 
-if [[ `uname` == "Linux" ]]; then
-    processor_num=`cat /proc/cpuinfo | grep processor | grep -v grep | wc -l`
-elif [[ `uname` == "Darwin" ]]; then
-    processor_num=`sysctl machdep.cpu | grep machdep.cpu.core_count | cut -d " " -f 2`
-else
-    processor_num=1
+if [ -z "$PPL_BUILD_THREAD_NUM" ]; then
+    PPL_BUILD_THREAD_NUM=32
+    echo -e "env 'PPL_BUILD_THREAD_NUM' is not set. use PPL_BUILD_THREAD_NUM=${PPL_BUILD_THREAD_NUM} by default."
 fi
 
 build_type='RelWithDebInfo'
@@ -16,6 +13,6 @@ options="-DCMAKE_BUILD_TYPE=${build_type} -DCMAKE_INSTALL_PREFIX=install $*"
 pplnn_build_dir="${workdir}/pplnn-build"
 mkdir ${pplnn_build_dir}
 cd ${pplnn_build_dir}
-cmd="cmake $options .. && cmake --build . -j ${processor_num} --config ${build_type} && cmake --build . --target install -j ${processor_num} --config ${build_type}"
+cmd="cmake $options .. && cmake --build . -j ${PPL_BUILD_THREAD_NUM} --config ${build_type} && cmake --build . --target install -j ${PPL_BUILD_THREAD_NUM} --config ${build_type}"
 echo "cmd -> $cmd"
 eval "$cmd"

cmake/deps.cmake

@@ -11,6 +11,9 @@ if(CMAKE_COMPILER_IS_GNUCC)
     if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9.0)
         message(FATAL_ERROR "gcc >= 4.9.0 is required.")
     endif()
+    if(CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 10.3.0)
+        message(FATAL_ERROR "gcc 10.3.0 has known bugs. use another version >= 9.4.0.")
+    endif()
 elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0.0)
         message(FATAL_ERROR "clang >= 6.0.0 is required.")
@@ -63,6 +66,7 @@ else()
     FetchContent_Declare(hpcc
         GIT_REPOSITORY ${PPLNN_DEP_HPCC_GIT}
         GIT_TAG ${__HPCC_COMMIT__}
+        GIT_SHALLOW TRUE
         SOURCE_DIR ${HPCC_DEPS_DIR}/hpcc
         BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/hpcc-build
         SUBBUILD_DIR ${HPCC_DEPS_DIR}/hpcc-subbuild)
@@ -131,7 +135,7 @@ if(PPLNN_DEP_FLATBUFFERS_PKG)
     hpcc_declare_pkg_dep(flatbuffers
         ${PPLNN_DEP_FLATBUFFERS_PKG})
 elseif(PPLNN_DEP_FLATBUFFERS_GIT)
-    hpcc_declare_git_dep(flatbuffers
+    hpcc_declare_git_dep_depth1(flatbuffers
         ${PPLNN_DEP_FLATBUFFERS_GIT}
         ${__FLATBUFFERS_TAG__})
 else()
@@ -163,19 +167,16 @@ endif()
 if(PPLNN_DEP_PROTOBUF_PKG)
     hpcc_declare_pkg_dep(protobuf
         ${PPLNN_DEP_PROTOBUF_PKG})
-elseif(PPLNN_DEP_PROTOBUF_GIT)
-    hpcc_declare_git_dep(protobuf
-        ${PPLNN_DEP_PROTOBUF_GIT}
-        ${__PROTOBUF_TAG__})
 else()
+    if(NOT PPLNN_DEP_PROTOBUF_GIT)
+        set(PPLNN_DEP_PROTOBUF_GIT "https://github.com/protocolbuffers/protobuf.git")
+    endif()
     if(PPLNN_PROTOBUF_VERSION)
-        hpcc_declare_git_dep(protobuf
-            ${PPLNN_DEP_PROTOBUF_GIT}
-            ${__PROTOBUF_TAG__})
-    else()
-        hpcc_declare_pkg_dep(protobuf
-            "https://github.com/protocolbuffers/protobuf/archive/refs/tags/${__PROTOBUF_TAG__}.zip")
+        set(__PROTOBUF_TAG__ ${PPLNN_PROTOBUF_VERSION})
     endif()
+    hpcc_declare_git_dep_depth1(protobuf
+        ${PPLNN_DEP_PROTOBUF_GIT}
+        ${__PROTOBUF_TAG__})
 endif()
 
 unset(__PROTOBUF_TAG__)
@@ -215,7 +216,7 @@ if(PPLNN_DEP_PYBIND11_PKG)
     hpcc_declare_pkg_dep(pybind11
         ${PPLNN_DEP_PYBIND11_PKG})
 elseif(PPLNN_DEP_PYBIND11_GIT)
-    hpcc_declare_git_dep(pybind11
+    hpcc_declare_git_dep_depth1(pybind11
         ${PPLNN_DEP_PYBIND11_GIT}
         ${__PYBIND11_TAG__})
 else()
@@ -238,7 +239,7 @@ if(PPLNN_DEP_GOOGLETEST_PKG)
     hpcc_declare_pkg_dep(googletest
         ${PPLNN_DEP_GOOGLETEST_PKG})
 elseif(PPLNN_DEP_GOOGLETEST_GIT)
-    hpcc_declare_git_dep(googletest
+    hpcc_declare_git_dep_depth1(googletest
         ${PPLNN_DEP_GOOGLETEST_GIT}
         ${__GOOGLETEST_TAG__})
 else()
@@ -258,7 +259,7 @@ if(PPLNN_USE_X86_64 OR PPLNN_USE_AARCH64 OR PPLNN_USE_ARMV7 OR PPLNN_USE_RISCV64
         if(NOT PPLNN_DEP_PPLCPUKERNEL_GIT)
             set(PPLNN_DEP_PPLCPUKERNEL_GIT "https://github.com/openppl-public/ppl.kernel.cpu.git")
         endif()
-        hpcc_declare_git_dep(ppl.kernel.cpu
+        hpcc_declare_git_dep_depth1(ppl.kernel.cpu
             ${PPLNN_DEP_PPLCPUKERNEL_GIT}
             master)
     endif()
@@ -274,8 +275,26 @@ if(PPLNN_USE_CUDA)
         if(NOT PPLNN_DEP_PPLCUDAKERNEL_GIT)
             set(PPLNN_DEP_PPLCUDAKERNEL_GIT "https://github.com/openppl-public/ppl.kernel.cuda.git")
         endif()
-        hpcc_declare_git_dep(ppl.kernel.cuda
+        hpcc_declare_git_dep_depth1(ppl.kernel.cuda
             ${PPLNN_DEP_PPLCUDAKERNEL_GIT}
             master)
     endif()
 endif()
+
+# --------------------------------------------------------------------------- #
+
+set(__LLM_KERNEL_CUDA_COMMIT__ master)
+
+if(PPLNN_DEP_PPL_LLM_KERNEL_CUDA_PKG)
+    hpcc_declare_pkg_dep(ppl.llm.kernel.cuda
+        ${PPLNN_DEP_PPL_LLM_KERNEL_CUDA_PKG})
+else()
+    if(NOT PPLNN_DEP_PPL_LLM_KERNEL_CUDA_GIT)
+        set(PPLNN_DEP_PPL_LLM_KERNEL_CUDA_GIT "https://github.com/openppl-public/ppl.llm.kernel.cuda.git")
+    endif()
+    hpcc_declare_git_dep_depth1(ppl.llm.kernel.cuda
+        ${PPLNN_DEP_PPL_LLM_KERNEL_CUDA_GIT}
+        ${__LLM_KERNEL_CUDA_COMMIT__})
+endif()
+
+unset(__LLM_KERNEL_CUDA_COMMIT__)

cmake/install.cmake

@@ -16,3 +16,5 @@ install(FILES include/ppl/nn/engines/engine.h DESTINATION include/ppl/nn/engines
 if(PPLNN_ENABLE_PMX_MODEL)
     install(DIRECTORY include/ppl/nn/models/pmx DESTINATION include/ppl/nn/models)
 endif()
+
+install(TARGETS ${PPLNN_ONNX_GENERATED_LIBS} DESTINATION lib)

cmake/llm_cuda.cmake

@@ -0,0 +1,51 @@
+hpcc_populate_dep(protobuf)
+
+if(NOT PPL_LLM_PROTOC_EXECUTABLE)
+    set(PPL_LLM_PROTOC_EXECUTABLE ${protobuf_BINARY_DIR}/protoc)
+endif()
+
+if(NOT PPLNN_ONNX_GENERATED_LIBS)
+    # generate new onnx.pb.* for pplnn
+    set(__LLM_GENERATED_DIR__ ${CMAKE_CURRENT_BINARY_DIR}/generated)
+    file(MAKE_DIRECTORY ${__LLM_GENERATED_DIR__})
+
+    set(__PROTO_DIR__ ${CMAKE_CURRENT_SOURCE_DIR}/src/ppl/nn/models/onnx/proto)
+    set(__ONNX_GENERATED_FILES__ "${__LLM_GENERATED_DIR__}/onnx.pb.h;${__LLM_GENERATED_DIR__}/onnx.pb.cc")
+    add_custom_command(
+        OUTPUT ${__ONNX_GENERATED_FILES__}
+        COMMAND ${PPL_LLM_PROTOC_EXECUTABLE}
+        ARGS --cpp_out ${__LLM_GENERATED_DIR__} -I ${__PROTO_DIR__}
+        ${__PROTO_DIR__}/onnx.proto
+        DEPENDS protoc ${__PROTO_DIR__}/onnx.proto)
+    add_library(pplnn_onnx_generated_static STATIC ${__ONNX_GENERATED_FILES__})
+    target_link_libraries(pplnn_onnx_generated_static PUBLIC libprotobuf)
+    target_include_directories(pplnn_onnx_generated_static PUBLIC ${__LLM_GENERATED_DIR__})
+    set(PPLNN_ONNX_GENERATED_LIBS pplnn_onnx_generated_static)
+
+    unset(__ONNX_GENERATED_FILES__)
+    unset(__PROTO_DIR__)
+    unset(__LLM_GENERATED_DIR__)
+endif()
+
+include(${HPCC_DEPS_DIR}/hpcc/cmake/cuda-common.cmake)
+
+hpcc_populate_dep(ppl.llm.kernel.cuda)
+
+file(GLOB_RECURSE __SRC__ src/ppl/nn/engines/llm_cuda/*.cc)
+add_library(ppl_llm_cuda_static ${__SRC__})
+target_link_libraries(ppl_llm_cuda_static PUBLIC pplnn_basic_static pplkernelcuda_static)
+target_include_directories(ppl_llm_cuda_static PUBLIC include src)
+target_compile_definitions(ppl_llm_cuda_static PUBLIC PPLNN_USE_LLM_CUDA)
+
+if(PPLNN_CUDA_ENABLE_NCCL)
+    target_compile_definitions(ppl_llm_cuda_static PUBLIC PPLNN_CUDA_ENABLE_NCCL)
+endif()
+
+unset(__SRC__)
+
+target_link_libraries(pplnn_static INTERFACE ppl_llm_cuda_static)
+
+if(PPLNN_INSTALL)
+    install(DIRECTORY include/ppl/nn/engines/llm_cuda DESTINATION include/ppl/nn/engines)
+    install(TARGETS ppl_llm_cuda_static DESTINATION lib)
+endif()

cmake/onnx_model.cmake

@@ -1,12 +1,22 @@
 file(GLOB_RECURSE __PPLNN_MODEL_ONNX_SRC__ src/ppl/nn/models/onnx/*.cc)
+list(REMOVE_ITEM __PPLNN_MODEL_ONNX_SRC__ ${PROJECT_SOURCE_DIR}/src/ppl/nn/models/onnx/generated/onnx.pb.cc)
 
 if(PPLNN_PROTOBUF_VERSION AND NOT PPLNN_ONNX_GENERATED_LIBS)
    message(FATAL_ERROR "`PPLNN_PROTOBUF_VERSION` is set to be [${PPLNN_PROTOBUF_VERSION}], but `PPLNN_ONNX_GENERATED_LIBS` is not set.")
 endif()
 
-# replace default *.pb.* files
-if(PPLNN_ONNX_GENERATED_LIBS)
-    list(REMOVE_ITEM __PPLNN_MODEL_ONNX_SRC__ ${PROJECT_SOURCE_DIR}/src/ppl/nn/models/onnx/generated/onnx.pb.cc)
+if(NOT TARGET libprotobuf)
+    include(cmake/protobuf.cmake)
+endif()
+
+# use default *.pb.* files
+if(NOT PPLNN_ONNX_GENERATED_LIBS)
+    add_library(pplnn_onnx_pb_generated_static ${PROJECT_SOURCE_DIR}/src/ppl/nn/models/onnx/generated/onnx.pb.cc)
+    target_link_libraries(pplnn_onnx_pb_generated_static PUBLIC libprotobuf)
+    target_include_directories(pplnn_onnx_pb_generated_static PUBLIC
+        ${protobuf_SOURCE_DIR}/src
+        ${PROJECT_SOURCE_DIR}/src/ppl/nn/models/onnx/generated)
+    set(PPLNN_ONNX_GENERATED_LIBS pplnn_onnx_pb_generated_static)
 endif()
 
 # if external sources are set, remove `default_register_resources.cc`
@@ -18,22 +28,13 @@ add_library(pplnn_onnx_static STATIC ${__PPLNN_MODEL_ONNX_SRC__} ${PPLNN_SOURCE_
 
 unset(__PPLNN_MODEL_ONNX_SRC__)
 
-if(PPLNN_ONNX_GENERATED_LIBS)
-    target_link_libraries(pplnn_onnx_static PRIVATE ${PPLNN_ONNX_GENERATED_LIBS})
-else()
-    target_include_directories(pplnn_onnx_static PRIVATE ${PROJECT_SOURCE_DIR}/src/ppl/nn/models/onnx/generated)
-endif()
-
 target_compile_definitions(pplnn_onnx_static PUBLIC PPLNN_ENABLE_ONNX_MODEL)
 target_link_libraries(pplnn_onnx_static PUBLIC pplnn_basic_static)
-
-include(cmake/protobuf.cmake)
-target_link_libraries(pplnn_onnx_static PUBLIC libprotobuf)
-target_include_directories(pplnn_onnx_static PRIVATE ${protobuf_SOURCE_DIR}/src)
+target_link_libraries(pplnn_onnx_static PRIVATE ${PPLNN_ONNX_GENERATED_LIBS})
 
 target_link_libraries(pplnn_static INTERFACE pplnn_onnx_static)
 
 if(PPLNN_INSTALL)
     install(DIRECTORY include/ppl/nn/models/onnx DESTINATION include/ppl/nn/models)
-    install(TARGETS pplnn_onnx_static DESTINATION lib)
+    install(TARGETS pplnn_onnx_static ${PPLNN_ONNX_GENERATED_LIBS} DESTINATION lib)
 endif()

cmake/pplnn-config.cmake.in

@@ -18,6 +18,7 @@ option(PPLNN_USE_X86_64 "" @PPLNN_USE_X86_64@)
 option(PPLNN_USE_AARCH64 "" @PPLNN_USE_AARCH64@)
 option(PPLNN_USE_RISCV64 "" @PPLNN_USE_RISCV64@)
 option(PPLNN_USE_CUDA "" @PPLNN_USE_CUDA@)
+option(PPLNN_USE_LLM_CUDA "" @PPLNN_USE_LLM_CUDA@)
 
 option(PPLNN_ENABLE_ONNX_MODEL "" @PPLNN_ENABLE_ONNX_MODEL@)
 
@@ -107,6 +108,30 @@ if(PPLNN_USE_RISCV64)
     list(APPEND PPLNN_LIBRARIES pplnn_riscv_static)
 endif()
 
+if(PPLNN_USE_LLM_CUDA)
+    if(NOT TARGET "pplkernelcuda_static")
+        include(${CMAKE_CURRENT_LIST_DIR}/pplkernelcuda-config.cmake)
+    endif()
+
+    set(__LINK_LIBS__ pplnn_basic_static pplkernelcuda_static)
+    if(PPLNN_CUDA_ENABLE_NCCL)
+        list(APPEND __LINK_LIBS__ ${NCCL_LIBRARIES})
+    endif()
+
+    get_filename_component(__PPLNN_LIB_PATH__ "${__PPLNN_PACKAGE_ROOTDIR__}/lib/@HPCC_STATIC_LIB_PREFIX@ppl_llm_cuda_static@HPCC_STATIC_LIB_SUFFIX@" ABSOLUTE)
+    add_library(ppl_llm_cuda_static STATIC IMPORTED)
+    set_target_properties(ppl_llm_cuda_static PROPERTIES
+        INTERFACE_LINK_LIBRARIES "${__LINK_LIBS__}"
+        IMPORTED_LOCATION "${__PPLNN_LIB_PATH__}"
+        IMPORTED_LOCATION_DEBUG "${__PPLNN_LIB_PATH__}"
+        IMPORTED_LOCATION_RELEASE "${__PPLNN_LIB_PATH__}")
+    unset(__PPLNN_LIB_PATH__)
+
+    unset(__LINK_LIBS__)
+
+    list(APPEND PPLNN_LIBRARIES ppl_llm_cuda_static)
+endif()
+
 get_filename_component(__PPLNN_LIB_PATH__ "${__PPLNN_PACKAGE_ROOTDIR__}/lib/@HPCC_STATIC_LIB_PREFIX@pplnn_basic_static@HPCC_STATIC_LIB_SUFFIX@" ABSOLUTE)
 set_target_properties(pplnn_basic_static PROPERTIES
     INTERFACE_LINK_LIBRARIES "pplcommon_static"
@@ -122,10 +147,18 @@ if(PPLNN_ENABLE_ONNX_MODEL)
         include(${__PPLNN_PACKAGE_ROOTDIR__}/lib/cmake/protobuf/protobuf-config.cmake)
     endif()
 
+    get_filename_component(__PPLNN_LIB_PATH__ "${__PPLNN_PACKAGE_ROOTDIR__}/lib/@HPCC_STATIC_LIB_PREFIX@@PPLNN_ONNX_GENERATED_LIBS@@HPCC_STATIC_LIB_SUFFIX@" ABSOLUTE)
+    add_library(pplnn_onnx_generated_static STATIC IMPORTED)
+    set_target_properties(pplnn_onnx_generated_static PROPERTIES
+        INTERFACE_LINK_LIBRARIES "protobuf::libprotobuf"
+        IMPORTED_LOCATION "${__PPLNN_LIB_PATH__}"
+        IMPORTED_LOCATION_DEBUG "${__PPLNN_LIB_PATH__}"
+        IMPORTED_LOCATION_RELEASE "${__PPLNN_LIB_PATH__}")
+
     get_filename_component(__PPLNN_LIB_PATH__ "${__PPLNN_PACKAGE_ROOTDIR__}/lib/@HPCC_STATIC_LIB_PREFIX@pplnn_onnx_static@HPCC_STATIC_LIB_SUFFIX@" ABSOLUTE)
     add_library(pplnn_onnx_static STATIC IMPORTED)
     set_target_properties(pplnn_onnx_static PROPERTIES
-        INTERFACE_LINK_LIBRARIES "pplnn_basic_static;protobuf::libprotobuf"
+        INTERFACE_LINK_LIBRARIES "pplnn_basic_static;pplnn_onnx_generated_static"
         IMPORTED_LOCATION "${__PPLNN_LIB_PATH__}"
         IMPORTED_LOCATION_DEBUG "${__PPLNN_LIB_PATH__}"
         IMPORTED_LOCATION_RELEASE "${__PPLNN_LIB_PATH__}")