diff --git a/CMakeLists.txt b/CMakeLists.txt index 4130a3f..95f0dbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,15 +82,36 @@ project(tritononnxruntimebackend LANGUAGES C CXX) # igpu. If not set, the current platform will be used. If building on # Jetpack, always set to igpu to avoid misdetection. # +# - If you want ROCm support set +# TRITON_ENABLE_ONNXRUNTIME_ROCM=ON and set +# TRITON_BUILD_ONNXRUNTIME_ROCM_VERSION to the ROCm stack +# version that is compatible with the specified version of ONNX +# Runtime. +# +# - If you want MIGraphX support set +# TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX=ON and set +# TRITON_BUILD_ONNXRUNTIME_MIGRAPHX_VERSION to the MIGraphX +# version that is compatible with the specified version of ONNX +# Runtime. Requires that ROCm Support also be set +# # - If you want to disable GPU usage, set TRITON_ENABLE_GPU=OFF. # This will make builds with CUDA and TensorRT flags to fail. # option(TRITON_ENABLE_GPU "Enable GPU support in backend" ON) +option(TRITON_ENABLE_ROCM "Enable AMD GPU support in backend" ON) option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON) option(TRITON_ENABLE_ONNXRUNTIME_TENSORRT "Enable TensorRT execution provider for ONNXRuntime backend in server" OFF) +option(TRITON_ENABLE_ONNXRUNTIME_ROCM + "Enable ROCm execution provider for ONNXRuntime backend in server" OFF) +option(TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX + "Enable MIGraphX execution provider for ONNXRuntime backend in server" OFF) option(TRITON_ENABLE_ONNXRUNTIME_OPENVINO "Enable OpenVINO execution provider for ONNXRuntime backend in server" OFF) +set(TRITON_BUILD_ROCM_VERSION "" CACHE STRING "Version of ROCm install") +set(TRITON_BUILD_ROCM_HOME "" CACHE PATH "Path to ROCm install") +set(TRITON_BUILD_MIGRAPHX_VERSION "" CACHE STRING "Version of MIGraphX install") +set(TRITON_BUILD_MIGRAPHX_HOME "" CACHE PATH "Path to MIGraphX install") set(TRITON_BUILD_CONTAINER "" CACHE STRING "Triton container to use a base for build") set(TRITON_BUILD_CONTAINER_VERSION "" CACHE STRING "Triton container version to target") set(TRITON_BUILD_ONNXRUNTIME_VERSION "" CACHE STRING "ONNXRuntime version to build") @@ -122,6 +143,12 @@ if (NOT TRITON_ENABLE_GPU) endif() # TRITON_ENABLE_ONNXRUNTIME_TENSORRT endif() # NOT TRITON_ENABLE_GPU +if (NOT TRITON_ENABLE_ROCM) + if (TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX) + message(FATAL_ERROR "TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX=ON requires TRITON_ENABLE_ROCM=ON") + endif() # TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX +endif() # NOT TRITON_ENABLE_ROCM + if(NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release) endif() @@ -149,7 +176,11 @@ else() endif() if(NOT TRITON_BUILD_CONTAINER) - set(TRITON_BUILD_CONTAINER "nvcr.io/nvidia/tritonserver:${TRITON_BUILD_CONTAINER_VERSION}-py3-min") + if (TRITON_ENABLE_ROCM) + set(TRITON_BUILD_CONTAINER "rocm/pytorch:rocm6.0.2_ubuntu22.04_py3.10_pytorch_2.1.2") + else() + set(TRITON_BUILD_CONTAINER "nvcr.io/nvidia/tritonserver:${TRITON_BUILD_CONTAINER_VERSION}-py3-min") + endif() endif() set(TRITON_ONNXRUNTIME_DOCKER_IMAGE "tritonserver_onnxruntime") @@ -201,6 +232,13 @@ if(${TRITON_ENABLE_GPU}) find_package(CUDAToolkit REQUIRED) endif() # TRITON_ENABLE_GPU +# +# ROCM +# +if(${TRITON_ENABLE_ROCM}) + find_package(hip REQUIRED) +endif() # TRITON_ENABLE_ROCM + # # Shared library implementing the Triton Backend API # @@ -234,6 +272,13 @@ target_compile_options( $<$:/Wall /D_WIN32_WINNT=0x0A00 /EHsc /Zc:preprocessor> ) +if(${TRITON_ENABLE_ROCM}) + target_compile_definitions( + triton-onnxruntime-backend + PRIVATE TRITON_ENABLE_ROCM=1 + ) +endif() # TRITON_ENABLE_ROCM + if(${TRITON_ENABLE_GPU}) target_compile_definitions( triton-onnxruntime-backend @@ -253,6 +298,20 @@ if(${TRITON_ENABLE_ONNXRUNTIME_OPENVINO}) ) endif() # TRITON_ENABLE_ONNXRUNTIME_OPENVINO +if(${TRITON_ENABLE_ONNXRUNTIME_ROCM}) + target_compile_definitions( + triton-onnxruntime-backend + PRIVATE TRITON_ENABLE_ONNXRUNTIME_ROCM=1 + ) +endif() # TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX + +if(${TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX}) + target_compile_definitions( + triton-onnxruntime-backend + PRIVATE TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX=1 + ) +endif() # TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX + if (WIN32) set_target_properties( triton-onnxruntime-backend @@ -305,6 +364,14 @@ if(${TRITON_ENABLE_GPU}) ) endif() # TRITON_ENABLE_GPU +if(${TRITON_ENABLE_ROCM}) + target_link_libraries( + triton-onnxruntime-backend + PRIVATE + hip::host + ) +endif() #TRITON_ENABLE_ROCM + if(${TRITON_ENABLE_ONNXRUNTIME_OPENVINO}) target_link_libraries( triton-onnxruntime-backend @@ -334,11 +401,29 @@ if(TRITON_ONNXRUNTIME_DOCKER_BUILD) set(_GEN_FLAGS ${_GEN_FLAGS} "--tensorrt-home=${TRITON_BUILD_TENSORRT_HOME}") endif() # TRITON_BUILD_TENSORRT_HOME if(${TRITON_ENABLE_ONNXRUNTIME_TENSORRT}) - set(_GEN_FLAGS ${_GEN_FLAGS} "--ort-tensorrt") + set(_GEN_FLAGS ${_GEN_FLAGS} "--ort-tensorrt --trt-version=${TRT_VERSION} --onnx-tensorrt-tag=${TRITON_ONNX_TENSORRT_REPO_TAG}") endif() # TRITON_ENABLE_ONNXRUNTIME_TENSORRT if(${TRITON_ENABLE_ONNXRUNTIME_OPENVINO}) set(_GEN_FLAGS ${_GEN_FLAGS} "--ort-openvino=${TRITON_BUILD_ONNXRUNTIME_OPENVINO_VERSION}") endif() # TRITON_ENABLE_ONNXRUNTIME_OPENVINO + if(NOT ${TRITON_BUILD_ROCM_VERSION} STREQUAL "") + set(_GEN_FLAGS ${_GEN_FLAGS} "--rocm-version=${TRITON_BUILD_ROCM_VERSION}") + endif() # TRITON_BUILD_ROCM_VERSION + if(NOT ${TRITON_BUILD_ROCM_HOME} STREQUAL "") + set(_GEN_FLAGS ${_GEN_FLAGS} "--migraphx-home=${TRITON_BUILD_ROCM_HOME}") + endif() # TRITON_BUILD_ROCM_HOME + if(${TRITON_ENABLE_ONNXRUNTIME_ROCM}) + set(_GEN_FLAGS ${_GEN_FLAGS} "--enable-rocm") + endif() # TRITON_ENABLE_ONNXRUNTIME_ROCM + if(NOT ${TRITON_BUILD_MIGRAPHX_VERSION} STREQUAL "") + set(_GEN_FLAGS ${_GEN_FLAGS} "--migraphx-version=${TRITON_BUILD_MIGRAPHX_VERSION}") + endif() # TRITON_BUILD_MIGRAPHX_VERSION + if(NOT ${TRITON_BUILD_MIGRAPHX_HOME} STREQUAL "") + set(_GEN_FLAGS ${_GEN_FLAGS} "--migraphx-home=${TRITON_BUILD_MIGRAPHX_HOME}") + endif() # TRITON_BUILD_MIGRAPHX_HOME + if(${TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX}) + set(_GEN_FLAGS ${_GEN_FLAGS} "--ort-migraphx") + endif() # TRITON_ENABLE_ONNXRUNTIME_MIGRAPHX set(ENABLE_GPU_EXTRA_ARGS "") if(${TRITON_ENABLE_GPU}) @@ -349,7 +434,7 @@ if(TRITON_ONNXRUNTIME_DOCKER_BUILD) add_custom_command( OUTPUT onnxruntime/lib/${ONNXRUNTIME_LIBRARY} - COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/gen_ort_dockerfile.py --triton-container="${TRITON_BUILD_CONTAINER}" --ort-version="${TRITON_BUILD_ONNXRUNTIME_VERSION}" --trt-version="${TRT_VERSION}" --onnx-tensorrt-tag="${TRITON_ONNX_TENSORRT_REPO_TAG}" ${_GEN_FLAGS} --output=Dockerfile.ort ${ENABLE_GPU_EXTRA_ARGS} + COMMAND python3 ${CMAKE_CURRENT_SOURCE_DIR}/tools/gen_ort_dockerfile.py --triton-container="${TRITON_BUILD_CONTAINER}" --ort-version="${TRITON_BUILD_ONNXRUNTIME_VERSION}" ${_GEN_FLAGS} --output=Dockerfile.ort ${ENABLE_GPU_EXTRA_ARGS} COMMAND docker build --memory ${TRITON_ONNXRUNTIME_DOCKER_MEMORY} --cache-from=${TRITON_ONNXRUNTIME_DOCKER_IMAGE} --cache-from=${TRITON_ONNXRUNTIME_DOCKER_IMAGE}_cache0 --cache-from=${TRITON_ONNXRUNTIME_DOCKER_IMAGE}_cache1 -t ${TRITON_ONNXRUNTIME_DOCKER_IMAGE} -f ./Dockerfile.ort ${CMAKE_CURRENT_SOURCE_DIR} COMMAND powershell.exe -noprofile -c "docker rm onnxruntime_backend_ort > $null 2>&1; if ($LASTEXITCODE) { 'error ignored...' }; exit 0" COMMAND docker create --name onnxruntime_backend_ort ${TRITON_ONNXRUNTIME_DOCKER_IMAGE} @@ -362,11 +447,14 @@ if(TRITON_ONNXRUNTIME_DOCKER_BUILD) add_custom_command( OUTPUT onnxruntime/lib/${ONNXRUNTIME_LIBRARY} - COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/gen_ort_dockerfile.py --ort-build-config="${CMAKE_BUILD_TYPE}" --triton-container="${TRITON_BUILD_CONTAINER}" --ort-version="${TRITON_BUILD_ONNXRUNTIME_VERSION}" --trt-version="${TRT_VERSION}" --onnx-tensorrt-tag="${TRITON_ONNX_TENSORRT_REPO_TAG}" ${_GEN_FLAGS} --output=Dockerfile.ort ${ENABLE_GPU_EXTRA_ARGS} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tools/gen_ort_dockerfile.py --ort-build-config="${CMAKE_BUILD_TYPE}" --triton-container="${TRITON_BUILD_CONTAINER}" --ort-version="${TRITON_BUILD_ONNXRUNTIME_VERSION}" ${_GEN_FLAGS} --output=Dockerfile.ort ${ENABLE_GPU_EXTRA_ARGS} + COMMAND cat Dockerfile.ort COMMAND docker build --cache-from=${TRITON_ONNXRUNTIME_DOCKER_IMAGE} --cache-from=${TRITON_ONNXRUNTIME_DOCKER_IMAGE}_cache0 --cache-from=${TRITON_ONNXRUNTIME_DOCKER_IMAGE}_cache1 -t ${TRITON_ONNXRUNTIME_DOCKER_IMAGE} -f ./Dockerfile.ort ${CMAKE_CURRENT_SOURCE_DIR} COMMAND docker rm onnxruntime_backend_ort || echo 'error ignored...' || true COMMAND docker create --name onnxruntime_backend_ort ${TRITON_ONNXRUNTIME_DOCKER_IMAGE} COMMAND rm -fr onnxruntime + COMMAND docker image list + COMMAND docker ps COMMAND docker cp onnxruntime_backend_ort:/opt/onnxruntime onnxruntime COMMAND docker rm onnxruntime_backend_ort COMMENT "Building ONNX Runtime" diff --git a/src/onnxruntime.cc b/src/onnxruntime.cc index decf297..790779b 100644 --- a/src/onnxruntime.cc +++ b/src/onnxruntime.cc @@ -43,6 +43,11 @@ #include #endif // TRITON_ENABLE_GPU +#ifdef TRITON_ENABLE_ROCM +#include +#endif // TRITON_ENABLE_ROCM + + // // ONNX Runtime Backend that implements the TRITONBACKEND API. // diff --git a/tools/gen_ort_dockerfile.py b/tools/gen_ort_dockerfile.py index 215f2c1..d3f08e8 100755 --- a/tools/gen_ort_dockerfile.py +++ b/tools/gen_ort_dockerfile.py @@ -122,6 +122,69 @@ def dockerfile_for_linux(output_file): ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so """ + if FLAGS.enable_rocm: + if FLAGS.rocm_version is not None: + df += """ARG ROCM_VERSION={}""".format(FLAGS.rocm_version) + else: + df += """ARG ROCM_VERSION=5.7""" + + df += """ +RUN apt-get clean && apt-get update && apt-get install -y locales +RUN locale-gen en_US.UTF-8 +RUN update-locale LANG=en_US.UTF-8 +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + +# Support multiarch +RUN dpkg --add-architecture i386 + +# Install rocm +RUN apt-get update && apt-get install -y gnupg2 --no-install-recommends curl && \ +curl -fsSL http://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor -o /etc/apt/trusted.gpg.d/rocm-keyring.gpg && \ +sh -c 'echo deb [arch=amd64] http://repo.radeon.com/rocm/apt/${ROCM_VERSION}/ jammy main > /etc/apt/sources.list.d/rocm.list' + +# From docs.amd.com for installing rocm. Needed to install properly +RUN sh -c \"echo 'Package: *\\nPin: release o=repo.radeon.com\\nPin-priority: 600' > /etc/apt/preferences.d/rocm-pin-600\" + +RUN apt-get update &&\ + apt-get install -y sudo git apt-utils bash build-essential curl doxygen gdb rocm-dev python3-dev python3-pip miopen-hip \ + rocblas half aria2 libnuma-dev pkg-config ccache software-properties-common wget libnuma-dev libssl-dev zlib1g-dev + +RUN aria2c -q -d /tmp -o cmake-3.27.3-linux-x86_64.tar.gz \ +https://github.com/Kitware/CMake/releases/download/v3.27.3/cmake-3.27.3-linux-x86_64.tar.gz &&\ +tar -zxf /tmp/cmake-3.27.3-linux-x86_64.tar.gz --strip=1 -C /usr + +# Install rbuild +RUN pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz numpy yapf==0.28.0 asciidoc CppHeaderParser setuptools wheel + +ENV PATH /opt/miniconda/bin:/code/cmake-3.27.3-linux-x86_64/bin:${PATH} +# Install rocm ep dependencies +RUN apt-get update &&\ + apt-get install -y rocrand rccl rccl-dev hipsparse hipfft hipcub hipblas rocthrust hip-base rocm-device-libs hipify-clang miopen-hip-dev rocm-cmake +""" + + if FLAGS.ort_migraphx: + if FLAGS.migraphx_version is not None: + df+= """ARG MIGRAPHX_VERSION={}""".format(FLAGS.migraphx_version) + else: + df+= """ARG MIGRAPHX_VERSION=develop""" + + df += """ +# Install MIGraphX from source +ARG GPU_TARGETS='gfx908;gfx90a;gfx1030;gfx1100;gfx1101;gfx1102;gfx940;gfx941;gfx942' + +# Workaround broken rocm packages +RUN ln -s /opt/rocm-* /opt/rocm +RUN echo "/opt/rocm/lib" > /etc/ld.so.conf.d/rocm.conf +RUN echo "/opt/rocm/llvm/lib" > /etc/ld.so.conf.d/rocm-llvm.conf +RUN ldconfig + +RUN mkdir /migraphx +RUN cd /migraphx && git clone --depth=1 --branch ${MIGRAPHX_VERSION} https://github.com/ROCm/AMDMIGraphX src && cd src && rbuild package --cxx /opt/rocm/llvm/bin/clang++ -d /migraphx/deps -B /migraphx/build -DPYTHON_EXECUTABLE=/usr/bin/python3 -DBUILD_DEV=On -DGPU_TARGETS=${GPU_TARGETS} && dpkg -i /migraphx/build/*.deb +RUN cd / && rm -rf /migraphx + """ + + if FLAGS.ort_openvino is not None: df += """ # Install OpenVINO @@ -165,7 +228,7 @@ def dockerfile_for_linux(output_file): RUN git clone -b tensorrt-8.0 --recursive ${ONNXRUNTIME_REPO} onnxruntime && \ (cd onnxruntime && git submodule update --init --recursive) - """ + """ # Use the tensorrt-8.5ea branch to use Tensor RT 8.5a to use the built-in tensorrt parser elif FLAGS.ort_version == "1.12.1": df += """ @@ -176,20 +239,34 @@ def dockerfile_for_linux(output_file): ARG ONNXRUNTIME_REPO ARG ONNXRUNTIME_BUILD_CONFIG - RUN git clone -b tensorrt-8.5ea --recursive ${ONNXRUNTIME_REPO} onnxruntime && \ - (cd onnxruntime && git submodule update --init --recursive) + RUN git clone -b tensorrt-8.5ea --recursive ${ONNXRUNTIME_REPO} onnxruntime && \ + (cd onnxruntime && git submodule update --init --recursive) - """ - else: - df += """ + """ + elif FLAGS.enable_rocm: + df += """ # - # ONNX Runtime build + # onnx runtime build # ARG ONNXRUNTIME_VERSION ARG ONNXRUNTIME_REPO ARG ONNXRUNTIME_BUILD_CONFIG - RUN git clone -b rel-${ONNXRUNTIME_VERSION} --recursive ${ONNXRUNTIME_REPO} onnxruntime && \ + run git clone -b ${ONNXRUNTIME_VERSION} --recursive ${ONNXRUNTIME_REPO} onnxruntime && \ + (cd onnxruntime && git submodule update --init --recursive) + + """ + + else: + df += """ + # + # onnx runtime build + # + arg onnxruntime_version + arg onnxruntime_repo + arg onnxruntime_build_config + + run git clone -b rel-${onnxruntime_version} --recursive ${onnxruntime_repo} onnxruntime && \ (cd onnxruntime && git submodule update --init --recursive) """ @@ -210,68 +287,90 @@ def dockerfile_for_linux(output_file): ep_flags += ' --cuda_home "{}"'.format(FLAGS.cuda_home) if FLAGS.cudnn_home is not None: ep_flags += ' --cudnn_home "{}"'.format(FLAGS.cudnn_home) - elif target_platform() == "igpu": - ep_flags += ' --cudnn_home "/usr/lib/aarch64-linux-gnu"' + elif target_platform() == "igpu": + ep_flags += ' --cudnn_home "/usr/lib/aarch64-linux-gnu"' if FLAGS.ort_tensorrt: ep_flags += " --use_tensorrt" if FLAGS.ort_version >= "1.12.1": ep_flags += " --use_tensorrt_builtin_parser" if FLAGS.tensorrt_home is not None: ep_flags += ' --tensorrt_home "{}"'.format(FLAGS.tensorrt_home) + cmake_defs = "CMAKE_CUDA_ARCHITECTURES" + cuda_archs = "\'60;61;70;75;80;86;90\'" - if os.name == "posix": - if os.getuid() == 0: - ep_flags += " --allow_running_as_root" + if FLAGS.enable_rocm: + ep_flags = "--use_rocm" + ep_flags += " --allow_running_as_root" + df += """ + RUN export PATH="/opt/cmake/bin:$PATH" + RUN export CXXFLAGS="-D__HIP_PLATFORM_AMD__=1 -w" + """ + if FLAGS.rocm_version is not None: + ep_flags += ' --rocm_version "{}"'.format(FLAGS.rocm_version) + if FLAGS.rocm_home is not None: + ep_flags += ' --rocm_home "{}"'.format(FLAGS.rocm_home) + if FLAGS.ort_migraphx: + ep_flags += " --use_migraphx" + if FLAGS.migraphx_home is not None: + ep_flags += ' --migraphx_home "{}"'.format(FLAGS.migraphx_home) + cmake_defs = "CMAKE_HIP_COMPILER" + cuda_archs = "/opt/rocm/llvm/bin/clang++" + ep_flags += " --allow_running_as_root" + + if os.name == "posix": + if os.getuid() == 0: + ep_flags += " --allow_running_as_root" - if FLAGS.ort_openvino is not None: - ep_flags += " --use_openvino CPU_FP32" + if FLAGS.ort_openvino is not None: + ep_flags += " --use_openvino CPU_FP32" if target_platform() == "igpu": ep_flags += ( " --skip_tests --cmake_extra_defines 'onnxruntime_BUILD_UNIT_TESTS=OFF'" - ) + ) cuda_archs = "53;62;72;87" - else: + elif not FLAGS.enable_rocm: cuda_archs = "60;61;70;75;80;86;90" df += """ -WORKDIR /workspace/onnxruntime -ARG COMMON_BUILD_ARGS="--config ${{ONNXRUNTIME_BUILD_CONFIG}} --skip_submodule_sync --parallel --build_shared_lib \ - --build_dir /workspace/build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES='{}' " -""".format( - cuda_archs - ) + WORKDIR /workspace/onnxruntime + ARG COMMON_BUILD_ARGS="--config ${{ONNXRUNTIME_BUILD_CONFIG}} --skip_submodule_sync --parallel --build_shared_lib \ + --build_dir /workspace/build --cmake_extra_defines {}={} " + """.format( + cmake_defs, + cuda_archs + ) df += """ -RUN ./build.sh ${{COMMON_BUILD_ARGS}} --update --build {} -""".format( - ep_flags - ) + RUN ./build.sh ${{COMMON_BUILD_ARGS}} --update --build {} + """.format( + ep_flags + ) df += """ -# -# Copy all artifacts needed by the backend to /opt/onnxruntime -# -WORKDIR /opt/onnxruntime - -RUN mkdir -p /opt/onnxruntime && \ - cp /workspace/onnxruntime/LICENSE /opt/onnxruntime && \ - cat /workspace/onnxruntime/cmake/external/onnx/VERSION_NUMBER > /opt/onnxruntime/ort_onnx_version.txt - -# ONNX Runtime headers, libraries and binaries -RUN mkdir -p /opt/onnxruntime/include && \ - cp /workspace/onnxruntime/include/onnxruntime/core/session/onnxruntime_c_api.h \ - /opt/onnxruntime/include && \ - cp /workspace/onnxruntime/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h \ - /opt/onnxruntime/include && \ - cp /workspace/onnxruntime/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h \ - /opt/onnxruntime/include - -RUN mkdir -p /opt/onnxruntime/lib && \ - cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_shared.so \ - /opt/onnxruntime/lib && \ - cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime.so \ - /opt/onnxruntime/lib + # + # Copy all artifacts needed by the backend to /opt/onnxruntime + # + WORKDIR /opt/onnxruntime + + RUN mkdir -p /opt/onnxruntime && \ + cp /workspace/onnxruntime/LICENSE /opt/onnxruntime && \ + cat /workspace/onnxruntime/cmake/external/onnx/VERSION_NUMBER > /opt/onnxruntime/ort_onnx_version.txt + + # ONNX Runtime headers, libraries and binaries + RUN mkdir -p /opt/onnxruntime/include && \ + cp /workspace/onnxruntime/include/onnxruntime/core/session/onnxruntime_c_api.h \ + /opt/onnxruntime/include && \ + cp /workspace/onnxruntime/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h \ + /opt/onnxruntime/include && \ + cp /workspace/onnxruntime/include/onnxruntime/core/providers/cpu/cpu_provider_factory.h \ + /opt/onnxruntime/include + + RUN mkdir -p /opt/onnxruntime/lib && \ + cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_shared.so \ + /opt/onnxruntime/lib && \ + cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime.so \ + /opt/onnxruntime/lib """ if target_platform() == "igpu": df += """ @@ -279,76 +378,90 @@ def dockerfile_for_linux(output_file): """ else: df += """ -RUN mkdir -p /opt/onnxruntime/bin && \ - cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/onnxruntime_perf_test \ - /opt/onnxruntime/bin && \ - cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/onnx_test_runner \ - /opt/onnxruntime/bin && \ - (cd /opt/onnxruntime/bin && chmod a+x *) -""" + RUN mkdir -p /opt/onnxruntime/bin && \ + cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/onnxruntime_perf_test \ + /opt/onnxruntime/bin && \ + cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/onnx_test_runner \ + /opt/onnxruntime/bin && \ + (cd /opt/onnxruntime/bin && chmod a+x *) + """ + if FLAGS.enable_gpu: + df += """ + RUN cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_cuda.so \ + /opt/onnxruntime/lib + """ + + if FLAGS.enable_rocm: + df += """ + RUN cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_rocm.so \ + /opt/onnxruntime/lib + """ - if FLAGS.enable_gpu: - df += """ -RUN cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_cuda.so \ - /opt/onnxruntime/lib -""" + if FLAGS.ort_tensorrt: + df += """ + # TensorRT specific headers and libraries + RUN cp /workspace/onnxruntime/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h \ + /opt/onnxruntime/include && \ + cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_tensorrt.so \ + /opt/onnxruntime/lib + """ + + if FLAGS.ort_migraphx: + df += """ + # MIGraphX specific headers and libraries + RUN cp /workspace/onnxruntime/onnxruntime/core/providers/migraphx/migraphx_provider_factory.h \ + /opt/onnxruntime/include && \ + cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_migraphx.so \ + /opt/onnxruntime/lib + """ - if FLAGS.ort_tensorrt: + if FLAGS.ort_openvino is not None: + df += """ + # OpenVino specific headers and libraries + RUN cp -r ${INTEL_OPENVINO_DIR}/docs/licensing /opt/onnxruntime/LICENSE.openvino + + RUN cp /workspace/onnxruntime/include/onnxruntime/core/providers/openvino/openvino_provider_factory.h \ + /opt/onnxruntime/include + + RUN cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_openvino.so \ + /opt/onnxruntime/lib && \ + cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino.so.${ONNXRUNTIME_OPENVINO_VERSION} \ + /opt/onnxruntime/lib && \ + cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_c.so.${ONNXRUNTIME_OPENVINO_VERSION} \ + /opt/onnxruntime/lib && \ + cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_intel_cpu_plugin.so \ + /opt/onnxruntime/lib && \ + cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_ir_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} \ + /opt/onnxruntime/lib && \ + cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_onnx_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} \ + /opt/onnxruntime/lib && \ + cp /usr/lib/x86_64-linux-gnu/libtbb.so.12 /opt/onnxruntime/lib && \ + cp /usr/lib/x86_64-linux-gnu/libpugixml.so.1 /opt/onnxruntime/lib + + RUN OV_SHORT_VERSION=`echo ${ONNXRUNTIME_OPENVINO_VERSION} | awk '{ split($0,a,"."); print substr(a[1],3) a[2] a[3] }'` && \ + (cd /opt/onnxruntime/lib && \ + chmod a-x * && \ + ln -s libopenvino.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino.so.${OV_SHORT_VERSION} && \ + ln -s libopenvino.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino.so && \ + ln -s libopenvino_c.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_c.so.${OV_SHORT_VERSION} && \ + ln -s libopenvino_c.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_c.so && \ + ln -s libopenvino_ir_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_ir_frontend.so.${OV_SHORT_VERSION} && \ + ln -s libopenvino_ir_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_ir_frontend.so && \ + ln -s libopenvino_onnx_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_onnx_frontend.so.${OV_SHORT_VERSION} && \ + ln -s libopenvino_onnx_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_onnx_frontend.so) + """ + # Linking compiled ONNX Runtime libraries to their corresponding versioned libraries df += """ -# TensorRT specific headers and libraries -RUN cp /workspace/onnxruntime/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_factory.h \ - /opt/onnxruntime/include && \ - cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_tensorrt.so \ - /opt/onnxruntime/lib -""" - - if FLAGS.ort_openvino is not None: + RUN cd /opt/onnxruntime/lib \ + && ln -s libonnxruntime.so libonnxruntime.so.${ONNXRUNTIME_VERSION} + """ df += """ -# OpenVino specific headers and libraries -RUN cp -r ${INTEL_OPENVINO_DIR}/docs/licensing /opt/onnxruntime/LICENSE.openvino - -RUN cp /workspace/onnxruntime/include/onnxruntime/core/providers/openvino/openvino_provider_factory.h \ - /opt/onnxruntime/include - -RUN cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libonnxruntime_providers_openvino.so \ - /opt/onnxruntime/lib && \ - cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino.so.${ONNXRUNTIME_OPENVINO_VERSION} \ - /opt/onnxruntime/lib && \ - cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_c.so.${ONNXRUNTIME_OPENVINO_VERSION} \ - /opt/onnxruntime/lib && \ - cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_intel_cpu_plugin.so \ - /opt/onnxruntime/lib && \ - cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_ir_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} \ - /opt/onnxruntime/lib && \ - cp ${INTEL_OPENVINO_DIR}/runtime/lib/intel64/libopenvino_onnx_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} \ - /opt/onnxruntime/lib && \ - cp /usr/lib/x86_64-linux-gnu/libtbb.so.12 /opt/onnxruntime/lib && \ - cp /usr/lib/x86_64-linux-gnu/libpugixml.so.1 /opt/onnxruntime/lib - -RUN OV_SHORT_VERSION=`echo ${ONNXRUNTIME_OPENVINO_VERSION} | awk '{ split($0,a,"."); print substr(a[1],3) a[2] a[3] }'` && \ - (cd /opt/onnxruntime/lib && \ - chmod a-x * && \ - ln -s libopenvino.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino.so.${OV_SHORT_VERSION} && \ - ln -s libopenvino.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino.so && \ - ln -s libopenvino_c.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_c.so.${OV_SHORT_VERSION} && \ - ln -s libopenvino_c.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_c.so && \ - ln -s libopenvino_ir_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_ir_frontend.so.${OV_SHORT_VERSION} && \ - ln -s libopenvino_ir_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_ir_frontend.so && \ - ln -s libopenvino_onnx_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_onnx_frontend.so.${OV_SHORT_VERSION} && \ - ln -s libopenvino_onnx_frontend.so.${ONNXRUNTIME_OPENVINO_VERSION} libopenvino_onnx_frontend.so) -""" - # Linking compiled ONNX Runtime libraries to their corresponding versioned libraries - df += """ -RUN cd /opt/onnxruntime/lib \ - && ln -s libonnxruntime.so libonnxruntime.so.${ONNXRUNTIME_VERSION} -""" - df += """ -RUN cd /opt/onnxruntime/lib && \ - for i in `find . -mindepth 1 -maxdepth 1 -type f -name '*\.so*'`; do \ - patchelf --set-rpath '$ORIGIN' $i; \ - done + RUN cd /opt/onnxruntime/lib && \ + for i in `find . -mindepth 1 -maxdepth 1 -type f -name '*\.so*'`; do \ + patchelf --set-rpath '$ORIGIN' $i; \ + done -# For testing copy ONNX custom op library and model + # For testing copy ONNX custom op library and model """ if target_platform() == "igpu": df += """ @@ -356,129 +469,173 @@ def dockerfile_for_linux(output_file): """ else: df += """ -RUN mkdir -p /opt/onnxruntime/test && \ - cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libcustom_op_library.so \ - /opt/onnxruntime/test && \ - cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/testdata/custom_op_library/custom_op_test.onnx \ - /opt/onnxruntime/test -""" + RUN mkdir -p /opt/onnxruntime/test && \ + cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/libcustom_op_library.so \ + /opt/onnxruntime/test && \ + cp /workspace/build/${ONNXRUNTIME_BUILD_CONFIG}/testdata/custom_op_library/custom_op_test.onnx \ + /opt/onnxruntime/test + """ - with open(output_file, "w") as dfile: - dfile.write(df) + with open(output_file, "w") as dfile: + dfile.write(df) -def dockerfile_for_windows(output_file): - df = dockerfile_common() + def dockerfile_for_windows(output_file): + df = dockerfile_common() - ## TEMPORARY: Using the tensorrt-8.0 branch until ORT 1.9 release to enable ORT backend with TRT 8.0 support. - # For ORT versions 1.8.0 and below the behavior will remain same. For ORT version 1.8.1 we will - # use tensorrt-8.0 branch instead of using rel-1.8.1 - # From ORT 1.9 onwards we will switch back to using rel-* branches - if FLAGS.ort_version == "1.8.1": - df += """ -SHELL ["cmd", "/S", "/C"] + ## TEMPORARY: Using the tensorrt-8.0 branch until ORT 1.9 release to enable ORT backend with TRT 8.0 support. + # For ORT versions 1.8.0 and below the behavior will remain same. For ORT version 1.8.1 we will + # use tensorrt-8.0 branch instead of using rel-1.8.1 + # From ORT 1.9 onwards we will switch back to using rel-* branches + if FLAGS.ort_version == "1.8.1": + df += """ + SHELL ["cmd", "/S", "/C"] -# -# ONNX Runtime build -# -ARG ONNXRUNTIME_VERSION -ARG ONNXRUNTIME_REPO + # + # ONNX Runtime build + # + ARG ONNXRUNTIME_VERSION + ARG ONNXRUNTIME_REPO -RUN git clone -b tensorrt-8.0 --recursive %ONNXRUNTIME_REPO% onnxruntime && \ - (cd onnxruntime && git submodule update --init --recursive) -""" - else: - df += """ -SHELL ["cmd", "/S", "/C"] + RUN git clone -b tensorrt-8.0 --recursive %ONNXRUNTIME_REPO% onnxruntime && \ + (cd onnxruntime && git submodule update --init --recursive) + """ + else: + df += """ + SHELL ["cmd", "/S", "/C"] + + # + # ONNX Runtime build + # + ARG ONNXRUNTIME_VERSION + ARG ONNXRUNTIME_REPO + RUN git clone -b %ONNXRUNTIME_VERSION% --recursive %ONNXRUNTIME_REPO% onnxruntime && \ + (cd onnxruntime && git submodule update --init --recursive) + """ + + if FLAGS.onnx_tensorrt_tag != "": + df += """ + RUN (cd \\workspace\\onnxruntime\\cmake\\external\\onnx-tensorrt && git fetch origin {}:ortrefbranch && git checkout ortrefbranch) + """.format( + FLAGS.onnx_tensorrt_tag + ) + + ep_flags = "" + if FLAGS.enable_gpu: + ep_flags = "--use_cuda --cmake_extra_defines \"CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80;86;90\" " + if FLAGS.cuda_version is not None: + ep_flags += ' --cuda_version "{}"'.format(FLAGS.cuda_version) + if FLAGS.cuda_home is not None: + ep_flags += ' --cuda_home "{}"'.format(FLAGS.cuda_home) + if FLAGS.cudnn_home is not None: + ep_flags += ' --cudnn_home "{}"'.format(FLAGS.cudnn_home) + if FLAGS.ort_tensorrt: + ep_flags += " --use_tensorrt" + if FLAGS.tensorrt_home is not None: + ep_flags += ' --tensorrt_home "{}"'.format(FLAGS.tensorrt_home) + + if FLAGS.enable_rocm: + df += """ + RUN sed -i 's/list(APPEND HIP_CLANG_FLAGS --amdgpu-target=gfx906 --amdgpu-target=gfx908)/list(APPEND HIP_CLANG_FLAGS --amdgpu-target=gfx906 --amdgpu-target=gfx908 --amdgpu-target=gfx1030)/g' onnxruntime/cmake/onnxruntime_providers.cmake && \ + sed -i 's/Version(torch.__version__) >= Version("1.11.0")/Version(torch.__version__).release >= Version("1.11.0").release/g' /workspace/onnxruntime/onnxruntime/python/tools/transformers/torch_onnx_export_helper.py; \ + RUN export PATH="/opt/cmake/bin:$PATH" + RUN export CXXFLAGS="-D__HIP_PLATFORM_AMD__=1 -w" + """ + ep_flags = "--cmake_extra_defines CMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ --use_rocm --skip_tests" + #if FLAGS.rocm_version is not None: + # ep_flags += ' --rocm_version={}'.format(FLAGS.rocm_version) + if FLAGS.rocm_home is not None: + ep_flags += ' --rocm_home {}'.format(FLAGS.rocm_home) + if FLAGS.ort_migraphx: + ep_flags += " --use_migraphx" + if FLAGS.migraphx_version is not None: + ep_flags += ' --migraphx_version {}'.format(FLAGS.migraphx_version) + if FLAGS.migraphx_home is not None: + ep_flags += ' --migraphx_home {}'.format(FLAGS.migraphx_home) + + ep_flags += " --allow_running_as_root" + + if FLAGS.ort_openvino is not None: + ep_flags += " --use_openvino CPU_FP32" -# -# ONNX Runtime build -# -ARG ONNXRUNTIME_VERSION -ARG ONNXRUNTIME_REPO -RUN git clone -b rel-%ONNXRUNTIME_VERSION% --recursive %ONNXRUNTIME_REPO% onnxruntime && \ - (cd onnxruntime && git submodule update --init --recursive) -""" - if FLAGS.onnx_tensorrt_tag != "": df += """ - RUN (cd \\workspace\\onnxruntime\\cmake\\external\\onnx-tensorrt && git fetch origin {}:ortrefbranch && git checkout ortrefbranch) + WORKDIR /workspace/onnxruntime + ARG VS_DEVCMD_BAT="\BuildTools\VC\Auxiliary\Build\vcvars64.bat" + RUN powershell Set-Content 'build.bat' -value 'call %VS_DEVCMD_BAT%',(Get-Content 'build.bat') + RUN build.bat --cmake_generator "Visual Studio 17 2022" --config Release --skip_submodule_sync --parallel --build_shared_lib --update --build --build_dir /workspace/build {} """.format( - FLAGS.onnx_tensorrt_tag + ep_flags ) - ep_flags = "" - if FLAGS.enable_gpu: - ep_flags = "--use_cuda" - if FLAGS.cuda_version is not None: - ep_flags += ' --cuda_version "{}"'.format(FLAGS.cuda_version) - if FLAGS.cuda_home is not None: - ep_flags += ' --cuda_home "{}"'.format(FLAGS.cuda_home) - if FLAGS.cudnn_home is not None: - ep_flags += ' --cudnn_home "{}"'.format(FLAGS.cudnn_home) - if FLAGS.ort_tensorrt: - ep_flags += " --use_tensorrt" - if FLAGS.tensorrt_home is not None: - ep_flags += ' --tensorrt_home "{}"'.format(FLAGS.tensorrt_home) - if FLAGS.ort_openvino is not None: - ep_flags += " --use_openvino CPU_FP32" + df += """ + # + # Copy all artifacts needed by the backend to /opt/onnxruntime + # + WORKDIR /opt/onnxruntime + RUN copy \\workspace\\onnxruntime\\LICENSE \\opt\\onnxruntime + RUN copy \\workspace\\onnxruntime\\cmake\\external\\onnx\\VERSION_NUMBER \\opt\\onnxruntime\\ort_onnx_version.txt + + # ONNX Runtime headers, libraries and binaries + WORKDIR /opt/onnxruntime/include + RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\session\\onnxruntime_c_api.h \\opt\\onnxruntime\\include + RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\session\\onnxruntime_session_options_config_keys.h \\opt\\onnxruntime\\include + RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\providers\\cpu\\cpu_provider_factory.h \\opt\\onnxruntime\\include + + WORKDIR /opt/onnxruntime/bin + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime.dll \\opt\\onnxruntime\\bin + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_shared.dll \\opt\\onnxruntime\\bin + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_perf_test.exe \\opt\\onnxruntime\\bin + RUN copy \\workspace\\build\\Release\\Release\\onnx_test_runner.exe \\opt\\onnxruntime\\bin + + WORKDIR /opt/onnxruntime/lib + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime.lib \\opt\\onnxruntime\\lib + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_shared.lib \\opt\\onnxruntime\\lib + """ + + if FLAGS.enable_gpu: + df += """ + WORKDIR /opt/onnxruntime/lib + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_cuda.lib \\opt\\onnxruntime\\lib + WORKDIR /opt/onnxruntime/bin + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_cuda.dll \\opt\\onnxruntime\\bin + """ + + if FLAGS.enable_rocm: + df += """ + WORKDIR /opt/onnxruntime/lib + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_rocm.lib \\opt\\onnxruntime\\lib + WORKDIR /opt/onnxruntime/bin + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_rocm.dll \\opt\\onnxruntime\\bin + """ - df += """ -WORKDIR /workspace/onnxruntime -ARG VS_DEVCMD_BAT="\BuildTools\VC\Auxiliary\Build\vcvars64.bat" -RUN powershell Set-Content 'build.bat' -value 'call %VS_DEVCMD_BAT%',(Get-Content 'build.bat') -RUN build.bat --cmake_generator "Visual Studio 17 2022" --config Release --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=60;61;70;75;80;86;90" --skip_submodule_sync --parallel --build_shared_lib --update --build --build_dir /workspace/build {} -""".format( - ep_flags - ) + if FLAGS.ort_tensorrt: + df += """ + # TensorRT specific headers and libraries + WORKDIR /opt/onnxruntime/include + RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\providers\\tensorrt\\tensorrt_provider_factory.h \\opt\\onnxruntime\\include - df += """ -# -# Copy all artifacts needed by the backend to /opt/onnxruntime -# -WORKDIR /opt/onnxruntime -RUN copy \\workspace\\onnxruntime\\LICENSE \\opt\\onnxruntime -RUN copy \\workspace\\onnxruntime\\cmake\\external\\onnx\\VERSION_NUMBER \\opt\\onnxruntime\\ort_onnx_version.txt - -# ONNX Runtime headers, libraries and binaries -WORKDIR /opt/onnxruntime/include -RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\session\\onnxruntime_c_api.h \\opt\\onnxruntime\\include -RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\session\\onnxruntime_session_options_config_keys.h \\opt\\onnxruntime\\include -RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\providers\\cpu\\cpu_provider_factory.h \\opt\\onnxruntime\\include - -WORKDIR /opt/onnxruntime/bin -RUN copy \\workspace\\build\\Release\\Release\\onnxruntime.dll \\opt\\onnxruntime\\bin -RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_shared.dll \\opt\\onnxruntime\\bin -RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_perf_test.exe \\opt\\onnxruntime\\bin -RUN copy \\workspace\\build\\Release\\Release\\onnx_test_runner.exe \\opt\\onnxruntime\\bin - -WORKDIR /opt/onnxruntime/lib -RUN copy \\workspace\\build\\Release\\Release\\onnxruntime.lib \\opt\\onnxruntime\\lib -RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_shared.lib \\opt\\onnxruntime\\lib -""" + WORKDIR /opt/onnxruntime/lib + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_tensorrt.dll \\opt\\onnxruntime\\bin - if FLAGS.enable_gpu: - df += """ -WORKDIR /opt/onnxruntime/lib -RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_cuda.lib \\opt\\onnxruntime\\lib -WORKDIR /opt/onnxruntime/bin -RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_cuda.dll \\opt\\onnxruntime\\bin -""" + WORKDIR /opt/onnxruntime/lib + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_tensorrt.lib \\opt\\onnxruntime\\lib + """ - if FLAGS.ort_tensorrt: - df += """ -# TensorRT specific headers and libraries -WORKDIR /opt/onnxruntime/include -RUN copy \\workspace\\onnxruntime\\include\\onnxruntime\\core\\providers\\tensorrt\\tensorrt_provider_factory.h \\opt\\onnxruntime\\include + if FLAGS.ort_migraphx: + df += """ + # MIGraphX specific headers and libraries + WORKDIR /opt/onnxruntime/include + RUN copy \\workspace\\onnxruntime\\onnxruntime\\core\\providers\\migraphx\\migraphx_provider_factory.h \\opt\\onnxruntime\\include -WORKDIR /opt/onnxruntime/lib -RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_tensorrt.dll \\opt\\onnxruntime\\bin + WORKDIR /opt/onnxruntime/lib + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_migraphx.dll \\opt\\onnxruntime\\bin -WORKDIR /opt/onnxruntime/lib -RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_tensorrt.lib \\opt\\onnxruntime\\lib -""" - with open(output_file, "w") as dfile: - dfile.write(df) + WORKDIR /opt/onnxruntime/lib + RUN copy \\workspace\\build\\Release\\Release\\onnxruntime_providers_migraphx.lib \\opt\\onnxruntime\\lib + """ + with open(output_file, "w") as dfile: + dfile.write(df) def preprocess_gpu_flags(): @@ -510,22 +667,31 @@ def preprocess_gpu_flags(): if FLAGS.tensorrt_home is None: FLAGS.tensorrt_home = "/tensorrt" else: - if "CUDNN_VERSION" in os.environ: - version = None - m = re.match(r"([0-9]\.[0-9])\.[0-9]\.[0-9]", os.environ["CUDNN_VERSION"]) - if m: - version = m.group(1) - if FLAGS.cudnn_home is None: - FLAGS.cudnn_home = "/usr/local/cudnn-{}/cuda".format(version) + if FLAGS.enable_gpu: + if "CUDNN_VERSION" in os.environ: + version = None + m = re.match(r"([0-9]\.[0-9])\.[0-9]\.[0-9]", os.environ["CUDNN_VERSION"]) + if m: + version = m.group(1) + if FLAGS.cudnn_home is None: + FLAGS.cudnn_home = "/usr/local/cudnn-{}/cuda".format(version) - if FLAGS.cuda_home is None: - FLAGS.cuda_home = "/usr/local/cuda" + if FLAGS.cuda_home is None: + FLAGS.cuda_home = "/usr/local/cuda" - if (FLAGS.cuda_home is None) or (FLAGS.cudnn_home is None): - print("error: linux build requires --cudnn-home and --cuda-home") + if (FLAGS.cuda_home is None) or (FLAGS.cudnn_home is None): + print("error: linux build requires --cudnn-home and --cuda-home") + + if FLAGS.tensorrt_home is None: + FLAGS.tensorrt_home = "/usr/src/tensorrt" + + if FLAGS.enable_rocm: + if FLAGS.rocm_home is None: + FLAGS.rocm_home = "/opt/rocm/" + + if FLAGS.migraphx_home is None: + FLAGS.migraphx_home = "/opt/rocm/" - if FLAGS.tensorrt_home is None: - FLAGS.tensorrt_home = "/usr/src/tensorrt" if __name__ == "__main__": @@ -544,6 +710,9 @@ def preprocess_gpu_flags(): parser.add_argument( "--enable-gpu", action="store_true", required=False, help="Enable GPU support" ) + parser.add_argument( + "--enable-rocm", action="store_true", required=False, help="Enable GPU support" + ) parser.add_argument( "--ort-build-config", type=str, @@ -555,7 +724,7 @@ def preprocess_gpu_flags(): "--target-platform", required=False, default=None, - help='Target for build, can be "linux", "windows" or "igpu". If not specified, build targets the current platform.', + help='Target for build, can be "ubuntu", "windows" or "jetpack". If not specified, build targets the current platform.', ) parser.add_argument( @@ -564,6 +733,13 @@ def preprocess_gpu_flags(): parser.add_argument( "--cuda-home", type=str, required=False, help="Home directory for CUDA." ) + parser.add_argument( + "--rocm-version", type=str, required=False, help="Version for ROCM." + ) + parser.add_argument( + "--rocm-home", type=str, required=False, help="Home directory for ROCM." + ) + parser.add_argument( "--cudnn-home", type=str, required=False, help="Home directory for CUDNN." ) @@ -580,15 +756,26 @@ def preprocess_gpu_flags(): help="Enable TensorRT execution provider.", ) parser.add_argument( - "--tensorrt-home", type=str, required=False, help="Home directory for TensorRT." + "--tensorrt-home", type=str, required=False, help="Home directory for TensorRT." ) parser.add_argument( "--onnx-tensorrt-tag", type=str, default="", help="onnx-tensorrt repo tag." ) parser.add_argument("--trt-version", type=str, default="", help="TRT version.") + parser.add_argument( + "--ort-migraphx", + action="store_true", + required=False, + help="Enable MIGraphX execution provider.", + ) + parser.add_argument( + "--migraphx-home", type=str, required=False, help="Home directory for MIGraphX." + ) + parser.add_argument("--migraphx-version", type=str, default="", help="MIGraphX version.") + FLAGS = parser.parse_args() - if FLAGS.enable_gpu: + if FLAGS.enable_gpu or FLAGS.enable_rocm: preprocess_gpu_flags() # if a tag is provided by the user, then simply use it @@ -612,6 +799,11 @@ def preprocess_gpu_flags(): if FLAGS.ort_openvino is not None: print("warning: OpenVINO not supported for windows, ignoring") FLAGS.ort_openvino = None + + print("Writing to output for Windows") dockerfile_for_windows(FLAGS.output) + print("Done") else: + print("Writing to output for Linux") dockerfile_for_linux(FLAGS.output) + print("Done")