Refix openmp on mac

HGuillemet · Jun 30, 2024 · 023ebaf · 023ebaf
1 parent 9070826
commit 023ebaf
Show file tree

Hide file tree

Showing 8 changed files with 40 additions and 58 deletions.
diff --git a/.github/actions/deploy-macosx/action.yml b/.github/actions/deploy-macosx/action.yml
@@ -24,13 +24,8 @@ runs:
           sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/MacOSX10*
         fi
 
-        brew uninstall --force --ignore-dependencies gcc gcc@7 gcc@8 gcc@9 gcc@10 gcc@11 gcc@12 gcc@13 little-cms2 maven openblas r # imagemagick libraw libuv
-        brew install boost ccache swig autoconf-archive automake cmake libtool libusb ant nasm xz pkg-config sdl2 gpg1 bison flex perl ragel binutils gradle gmp isl libmpc mpfr geoip pcre ssdeep yajl libomp llvm@18
-        
-        # Install version of libomp for llvm 15
-        # brew uninstall -f libomp
-        #curl -LO https://mac.r-project.org/openmp/openmp-15.0.7-darwin20-Release.tar.gz
-        #tar fvxz openmp-15.0.7-darwin20-Release.tar.gz -C /
+        brew uninstall --force --ignore-dependencies gcc gcc@7 gcc@8 gcc@9 gcc@10 gcc@11 gcc@12 gcc@13 little-cms2 maven openblas r
+        brew install boost ccache swig autoconf-archive automake cmake libomp libtool libusb ant nasm xz pkg-config sdl2 gpg1 bison flex perl ragel binutils gradle gmp isl libmpc mpfr geoip pcre ssdeep yajl
         
         # echo Installing an older less buggy version of CMake
         # curl -LO https://raw.githubusercontent.com/Homebrew/homebrew-core/30a3e5e02420a515f9bad02e3cfcde95316db529/Formula/cmake.rb
@@ -61,9 +56,15 @@ runs:
           brew link libomp --force -v
         fi
 
-        sudo install_name_tool -add_rpath @loader_path/. -id @rpath/libomp.dylib `brew ls libomp|grep libomp.dylib`
-        sudo install_name_tool -add_rpath @loader_path/. -id @rpath/libSDL2-2.0.0.dylib `brew ls sdl2|grep libSDL2-2.0.0.dylib`
-        sudo install_name_tool -add_rpath @loader_path/. -id @rpath/libusb-1.0.0.dylib `brew ls libusb|grep libusb-1.0.0.dylib`
+        # Change rpath and resign all brew libraries that could be bundled in presets
+        for i in libomp sdl2 libusb; do
+          for l in $(brew ls $i|grep .dylib); do
+            if [ ! -L $l ]; then 
+              install_name_tool -add_rpath @loader_path/. -id @rpath/$(basename $l) $l
+              codesign --force -s - $l
+            fi
+          done
+        done
 
         curl -LO https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz
         sudo tar -xzf apache-maven-3.6.3-bin.tar.gz -C /usr/local/opt
@@ -109,10 +110,12 @@ runs:
     - name: Build project
       shell: bash
       run: |
-        export LIBRARY_PATH="/usr/local/lib/gcc/10/"
-        export LD_LIBRARY_PATH="$LIBRARY_PATH"
-        export DYLD_LIBRARY_PATH="$LIBRARY_PATH"
-        export DYLD_FALLBACK_LIBRARY_PATH="$LIBRARY_PATH"
+        if [[ -n ${CI_DEPLOY_NEED_GCC:-} ]]; then
+          export LIBRARY_PATH="/usr/local/lib/gcc/10/"
+          export LD_LIBRARY_PATH="$LIBRARY_PATH"
+          export DYLD_LIBRARY_PATH="$LIBRARY_PATH"
+          export DYLD_FALLBACK_LIBRARY_PATH="$LIBRARY_PATH"
+        fi
 
         # Change default JVM on macos-14 runner
         if [[ -n ${JAVA_HOME_17_arm64:-} ]]; then
@@ -122,7 +125,6 @@ runs:
         git --version
         clang --version
         /usr/local/bin/gcc-* --version || true
-        which cmake
         cmake --version
         gradle --version
         mvn -version

diff --git a/.github/workflows/cpython.yml b/.github/workflows/cpython.yml
@@ -44,22 +44,22 @@ jobs:
   macosx-x86_64:
     runs-on: macos-12
     steps:
-      - uses: bytedeco/javacpp-presets/.github/actions/deploy-macosx@actions
+      - uses: HGuillemet/javacpp-presets/.github/actions/deploy-macosx@apteryx_pytorch
   macosx-arm64:
     runs-on: macos-14
     steps:
-      - uses: HGuillemet/javacpp-presets/.github/actions/deploy-macosx@cpython-macosx-arm64
+      - uses: HGuillemet/javacpp-presets/.github/actions/deploy-macosx@apteryx_pytorch
 #  windows-x86:
 #    runs-on: windows-2019
 #    steps:
 #      - uses: bytedeco/javacpp-presets/.github/actions/deploy-windows@actions
   windows-x86_64:
     runs-on: windows-2019
     steps:
-      - uses: bytedeco/javacpp-presets/.github/actions/deploy-windows@actions
+      - uses: HGuillemet/javacpp-presets/.github/actions/deploy-windows@apteryx_pytorch
   redeploy:
     needs: [linux-x86_64, macosx-arm64, macosx-x86_64, windows-x86_64]
 #    needs: [linux-armhf, linux-arm64, linux-ppc64le, linux-x86, linux-x86_64, macosx-x86_64, windows-x86, windows-x86_64]
     runs-on: ubuntu-20.04
     steps:
-      - uses: bytedeco/javacpp-presets/.github/actions/redeploy@actions
+      - uses: HGuillemet/javacpp-presets/.github/actions/redeploy@apteryx_pytorch
diff --git a/.github/workflows/pytorch.yml b/.github/workflows/pytorch.yml
@@ -33,6 +33,10 @@ jobs:
       - uses: HGuillemet/javacpp-presets/.github/actions/deploy-ubuntu@apteryx_pytorch
         timeout-minutes: 350
   macosx-arm64:
+    runs-on: macos-14
+    steps:
+      - uses: HGuillemet/javacpp-presets/.github/actions/deploy-macosx@apteryx_pytorch
+  macosx-x86_64:
     runs-on: macos-12
     steps:
       - uses: HGuillemet/javacpp-presets/.github/actions/deploy-macosx@apteryx_pytorch
@@ -58,7 +62,7 @@ jobs:
       - uses: HGuillemet/javacpp-presets/.github/actions/deploy-windows@apteryx_pytorch
         timeout-minutes: 350
   redeploy:
-    needs: [linux-x86_64, macosx-arm64, windows-x86_64]
+    needs: [linux-x86_64, macosx-arm64, macosx-x86_64, windows-x86_64]
     runs-on: ubuntu-20.04
     steps:
       - uses: HGuillemet/javacpp-presets/.github/actions/redeploy@apteryx_pytorch
diff --git a/cpython/platform/pom.xml b/cpython/platform/pom.xml
@@ -10,7 +10,7 @@
     <relativePath>../../</relativePath>
   </parent>
 
-  <groupId>org.bytedeco</groupId>
+  <groupId>fr.apteryx</groupId>
   <artifactId>cpython-platform</artifactId>
   <version>3.12.4-${project.parent.version}</version>
   <name>JavaCPP Presets Platform for CPython</name>

diff --git a/cpython/pom.xml b/cpython/pom.xml
@@ -9,7 +9,7 @@
     <version>1.5.11-SNAPSHOT</version>
   </parent>
 
-  <groupId>org.bytedeco</groupId>
+  <groupId>fr.apteryx</groupId>
   <artifactId>cpython</artifactId>
   <version>3.12.4-${project.parent.version}</version>
   <name>JavaCPP Presets for CPython</name>

diff --git a/pom.xml b/pom.xml
@@ -1481,7 +1481,6 @@
         <module>tesseract</module>
         <module>pytorch</module>
         <module>sentencepiece</module>
-        <module>cpython</module>
       </modules>
       <properties>
         <javacpp.platform.android-arm></javacpp.platform.android-arm>

diff --git a/pytorch/cppbuild.sh b/pytorch/cppbuild.sh
@@ -143,16 +143,16 @@ case $PLATFORM in
     macosx-arm64)
         export CC="clang"
         export CXX="clang++"
-        export PATH=$(brew --prefix llvm@18)/bin:$PATH # Use brew LLVM 15 instead of Xcode LLVM 14
-        export CMAKE_OSX_ARCHITECTURES=arm64 # enable cross-compilation on a x86_64 host machine
+        # export PATH=$(brew --prefix llvm@18)/bin:$PATH # Use brew LLVM instead of Xcode LLVM 14
         export USE_MKLDNN=OFF
         export USE_QNNPACK=OFF # not compatible with arm64 as of PyTorch 2.1.2
         export CMAKE_OSX_DEPLOYMENT_TARGET=11.00 # minimum needed for arm64 support
         ;;
     macosx-x86_64)
         export CC="clang"
         export CXX="clang++"
-        export PATH=$(brew --prefix llvm@18)/bin:$PATH # Use brew LLVM 15 instead of Xcode LLVM 14
+        export USE_MKLDNN=OFF
+        # export PATH=$(brew --prefix llvm@18)/bin:$PATH # Use brew LLVM instead of Xcode LLVM 14
         ;;
     windows-x86_64)
         if which ccache.exe; then
@@ -212,36 +212,9 @@ sedinplace 's/const std::string& interface)/const std::string\& interface_name)/
 # Wait for eventual upstream fix, or for cmake 2.30 that allows to choose between -openmp and -openmp:experimental
 # and see if choosing experimental works. See Issue #1503.
 # On Linux, pytorch FindOpenMP.cmake picks llvm libomp over libgomp. See Issue #1504.
-# Keep it on MacOS since it appends the correct -Xpreprocessor -fopenmp -I/usr/local/include flags
-if [[ ! $PLATFORM == macosx-* ]]; then
-  rm cmake/Modules/FindOpenMP.cmake
-  sedinplace 's/include(${CMAKE_CURRENT_LIST_DIR}\/Modules\/FindOpenMP.cmake)/find_package(OpenMP)/g' cmake/Dependencies.cmake
-fi
-if [[ $PLATFORM == macosx-* ]]; then
-  brew ls libomp
-  #export CMAKE_INCLUDE_PATH=/usr/local/Cellar/libomp/18.1.7/include
-  #export CMAKE_LIBRARY_PATH=/usr/local/Cellar/libomp/18.1.7/lib
-  #if [[ ! -e $CMAKE_INCLUDE_PATH ]]; then
-  #    echo libomp 18.1.7 not found
-  #    exit 1
-  #fi
-  export CMAKE_INCLUDE_PATH=/usr/local/include
-  export CMAKE_LIBRARY_PATH=/usr/local/lib
-  #echo Setting CMAKE_INCLUDE_PATH=$CMAKE_INCLUDE_PATH
-  #echo Setting CMAKE_LIBRARY_PATH=$CMAKE_LIBRARY_PATH
-  export LD_LIBRARY_PATH=$CMAKE_LIBRARY_PATH
-  export DYLD_LIBRARY_PATH=$CMAKE_LIBRARY_PATH
-  export OpenMP_C_INCLUDE_DIR=$CMAKE_INCLUDE_PATH
-  export OpenMP_CXX_INCLUDE_DIR=$CMAKE_INCLUDE_PATH
-  export CXXFLAGS="-I$CMAKE_INCLUDE_PATH ${CXXFLAGS:-}"
-  export CFLAGS="-I$CMAKE_INCLUDE_PATH ${CFLAGS:-}"
-  export LDFLAGS="-L$CMAKE_LIBRARY_PATH ${LDFLAGS:-}"
-  #echo Setting CFLAGS=$CFLAGS
-  #echo Setting CXXFLAGS=$CXXFLAGS
-
-  #curl -O https://mac.r-project.org/openmp/openmp-14.0.6-darwin20-Release.tar.gz
-  #tar fvxz openmp-14.0.6-darwin20-Release.tar.gz -C /
-fi
+# On MacOS CMake standard version works tooL
+rm cmake/Modules/FindOpenMP.cmake
+sedinplace 's/include(${CMAKE_CURRENT_LIST_DIR}\/Modules\/FindOpenMP.cmake)/find_package(OpenMP)/g' cmake/Dependencies.cmake
 
 #USE_FBGEMM=0 USE_KINETO=0 USE_GLOO=0 USE_MKLDNN=0 \
 "$PYTHON_BIN_PATH" setup.py build
@@ -256,7 +229,7 @@ ln -sf pytorch/torch/bin ../bin
 
 case $PLATFORM in
     macosx-*)
-        cp /usr/local/lib/libomp.dylib ../lib
+        cp $(brew ls libomp|grep libomp.dylib) ../lib
         ;;
     windows-*)
         cp ../libuv/dist/lib/Release/* ../lib

diff --git a/pytorch/src/main/java/org/bytedeco/pytorch/presets/torch.java b/pytorch/src/main/java/org/bytedeco/pytorch/presets/torch.java
@@ -109,9 +109,13 @@
             extension = "-gpu"
         ),
         @Platform(
-            value = {"linux", "macosx"},
+            value = {"linux"},
             link = { "c10", "torch", "torch_cpu" }
         ),
+        @Platform(
+            value = {"macosx"},
+            link = { "c10", "torch", "torch_cpu", "omp" }
+        ),
         @Platform(
             value = "windows",
             link = { "c10", "torch", "torch_cpu", "uv" }