Merge pull request #1469 from 0x3878f/pipeline

Enhance monitoring for macOS and Windows platforms using github action
PaddlePaddle · Jan 20, 2025 · df6d78a · df6d78a
2 parents df05e23 + 6bf9d26
commit df6d78a
Show file tree

Hide file tree

Showing 14 changed files with 367 additions and 277 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -2,9 +2,9 @@ name: Build and Test
 
 on:
   push:
-    branches: [ "develop" ]
+    branches: [ "test_pir" ]
   pull_request:
-    branches: [ "develop" ]
+    branches: [ "test_pir" ]
 
 permissions:
   contents: read
@@ -15,20 +15,19 @@ jobs:
     strategy:
       matrix:
         python-version: [ '3.8']
-        architecture: [ 'x64' ]
-
+        architecture: [ 'x86_64']
     steps:
       # Checkout the latest branch of Paddle2ONNX.
       - name: Checkout Paddle2ONNX
         uses: actions/checkout@v4
         with:
           submodules: true
 
-      - name: Build on manylinux2014_x86_64
-        uses: docker://quay.io/pypa/manylinux2014_x86_64:latest
+      - name: Build on manylinux_2_28_x86_64
+        uses: docker://quay.io/pypa/manylinux_2_28_x86_64:latest
         with:
           entrypoint: bash
-          args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_x86_64 CentOS
+          args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }}
 
       - name: Setup Python 3.8
         uses: actions/setup-python@v5
@@ -50,49 +49,62 @@ jobs:
     runs-on: windows-2019
     strategy:
       matrix:
-        python-version: [ '3.8' ]
+        python-version: [ '3.10' ]
         architecture: [ 'x64' ]
 
     steps:
-    # Checkout the latest branch of Paddle2ONNX.
-    - name: Checkout Paddle2ONNX
-      uses: actions/checkout@v4
-      with:
-        submodules: true
-
-    # Download and extract protobuf
-    - name: Download and extract protobuf
-      run: |
-        curl -L -o protobuf.zip https://bj.bcebos.com/fastdeploy/third_libs/protobuf-win-x64-3.16.0.zip
-        powershell -Command "Expand-Archive -Path protobuf.zip -DestinationPath $PWD/protobuf"
-
-    # Setup Python
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    # Install Python dependencies
-    - name: Install Python dependencies
-      run: |
-        python -m pip install -q --upgrade pip
-        python -m pip install setuptools wheel auditwheel auditwheel-symbols build
-
-    # Build package
-    - name: Build package
-      run: |
-        $Env:PATH = "${{ github.workspace }}\protobuf\bin;" + $Env:PATH
-        python -m build --wheel
-
-    # Install Paddle2ONNX
-    - name: Install Paddle2ONNX
-      run: |
-        Get-ChildItem -Path dist/*.whl | foreach {python -m pip install --upgrade $_.fullname}
-
-    # Install Test
-    - name: Run Test
-      working-directory: ./tests
-      run: |
-        .\run.bat python
+      # Checkout the latest branch of Paddle2ONNX.
+      - name: Checkout Paddle2ONNX
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      # Clone protobuf repository and checkout to v21.12
+      - name: Clone protobuf
+        run: |
+          git clone https://github.com/protocolbuffers/protobuf.git
+          cd protobuf
+          git checkout v21.12
+
+      # Build and install protobuf
+      - name: Build and install protobuf
+        run: |
+          cd protobuf
+          git submodule update --init --recursive
+          mkdir build
+          cd build
+          cmake -G "Visual Studio 16 2019"  -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}\protobuf\protobuf_install\ -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_EXAMPLES=OFF ..
+          cmake --build . --config Release --target install
+
+      # Setup Python
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # Install Python dependencies
+      - name: Install Python dependencies
+        run: |
+          python -m pip install -q --upgrade pip
+          python -m pip install setuptools wheel auditwheel auditwheel-symbols build
+
+      # Build package
+      - name: Build package
+        run: |
+          $Env:PATH = "${{ github.workspace }}\protobuf\protobuf_install\bin;" + $Env:PATH
+          $Env:PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/"
+          python -m build --wheel
+
+      # Install Paddle2ONNX
+      - name: Install Paddle2ONNX
+        run: |
+          Get-ChildItem -Path dist/*.whl | foreach {python -m pip install --upgrade $_.fullname}
+
+      # Install Test
+      - name: Run Test
+        working-directory: ./tests
+        run: |
+          .\run.bat python
 
   build_on_macos:
     # Use MACOSX_DEPLOYMENT_TARGET=12.0 to produce compatible wheel
@@ -103,7 +115,6 @@ jobs:
       matrix:
         python-version: [ '3.8']
         architecture: [ 'arm64' ]
-
     steps:
       # Checkout the latest branch of Paddle2ONNX.
       - name: Checkout Paddle2ONNX
@@ -117,10 +128,22 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      # Download protobuf
-      - name: Download protobuf
+      # Clone protobuf repository and checkout to v21.12
+      - name: Clone protobuf
+        run: |
+          git clone https://github.com/protocolbuffers/protobuf.git
+          cd protobuf
+          git checkout v21.12
+
+      # Build and install protobuf
+      - name: Build and install protobuf
         run: |
-          source .github/workflows/scripts/download_protobuf.sh
+          cd protobuf
+          git submodule update --init --recursive
+          mkdir build
+          cd build
+          cmake ../cmake -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed_protobuf -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=14
+          cmake --build . --target install
 
       # Install Python dependencies
       - name: Install Python dependencies
@@ -132,6 +155,7 @@ jobs:
       - name: Build package
         run: |
           export PATH="${{ github.workspace }}/installed_protobuf/bin:$PATH"
+          export PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/"
           python -m build --wheel
 
       # Install Paddle2ONNX

diff --git a/.github/workflows/release_linux_aarch64.yml b/.github/workflows/release_linux_aarch64.yml
@@ -14,7 +14,7 @@ jobs:
       matrix:
         python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12']
         architecture: [ 'x64' ]
-    
+
     steps:
       # Checkout the latest branch of Paddle2ONNX.
       - name: Checkout Paddle2ONNX
@@ -25,11 +25,11 @@ jobs:
       # setting up qemu for enabling aarch64 binary execution on x86 machine
       - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
 
-      - name: Build on manylinux2014_aarch64
-        uses: docker://quay.io/pypa/manylinux2014_aarch64:latest
+      - name: Build on manylinux_2_28_aarch64
+        uses: docker://quay.io/pypa/manylinux_2_28_aarch64
         with:
           entrypoint: bash
-          args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_aarch64 CentOS
+          args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }}
 
       - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
         with:
@@ -40,4 +40,4 @@ jobs:
         uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
         with:
           user: __token__
-          password: ${{ secrets.PADDLE2ONNX_API_TOKEN }}
+          password: ${{ secrets.PADDLE2ONNX_API_TOKEN }}
diff --git a/.github/workflows/release_linux_x86_64.yml b/.github/workflows/release_linux_x86_64.yml
@@ -22,11 +22,11 @@ jobs:
         with:
           submodules: true
 
-      - name: Build on manylinux2014_x86_64
-        uses: docker://quay.io/pypa/manylinux2014_x86_64:latest
+      - name: Build on manylinux_2_28_x86_64
+        uses: docker://quay.io/pypa/manylinux_2_28_x86_64:latest
         with:
           entrypoint: bash
-          args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }} manylinux2014_x86_64 CentOS
+          args: .github/workflows/scripts/entrypoint.sh ${{ matrix.python-version }}
 
       - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32
         with:
@@ -37,4 +37,4 @@ jobs:
         uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
         with:
           user: __token__
-          password: ${{ secrets.PADDLE2ONNX_API_TOKEN }}
+          password: ${{ secrets.PADDLE2ONNX_API_TOKEN }}
diff --git a/.github/workflows/release_mac.yml b/.github/workflows/release_mac.yml
@@ -26,22 +26,28 @@ jobs:
         with:
           submodules: true
 
-      # Checkout the latest branch of Paddle2ONNX.
-      - name: Checkout Paddle2ONNX
-        uses: actions/checkout@v4
-        with:
-          submodules: true
-
       # Setup Python
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: ${{ matrix.python-version }}
 
-      # Download protobuf
-      - name: Download protobuf
+      # Clone protobuf repository and checkout to v21.12
+      - name: Clone protobuf
+        run: |
+          git clone https://github.com/protocolbuffers/protobuf.git
+          cd protobuf
+          git checkout v21.12
+
+      # Build and install protobuf
+      - name: Build and install protobuf
         run: |
-          source .github/workflows/scripts/download_protobuf.sh
+          cd protobuf
+          git submodule update --init --recursive
+          mkdir build
+          cd build
+          cmake ../cmake -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/installed_protobuf -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=14
+          cmake --build . --target install
 
       # Install Python dependencies
       - name: Install Python dependencies
@@ -53,6 +59,7 @@ jobs:
       - name: Build package
         run: |
           export PATH="${{ github.workspace }}/installed_protobuf/bin:$PATH"
+          export PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/"
           python -m build --wheel
 
       - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32

diff --git a/.github/workflows/release_win_amd64.yml b/.github/workflows/release_win_amd64.yml
@@ -12,7 +12,8 @@ jobs:
     runs-on: windows-2019
     strategy:
       matrix:
-        python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12']
+        # python-version: [ '3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.10']
         architecture: [ 'x64' ]
 
     steps:
@@ -22,17 +23,29 @@ jobs:
         with:
           submodules: true
 
-      # Download and extract protobuf
-      - name: Download and extract protobuf
+      # Clone protobuf repository and checkout to v21.12
+      - name: Clone protobuf
         run: |
-          curl -L -o protobuf.zip https://bj.bcebos.com/fastdeploy/third_libs/protobuf-win-x64-3.16.0.zip
-          powershell -Command "Expand-Archive -Path protobuf.zip -DestinationPath $PWD/protobuf"
+          git clone https://github.com/protocolbuffers/protobuf.git
+          cd protobuf
+          git checkout v21.12
+
+      # Build and install protobuf
+      - name: Build and install protobuf
+        run: |
+          cd protobuf
+          git submodule update --init --recursive
+          mkdir build
+          cd build
+          cmake -G "Visual Studio 16 2019"  -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}\protobuf\protobuf_install\ -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_BUILD_EXAMPLES=OFF ..
+          cmake --build . --config Release --target install
 
       # Setup Python
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: ${{ matrix.python-version }}
+
       # Install Python dependencies
       - name: Install Python dependencies
         run: |
@@ -41,7 +54,8 @@ jobs:
 
       - name: Build package
         run: |
-          $Env:PATH = "${{ github.workspace }}\protobuf\bin;" + $Env:PATH
+          $Env:PATH = "${{ github.workspace }}\protobuf\protobuf_install\bin;" + $Env:PATH
+          $Env:PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/"
           python -m build --wheel
 
       - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32

diff --git a/.github/workflows/scripts/entrypoint.sh b/.github/workflows/scripts/entrypoint.sh
@@ -1,5 +1,19 @@
 #!/bin/bash
 
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Copyright (c) ONNX Project Contributors
 #
 # SPDX-License-Identifier: Apache-2.0
@@ -24,20 +38,31 @@ PYTHON_COMMAND="/opt/python/${PY_VER}/bin/python"
 $PIP_INSTALL_COMMAND --upgrade pip
 $PIP_INSTALL_COMMAND cmake
 
-# Build protobuf from source
-if [[ "$SYSTEM_NAME" == "CentOS" ]]; then
-    yum install -y wget
-fi
-source .github/workflows/scripts/download_protobuf.sh
+# Build and install protobuf
+original_dir=$(pwd)
+git clone https://github.com/protocolbuffers/protobuf.git
+cd protobuf
+git checkout v21.12
+git submodule update --init
+mkdir build_source && cd build_source
+cmake ../cmake -DCMAKE_INSTALL_PREFIX=`pwd`/installed_protobuf_lib -Dprotobuf_BUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release
+cmake --build . --target install
+export PATH=`pwd`/installed_protobuf_lib/bin:${PATH}
+cd $original_dir
+
+export PIP_EXTRA_INDEX_URL="https://www.paddlepaddle.org.cn/packages/nightly/cpu/"
 
 # Build Paddle2ONNX wheels
 $PYTHON_COMMAND -m build --wheel || { echo "Building wheels failed."; exit 1; }
 
+#  ============================================================================
+#   Due to libpaddle’s limitations, it can only maintain a restricted platform tag of ‘linux_x86_64’.
+#  ============================================================================
 # Bundle external shared libraries into the wheels
 # find -exec does not preserve failed exit codes, so use an output file for failures
 failed_wheels=$PWD/failed-wheels
 rm -f "$failed_wheels"
-find . -type f -iname "*-linux*.whl" -exec sh -c "auditwheel repair '{}' -w \$(dirname '{}') --plat '${PLAT}' || { echo 'Repairing wheels failed.'; auditwheel show '{}' >> '$failed_wheels'; }" \;
+find . -type f -iname "*-linux*.whl" -exec sh -c "auditwheel repair '{}' -w \$(dirname '{}') --exclude libpaddle.so || { echo 'Repairing wheels failed.'; auditwheel show '{}' >> '$failed_wheels'; }" \;
 
 if [[ -f "$failed_wheels" ]]; then
     echo "Repairing wheels failed:"

diff --git a/VERSION_NUMBER b/VERSION_NUMBER
@@ -1 +1 @@
-1.3.1
+2.0.0