From 64441616db8780be175bc32dbc7fa5eecdcc0fbf Mon Sep 17 00:00:00 2001 From: zyy17 Date: Sat, 18 Mar 2023 16:16:15 +0800 Subject: [PATCH] ci: refactor compile-python.sh and use the python310 to build amd64 binary (#1199) --- .github/workflows/release.yml | 47 ++++++++++++-- docker/aarch64/compile-python.sh | 103 +++++++++++++++++++++---------- 2 files changed, 113 insertions(+), 37 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7e0e496239a4..540396fab9bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -121,11 +121,12 @@ jobs: sudo apt-get -y update sudo apt-get -y install libssl-dev pkg-config g++-aarch64-linux-gnu gcc-aarch64-linux-gnu binutils-aarch64-linux-gnu wget - - name: Compile Python 3.10.10 from source for aarch64-linux - if: contains(matrix.arch, 'aarch64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend') + # FIXME(zyy17): Should we specify the version of python when building binary for darwin? + - name: Compile Python 3.10.10 from source for linux + if: contains(matrix.arch, 'linux') && contains(matrix.opts, 'pyo3_backend') run: | sudo chmod +x ./docker/aarch64/compile-python.sh - sudo ./docker/aarch64/compile-python.sh + sudo ./docker/aarch64/compile-python.sh ${{ matrix.arch }} - name: Install rust toolchain uses: dtolnay/rust-toolchain@master @@ -144,13 +145,47 @@ jobs: if: contains(matrix.arch, 'aarch64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend') run: | # TODO(zyy17): We should make PYO3_CROSS_LIB_DIR configurable. - export PYO3_CROSS_LIB_DIR=$(pwd)/python_arm64_build/lib + export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64 + export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH + export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH + export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH + + export PYO3_CROSS_LIB_DIR=${PWD}/python-3.10.10/aarch64 echo "PYO3_CROSS_LIB_DIR: $PYO3_CROSS_LIB_DIR" - alias python=python3 + alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3 + alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip + + cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }} + + - name: Run cargo build with pyo3 for amd64-linux + if: contains(matrix.arch, 'x86_64-unknown-linux-gnu') && contains(matrix.opts, 'pyo3_backend') + run: | + export PYTHON_INSTALL_PATH_AMD64=${PWD}/python-3.10.10/amd64 + export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH + export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH + export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH + + echo "implementation=CPython" >> pyo3.config + echo "version=3.10" >> pyo3.config + echo "implementation=CPython" >> pyo3.config + echo "shared=true" >> pyo3.config + echo "abi3=true" >> pyo3.config + echo "lib_name=python3.10" >> pyo3.config + echo "lib_dir=$PYTHON_INSTALL_PATH_AMD64/lib" >> pyo3.config + echo "executable=$PYTHON_INSTALL_PATH_AMD64/bin/python3" >> pyo3.config + echo "pointer_width=64" >> pyo3.config + echo "build_flags=" >> pyo3.config + echo "suppress_build_script_link_lines=false" >> pyo3.config + + cat pyo3.config + export PYO3_CONFIG_FILE=${PWD}/pyo3.config + alias python=$PYTHON_INSTALL_PATH_AMD64/bin/python3 + alias pip=$PYTHON_INSTALL_PATH_AMD64/bin/python3-pip + cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }} - name: Run cargo build - if: contains(matrix.arch, 'aarch64-unknown-linux-gnu') == false || contains(matrix.opts, 'pyo3_backend') == false + if: contains(matrix.arch, 'darwin') || contains(matrix.opts, 'pyo3_backend') == false run: cargo build --profile ${{ env.CARGO_PROFILE }} --locked --target ${{ matrix.arch }} ${{ matrix.opts }} - name: Calculate checksum and rename binary diff --git a/docker/aarch64/compile-python.sh b/docker/aarch64/compile-python.sh index eaabb092658a..ba0bef763de9 100644 --- a/docker/aarch64/compile-python.sh +++ b/docker/aarch64/compile-python.sh @@ -1,9 +1,36 @@ +#!/usr/bin/env bash + +set -e + # this script will download Python source code, compile it, and install it to /usr/local/lib # then use this python to compile cross-compiled python for aarch64 +ARCH=$1 +PYTHON_VERSION=3.10.10 +PYTHON_SOURCE_DIR=Python-${PYTHON_VERSION} +PYTHON_INSTALL_PATH_AMD64=${PWD}/python-${PYTHON_VERSION}/amd64 +PYTHON_INSTALL_PATH_AARCH64=${PWD}/python-${PYTHON_VERSION}/aarch64 + +function download_python_source_code() { + wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz + tar -xvf Python-$PYTHON_VERSION.tgz +} + +function compile_for_amd64_platform() { + mkdir -p "$PYTHON_INSTALL_PATH_AMD64" + + echo "Compiling for amd64 platform..." + + ./configure \ + --prefix="$PYTHON_INSTALL_PATH_AMD64" \ + --enable-shared \ + ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \ + ac_cv_have_long_long_format=yes \ + --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no + + make + make install +} -wget https://www.python.org/ftp/python/3.10.10/Python-3.10.10.tgz -tar -xvf Python-3.10.10.tgz -cd Python-3.10.10 # explain Python compile options here a bit:s # --enable-shared: enable building a shared Python library (default is no) but we do need it for calling from rust # CC, CXX, AR, LD, RANLIB: set the compiler, archiver, linker, and ranlib programs to use @@ -14,33 +41,47 @@ cd Python-3.10.10 # ac_cv_have_long_long_format=yes: target platform supports long long type # disable-ipv6: disable ipv6 support, we don't need it in here # ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no: disable pty support, we don't need it in here +function compile_for_aarch64_platform() { + export LD_LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LD_LIBRARY_PATH + export LIBRARY_PATH=$PYTHON_INSTALL_PATH_AMD64/lib:$LIBRARY_PATH + export PATH=$PYTHON_INSTALL_PATH_AMD64/bin:$PATH + + mkdir -p "$PYTHON_INSTALL_PATH_AARCH64" + + echo "Compiling for aarch64 platform..." + echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH" + echo "LIBRARY_PATH: $LIBRARY_PATH" + echo "PATH: $PATH" + + ./configure --build=x86_64-linux-gnu --host=aarch64-linux-gnu \ + --prefix="$PYTHON_INSTALL_PATH_AARCH64" --enable-optimizations \ + CC=aarch64-linux-gnu-gcc \ + CXX=aarch64-linux-gnu-g++ \ + AR=aarch64-linux-gnu-ar \ + LD=aarch64-linux-gnu-ld \ + RANLIB=aarch64-linux-gnu-ranlib \ + --enable-shared \ + ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \ + ac_cv_have_long_long_format=yes \ + --disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no + + make + make altinstall +} + +# Main script starts here. +download_python_source_code + +# Enter the python source code directory. +cd $PYTHON_SOURCE_DIR || exit 1 # Build local python first, then build cross-compiled python. -./configure \ ---enable-shared \ -ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \ -ac_cv_have_long_long_format=yes \ ---disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no && \ -make -make install -cd .. -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/ -export LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib/ -export PY_INSTALL_PATH=$(pwd)/python_arm64_build -cd Python-3.10.10 && \ -make clean && \ -make distclean && \ -alias python=python3 && \ -./configure --build=x86_64-linux-gnu --host=aarch64-linux-gnu \ ---prefix=$PY_INSTALL_PATH --enable-optimizations \ -CC=aarch64-linux-gnu-gcc \ -CXX=aarch64-linux-gnu-g++ \ -AR=aarch64-linux-gnu-ar \ -LD=aarch64-linux-gnu-ld \ -RANLIB=aarch64-linux-gnu-ranlib \ ---enable-shared \ -ac_cv_pthread_is_default=no ac_cv_pthread=yes ac_cv_cxx_thread=yes \ -ac_cv_have_long_long_format=yes \ ---disable-ipv6 ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no && \ -make && make altinstall && \ -cd .. +compile_for_amd64_platform + +# Clean the build directory. +make clean && make distclean + +# Cross compile python for aarch64. +if [ "$ARCH" = "aarch64-unknown-linux-gnu" ]; then + compile_for_aarch64_platform +fi