diff --git a/.github/workflows/build-python-wrapper/action.yml b/.github/workflows/build-python-wrapper/action.yml index 5406fba855..0b686db40e 100644 --- a/.github/workflows/build-python-wrapper/action.yml +++ b/.github/workflows/build-python-wrapper/action.yml @@ -38,7 +38,7 @@ runs: - name: Install Python software dependencies shell: bash run: | - python3 -m ensurepip --upgrade + python3 -m ensurepip --upgrade || true python3 -m pip install --upgrade pip python3 -m pip install virtualenv mypy-protobuf @@ -53,11 +53,11 @@ runs: - name: Build Pybushka shell: bash working-directory: ./python + if: ${{ inputs.publish != 'true' }} run: | source "$HOME/.cargo/env" python3 -m venv .env source .env/bin/activate python3 -m pip install --upgrade pip python3 -m pip install -r requirements.txt - RELEASE_FLAG=`if [ ${{ inputs.publish }} = 'true' ]; then echo --release; fi` - maturin develop ${RELEASE_FLAG} + maturin develop diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 80d9281d6c..9715319862 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,12 +1,13 @@ +# The cross platform build was created based on the [Packaging Rust Applications for the NPM Registry blog](https://blog.orhun.dev/packaging-rust-for-npm/). + name: Continuous Deployment on: push: tags: - "v*.*" - jobs: - publish-npm-binaries: + publish-binaries: name: Publish NPM packages runs-on: ${{ matrix.build.RUNNER }} strategy: @@ -19,6 +20,8 @@ jobs: RUNNER: ubuntu-latest, ARCH: x64, TARGET: x86_64-unknown-linux-gnu, + NPM_PUBLISH: true, + PYPI_PUBLISH: true, } - { OS: ubuntu-latest, @@ -26,6 +29,9 @@ jobs: RUNNER: [self-hosted, Linux, ARM64], ARCH: arm64, TARGET: aarch64-unknown-linux-gnu, + NPM_PUBLISH: true, + PYPI_PUBLISH: true, + CONTAINER: "2_28", } - { OS: macos-latest, @@ -33,6 +39,8 @@ jobs: RUNNER: macos-latest, ARCH: x64, TARGET: x86_64-apple-darwin, + NPM_PUBLISH: true, + PYPI_PUBLISH: true, } - { OS: macos-latest, @@ -40,6 +48,8 @@ jobs: RUNNER: macos-13-xlarge, arch: arm64, TARGET: aarch64-apple-darwin, + NPM_PUBLISH: true, + PYPI_PUBLISH: true, } steps: - name: Checkout @@ -52,7 +62,10 @@ jobs: run: | echo "RELEASE_VERSION=${GITHUB_REF:11}" >> $GITHUB_ENV + ###### NODEJS ##### + - name: Setup node + if: matrix.build.NPM_PUBLISH == true uses: actions/setup-node@v3 with: node-version: "16" @@ -63,6 +76,7 @@ jobs: token: ${{ secrets.NPM_AUTH_TOKEN }} - name: Build Node wrapper + if: matrix.build.NPM_PUBLISH == true uses: ./.github/workflows/build-node-wrapper with: os: ${{ matrix.build.OS }} @@ -74,6 +88,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - name: Publish to NPM + if: matrix.build.NPM_PUBLISH == true shell: bash working-directory: ./node run: | @@ -81,7 +96,8 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} - - name: Pack the package + - name: Pack the Node package + if: matrix.build.NPM_PUBLISH == true shell: bash working-directory: ./node run: | @@ -93,16 +109,113 @@ jobs: ls ./bin env: NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} - - name: Upload the package + + - name: Upload the Node package + if: matrix.build.NPM_PUBLISH == true uses: actions/upload-artifact@v3 with: name: ${{ matrix.build.TARGET }} path: ./node/bin if-no-files-found: error - publish-npm-base: + ###### PYTHON ##### + + - name: Set the package version + if: matrix.build.PYPI_PUBLISH == true + working-directory: ./python + run: | + sed -i "s|255.255.255|${{ env.RELEASE_VERSION }}|g" Cargo.toml + cat Cargo.toml + + - name: Set up Python + if: matrix.build.PYPI_PUBLISH == true && !contains(matrix.build.RUNNER, 'self-hosted') + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Setup Python for self-hosted Ubuntu runners + if: matrix.build.PYPI_PUBLISH == true && contains(matrix.build.OS, 'ubuntu') && contains(matrix.build.RUNNER, 'self-hosted') + run: | + sudo apt update -y + sudo apt upgrade -y + sudo apt install python3 python3-venv python3-pip -y + + - name: Build Python wrapper + if: matrix.build.PYPI_PUBLISH == true + uses: ./.github/workflows/build-python-wrapper + with: + os: ${{ matrix.build.OS }} + target: ${{ matrix.build.TARGET }} + publish: "true" + github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Test test + if: matrix.build.PYPI_PUBLISH == true + run: | + which python3 + ls -ls /usr/bin/python* + - name: Build Python wheels (linux) + if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'linux') + uses: PyO3/maturin-action@v1 + with: + working-directory: ./python + target: ${{ matrix.build.TARGET }} + args: --release --strip --out wheels -i python3.8 python3.9 python3.10 python3.11 python3.12 + manylinux: auto + container: ${{ matrix.build.CONTAINER != '' && matrix.build.CONTAINER || '2014' }} + before-script-linux: | + # Install protobuf compiler + if [[ $(`which apt`) != '' ]] + then + apt install protobuf-compiler -y + else + PB_REL="https://github.com/protocolbuffers/protobuf/releases" + curl -LO $PB_REL/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip + unzip protoc-3.15.8-linux-x86_64.zip -d $HOME/.local + export PATH="$PATH:$HOME/.local/bin" + fi + # Don't ignore protobuf file when packaging the project + sed -i '/pb2.py/d' .gitignore + + - name: Build Python wheels (macos) + if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'darwin') + uses: PyO3/maturin-action@v1 + with: + working-directory: ./python + target: ${{ matrix.build.TARGET }} + args: --release --strip --out wheels -i python3.10 python3.11 python3.12 + before-script-linux: | + # Don't ignore protobuf file when packaging the project + sed -i '' '/pb2.py/d' .gitignore + + - name: Upload Python wheels + if: matrix.build.PYPI_PUBLISH == true + uses: actions/upload-artifact@v3 + with: + name: wheels + path: python/wheels + if-no-files-found: error + + publish-to-pypi: + name: Publish PyPI package + runs-on: ubuntu-latest + needs: publish-binaries + steps: + - uses: actions/download-artifact@v3 + with: + path: python/wheels + name: wheels + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.TESTPYPI_API_TOKEN }} + MATURIN_REPOSITORY: testpypi + with: + command: upload + args: --skip-existing python/wheels/* + + publish-base-to-npm: name: Publish the base NPM package - needs: publish-npm-binaries + needs: publish-binaries runs-on: ubuntu-latest steps: - name: Checkout diff --git a/node/npm/README.md b/node/npm/README.md deleted file mode 100644 index fff3f0529b..0000000000 --- a/node/npm/README.md +++ /dev/null @@ -1 +0,0 @@ -The cross platform build was created based on the [Packaging Rust Applications for the NPM Registry blog](https://blog.orhun.dev/packaging-rust-for-npm/). diff --git a/python/Cargo.toml b/python/Cargo.toml index ed76d199f8..bd0a78bacd 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pybushka" -version = "0.1.0" +version = "255.255.255" edition = "2021" license = "Apache-2.0" authors = ["Amazon Web Services"] diff --git a/python/pyproject.toml b/python/pyproject.toml index 195a896269..354e363c8a 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -2,6 +2,21 @@ requires = ["maturin>=0.13,<0.14"] build-backend = "maturin" +[project] +name = "babushka" +requires-python = ">=3.8" +classifiers = [ + "Development Status :: 4 - Beta", + "Topic :: Database", + "Topic :: Utilities", + "License :: OSI Approved :: Apache Software License", + "Intended Audience :: Developers", + "Topic :: Software Development", + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + [tool.isort] profile = "black" @@ -11,19 +26,4 @@ max-line-length = 127 extend-ignore = ['E203'] [tool.black] -target-version = ['py38', 'py39', 'py310', 'py311'] - -[project.option-dependencies] -test = [ - "hypothesis", - "sympy" -] - -[project] -name = "babushka" -requires-python = ">=3.8" -classifiers = [ - "Programming Language :: Rust", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", -] +target-version = ['py38', 'py39', 'py310', 'py311', '3.12']