Added CD workflow for publishing to PyPi

valkey-io · Dec 14, 2023 · bb264eb · bb264eb
1 parent aa0210a
commit bb264eb
Show file tree

Hide file tree

Showing 5 changed files with 139 additions and 27 deletions.
diff --git a/.github/workflows/build-python-wrapper/action.yml b/.github/workflows/build-python-wrapper/action.yml
@@ -38,7 +38,7 @@ runs:
         - name: Install Python software dependencies
           shell: bash
           run: |
-              python3 -m ensurepip --upgrade
+              python3 -m ensurepip --upgrade || true
               python3 -m pip install --upgrade pip
               python3 -m pip install virtualenv mypy-protobuf
 
@@ -53,11 +53,11 @@ runs:
         - name: Build Pybushka
           shell: bash
           working-directory: ./python
+          if: ${{ inputs.publish != 'true' }}
           run: |
               source "$HOME/.cargo/env"
               python3 -m venv .env
               source .env/bin/activate
               python3 -m pip install --upgrade pip
               python3 -m pip install -r requirements.txt
-              RELEASE_FLAG=`if [ ${{ inputs.publish }} = 'true'  ]; then echo --release; fi`
-              maturin develop ${RELEASE_FLAG}
+              maturin develop
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -1,12 +1,13 @@
+# The cross platform build was created based on the [Packaging Rust Applications for the NPM Registry blog](https://blog.orhun.dev/packaging-rust-for-npm/).
+
 name: Continuous Deployment
 
 on:
     push:
         tags:
             - "v*.*"
-
 jobs:
-    publish-npm-binaries:
+    publish-binaries:
         name: Publish NPM packages
         runs-on: ${{ matrix.build.RUNNER }}
         strategy:
@@ -19,27 +20,36 @@ jobs:
                           RUNNER: ubuntu-latest,
                           ARCH: x64,
                           TARGET: x86_64-unknown-linux-gnu,
+                          NPM_PUBLISH: true,
+                          PYPI_PUBLISH: true,
                       }
                     - {
                           OS: ubuntu-latest,
                           NAMED_OS: linux,
                           RUNNER: [self-hosted, Linux, ARM64],
                           ARCH: arm64,
                           TARGET: aarch64-unknown-linux-gnu,
+                          NPM_PUBLISH: true,
+                          PYPI_PUBLISH: true,
+                          CONTAINER: "2_28",
                       }
                     - {
                           OS: macos-latest,
                           NAMED_OS: darwin,
                           RUNNER: macos-latest,
                           ARCH: x64,
                           TARGET: x86_64-apple-darwin,
+                          NPM_PUBLISH: true,
+                          PYPI_PUBLISH: true,
                       }
                     - {
                           OS: macos-latest,
                           NAMED_OS: darwin,
                           RUNNER: macos-13-xlarge,
                           arch: arm64,
                           TARGET: aarch64-apple-darwin,
+                          NPM_PUBLISH: true,
+                          PYPI_PUBLISH: true,
                       }
         steps:
             - name: Checkout
@@ -52,7 +62,10 @@ jobs:
               run: |
                   echo "RELEASE_VERSION=${GITHUB_REF:11}" >> $GITHUB_ENV
 
+            ###### NODEJS #####
+
             - name: Setup node
+              if: matrix.build.NPM_PUBLISH == true
               uses: actions/setup-node@v3
               with:
                   node-version: "16"
@@ -63,6 +76,7 @@ jobs:
                   token: ${{ secrets.NPM_AUTH_TOKEN }}
 
             - name: Build Node wrapper
+              if: matrix.build.NPM_PUBLISH == true
               uses: ./.github/workflows/build-node-wrapper
               with:
                   os: ${{ matrix.build.OS }}
@@ -74,14 +88,16 @@ jobs:
                   github-token: ${{ secrets.GITHUB_TOKEN }}
 
             - name: Publish to NPM
+              if: matrix.build.NPM_PUBLISH == true
               shell: bash
               working-directory: ./node
               run: |
                   npm publish
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
 
-            - name: Pack the package
+            - name: Pack the Node package
+              if: matrix.build.NPM_PUBLISH == true
               shell: bash
               working-directory: ./node
               run: |
@@ -93,16 +109,113 @@ jobs:
                   ls ./bin
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
-            - name: Upload the package
+
+            - name: Upload the Node package
+              if: matrix.build.NPM_PUBLISH == true
               uses: actions/upload-artifact@v3
               with:
                   name: ${{ matrix.build.TARGET }}
                   path: ./node/bin
                   if-no-files-found: error
 
-    publish-npm-base:
+            ###### PYTHON #####
+
+            - name: Set the package version
+              if: matrix.build.PYPI_PUBLISH == true
+              working-directory: ./python
+              run: |
+                  sed -i "s|255.255.255|${{ env.RELEASE_VERSION }}|g" Cargo.toml
+                  cat Cargo.toml
+
+            - name: Set up Python
+              if: matrix.build.PYPI_PUBLISH == true && !contains(matrix.build.RUNNER, 'self-hosted')
+              uses: actions/setup-python@v4
+              with:
+                  python-version: "3.10"
+
+            - name: Setup Python for self-hosted Ubuntu runners
+              if: matrix.build.PYPI_PUBLISH == true && contains(matrix.build.OS, 'ubuntu') && contains(matrix.build.RUNNER, 'self-hosted')
+              run: |
+                  sudo apt update -y
+                  sudo apt upgrade -y
+                  sudo apt install python3 python3-venv python3-pip -y
+
+            - name: Build Python wrapper
+              if: matrix.build.PYPI_PUBLISH == true
+              uses: ./.github/workflows/build-python-wrapper
+              with:
+                  os: ${{ matrix.build.OS }}
+                  target: ${{ matrix.build.TARGET }}
+                  publish: "true"
+                  github-token: ${{ secrets.GITHUB_TOKEN }}
+            - name: Test test
+              if: matrix.build.PYPI_PUBLISH == true
+              run: |
+                  which python3
+                  ls -ls /usr/bin/python*
+            - name: Build Python wheels (linux)
+              if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'linux')
+              uses: PyO3/maturin-action@v1
+              with:
+                  working-directory: ./python
+                  target: ${{ matrix.build.TARGET }}
+                  args: --release --strip --out wheels -i python3.8 python3.9 python3.10 python3.11 python3.12
+                  manylinux: auto
+                  container: ${{ matrix.build.CONTAINER != '' && matrix.build.CONTAINER || '2014' }}
+                  before-script-linux: |
+                      # Install protobuf compiler
+                      if [[ $(`which apt`) != '' ]]
+                      then 
+                        apt install protobuf-compiler -y
+                      else
+                        PB_REL="https://github.com/protocolbuffers/protobuf/releases"
+                        curl -LO $PB_REL/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
+                        unzip protoc-3.15.8-linux-x86_64.zip -d $HOME/.local
+                        export PATH="$PATH:$HOME/.local/bin"
+                      fi
+                      # Don't ignore protobuf file when packaging the project
+                      sed -i '/pb2.py/d' .gitignore
+
+            - name: Build Python wheels (macos)
+              if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'darwin')
+              uses: PyO3/maturin-action@v1
+              with:
+                  working-directory: ./python
+                  target: ${{ matrix.build.TARGET }}
+                  args: --release --strip --out wheels -i python3.10 python3.11 python3.12
+                  before-script-linux: |
+                      # Don't ignore protobuf file when packaging the project
+                      sed -i '' '/pb2.py/d' .gitignore
+
+            - name: Upload Python wheels
+              if: matrix.build.PYPI_PUBLISH == true
+              uses: actions/upload-artifact@v3
+              with:
+                  name: wheels
+                  path: python/wheels
+                  if-no-files-found: error
+
+    publish-to-pypi:
+        name: Publish PyPI package
+        runs-on: ubuntu-latest
+        needs: publish-binaries
+        steps:
+            - uses: actions/download-artifact@v3
+              with:
+                  path: python/wheels
+                  name: wheels
+            - name: Publish to PyPI
+              uses: PyO3/maturin-action@v1
+              env:
+                  MATURIN_PYPI_TOKEN: ${{ secrets.TESTPYPI_API_TOKEN }}
+                  MATURIN_REPOSITORY: testpypi
+              with:
+                  command: upload
+                  args: --skip-existing python/wheels/*
+
+    publish-base-to-npm:
         name: Publish the base NPM package
-        needs: publish-npm-binaries
+        needs: publish-binaries
         runs-on: ubuntu-latest
         steps:
             - name: Checkout

diff --git a/node/npm/README.md b/node/npm/README.md
diff --git a/python/Cargo.toml b/python/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "pybushka"
-version = "0.1.0"
+version = "255.255.255"
 edition = "2021"
 license = "Apache-2.0"
 authors = ["Amazon Web Services"]

diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -2,6 +2,21 @@
 requires = ["maturin>=0.13,<0.14"]
 build-backend = "maturin"
 
+[project]
+name = "babushka"
+requires-python = ">=3.8"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Topic :: Database",
+    "Topic :: Utilities",
+    "License :: OSI Approved :: Apache Software License",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development",
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+]
+
 [tool.isort]
 profile = "black"
 
@@ -11,19 +26,4 @@ max-line-length = 127
 extend-ignore = ['E203']
 
 [tool.black]
-target-version = ['py38', 'py39', 'py310', 'py311']
-
-[project.option-dependencies]
-test = [
-    "hypothesis",
-    "sympy"
-]
-
-[project]
-name = "babushka"
-requires-python = ">=3.8"
-classifiers = [
-    "Programming Language :: Rust",
-    "Programming Language :: Python :: Implementation :: CPython",
-    "Programming Language :: Python :: Implementation :: PyPy",
-]
+target-version = ['py38', 'py39', 'py310', 'py311', '3.12']