Added CD workflow for publishing to PyPi

.github/workflows/build-python-wrapper/action.yml

@@ -38,7 +38,7 @@ runs:
         - name: Install Python software dependencies
           shell: bash
           run: |
-              python3 -m ensurepip --upgrade
+              python3 -m ensurepip --upgrade || true
               python3 -m pip install --upgrade pip
               python3 -m pip install virtualenv mypy-protobuf
 
@@ -53,11 +53,11 @@ runs:
         - name: Build Pybushka
           shell: bash
           working-directory: ./python
+          if: ${{ inputs.publish != 'true' }}
           run: |
               source "$HOME/.cargo/env"
               python3 -m venv .env
               source .env/bin/activate
               python3 -m pip install --upgrade pip
               python3 -m pip install -r requirements.txt
-              RELEASE_FLAG=`if [ ${{ inputs.publish }} = 'true'  ]; then echo --release; fi`
-              maturin develop ${RELEASE_FLAG}
+              maturin develop

.github/workflows/cd.yml

@@ -1,13 +1,14 @@
+# The cross platform build was created based on the [Packaging Rust Applications for the NPM Registry blog](https://blog.orhun.dev/packaging-rust-for-npm/).
+
 name: Continuous Deployment
 
 on:
     push:
         tags:
             - "v*.*"
-
 jobs:
-    publish-npm-binaries:
-        name: Publish NPM packages
+    publish-binaries:
+        name: Publish packages
         runs-on: ${{ matrix.build.RUNNER }}
         strategy:
             fail-fast: false
@@ -19,27 +20,36 @@ jobs:
                           RUNNER: ubuntu-latest,
                           ARCH: x64,
                           TARGET: x86_64-unknown-linux-gnu,
+                          NPM_PUBLISH: true,
+                          PYPI_PUBLISH: true,
                       }
                     - {
                           OS: ubuntu-latest,
                           NAMED_OS: linux,
                           RUNNER: [self-hosted, Linux, ARM64],
                           ARCH: arm64,
                           TARGET: aarch64-unknown-linux-gnu,
+                          NPM_PUBLISH: true,
+                          PYPI_PUBLISH: true,
+                          CONTAINER: "2_28",
                       }
                     - {
                           OS: macos-latest,
                           NAMED_OS: darwin,
                           RUNNER: macos-latest,
                           ARCH: x64,
                           TARGET: x86_64-apple-darwin,
+                          NPM_PUBLISH: true,
+                          PYPI_PUBLISH: true,
                       }
                     - {
                           OS: macos-latest,
                           NAMED_OS: darwin,
                           RUNNER: macos-13-xlarge,
                           arch: arm64,
                           TARGET: aarch64-apple-darwin,
+                          NPM_PUBLISH: true,
+                          PYPI_PUBLISH: true,
                       }
         steps:
             - name: Checkout
@@ -52,7 +62,10 @@ jobs:
               run: |
                   echo "RELEASE_VERSION=${GITHUB_REF:11}" >> $GITHUB_ENV
 
+            ###### NODEJS #####
+
             - name: Setup node
+              if: matrix.build.NPM_PUBLISH == true
               uses: actions/setup-node@v3
               with:
                   node-version: "16"
@@ -63,6 +76,7 @@ jobs:
                   token: ${{ secrets.NPM_AUTH_TOKEN }}
 
             - name: Build Node wrapper
+              if: matrix.build.NPM_PUBLISH == true
               uses: ./.github/workflows/build-node-wrapper
               with:
                   os: ${{ matrix.build.OS }}
@@ -74,14 +88,16 @@ jobs:
                   github-token: ${{ secrets.GITHUB_TOKEN }}
 
             - name: Publish to NPM
+              if: matrix.build.NPM_PUBLISH == true
               shell: bash
               working-directory: ./node
               run: |
                   npm publish
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
 
-            - name: Pack the package
+            - name: Pack the Node package
+              if: matrix.build.NPM_PUBLISH == true
               shell: bash
               working-directory: ./node
               run: |
@@ -93,16 +109,115 @@ jobs:
                   ls ./bin
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
-            - name: Upload the package
+
+            - name: Upload the Node package
+              if: matrix.build.NPM_PUBLISH == true
               uses: actions/upload-artifact@v3
               with:
                   name: ${{ matrix.build.TARGET }}
                   path: ./node/bin
                   if-no-files-found: error
 
-    publish-npm-base:
+            ###### PYTHON #####
+
+            - name: Set the package version for Python
+              if: matrix.build.PYPI_PUBLISH == true
+              working-directory: ./python
+              run: |
+                  SED_FOR_MACOS=`if [[ "${{ matrix.build.OS }}" =~ .*"macos".*  ]]; then echo "''"; fi`
+                  sed -i $SED_FOR_MACOS "s|255.255.255|${{ env.RELEASE_VERSION }}|g" ./Cargo.toml
+                  # Log the edited Cargo.toml file
+                  cat Cargo.toml
+
+            - name: Set up Python
+              if: matrix.build.PYPI_PUBLISH == true && !contains(matrix.build.RUNNER, 'self-hosted')
+              uses: actions/setup-python@v4
+              with:
+                  python-version: "3.10"
+
+            - name: Setup Python for self-hosted Ubuntu runners
+              if: matrix.build.PYPI_PUBLISH == true && contains(matrix.build.OS, 'ubuntu') && contains(matrix.build.RUNNER, 'self-hosted')
+              run: |
+                  sudo apt update -y
+                  sudo apt upgrade -y
+                  sudo apt install python3 python3-venv python3-pip -y
+
+            - name: Build Python wrapper
+              if: matrix.build.PYPI_PUBLISH == true
+              uses: ./.github/workflows/build-python-wrapper
+              with:
+                  os: ${{ matrix.build.OS }}
+                  target: ${{ matrix.build.TARGET }}
+                  publish: "true"
+                  github-token: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Include protobuf files in the package
+              working-directory: ./python
+              if: matrix.build.PYPI_PUBLISH == true
+              run: |
+                  SED_FOR_MACOS=`if [[ "${{ matrix.build.OS }}" =~ .*"macos".*  ]]; then echo "''"; fi`
+                  sed -i $SED_FOR_MACOS '/pb2.py/d' .gitignore
+                  # Log the edited .gitignore file
+                  cat .gitignore
+
+            - name: Build Python wheels (linux)
+              if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'linux')
+              uses: PyO3/maturin-action@v1
+              with:
+                  working-directory: ./python
+                  target: ${{ matrix.build.TARGET }}
+                  args: --release --strip --out wheels -i python3.8 python3.9 python3.10 python3.11 python3.12
+                  manylinux: auto
+                  container: ${{ matrix.build.CONTAINER != '' && matrix.build.CONTAINER || '2014' }}
+                  before-script-linux: |
+                      # Install protobuf compiler
+                      if [[ $(`which apt`) != '' ]]
+                      then 
+                        apt install protobuf-compiler -y
+                      else
+                        PB_REL="https://github.com/protocolbuffers/protobuf/releases"
+                        curl -LO $PB_REL/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip
+                        unzip protoc-3.15.8-linux-x86_64.zip -d $HOME/.local
+                        export PATH="$PATH:$HOME/.local/bin"
+                      fi
+
+            - name: Build Python wheels (macos)
+              if: matrix.build.PYPI_PUBLISH == true && startsWith(matrix.build.NAMED_OS, 'darwin')
+              uses: PyO3/maturin-action@v1
+              with:
+                  working-directory: ./python
+                  target: ${{ matrix.build.TARGET }}
+                  args: --release --strip --out wheels -i python3.10 python3.11 python3.12
+
+            - name: Upload Python wheels
+              if: matrix.build.PYPI_PUBLISH == true
+              uses: actions/upload-artifact@v3
+              with:
+                  name: wheels
+                  path: python/wheels
+                  if-no-files-found: error
+
+    publish-to-pypi:
+        name: Publish PyPI package
+        runs-on: ubuntu-latest
+        needs: publish-binaries
+        steps:
+            - uses: actions/download-artifact@v3
+              with:
+                  path: python/wheels
+                  name: wheels
+            - name: Publish to PyPI
+              uses: PyO3/maturin-action@v1
+              env:
+                  MATURIN_PYPI_TOKEN: ${{ secrets.TESTPYPI_API_TOKEN }}
+                  MATURIN_REPOSITORY: testpypi
+              with:
+                  command: upload
+                  args: --skip-existing python/wheels/*
+
+    publish-base-to-npm:
         name: Publish the base NPM package
-        needs: publish-npm-binaries
+        needs: publish-binaries
         runs-on: ubuntu-latest
         steps:
             - name: Checkout
@@ -160,8 +275,11 @@ jobs:
                   packages_list=`find ${{steps.download.outputs.download-path}} -type f -follow -print`
                   for package in $packages_list
                   do
-                    echo "Installing package $package"
-                    npm i --no-save "$package"
+                    if [[ "${package}" == *.tgz ]]
+                    then
+                        echo "Installing package $package"
+                        npm i --no-save "$package"
+                    fi
                   done
 
             - name: Publish the base package

python/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pybushka"
-version = "0.1.0"
+version = "255.255.255"
 edition = "2021"
 license = "Apache-2.0"
 authors = ["Amazon Web Services"]

python/pyproject.toml

@@ -2,6 +2,26 @@
 requires = ["maturin>=0.13,<0.14"]
 build-backend = "maturin"
 
+[project]
+name = "babushka"
+requires-python = ">=3.8"
+dependencies = [
+    "async-timeout>=4.0.2",
+    "typing-extensions>=4.8.0",
+    "google-api-python-client==2.85.0"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Topic :: Database",
+    "Topic :: Utilities",
+    "License :: OSI Approved :: Apache Software License",
+    "Intended Audience :: Developers",
+    "Topic :: Software Development",
+    "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
+]
+
 [tool.isort]
 profile = "black"
 
@@ -11,19 +31,4 @@ max-line-length = 127
 extend-ignore = ['E203']
 
 [tool.black]
-target-version = ['py38', 'py39', 'py310', 'py311']
-
-[project.option-dependencies]
-test = [
-    "hypothesis",
-    "sympy"
-]
-
-[project]
-name = "babushka"
-requires-python = ">=3.8"
-classifiers = [
-    "Programming Language :: Rust",
-    "Programming Language :: Python :: Implementation :: CPython",
-    "Programming Language :: Python :: Implementation :: PyPy",
-]
+target-version = ['py38', 'py39', 'py310', 'py311', 'py312']