Merge branch 'main' into MSM--acceleration

.github/changed-files.yml

@@ -20,4 +20,5 @@ cpp:
   - .github/workflows/cpp-golang-rust.yml
 examples:
   - examples/**/*
-  - .github/workflows/cpp-golang-rust.yml
+  - .github/workflows/cpp-golang-rust.yml
+  - .github/workflows/rust-examples.yml

.github/pull_request_template.md

@@ -5,3 +5,6 @@ This PR...
 ## Linked Issues
 
 Resolves #
+
+## (optional) CUDA backend branch
+# cuda-backend-branch: main # specify the branch here

.github/workflows/cpp-golang-rust.yml

@@ -1,18 +1,29 @@
-name: C++/CUDA/Go/RUST & Examples
+name: Spell Check, C++/CUDA/Go/RUST & Examples
 
 on:
   pull_request:
     branches:
       - main
-  push:
-    branches:
-      - main
+
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+
+  spelling-checker:
+    name: Check Spelling
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: codespell-project/actions-codespell@v2
+        with:
+          # https://github.com/codespell-project/actions-codespell?tab=readme-ov-file#parameter-skip
+          skip: ./**/target,./**/build,./docs/*.js,./docs/*.json,./*.svg
+          # https://github.com/codespell-project/actions-codespell?tab=readme-ov-file#parameter-ignore_words_file
+          ignore_words_file: .codespellignore
+
   check-changed-files:
     uses: ./.github/workflows/check-changed-files.yml
 
@@ -23,9 +34,26 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+    - name: Setup go
+      if: needs.check-changed-files.outputs.golang == 'true'
+      uses: actions/setup-go@v5
+      with:
+        go-version: '1.22.0'
     - name: Check clang-format
       if: needs.check-changed-files.outputs.cpp == 'true'
       run: ./scripts/format_all.sh . --check --exclude "build|wrappers"
+    - name: Check gofmt
+      if: needs.check-changed-files.outputs.golang == 'true'
+      run: if [[ $(go list ./... | xargs go fmt) ]]; then echo "Please run go fmt"; exit 1; fi
+    - name: Check rustfmt
+      if: needs.check-changed-files.outputs.rust == 'true'
+      working-directory: ./wrappers/rust
+      # "-name target -prune" removes searching in any directory named "target"
+      # Formatting by single file is necessary due to generated files not being present
+      # before building the project.
+      # e.g. icicle-cuda-runtime/src/bindings.rs is generated and icicle-cuda-runtime/src/lib.rs includes that module
+      # causing rustfmt to fail.
+      run: if [[ $(find . -path ./icicle-curves/icicle-curve-template -prune -o -name target -prune -o -iname *.rs -print | xargs cargo fmt --check --) ]]; then echo "Please run cargo fmt"; exit 1; fi
 
   extract-cuda-backend-branch:
     uses: ./.github/workflows/extract-backends.yml
@@ -132,8 +160,10 @@ jobs:
         CURVE_DIR=icicle-${CURVE//_/-}
         export ICICLE_BACKEND_INSTALL_DIR=${{ steps.cuda-flag.outputs.INSTALL_PATH }}
         cd ./$CURVE_DIR
-        cargo test --release --verbose
-    - name: Run C++ curve Tests
+        cargo test --release --verbose -- --skip phase
+        cargo test phase2 --release
+        cargo test phase3 --release
+    - name: Run C++ Curve Tests
       working-directory: ./icicle/build/tests
       if: needs.check-changed-files.outputs.cpp == 'true'
       run: |
@@ -178,6 +208,8 @@ jobs:
             build_args: -DEXT_FIELD=OFF
           - name: m31
             build_args: -DEXT_FIELD=ON
+          - name: koalabear
+            build_args: -DEXT_FIELD=ON
     steps:
     - name: Checkout Repo
       uses: actions/checkout@v4
@@ -260,7 +292,9 @@ jobs:
         FIELD_DIR=icicle-${FIELD//_/-}
         export ICICLE_BACKEND_INSTALL_DIR=${{ steps.cuda-flag.outputs.INSTALL_PATH }}
         cd ./$FIELD_DIR
-        cargo test --release --verbose
+        cargo test --release --verbose -- --skip phase
+        cargo test phase2 --release
+        cargo test phase3 --release
     - name: Run C++ field Tests
       working-directory: ./icicle/build/tests
       if: needs.check-changed-files.outputs.cpp == 'true'
@@ -283,7 +317,7 @@ jobs:
             fi
         done
 
-    - name: Run Golang field Tests (babybear only)
+    - name: Run Golang field Tests
       working-directory: ./wrappers/golang/fields
       if: needs.check-changed-files.outputs.golang == 'true' || needs.check-changed-files.outputs.cpp == 'true'
       # if: matrix.field.name == 'babybear' #&& matrix.field.name == 'babybear'
@@ -383,7 +417,9 @@ jobs:
       if: needs.check-changed-files.outputs.rust == 'true' || needs.check-changed-files.outputs.cpp == 'true'
       run: |
         export ICICLE_BACKEND_INSTALL_DIR=${{ steps.cuda-flag.outputs.INSTALL_PATH }}
-        cargo test --release --verbose
+        cargo test --release --verbose -- --skip phase
+        cargo test phase2 --release
+        cargo test phase3 --release
     - name: Test GoLang Hashes
       working-directory: ./wrappers/golang/hash
       if: needs.check-changed-files.outputs.golang == 'true' || needs.check-changed-files.outputs.cpp == 'true'
@@ -399,4 +435,82 @@ jobs:
         export ICICLE_BACKEND_INSTALL_DIR=${{ steps.cuda-flag.outputs.ICICLE_BACKEND_INSTALL_DIR }}
         export LD_LIBRARY_PATH=${{ steps.cuda-flag.outputs.INSTALL_PATH }}/lib
         export CGO_LDFLAGS="-L$LD_LIBRARY_PATH -licicle_hash -lstdc++ -Wl,-rpath=LD_LIBRARY_PATH"
-        go test ./tests -count=1 -failfast -p 2 -timeout 60m -v
+        go test ./tests -count=1 -failfast -p 2 -timeout 60m -v
+
+
+
+  run-rust-examples:
+    name: Run Rust Examples
+    runs-on: [self-hosted, Linux, X64, icicle]
+    needs: [check-changed-files, check-format, extract-cuda-backend-branch]
+    steps:
+    - name: Checkout Repo
+      uses: actions/checkout@v4
+    - name: Checkout CUDA Backend
+      uses: actions/checkout@v4
+      with:
+        repository: ingonyama-zk/icicle-cuda-backend
+        path: ./icicle/backend/cuda
+        ssh-key: ${{ secrets.CUDA_PULL_KEY }}
+        ref: ${{ needs.extract-cuda-backend-branch.outputs.cuda-backend-branch }}
+    - name: Get CUDA Backend Commit SHA
+      if: needs.check-changed-files.outputs.golang == 'true' || needs.check-changed-files.outputs.cpp == 'true'
+      working-directory: ./icicle/backend/cuda
+      id: extract-cuda-sha
+      run: |
+        CUDA_BACKEND_SHA=$(git rev-parse HEAD)
+        echo "CUDA Backend Commit SHA: $CUDA_BACKEND_SHA"
+        echo "cuda-backend-sha=$CUDA_BACKEND_SHA" >> $GITHUB_OUTPUT
+    - name: Set CUDA backend flag
+      if: needs.check-changed-files.outputs.golang == 'true' || needs.check-changed-files.outputs.cpp == 'true'
+      id: cuda-flag
+      run: |
+        CUDA_BACKEND_SHA=${{ steps.extract-cuda-sha.outputs.cuda-backend-sha }}
+        COMMIT_FILE="gh_commit_sha_hash_${CUDA_BACKEND_SHA}"
+        if [ "${{ needs.extract-cuda-backend-branch.outputs.cuda-backend-branch }}" == "main" ]; then
+          INSTALL_PATH=${{ github.workspace }}/../../main_lib/rust-examples
+          echo "INSTALL_PATH=${{ github.workspace }}/../../main_lib/rust-examples" >> $GITHUB_OUTPUT
+          COMMIT_FILE_PATH=${INSTALL_PATH}/lib/${COMMIT_FILE}
+          echo "COMMIT_FILE_PATH=${INSTALL_PATH}/lib/${COMMIT_FILE}" >> $GITHUB_OUTPUT
+
+          if [ -f "$COMMIT_FILE_PATH" ]; then
+            echo "${COMMIT_FILE} exists in ${INSTALL_PATH}/lib. Setting CUDA_FLAG="
+            echo "CUDA_FLAG=" >> $GITHUB_OUTPUT
+          else
+            echo "${COMMIT_FILE} does not exist in ${INSTALL_PATH}/lib. Setting CUDA_FLAG=-DCUDA_BACKEND=local"
+            echo "CUDA_FLAG=-DCUDA_BACKEND=local" >> $GITHUB_OUTPUT
+            rm -rf ${INSTALL_PATH}
+          fi
+          echo "CMAKE_INSTALL_PREFIX=-DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}" >> $GITHUB_OUTPUT
+          echo "ICICLE_BACKEND_INSTALL_DIR=${INSTALL_PATH}/lib" >> $GITHUB_OUTPUT
+        else
+          INSTALL_PATH=${{ github.workspace }}/../../temp_lib/rust-examples
+          echo "INSTALL_PATH=${{ github.workspace }}/../../temp_lib/rust-examples" >> $GITHUB_OUTPUT
+          COMMIT_FILE_PATH=${INSTALL_PATH}/lib/${COMMIT_FILE}
+          echo "COMMIT_FILE_PATH=${INSTALL_PATH}/lib/${COMMIT_FILE}" >> $GITHUB_OUTPUT
+          if [ -f "$COMMIT_FILE_PATH" ]; then
+            echo "${COMMIT_FILE} exists in ${INSTALL_PATH}/lib. Setting CUDA_FLAG="
+            echo "CUDA_FLAG=" >> $GITHUB_OUTPUT
+          else
+            echo "${COMMIT_FILE} does not exist in ${INSTALL_PATH}/lib. Setting CUDA_FLAG=-DCUDA_BACKEND=local"
+            echo "CUDA_FLAG=-DCUDA_BACKEND=local" >> $GITHUB_OUTPUT
+            rm -rf ${INSTALL_PATH}
+          fi
+          echo "CMAKE_INSTALL_PREFIX=-DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}" >> $GITHUB_OUTPUT
+          echo "ICICLE_BACKEND_INSTALL_DIR=${INSTALL_PATH}/lib" >> $GITHUB_OUTPUT
+        fi
+    - name: Rust examples
+      working-directory: ./examples/rust
+      if: needs.check-changed-files.outputs.rust == 'true' || needs.check-changed-files.outputs.examples == 'true'
+      run: |        
+        export ICICLE_BACKEND_INSTALL_DIR=${{ steps.cuda-flag.outputs.ICICLE_BACKEND_INSTALL_DIR }}
+        export RUST_ICICLE_BACKEND_INSTALL_DIR=$ICICLE_BACKEND_INSTALL_DIR/backend
+        # loop over all directories in the current directory
+        for dir in $(find . -mindepth 1 -maxdepth 1 -type d); do
+          if [ -d "$dir" ]; then
+            echo "Running command in $dir"
+            cd $dir
+            ./run.sh -d CUDA
+            cd -
+          fi
+        done

README.md

@@ -130,7 +130,7 @@ chmod +x build.sh
 ```
 
 > [!NOTE]
-> To specify the field, use the flag -field=<field>, where <field> can be one of the following: babybear, stark252, m31.
+> To specify the field, use the flag -field=<field>, where <field> can be one of the following: babybear, stark252, m31, koalabear.
 > To specify a curve, use the flag -curve=<curve>, where <curve> can be one of the following: bn254, bls12_377, bls12_381, bw6_761, grumpkin.
 
 Once ICICLE has been built, you can add specific packages when you need them in your application:
@@ -163,7 +163,7 @@ cmake -S icicle -B build -DFIELD=babybear
 ```
 
 > [!NOTE]
-> To specify the field, use the flag -DFIELD=field, where field can be one of the following: babybear, stark252, m31.
+> To specify the field, use the flag -DFIELD=field, where field can be one of the following: babybear, stark252, m31, koalabear.
 > To specify a curve, use the flag -DCURVE=curve, where curve can be one of the following: bn254, bls12_377, bls12_381, bw6_761, grumpkin.
 
 **Build the project:**
@@ -333,6 +333,7 @@ See [LICENSE-MIT][LMIT] for details.
 [babybear]: ./icicle/fields/
 [stark252]: ./icicle/fields/
 [m31]: ./icicle/fields/
+[koalabear]: ./icicle/fields/
 [LMIT]: ./LICENSE
 [DISCORD]: https://discord.gg/Y4SkbDf2Ff
 [googletest]: https://github.com/google/googletest/

docs/docs/icicle/build_from_source.md

@@ -24,7 +24,7 @@ ICICLE can be built and tested in C++ using CMake. The build process is straight
    ```
 
 :::info
-To specify the field, use the flag -DFIELD=field, where field can be one of the following: babybear, stark252, m31.
+To specify the field, use the flag -DFIELD=field, where field can be one of the following: babybear, stark252, m31, koalabear.
 
 To specify a curve, use the flag -DCURVE=curve, where curve can be one of the following: bn254, bls12_377, bls12_381, bw6_761, grumpkin.
 :::

docs/docs/icicle/libraries.md

@@ -37,14 +37,14 @@ Each library has a corresponding crate. See [programmers guide](./programmers_gu
 
 #### Supported fields and operations
 
-| Operation\Field                           | [babybear](https://eprint.iacr.org/2023/824.pdf) | [Stark252](https://docs.starknet.io/documentation/architecture_and_concepts/Cryptography/p-value/) |  m31  |
-| ----------------------------------------- | :----------------------------------------------: | :------------------------------------------------------------------------------------------------: | :---: |
-| [Vector operations](./primitives/vec_ops) |                        ✅                         |                                                 ✅                                                  |   ✅   |
-| [Polynomials](./polynomials/overview)     |                        ✅                         |                                                 ✅                                                  |   ❌   |
-| [NTT](primitives/ntt)                     |                        ✅                         |                                                 ✅                                                  |   ❌   |
-| Extension Field                           |                        ✅                         |                                                 ❌                                                  |   ✅   |
-| [Poseidon](primitives/hash#poseidon)      |                        ✅                         |                                                 ✅                                                  |   ✅   |
-| [Poseidon2](primitives/hash#poseidon2)    |                        ✅                         |                                                 ✅                                                  |   ✅   |
+| Operation\Field                           | [babybear](https://eprint.iacr.org/2023/824.pdf) | [Stark252](https://docs.starknet.io/documentation/architecture_and_concepts/Cryptography/p-value/) |  m31  |  Koalabear  |
+| ----------------------------------------- | :----------------------------------------------: | :------------------------------------------------------------------------------------------------: | :---: | :---: |
+| [Vector operations](./primitives/vec_ops) |                        ✅                         |                                                 ✅                                                  |   ✅     |   ✅   |
+| [Polynomials](./polynomials/overview)     |                        ✅                         |                                                 ✅                                                  |   ❌     |   ✅   |
+| [NTT](primitives/ntt)                     |                        ✅                         |                                                 ✅                                                  |   ❌     |   ✅   |
+| Extension Field                           |                        ✅                         |                                                 ❌                                                  |   ✅     |   ✅   |
+| [Poseidon](primitives/hash#poseidon)      |                        ✅                         |                                                 ✅                                                  |   ✅     |   ✅   |
+| [Poseidon2](primitives/hash#poseidon2)    |                        ✅                         |                                                 ✅                                                  |   ✅     |   ✅   |
 
 ### Misc
 

docs/docs/icicle/primitives/poseidon2.md

@@ -32,6 +32,7 @@ For ICICLE supported curves/fields
 * BN254 $\alpha = 5$
 * Grumpkin $\alpha = 5$
 * Stark252 $\alpha=3$
+* Koalabear $\alpha=3$
 
 ### MDS matrix structure
 

docs/docs/icicle/programmers_guide/cpp.md

@@ -32,6 +32,21 @@ eIcicleError result = icicle_set_device(device);
 eIcicleError result = icicle_get_active_device(device);
 ```
 
+### Setting and Getting the Default Device
+
+You can set the default device for all threads:
+
+```cpp
+icicle::Device device = {"CUDA", 0}; // or other
+eIcicleError result = icicle_set_default_device(device);
+```
+
+:::caution
+
+Setting a default device should be done **once** from the main thread of the application. If another device or backend is needed for a specific thread [icicle_set_device](#setting-and-getting-active-device) should be used instead.
+
+:::
+
 ### Querying Device Information
 
 Retrieve the number of available devices and check if a pointer is allocated on the host or on the active device:

docs/docs/icicle/programmers_guide/general.md

@@ -85,6 +85,7 @@ ICICLE provides a device abstraction layer that allows you to interact with diff
 
 - **Loading Backends**: Backends are loaded dynamically based on the environment configuration or a specified path.
 - **Setting Active Device**: The active device for a thread can be set, allowing for targeted computation on a specific device.
+- **Setting Default Device**: The default device for any thread without an active device can be set, removing the need to specify an alternative device on each thread. This is especially useful when running on a backend that is not the built-in CPU backend which is the default device to start.
 
 ## Streams
 

docs/docs/icicle/programmers_guide/go.md

@@ -27,12 +27,27 @@ result := runtime.LoadBackend("/path/to/backend/installdir", true)
 You can set the active device for the current thread and retrieve it when needed:
 
 ```go
-device = runtime.CreateDevice("CUDA", 0) // or other
+device := runtime.CreateDevice("CUDA", 0) // or other
 result := runtime.SetDevice(device)
 // or query current (thread) device 
 activeDevice := runtime.GetActiveDevice()
 ```
 
+### Setting and Getting the Default Device
+
+You can set the default device for all threads:
+
+```go
+device := runtime.CreateDevice("CUDA", 0) // or other
+defaultDevice := runtime.SetDefaultDevice(device);
+```
+
+:::caution
+
+Setting a default device should be done **once** from the main thread of the application. If another device or backend is needed for a specific thread [runtime.SetDevice](#setting-and-getting-active-device) should be used instead.
+
+:::
+
 ### Querying Device Information
 
 Retrieve the number of available devices and check if a pointer is allocated on the host or on the active device:

docs/docs/icicle/programmers_guide/rust.md

@@ -55,6 +55,21 @@ icicle_runtime::set_device(&device).unwrap();
 let active_device = icicle_runtime::get_active_device().unwrap();
 ```
 
+### Setting and Getting the Default Device
+
+You can set the default device for all threads:
+
+```caution
+let device = Device::new("CUDA", 0); // or other
+let default_device = icicle_runtime::set_default_device(device);
+```
+
+:::note
+
+Setting a default device should be done **once** from the main thread of the application. If another device or backend is needed for a specific thread [icicle_runtime::set_device](#setting-and-getting-active-device) should be used instead.
+
+:::
+
 ### Querying Device Information
 
 Retrieve the number of available devices and check if a pointer is allocated on the host or on the active device:

docs/docusaurus.config.ts

@@ -175,7 +175,7 @@ const config: Config = {
     announcementBar: {
       id: 'announcement', // Any value that will identify this message.
       content:
-        '<strong>❄️🎉 New Release! ICICLE v3.2! 🎉❄️</strong>',
+        '<strong>❄️🎉 New Release! ICICLE v3.3! 🎉❄️</strong>',
       backgroundColor: '#64f5ef', // Light blue background color.
       textColor: '#000000', // Black text color.
       isCloseable: true, // Defaults to `true`.