llama.yml

name: ggml llama2 examples

on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:
    inputs:
      logLevel:
        description: 'Log level'
        required: true
        default: 'info'
  push:
    branches: [ '*' ]
    paths:
      - ".github/workflows/llama.yml"
      - "wasmedge-ggml/**"
  pull_request:
    branches: [ '*' ]
    paths:
      - ".github/workflows/llama.yml"
      - "wasmedge-ggml/**"
  merge_group:

jobs:
  build:
    strategy:
      matrix:
        runner: [ubuntu-latest, macos-m1]
        wasmedge: ["0.14.1"]
        plugin: [wasi_nn-ggml]
        job:
          - name: "Tiny Llama"
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/llama
              curl -LO https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/resolve/main/tinyllama-1.1b-chat-v0.3.Q5_K_M.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --nn-preload default:GGML:AUTO:tinyllama-1.1b-chat-v0.3.Q5_K_M.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-llama.wasm \
                default \
                $'<|im_start|>system\nYou are an AI assistant<|im_end|>\n<|im_start|>user\nWhere is the capital of Japan?<|im_end|>\n<|im_start|>assistant'

          - name: Gemma 2B
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/gemma
              curl -LO https://huggingface.co/second-state/Gemma-2b-it-GGUF/resolve/main/gemma-2b-it-Q5_K_M.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --nn-preload default:GGML:AUTO:gemma-2b-it-Q5_K_M.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-gemma.wasm \
                default \
                '<start_of_turn>user Where is the capital of Japan? <end_of_turn><start_of_turn>model'

          - name: Llava v1.6 7B
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/llava
              curl -LO https://huggingface.co/cmp-nct/llava-1.6-gguf/resolve/main/vicuna-7b-q5_k.gguf
              curl -LO https://huggingface.co/cmp-nct/llava-1.6-gguf/resolve/main/mmproj-vicuna7b-f16.gguf
              curl -LO https://llava-vl.github.io/static/images/monalisa.jpg
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env mmproj=mmproj-vicuna7b-f16.gguf \
                --env image=monalisa.jpg \
                --env ctx_size=4096 \
                --env n_gpu_layers="$NGL" \
                --nn-preload default:GGML:AUTO:vicuna-7b-q5_k.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-llava.wasm \
                default \
                $'You are a helpful, respectful and honest assistant. Always answer as short as possible, while being safe.\nUSER:<image>\nDo you know who drew this painting?\nASSISTANT:'

          - name: Llama3 8B
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/llama
              curl -LO https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct.Q5_K_M.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --env llama3=true \
                --nn-preload default:GGML:AUTO:Meta-Llama-3-8B-Instruct.Q5_K_M.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-llama.wasm \
                default \
                $"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.<|eot_id|>\n<|start_header_id|>user<|end_header_id|>\n\nWhat's the capital of Japan?<|eot_id|>\n<|start_header_id|>assistant<|end_header_id|>\n\n"

          - name: Llama3 8B (Streaming)
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/llama-stream
              curl -LO https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct.Q5_K_M.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --env llama3=true \
                --nn-preload default:GGML:AUTO:Meta-Llama-3-8B-Instruct.Q5_K_M.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-llama-stream.wasm \
                default \
                $"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.<|eot_id|>\n<|start_header_id|>user<|end_header_id|>\n\nWhat's the capital of Japan?<|eot_id|>\n<|start_header_id|>assistant<|end_header_id|>\n\n"

          - name: Multiple Models Example
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/multimodel
              curl -LO https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf
              curl -LO https://huggingface.co/cmp-nct/llava-1.6-gguf/resolve/main/vicuna-7b-q5_k.gguf
              curl -LO https://huggingface.co/cmp-nct/llava-1.6-gguf/resolve/main/mmproj-vicuna7b-f16.gguf
              curl -LO https://llava-vl.github.io/static/images/monalisa.jpg
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --env image=monalisa.jpg \
                --env mmproj=mmproj-vicuna7b-f16.gguf \
                --nn-preload llama2:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \
                --nn-preload llava:GGML:AUTO:vicuna-7b-q5_k.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-multimodel.wasm \
                'describe this picture please'

          - name: Embedding Example (All-MiniLM)
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/embedding
              curl -LO https://huggingface.co/second-state/All-MiniLM-L6-v2-Embedding-GGUF/resolve/main/all-MiniLM-L6-v2-ggml-model-f16.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --nn-preload default:GGML:AUTO:all-MiniLM-L6-v2-ggml-model-f16.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-llama-embedding.wasm \
                default \
                'hello world'

          - name: Embedding Example (Llama-2)
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/embedding
              curl -LO https://huggingface.co/TheBloke/Llama-2-7b-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --nn-preload default:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-llama-embedding.wasm \
                default \
                'hello world'

          - name: RPC Example
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/nnrpc
              curl -LO https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --nn-preload default:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-nnrpc.wasm \
                default \
                $'[INST] <<SYS>>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.\n<</SYS>>\nWhat is the capital of Japan?[/INST]'

          - name: Set Input Twice
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/test/set-input-twice
              curl -LO https://huggingface.co/second-state/Gemma-2b-it-GGUF/resolve/main/gemma-2b-it-Q5_K_M.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --nn-preload default:GGML:AUTO:gemma-2b-it-Q5_K_M.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-set-input-twice.wasm \
                default \
                '<start_of_turn>user Where is the capital of Japan? <end_of_turn><start_of_turn>model'

          - name: Grammar Example
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/grammar
              curl -LO https://huggingface.co/TheBloke/Llama-2-7b-GGUF/resolve/main/llama-2-7b.Q5_K_M.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --nn-preload default:GGML:AUTO:llama-2-7b.Q5_K_M.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-grammar.wasm \
                default \
                'JSON object with 5 country names as keys and their capitals as values: '

          - name: Model Not Found
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/test/model-not-found
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --nn-preload default:GGML:AUTO:model-not-found.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-model-not-found.wasm \
                default

          - name: Unload
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/test/unload
              curl -LO https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --nn-preload default:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-unload.wasm \
                default \
                $'[INST] <<SYS>>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.\n<</SYS>>\nWhat is the capital of Japan?[/INST]'

          - name: JSON Schema
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/json-schema
              curl -LO https://huggingface.co/TheBloke/Llama-2-7b-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --nn-preload default:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-json-schema.wasm \
                default \
                $'[INST] <<SYS>>\nYou are a helpful, respectful and honest assistant. Always output JSON format string.\n<</SYS>>\nGive me a JSON array of Apple products.[/INST]'

          - name: Qwen2-VL
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/qwen2vl
              curl -LO https://huggingface.co/second-state/Qwen2-VL-2B-Instruct-GGUF/resolve/main/Qwen2-VL-2B-Instruct-vision-encoder.gguf
              curl -LO https://huggingface.co/second-state/Qwen2-VL-2B-Instruct-GGUF/resolve/main/Qwen2-VL-2B-Instruct-Q5_K_M.gguf
              curl -LO https://llava-vl.github.io/static/images/monalisa.jpg
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --nn-preload default:GGML:AUTO:Qwen2-VL-2B-Instruct-Q5_K_M.gguf \
                --env mmproj=Qwen2-VL-2B-Instruct-vision-encoder.gguf \
                --env image=monalisa.jpg \
                target/wasm32-wasip1/release/wasmedge-ggml-qwen2vl.wasm \
                default \
                $'<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><image><|vision_end|>what is in this picture?<|im_end|>\n<|im_start|>assistant\n'

          - name: Text-to-speech
            shell: bash
            run: |
              test -f ~/.wasmedge/env && source ~/.wasmedge/env
              cd wasmedge-ggml/tts
              curl -LO https://huggingface.co/second-state/OuteTTS-0.2-500M-GGUF/resolve/main/OuteTTS-0.2-500M-Q5_K_M.gguf
              curl -LO https://huggingface.co/second-state/OuteTTS-0.2-500M-GGUF/resolve/main/wavtokenizer-large-75-ggml-f16.gguf
              curl -LO https://raw.githubusercontent.com/edwko/OuteTTS/refs/heads/main/outetts/version/v1/default_speakers/en_male_1.json
              cargo build --target wasm32-wasip1 --release
              time wasmedge --dir .:. \
                --env n_gpu_layers="$NGL" \
                --nn-preload default:GGML:AUTO:OuteTTS-0.2-500M-Q5_K_M.gguf \
                --env tts=true \
                --env tts_output_file=output.wav \
                --env tts_speaker_file=en_male_1.json \
                --env model_vocoder=wavtokenizer-large-75-ggml-f16.gguf \
                target/wasm32-wasip1/release/wasmedge-ggml-tts.wasm \
                default \
                'Hello, world.'
              sha1sum *.wav

          - name: Build llama-stream
            run: |
              cd wasmedge-ggml/llama-stream
              cargo build --target wasm32-wasip1 --release

          - name: Build llava-base64-stream
            run: |
              cd wasmedge-ggml/llava-base64-stream
              cargo build --target wasm32-wasip1 --release

    name: ${{ matrix.runner == 'ubuntu-latest' && 'ubuntu:20.04' || matrix.runner }} - ${{ matrix.job.name }} - ${{ matrix.wasmedge }} - ${{ matrix.plugin }}
    runs-on: ${{ matrix.runner }}
    # set image to `ubuntu:20.04` if runner is `ubuntu-latest`
    container: ${{ matrix.runner == 'ubuntu-latest' && fromJSON('{"image":"ubuntu:20.04"}') || null }}
    steps:
    - uses: actions/checkout@v4

    - if: ${{ matrix.runner == 'ubuntu-latest' }}
      name: Install apt-get packages
      run: |
        ACCEPT_EULA=Y apt-get update
        ACCEPT_EULA=Y apt-get upgrade -y
        apt-get install -y wget git curl software-properties-common build-essential
      env:
        DEBIAN_FRONTEND: noninteractive

    - name: Install Rust target for wasm
      uses: dtolnay/rust-toolchain@stable
      with:
        target: wasm32-wasip1

    - name: Install WasmEdge + WASI-NN + GGML
      run: |
        curl -sSf https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh | bash -s -- -v ${{ matrix.wasmedge }} --plugins ${{ matrix.plugin }}

    - name: Set environment variable
      run: echo "NGL=${{ matrix.ngl || 0 }}" >> $GITHUB_ENV

    - name: ${{ matrix.job.name }}
      run: ${{ matrix.job.run }}
      shell: bash