ggml llama2 examples #821
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ggml llama2 examples | |
| on: | |
| schedule: | |
| - cron: "0 0 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| logLevel: | |
| description: 'Log level' | |
| required: true | |
| default: 'info' | |
| push: | |
| branches: [ '*' ] | |
| paths: | |
| - ".github/workflows/llama.yml" | |
| - "wasmedge-ggml/**" | |
| pull_request: | |
| branches: [ '*' ] | |
| paths: | |
| - ".github/workflows/llama.yml" | |
| - "wasmedge-ggml/**" | |
| merge_group: | |
| jobs: | |
| build: | |
| strategy: | |
| matrix: | |
| runner: [ubuntu-latest, macos-m1] | |
| wasmedge: ["0.14.1"] | |
| plugin: [wasi_nn-ggml] | |
| job: | |
| - name: "Tiny Llama" | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/llama | |
| curl -LO https://huggingface.co/TheBloke/TinyLlama-1.1B-Chat-v0.3-GGUF/resolve/main/tinyllama-1.1b-chat-v0.3.Q5_K_M.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --nn-preload default:GGML:AUTO:tinyllama-1.1b-chat-v0.3.Q5_K_M.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-llama.wasm \ | |
| default \ | |
| $'<|im_start|>system\nYou are an AI assistant<|im_end|>\n<|im_start|>user\nWhere is the capital of Japan?<|im_end|>\n<|im_start|>assistant' | |
| - name: Gemma 2B | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/gemma | |
| curl -LO https://huggingface.co/second-state/Gemma-2b-it-GGUF/resolve/main/gemma-2b-it-Q5_K_M.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --nn-preload default:GGML:AUTO:gemma-2b-it-Q5_K_M.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-gemma.wasm \ | |
| default \ | |
| '<start_of_turn>user Where is the capital of Japan? <end_of_turn><start_of_turn>model' | |
| - name: Llava v1.6 7B | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/llava | |
| curl -LO https://huggingface.co/cmp-nct/llava-1.6-gguf/resolve/main/vicuna-7b-q5_k.gguf | |
| curl -LO https://huggingface.co/cmp-nct/llava-1.6-gguf/resolve/main/mmproj-vicuna7b-f16.gguf | |
| curl -LO https://llava-vl.github.io/static/images/monalisa.jpg | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env mmproj=mmproj-vicuna7b-f16.gguf \ | |
| --env image=monalisa.jpg \ | |
| --env ctx_size=4096 \ | |
| --env n_gpu_layers="$NGL" \ | |
| --nn-preload default:GGML:AUTO:vicuna-7b-q5_k.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-llava.wasm \ | |
| default \ | |
| $'You are a helpful, respectful and honest assistant. Always answer as short as possible, while being safe.\nUSER:<image>\nDo you know who drew this painting?\nASSISTANT:' | |
| - name: Llama3 8B | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/llama | |
| curl -LO https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct.Q5_K_M.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --env llama3=true \ | |
| --nn-preload default:GGML:AUTO:Meta-Llama-3-8B-Instruct.Q5_K_M.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-llama.wasm \ | |
| default \ | |
| $"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.<|eot_id|>\n<|start_header_id|>user<|end_header_id|>\n\nWhat's the capital of Japan?<|eot_id|>\n<|start_header_id|>assistant<|end_header_id|>\n\n" | |
| - name: Llama3 8B (Streaming) | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/llama-stream | |
| curl -LO https://huggingface.co/QuantFactory/Meta-Llama-3-8B-Instruct-GGUF/resolve/main/Meta-Llama-3-8B-Instruct.Q5_K_M.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --env llama3=true \ | |
| --nn-preload default:GGML:AUTO:Meta-Llama-3-8B-Instruct.Q5_K_M.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-llama-stream.wasm \ | |
| default \ | |
| $"<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.<|eot_id|>\n<|start_header_id|>user<|end_header_id|>\n\nWhat's the capital of Japan?<|eot_id|>\n<|start_header_id|>assistant<|end_header_id|>\n\n" | |
| - name: Multiple Models Example | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/multimodel | |
| curl -LO https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf | |
| curl -LO https://huggingface.co/cmp-nct/llava-1.6-gguf/resolve/main/vicuna-7b-q5_k.gguf | |
| curl -LO https://huggingface.co/cmp-nct/llava-1.6-gguf/resolve/main/mmproj-vicuna7b-f16.gguf | |
| curl -LO https://llava-vl.github.io/static/images/monalisa.jpg | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --env image=monalisa.jpg \ | |
| --env mmproj=mmproj-vicuna7b-f16.gguf \ | |
| --nn-preload llama2:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \ | |
| --nn-preload llava:GGML:AUTO:vicuna-7b-q5_k.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-multimodel.wasm \ | |
| 'describe this picture please' | |
| - name: Embedding Example (All-MiniLM) | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/embedding | |
| curl -LO https://huggingface.co/second-state/All-MiniLM-L6-v2-Embedding-GGUF/resolve/main/all-MiniLM-L6-v2-ggml-model-f16.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --nn-preload default:GGML:AUTO:all-MiniLM-L6-v2-ggml-model-f16.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-llama-embedding.wasm \ | |
| default \ | |
| 'hello world' | |
| - name: Embedding Example (Llama-2) | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/embedding | |
| curl -LO https://huggingface.co/TheBloke/Llama-2-7b-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --nn-preload default:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-llama-embedding.wasm \ | |
| default \ | |
| 'hello world' | |
| - name: RPC Example | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/nnrpc | |
| curl -LO https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --nn-preload default:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-nnrpc.wasm \ | |
| default \ | |
| $'[INST] <<SYS>>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.\n<</SYS>>\nWhat is the capital of Japan?[/INST]' | |
| - name: Set Input Twice | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/test/set-input-twice | |
| curl -LO https://huggingface.co/second-state/Gemma-2b-it-GGUF/resolve/main/gemma-2b-it-Q5_K_M.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --nn-preload default:GGML:AUTO:gemma-2b-it-Q5_K_M.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-set-input-twice.wasm \ | |
| default \ | |
| '<start_of_turn>user Where is the capital of Japan? <end_of_turn><start_of_turn>model' | |
| - name: Grammar Example | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/grammar | |
| curl -LO https://huggingface.co/TheBloke/Llama-2-7b-GGUF/resolve/main/llama-2-7b.Q5_K_M.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --nn-preload default:GGML:AUTO:llama-2-7b.Q5_K_M.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-grammar.wasm \ | |
| default \ | |
| 'JSON object with 5 country names as keys and their capitals as values: ' | |
| - name: Model Not Found | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/test/model-not-found | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --nn-preload default:GGML:AUTO:model-not-found.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-model-not-found.wasm \ | |
| default | |
| - name: Unload | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/test/unload | |
| curl -LO https://huggingface.co/second-state/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --nn-preload default:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-unload.wasm \ | |
| default \ | |
| $'[INST] <<SYS>>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you do not know the answer to a question, please do not share false information.\n<</SYS>>\nWhat is the capital of Japan?[/INST]' | |
| - name: JSON Schema | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/json-schema | |
| curl -LO https://huggingface.co/TheBloke/Llama-2-7b-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --nn-preload default:GGML:AUTO:llama-2-7b-chat.Q5_K_M.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-json-schema.wasm \ | |
| default \ | |
| $'[INST] <<SYS>>\nYou are a helpful, respectful and honest assistant. Always output JSON format string.\n<</SYS>>\nGive me a JSON array of Apple products.[/INST]' | |
| - name: Qwen2-VL | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/qwen2vl | |
| curl -LO https://huggingface.co/second-state/Qwen2-VL-2B-Instruct-GGUF/resolve/main/Qwen2-VL-2B-Instruct-vision-encoder.gguf | |
| curl -LO https://huggingface.co/second-state/Qwen2-VL-2B-Instruct-GGUF/resolve/main/Qwen2-VL-2B-Instruct-Q5_K_M.gguf | |
| curl -LO https://llava-vl.github.io/static/images/monalisa.jpg | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --nn-preload default:GGML:AUTO:Qwen2-VL-2B-Instruct-Q5_K_M.gguf \ | |
| --env mmproj=Qwen2-VL-2B-Instruct-vision-encoder.gguf \ | |
| --env image=monalisa.jpg \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-qwen2vl.wasm \ | |
| default \ | |
| $'<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|><image><|vision_end|>what is in this picture?<|im_end|>\n<|im_start|>assistant\n' | |
| - name: Text-to-speech | |
| shell: bash | |
| run: | | |
| test -f ~/.wasmedge/env && source ~/.wasmedge/env | |
| cd wasmedge-ggml/tts | |
| curl -LO https://huggingface.co/second-state/OuteTTS-0.2-500M-GGUF/resolve/main/OuteTTS-0.2-500M-Q5_K_M.gguf | |
| curl -LO https://huggingface.co/second-state/OuteTTS-0.2-500M-GGUF/resolve/main/wavtokenizer-large-75-ggml-f16.gguf | |
| cargo build --target wasm32-wasip1 --release | |
| time wasmedge --dir .:. \ | |
| --env n_gpu_layers="$NGL" \ | |
| --nn-preload default:GGML:AUTO:OuteTTS-0.2-500M-Q5_K_M.gguf \ | |
| --env tts=true \ | |
| --env tts_output_file=output.wav \ | |
| --env model_vocoder=wavtokenizer-large-75-ggml-f16.gguf \ | |
| target/wasm32-wasip1/release/wasmedge-ggml-tts.wasm \ | |
| default \ | |
| 'Hello, world.' | |
| - name: Build llama-stream | |
| run: | | |
| cd wasmedge-ggml/llama-stream | |
| cargo build --target wasm32-wasip1 --release | |
| - name: Build llava-base64-stream | |
| run: | | |
| cd wasmedge-ggml/llava-base64-stream | |
| cargo build --target wasm32-wasip1 --release | |
| name: ${{ matrix.runner == 'ubuntu-latest' && 'ubuntu:20.04' || matrix.runner }} - ${{ matrix.job.name }} - ${{ matrix.wasmedge }} - ${{ matrix.plugin }} | |
| runs-on: ${{ matrix.runner }} | |
| # set image to `ubuntu:20.04` if runner is `ubuntu-latest` | |
| container: ${{ matrix.runner == 'ubuntu-latest' && fromJSON('{"image":"ubuntu:20.04"}') || null }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - if: ${{ matrix.runner == 'ubuntu-latest' }} | |
| name: Install apt-get packages | |
| run: | | |
| ACCEPT_EULA=Y apt-get update | |
| ACCEPT_EULA=Y apt-get upgrade -y | |
| apt-get install -y wget git curl software-properties-common build-essential | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| - name: Install Rust target for wasm | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| target: wasm32-wasip1 | |
| - name: Install WasmEdge + WASI-NN + GGML | |
| run: | | |
| curl -sSf https://raw.githubusercontent.com/WasmEdge/WasmEdge/master/utils/install.sh | bash -s -- -v ${{ matrix.wasmedge }} --plugins ${{ matrix.plugin }} | |
| - name: Set environment variable | |
| run: echo "NGL=${{ matrix.ngl || 0 }}" >> $GITHUB_ENV | |
| - name: ${{ matrix.job.name }} | |
| run: ${{ matrix.job.run }} | |
| shell: bash |