From 74c3310cac4a030f4bacc4afd2b2691a06afddf1 Mon Sep 17 00:00:00 2001 From: minmingzhu Date: Fri, 15 Mar 2024 01:48:58 +0000 Subject: [PATCH] support mistralai/Mixtral-8x7B-Instruct-v0.1 for CPU Signed-off-by: minmingzhu --- .github/workflows/workflow_finetune.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/workflow_finetune.yml b/.github/workflows/workflow_finetune.yml index 6026c293f..05447c613 100644 --- a/.github/workflows/workflow_finetune.yml +++ b/.github/workflows/workflow_finetune.yml @@ -34,7 +34,7 @@ jobs: name: finetune strategy: matrix: - model: [ EleutherAI/gpt-j-6b, meta-llama/Llama-2-7b-chat-hf, gpt2, bigscience/bloom-560m, facebook/opt-125m, mosaicml/mpt-7b-chat, huggyllama/llama-7b, mistralai/Mistral-7B-v0.1 ] + model: [ EleutherAI/gpt-j-6b, meta-llama/Llama-2-7b-chat-hf, gpt2, bigscience/bloom-560m, facebook/opt-125m, mosaicml/mpt-7b-chat, huggyllama/llama-7b, mistralai/Mistral-7B-v0.1, mistralai/Mixtral-8x7B-Instruct-v0.1 ] isPR: - ${{inputs.ci_type == 'pr'}} @@ -44,6 +44,7 @@ jobs: - { model: "EleutherAI/gpt-j-6b"} - { model: "meta-llama/Llama-2-7b-chat-hf"} - { model: "mistralai/Mistral-7B-v0.1"} + - { model: "mistralai/Mixtral-8x7B-Instruct-v0.1"} runs-on: self-hosted @@ -131,6 +132,8 @@ jobs: } if "${{ matrix.model }}" == "mistralai/Mistral-7B-v0.1": result['General']['lora_config']['target_modules'] = ["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "lm_head",] + elif "${{ matrix.model }}" == "mistralai/Mixtral-8x7B-Instruct-v0.1": + result['General']['lora_config']['target_modules'] = ["k_proj", "v_proj"] else: result['General']['lora_config']['target_modules'] = None with open(conf_path, 'w') as output: