mistral-7b-Instruct-v0.2.yaml

port: 8000
name: mistral-7b-instruct-v0.2
route_prefix: /mistral-7b-instruct-v0.2
device: cpu
cpus_per_worker: 24
vllm:
  enabled: true
  precision: bf16
model_description:
  model_id_or_path: mistralai/Mistral-7B-Instruct-v0.2
  ipexllm: false
  tokenizer_name_or_path: mistralai/Mistral-7B-Instruct-v0.2
  config:
    trust_remote_code: true