llama-2-7b-chat-hf.yaml

port: 8000
name: llama-2-7b-chat-hf
route_prefix: /llama-2-7b-chat-hf
cpus_per_worker: 24
gpus_per_worker: 0
deepspeed: false
workers_per_group: 2
device: "cpu"
ipex:
  enabled: false
  precision: bf16
model_description:
  model_id_or_path: meta-llama/Llama-2-7b-chat-hf
  tokenizer_name_or_path: meta-llama/Llama-2-7b-chat-hf
  chat_processor: ChatModelLLama
  prompt:
    intro: ''
    human_id: '[INST] {msg} [/INST]

      '
    bot_id: ''
    stop_words: []
  config:
    use_auth_token: ''