llama-2-7b-chat-hf-autoscaling.yaml

port: 8000
name: llama-2-7b-chat-hf
route_prefix: /llama-2-7b-chat-hf
max_ongoing_requests: 64
autoscaling_config:
    min_replicas: 1
    initial_replicas: 1
    max_replicas: 2
    target_ongoing_requests: 24
    downscale_delay_s: 30
    upscale_delay_s: 10
device: cpu
cpus_per_worker: 24
vllm:
  enabled: true
  max_num_seqs: 64
  precision: bf16
model_description:
  model_id_or_path: NousResearch/Llama-2-7b-chat-hf
  tokenizer_name_or_path: NousResearch/Llama-2-7b-chat-hf
  config:
    use_auth_token: ''