finetune_intel_gpu.yaml

General:
  base_model: EleutherAI/gpt-j-6b
  gpt_base_model: true
  output_dir: /tmp/llm-ray/output
  save_strategy: no
  config:
    trust_remote_code: false
    use_auth_token: null
  lora_config:
    task_type: CAUSAL_LM
    r: 8
    lora_alpha: 32
    lora_dropout: 0.1
  enable_gradient_checkpointing: false
Dataset:
  train_file: examples/data/sample_finetune_data.jsonl
  validation_file: null
  validation_split_percentage: 5
Training:
  optimizer: AdamW
  batch_size: 4
  epochs: 3
  learning_rate: 1.0e-05
  lr_scheduler: linear
  weight_decay: 0.0
  mixed_precision: bf16
  device: gpu
  num_training_workers: 2
  accelerate_mode: DDP
  resources_per_worker:
    CPU: 1
    GPU: 1
  gradient_accumulation_steps: 1
  logging_steps: 10