From 18db9238840d63e1a3ed7d1b129761ecbc600f70 Mon Sep 17 00:00:00 2001
From: samsja <55492238+samsja@users.noreply.github.com>
Date: Mon, 30 Sep 2024 17:43:10 -0700
Subject: [PATCH] update 7b config (#27)

* align model seq len with data seq len

* update 7b config to 2k seq len

---------

Co-authored-by: Sami jaghouar <sami@primeintellect.ai>
---
 configs/7B/H100.toml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/configs/7B/H100.toml b/configs/7B/H100.toml
index c1272c34..fb6816ce 100644
--- a/configs/7B/H100.toml
+++ b/configs/7B/H100.toml
@@ -2,11 +2,14 @@ name_model = "7B"
 project = "debug_7B_zero_band"
 
 [train]
-micro_bs = 6  
+micro_bs = 1
 sharding_strategy = "SHARD_GRAD_OP"
 
 [optim]
-batch_size = 3840
+batch_size = 1024 #2M tokens bs
 warmup_steps = 1000
 total_steps = 88_000
-lr = 6e-4
\ No newline at end of file
+lr = 3e-4
+
+[data]
+seq_length = 2048
\ No newline at end of file