From 582dda62e0fc1c641b5430a389376dc4003317e8 Mon Sep 17 00:00:00 2001
From: yurakuratov <9271630+yurakuratov@users.noreply.github.com>
Date: Fri, 9 Jun 2023 14:04:19 +0300
Subject: [PATCH] add Suplementary_Table_2.csv

---
 manuscript_data/Suplementary_Table_2.csv | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/manuscript_data/Suplementary_Table_2.csv b/manuscript_data/Suplementary_Table_2.csv
index 92834de..acb0296 100644
--- a/manuscript_data/Suplementary_Table_2.csv
+++ b/manuscript_data/Suplementary_Table_2.csv
@@ -1 +1,10 @@
-# TODO provide GENA LM paramteres here
+Trained in this study,Public name,Layers/Heads/Hiddens,Number of parameters,Architechture,Positional information,Pre-LN,Pre-training SeqLen (tokens),Pre-training task,Vocabulary size,Tokenizer type,Training dataset,Learning rate,Warm-up steps,Optimizer,LR Scheduler,init from,Public link
+no,DNABERT,12/12/768,,BERT - Full Attention,BERT absolute position embeddings,FALSE,512,,-,kmer,GRCh38.p13,,,,,,"https://academic.oup.com/bioinformatics/article/37/15/2112/6128680 , trained by authors"
+yes,gena-lm-bert-base,12/12/768,110M,BERT - Full Attention,BERT absolute position embeddings,"TRUE, w/o the last layer norm",512,MLM+NSP,32000,BPE,"T2T, spit v1",1e-04,10000,AdamW,constant,,https://huggingface.co/AIRI-Institute/gena-lm-bert-base
+yes,gena-lm-bigbird-base-sparse,12/12/768,110M,BigBird - Sparse Attention (DeepSpeed),RoPE position embeddings,"TRUE, w/o the last layer norm",4096,MLM+NSP,32000,BPE,"T2T, spit v1",1e-04,10000,FusedAdamW,constant,,https://huggingface.co/AIRI-Institute/gena-lm-bigbird-base-sparse
+yes,gena-lm-bert-base-t2t,12/12/768,110M,BERT - Full Attention,BERT absolute position embeddings,"TRUE, w/o the last layer norm",512,MLM,32000,BPE,"T2T, augment. 1000G SNPs",1e-04,10000,FusedAdamW,constant,,https://huggingface.co/AIRI-Institute/gena-lm-bert-base-t2t
+yes,gena-lm-bert-base-t2t-multi,12/12/768,110M,BERT - Full Attention,BERT absolute position embeddings,"TRUE, w/o the last layer norm",512,MLM,32000,BPE,"T2T, augment. 1000G SNPs, Multispieces",1e-04,0,FusedAdamW,constant,gena-lm-bert-base-t2t,https://huggingface.co/AIRI-Institute/gena-lm-bert-base-t2t-multi
+yes,gena-lm-bigbird-base-sparse-t2t,12/12/768,110M,BigBird - Sparse Attention (DeepSpeed),RoPE position embeddings,TRUE,4096,MLM,32000,BPE,"T2T, augment. 1000G SNPs",1e-04,10000,FusedAdamW,linear,,https://huggingface.co/AIRI-Institute/gena-lm-bigbird-base-sparse-t2t
+yes,gena-lm-bigbird-base-t2t,12/12/768,110M,BigBird - Sparse Attention (HuggingFace),BERT absolute position embeddings,FALSE,4096,MLM,32000,BPE,"T2T, augment. 1000G SNPs",1e-04,10000,FusedAdamW,linear,,https://huggingface.co/AIRI-Institute/gena-lm-bigbird-base-t2t
+yes,gena-lm-bert-large-t2t,24/16/1024,336M,BERT-large - Full Attention,BERT absolute position embeddings,TRUE,512,MLM,32000,BPE,"T2T, augment. 1000G SNPs",1e-04,10000,FusedAdamW,constant,,https://huggingface.co/AIRI-Institute/gena-lm-bert-large-t2t
+yes,gena-lm-bert-base-lastln-t2t,12/12/768,110M,BERT - Full Attention,BERT absolute position embeddings,TRUE,512,MLM,32000,BPE,"T2T, augment. 1000G SNPs",1e-04,0,FusedAdamW,linear,,https://huggingface.co/AIRI-Institute/gena-lm-bert-base-lastln-t2t