c4_original-open_lm_7b_swiglutorch-warm=5000-lr=0p0003-wd=0p33-cd=3e-05-bs=2048-mult=1-seed=124-tokens=137788211200.json

{
    "name": "c4_original-open_lm_7b_swiglutorch-warm=5000-lr=0p0003-wd=0p33-cd=3e-05-bs=2048-mult=1-seed=124-tokens=137788211200",
    "dataset_name": "c4_original",
    "dataset_uuid": "7e0f5507-aa36-4d8c-9026-d049f885adf1", 
    "hyperparameters": {
        "model": "training/open_lm_configs/open_lm_7b_swiglutorch.json",
        "tokens": 137788211200,
        "warmup": 5000,
        "lr": 0.0003,
        "wd": 0.33,
        "cd": 3e-05,
        "global_bs": 2048,
        "acc": 4,
        "qk_norm": true,
        "z_loss": 0.0001,
        "grad_checkpointing": false,
        "params": 6889410560,
        "params_no_embed": 6682841088,
        "fsdp_flags": [
            "--fsdp",
            "--fsdp-limit-all-gathers",
            "--fsdp-pure-bf16",
            "--fsdp-use-orig-params"
        ],
        "chinchilla_multiplier": 1,
        "seed": 124,
        "vocab_size": 50432,
        "norm": "gain_only_lp_layer_norm"
    },
    "checkpoint_url": "s3://***REMOVED***/users/jeffreyli/mlr/open_lm/dcnlp/checkpoints/c4_original-open_lm_7b_swiglutorch-warm=5000-lr=0p0003-wd=0p33-cd=3e-05-bs=2048-mult=1-seed=124-tokens=137788211200/checkpoints/epoch_12.pt",
    "open_lm_version": "0.0.34",
    "open_lm_args": [
        "--workers",
        "1",
        "--precision",
        "amp_bfloat16",
        "--global-batch-size",
        "2048",
        "--log-every-n-steps",
        "20",
        "--grad-clip-norm",
        "1",
        "--lr",
        "0.0003",
        "--warmup",
        "5000",
        "--model",
        "training/open_lm_configs/open_lm_7b_swiglutorch.json",
        "--wd",
        "0.33",
        "--beta2",
        "0.95",
        "--epochs",
        "10",
        "--resume",
        "latest",
        "--seed",
        "124",
        "--accum-freq",
        "4",
        "--model-norm",
        "gain_only_lp_layer_norm",
        "--delete-previous-checkpoint",
        "--lr-cooldown-end",
        "3e-05",
        "--logs",
        "67db6b77-c7c4-48ae-b431-57254587ed43_training_logs/",
        "--attn-name",
        "torch_attn",
        "--log-logit-mean",
        "--data-tolerate-error-p",
        "0.95",
        "--data-tolerate-num-ckpts",
        "5",
        "--multiple-data-passes",
        "--train-num-samples",
        "13778821120",
        "--dataset-manifest",
        "s3://***REMOVED***/users/jeffreyli/mlr/dcnlp_data/tokenized/rpj_original/manifest.jsonl",
        "--data-key",
        "json.gz",
        "--name",
        "rpj_original-open_lm_7b_swiglutorch-warm=5000-lr=0p0003-wd=0p33-cd=3e-05-bs=2048-mult=1-seed=124-tokens=137788211200",
        "--fsdp",
        "--fsdp-limit-all-gathers",
        "--fsdp-pure-bf16",
        "--fsdp-use-orig-params",
        "--report-to",
        "wandb",
        "--wandb-project-name",
        "dcnlp",
        "--qk-norm",
        "--z-loss",
        "0.0001",
        "--remote-sync",
        "s3://***REMOVED***/users/jeffreyli/mlr/open_lm/dcnlp/checkpoints",
        "--torchcompile"
    ],
    "results": [],
    "params_url": "s3://***REMOVED***/users/jeffreyli/mlr/open_lm/dcnlp/checkpoints/c4_original-open_lm_7b_swiglutorch-warm=5000-lr=0p0003-wd=0p33-cd=3e-05-bs=2048-mult=1-seed=124-tokens=137788211200/params.txt",
    "uuid": "4950f179-c534-436a-a6e0-7d27dab92f48",
    "creation_date": "2024_05_31-02_54_20",
    "failed": false,
    "error": ""
}