dolma_v1_no_resample-open_lm_1b-warm=5000-lr=0p003-wd=0p033-cd=3e-05-bs=256-mult=1-seed=124.json

{
    "name": "dolma_v1_no_resample-open_lm_1b-warm=5000-lr=0p003-wd=0p033-cd=3e-05-bs=256-mult=1-seed=124",
    "dataset_name": "dolma_v1_no_resample",
    "dataset_uuid": "bdff90b2-5557-4699-a4c5-8d7b287ddc1b",
    "hyperparameters": {
        "model": "open_lm_1b",
        "tokens": 28795904000,
        "warmup": 5000,
        "lr": 0.003,
        "wd": 0.033,
        "cd": 3e-05,
        "global_bs": 256,
        "acc": 1,
        "qk_norm": true,
        "z_loss": 0.0001,
        "grad_checkpointing": false,
        "params": 1439795200,
        "params_no_embed": 1336510464,
        "fsdp_flags": [
            "--fsdp",
            "--fsdp-amp",
            "--fsdp-limit-all-gathers"
        ],
        "chinchilla_multiplier": 1,
        "seed": 124
    },
    "checkpoint_url": "s3://dcnlp-west/dcnlp_experiments_tri/openlm/dcnlp/experiments/1b_1x_dolma-v1-no-resample/dolma_v1_no_resample-open_lm_1b-warm=5000-lr=0p003-wd=0p033-cd=3e-05-bs=256-mult=1-seed=124/checkpoints/epoch_7.pt",
    "open_lm_version": "0.0.21",
    "open_lm_args": [
        "--workers",
        "2",
        "--precision",
        "amp_bfloat16",
        "--global-batch-size",
        "256",
        "--log-every-n-steps",
        "20",
        "--grad-clip-norm",
        "1",
        "--lr",
        "0.003",
        "--warmup",
        "5000",
        "--model",
        "open_lm_1b",
        "--wd",
        "0.033",
        "--beta2",
        "0.95",
        "--epochs",
        "5",
        "--resume",
        "latest",
        "--seed",
        "124",
        "--accum-freq",
        "1",
        "--model-norm",
        "gain_only_lp_layer_norm",
        "--delete-previous-checkpoint",
        "--lr-cooldown-end",
        "3e-05",
        "--logs",
        "./logs/30011",
        "--train-num-samples",
        "5759180800",
        "--dataset-manifest",
        "s3://***REMOVED***/openlm/dcnlp/datasets/dolma_v1/manifest.jsonl",
        "--data-key",
        "json.gz",
        "--name",
        "dolma_v1_no_resample-open_lm_1b-warm=5000-lr=0p003-wd=0p033-cd=3e-05-bs=256-mult=1-seed=124",
        "--fsdp",
        "--fsdp-amp",
        "--fsdp-limit-all-gathers",
        "--val-data",
        "/home/ubuntu/research/openlm/dcnlp/training/eval_data/open_lm_val/shard_00000000.tar",
        "/home/ubuntu/research/openlm/dcnlp/training/eval_data/c4_val/shard-0000000.tar",
        "--val-frequency",
        "5",
        "--val-data-key",
        "json",
        "txt",
        "--val-tok-ci",
        "--val-seq-ci",
        "--val-max-pop-ci",
        "300000",
        "--qk-norm",
        "--z-loss",
        "0.0001",
        "--remote-sync",
        "s3://***REMOVED***/openlm/dcnlp/experiments/1b_1x_dolma-v1-no-resample/"
    ],
    "results": [],
    "params_url": "s3://dcnlp-west/dcnlp_experiments_tri/openlm/dcnlp/experiments/1b_1x_dolma-v1-no-resample/dolma_v1_no_resample-open_lm_1b-warm=5000-lr=0p003-wd=0p033-cd=3e-05-bs=256-mult=1-seed=124/params.txt",
    "uuid": "310cd6c7-6409-46bf-9cc1-9aea7f41c476",
    "creation_date": "2024_01_29-11_21_56"
}