decapoda-research_llama-65b-hf-a100-sxm-80gb-w16a16e16-tp2-pp1-dp1-sp1-fe0.7-hbme0.9-summary.json

{
    "batch_size_per_gpu": 1,
    "seq_len": 512,
    "tp_size": 2,
    "pp_size": 1,
    "num_tokens_to_generate": 32,
    "use_kv_cache": true,
    "kv_cache_memory_per_gpu": 713031680.0,
    "kv_cache_latency": 0.0007771038962454361,
    "layernorm_dtype_bytes": 2,
    "embedding_memory_per_gpu": 262144000.0,
    "weight_memory_per_gpu": 64686653440.0,
    "prefill_max_batch_size_per_gpu": 1,
    "prefill_activation_memory_per_gpu": 9395240960.0,
    "prefill_num_flops_fwd_total": 66926327889920,
    "decode_max_batch_size_per_gpu": 21,
    "decode_activation_memory_per_gpu": 13117440.0,
    "decode_num_flops_fwd_total": 129375928320,
    "prefill_latency": 0.1594699851924863,
    "prefill_latency_fwd_attn": 0.051917187094505504,
    "prefill_latency_fwd_mlp": 0.10068787800146521,
    "prefill_latency_fwd_layernorm": 0.0014627838046972915,
    "prefill_latency_fwd_tp_comm": 0.004473924266666667,
    "prefill_latency_fwd_input_embedding": 0.00031366198852160644,
    "prefill_latency_fwd_output_embedding_loss": 0.0006145500366300367,
    "decode_latency": 0.03746739204280868,
    "decode_latency_fwd_attn": 0.011704418767369622,
    "decode_latency_fwd_mlp": 0.02340811212467985,
    "decode_latency_fwd_layernorm": 2.8569996185493974e-06,
    "decode_latency_fwd_tp_comm": 0.0012799999999999999,
    "decode_latency_fwd_input_embedding": 0.00029369996185493974,
    "decode_latency_fwd_output_embedding_loss": 1.2002930402930405e-06,
    "total_decode_latency": 1.1989565453698778,
    "total_latency": 1.3584265305623642,
    "total_per_token_latency": 0.04245082908007388
}