diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_13b/pretrain-llama2_13b-auto_tuner.json b/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_13b/pretrain-llama2_13b-auto_tuner.json
index 0a39212b492f..d73ee4c93280 100644
--- a/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_13b/pretrain-llama2_13b-auto_tuner.json
+++ b/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_13b/pretrain-llama2_13b-auto_tuner.json
@@ -16,6 +16,7 @@
     "sequence_parallel": 0,   
     "use_flash_attention": true,
     "use_fused_rms_norm": true,
+    "use_fast_layer_norm": true,
     "fuse_attention_ffn": true,
     "fuse_attention_qkv": true,
     "use_fused_rope": true,
diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_70b/pretrain-llama2_70b-auto_tuner.json b/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_70b/pretrain-llama2_70b-auto_tuner.json
index b3d234a484a4..8ec645fe8056 100644
--- a/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_70b/pretrain-llama2_70b-auto_tuner.json
+++ b/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_70b/pretrain-llama2_70b-auto_tuner.json
@@ -16,6 +16,7 @@
     "sequence_parallel": 1,   
     "use_flash_attention": true,
     "use_fused_rms_norm": true,
+    "use_fast_layer_norm": true,
     "fuse_attention_ffn": true,
     "fuse_attention_qkv": true,
     "use_fused_rope": true,
diff --git a/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_7b/pretrain-llama2_7b-auto_tuner.json b/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_7b/pretrain-llama2_7b-auto_tuner.json
index ddf5e30f9bea..52b48e8d897a 100644
--- a/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_7b/pretrain-llama2_7b-auto_tuner.json
+++ b/tests/test_tipc/dygraph/hybrid_parallelism/llama2/auto_config_llama2_7b/pretrain-llama2_7b-auto_tuner.json
@@ -16,6 +16,7 @@
     "sequence_parallel": 0,   
     "use_flash_attention": true,
     "use_fused_rms_norm": true,
+    "use_fast_layer_norm": true,
     "fuse_attention_ffn": true,
     "fuse_attention_qkv": true,
     "use_fused_rope": true,