fix for qwen2moe

DrownFish19 · Feb 25, 2025 · f1676df · f1676df
1 parent e969104
commit f1676df
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/paddlenlp/transformers/moe_gate.py b/paddlenlp/transformers/moe_gate.py
@@ -520,7 +520,7 @@ def topkgating(
 
         # get topk mask
         mask = paddle.zeros_like(gates).put_along_axis(top_idx, paddle.to_tensor(1.0), axis=1)
-        if self.config.seq_aux:
+        if hasattr(self.config, "seq_aux") and self.config.seq_aux:
             l_aux = self._cal_seq_aux_loss(gates_ori, self.top_k, top_idx)
         else:
             l_aux = self._cal_aux_loss(gates, mask)

diff --git a/paddlenlp/transformers/qwen2_moe/modeling.py b/paddlenlp/transformers/qwen2_moe/modeling.py
@@ -765,7 +765,7 @@ def forward(self, hidden_states):
         Args:
             hidden_states (_type_): [batch_size * seq_len, hidden_size]
         """
-        _, h_dim = hidden_states.shape
+        _, _, h_dim = hidden_states.shape
 
         # compute gating score
         logits = F.linear(hidden_states, self.weight, None)