update RotaryEmbedding for multiple models (#3076)

Author: cjw-d

paddleformers/transformers/ernie4_5/modeling.py

@@ -114,6 +114,8 @@ def apply_fused_rope(query_states, key_states, rope_theta):
 class Ernie4_5RotaryEmbedding(nn.Layer):
     def __init__(self, config):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
         self.config = config
         self.head_dim = config.head_dim
         self.base = config.rope_theta

paddleformers/transformers/ernie4_5_moe/modeling.py

@@ -92,6 +92,8 @@ def mtp_hidden_states_set_zero(hidden_states, inbatch_pack_offset):
 class Ernie4_5_MoeRotaryEmbedding(nn.Layer):
     def __init__(self, config):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
         self.config = config
         self.head_dim = config.head_dim
         self.base = config.rope_theta

paddleformers/transformers/gemma3_text/modeling.py

@@ -123,6 +123,8 @@ def forward(self, x):
 class Gemma3RotaryEmbedding(nn.Layer):
     def __init__(self, config):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
         self.config = config
         base = config.rope_theta
         partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0

paddleformers/transformers/glm4_moe/modeling.py

@@ -1201,6 +1201,8 @@ def _gen_inv_aoa_config(cls, config: Glm4MoeConfig):
 class Glm4MoeRotaryEmbedding(nn.Layer):
     def __init__(self, config: Glm4MoeConfig, device=None):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
         self.config = config
         base = config.rope_theta
         partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0

paddleformers/transformers/llama/modeling.py

@@ -289,6 +289,8 @@ def _compute_llama3_parameters(config):
 class LlamaRotaryEmbedding(nn.Layer):
     def __init__(self, config):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
         self.config = config
         self.head_dim = getattr(config, "head_dim", config.hidden_size // config.num_attention_heads)
 

paddleformers/transformers/phi3/modeling.py

@@ -255,6 +255,8 @@ def forward(
 class Phi3RotaryEmbedding(nn.Layer):
     def __init__(self, config: Phi3Config, device=None):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
         self.config = config
         base = config.rope_theta
         partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0

paddleformers/transformers/qwen2/modeling.py

@@ -457,6 +457,8 @@ def _gen_inv_aoa_config(cls, config: Qwen2Config):
 class Qwen2RotaryEmbedding(nn.Layer):
     def __init__(self, config: Qwen2Config):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
         self.config = config
         base = config.rope_theta
         partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0

paddleformers/transformers/qwen2_5_vl/modeling.py

@@ -705,6 +705,8 @@ class Qwen2_5_VLRotaryEmbedding(nn.Layer):
 
     def __init__(self, config: Qwen2_5_VLTextConfig):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
         self.config = config
         base = config.rope_theta
         partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0

paddleformers/transformers/qwen2_moe/modeling.py

@@ -436,6 +436,8 @@ def forward(
 class Qwen2MoeRotaryEmbedding(nn.Layer):
     def __init__(self, config: Qwen2MoeConfig):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
         self.config = config
         base = config.rope_theta
         partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0

paddleformers/transformers/qwen3/modeling.py

@@ -479,6 +479,8 @@ def _gen_inv_aoa_config(cls, config: Qwen3Config):
 class Qwen3RotaryEmbedding(nn.Layer):
     def __init__(self, config: Qwen3Config):
         super().__init__()
+        self.max_seq_len_cached = config.max_position_embeddings
+        self.original_max_seq_len = config.max_position_embeddings
         self.config = config
         base = config.rope_theta
         partial_rotary_factor = config.partial_rotary_factor if hasattr(config, "partial_rotary_factor") else 1.0