Add Qwen2MoE

tests/models/qwen2_moe/test_modeling_qwen2_moe.py

@@ -509,6 +509,7 @@ def test_load_balancing_loss(self):
         config, input_dict = self.model_tester.prepare_config_and_inputs_for_common()
         config.num_labels = 3
         config.num_experts = 8
+        config.expert_interval = 2
         config.output_router_logits = True
         input_ids = input_dict["input_ids"]
         attention_mask = input_ids.ne(1).to(torch_device)