TST Make tests pass on Cambricon MLUs (#1747)

Small adjustments to tests to make them pass on Cambricon MLUs (mostly
tolerances). Note that we have no MLU test runners for PEFT, so have to
rely on others to run these tests.

tests/test_custom_models.py

@@ -957,6 +957,9 @@ def test_disable_adapters_with_merging(self, test_name, model_id, config_cls, co
         # check that there is a difference in results after training
         assert not torch.allclose(outputs_before, outputs_after, atol=atol, rtol=rtol)
 
+        if self.torch_device in ["mlu"] and model_id in ["Conv2d"]:
+            atol, rtol = 1e-3, 1e-2  # MLU
+
         # unmerged or merged should make no difference
         assert torch.allclose(outputs_after, outputs_unmerged, atol=atol, rtol=rtol)
 

tests/test_lora_megatron.py

@@ -24,6 +24,8 @@
 
 from peft import LoraConfig, PeftModel, get_peft_model, get_peft_model_state_dict
 
+from .testing_utils import require_torch_gpu
+
 
 def is_megatron_available() -> bool:
     return importlib.util.find_spec("megatron") is not None
@@ -93,6 +95,7 @@ def forward(self, input):
             x = self.lm_head(x)[0]
             return x
 
+    @require_torch_gpu
     class TestMegatronLora(unittest.TestCase):
         def setUp(self):
             initialize_model_parallel(1, 1)

tests/testing_common.py

@@ -24,6 +24,7 @@
 import torch
 import yaml
 from diffusers import StableDiffusionPipeline
+from packaging import version
 
 from peft import (
     AdaLoraConfig,
@@ -464,13 +465,16 @@ def _test_merge_layers_fp16(self, model_id, config_cls, config_kwargs):
         if ("gpt2" in model_id.lower()) and (config_cls != LoraConfig):
             self.skipTest("Merging GPT2 adapters not supported for IA³ (yet)")
 
+        if (self.torch_device in ["cpu"]) and (version.parse(torch.__version__) <= version.parse("2.1")):
+            self.skipTest("PyTorch 2.1 not supported for Half of addmm_impl_cpu_ ")
+
         model = self.transformers_class.from_pretrained(model_id, torch_dtype=torch.float16)
         config = config_cls(
             base_model_name_or_path=model_id,
             **config_kwargs,
         )
         model = get_peft_model(model, config)
-        model = model.to(device="cpu", dtype=torch.float16)
+        model = model.to(device=self.torch_device, dtype=torch.float16)
 
         model.eval()
 
@@ -561,6 +565,8 @@ def _test_merge_layers(self, model_id, config_cls, config_kwargs):
         logits_merged_unloaded = model(**dummy_input)[0]
 
         atol, rtol = 1e-4, 1e-4
+        if self.torch_device in ["mlu"]:
+            atol, rtol = 1e-3, 1e-3  # MLU
         if (config.peft_type == "IA3") and (model_id == "Conv2d"):
             # for some reason, the IA³ Conv2d introduces a larger error
             atol, rtol = 0.3, 0.01
@@ -689,16 +695,19 @@ def _test_safe_merge(self, model_id, config_cls, config_kwargs):
         model = get_peft_model(model, config).eval()
         logits_peft = model(**inputs)[0]
 
+        atol, rtol = 1e-6, 1e-6  # default
         # Initializing with LN tuning cannot be configured to change the outputs (unlike init_lora_weights=False)
         if not issubclass(config_cls, LNTuningConfig):
             # sanity check that the logits are different
-            assert not torch.allclose(logits_base, logits_peft, atol=1e-6, rtol=1e-6)
+            assert not torch.allclose(logits_base, logits_peft, atol=atol, rtol=rtol)
 
         model_unloaded = model.merge_and_unload(safe_merge=True)
         logits_unloaded = model_unloaded(**inputs)[0]
 
+        if self.torch_device in ["mlu"]:
+            atol, rtol = 1e-3, 1e-3  # MLU
         # check that the logits are the same after unloading
-        assert torch.allclose(logits_peft, logits_unloaded, atol=1e-6, rtol=1e-6)
+        assert torch.allclose(logits_peft, logits_unloaded, atol=atol, rtol=rtol)
 
     def _test_mixed_adapter_batches(self, model_id, config_cls, config_kwargs):
         # Test for mixing different adapters in a single batch by passing the adapter_names argument