fix device bug (#873)

Signed-off-by: He, Xin3 <xin3.he@intel.com>

Author: xin3he

auto_round/compressors/base.py

@@ -1420,7 +1420,7 @@ def _quantize_layer_via_rtn(self, name: str) -> None:
             m.zp = None
         else:
             try:
-                m.to(self.device)
+                m = m.to(self.device)
                 m = WrapperLinear(
                     m,
                     enable_minmax_tuning=False,
@@ -1858,6 +1858,7 @@ def _quantize_layers(self, layer_names: list, layer_inputs: dict) -> None:
                 from auto_round.data_type import QUANT_FUNC_WITH_DTYPE
 
                 layer = get_module(self.model, layer_name)
+                layer = layer.to(self.device)
                 if _is_fp8_model(self.model):
                     new_layer = convert_fp8_layer_to_linear(layer, self.amp_dtype).to(self.device)
                     set_module(self.model, layer_name, new_layer)