Fix wrong scale eps applied

Author: alexsamardzic

torchao/quantization/quant_primitives.py

@@ -944,10 +944,16 @@ def _choose_qparams_affine(
         else:
             zero_point = torch.full_like(scale, int((quant_max + quant_min + 1) / 2))
         scale = torch.clamp(scale, min=eps)
+        if torch.is_floating_point(scale):
+            # Prevent 1.0 / scale to become Inf.
+            scale = torch.clamp(scale, min=2 * torch.finfo(scale.dtype).tiny)
     else:
         assert mapping_type == MappingType.ASYMMETRIC.name
         scale = (max_val_pos - min_val_neg) / float(quant_max - quant_min)
         scale = torch.clamp(scale, min=eps)
+        if torch.is_floating_point(scale):
+            # Prevent 1.0 / scale to become Inf.
+            scale = torch.clamp(scale, min=2 * torch.finfo(scale.dtype).tiny)
         if zero_point_domain == ZeroPointDomain.NONE.name:
             zero_point = None
         else: