fix int overflow in quantization

szcompressor · May 27, 2024 · f8fc2cb · f8fc2cb
1 parent f6cae37
commit f8fc2cb
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/include/SZ3/quantizer/IntegerQuantizer.hpp b/include/SZ3/quantizer/IntegerQuantizer.hpp
@@ -33,7 +33,7 @@ namespace SZ3 {
         // quantize the data with a prediction value, and returns the quantization index
         int quantize(T data, T pred) {
             T diff = data - pred;
-            int quant_index = (int) (fabs(diff) * this->error_bound_reciprocal) + 1;
+            auto quant_index = (int64_t) (fabs(diff) * this->error_bound_reciprocal) + 1;
             if (quant_index < this->radius * 2) {
                 quant_index >>= 1;
                 int half_index = quant_index;
@@ -60,7 +60,7 @@ namespace SZ3 {
         // int quantize(T data, T pred, T& dec_data);
         int quantize_and_overwrite(T &data, T pred) {
             T diff = data - pred;
-            int quant_index = (int) (fabs(diff) * this->error_bound_reciprocal) + 1;
+            auto quant_index = (int64_t) (fabs(diff) * this->error_bound_reciprocal) + 1;
             if (quant_index < this->radius * 2) {
                 quant_index >>= 1;
                 int half_index = quant_index;
@@ -88,7 +88,7 @@ namespace SZ3 {
 
         int quantize_and_overwrite(T ori, T pred, T &dest) {
             T diff = ori - pred;
-            int quant_index = (int) (fabs(diff) * this->error_bound_reciprocal) + 1;
+            auto quant_index = (int64_t) (fabs(diff) * this->error_bound_reciprocal) + 1;
             if (quant_index < this->radius * 2) {
                 quant_index >>= 1;
                 int half_index = quant_index;