Skip to content

Commit

Permalink
[ntuple] simpler unquantization formula
Browse files Browse the repository at this point in the history
  • Loading branch information
silverweed committed Sep 24, 2024
1 parent d6c9c0f commit 64700ac
Showing 1 changed file with 1 addition and 2 deletions.
3 changes: 1 addition & 2 deletions tree/ntuple/v7/src/RColumnElement.hxx
Original file line number Diff line number Diff line change
Expand Up @@ -818,7 +818,6 @@ int UnquantizeReals(T *dst, const Quantized_t *src, std::size_t count, double mi

const std::size_t quantMax = (1ull << nQuantBits) - 1;
const double scale = (max - min) / quantMax;
const double bias = min * quantMax / (max - min);
const std::size_t unusedBits = sizeof(Quantized_t) * 8 - nQuantBits;

int nOutOfRange = 0;
Expand All @@ -831,7 +830,7 @@ int UnquantizeReals(T *dst, const Quantized_t *src, std::size_t count, double mi
ByteSwapIfNecessary(elem);

const double fq = static_cast<double>(elem);
const double e = (fq + bias) * scale;
const double e = fq * scale + min;
dst[i] = static_cast<T>(e);

nOutOfRange += !(min <= dst[i] && dst[i] <= max);
Expand Down

0 comments on commit 64700ac

Please sign in to comment.