-
Notifications
You must be signed in to change notification settings - Fork 37
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
fix for floating point rounding issue #34
base: master
Are you sure you want to change the base?
Conversation
This trips up for sufficiently large values: > for(var x = Number.EPSILON; Number.isFinite(x); x *= 10) console.log(x, Math.round(x*100)/100, rnd(x, 2));
2.220446049250313e-16 0 '0'
2.220446049250313e-15 0 '0'
2.220446049250313e-14 0 '0'
2.220446049250313e-13 0 '0'
2.220446049250313e-12 0 '0'
2.220446049250313e-11 0 '0'
2.220446049250313e-10 0 '0'
2.220446049250313e-9 0 '0'
2.220446049250313e-8 0 '0'
2.220446049250313e-7 0 '0'
0.000002220446049250313 0 '0'
0.00002220446049250313 0 '0'
0.0002220446049250313 0 '0'
0.002220446049250313 0 '0'
0.02220446049250313 0.02 '0.02'
0.2220446049250313 0.22 '0.22'
2.220446049250313 2.22 '2.22'
22.20446049250313 22.2 '22.2'
222.0446049250313 222.04 '222.04'
2220.446049250313 2220.45 '2220.45'
22204.46049250313 22204.46 '22204.46'
222044.6049250313 222044.6 '222044.6'
2220446.049250313 2220446.05 '2220446.05'
22204460.49250313 22204460.49 '22204460.49'
222044604.9250313 222044604.93 '222044604.93'
2220446049.250313 2220446049.25 '2220446049.25'
22204460492.503128 22204460492.5 '22204460492.5'
222044604925.03128 222044604925.03 '222044604925.03'
2220446049250.313 2220446049250.31 '2220446049250.31'
22204460492503.13 22204460492503.13 '22204460492503.13'
222044604925031.28 222044604925031.28 '222044604925031.3'
2220446049250312.8 2220446049250312.8 '2220446049250313'
22204460492503130 22204460492503130 '22204460492503130'
222044604925031300 222044604925031300 '222044604925031300'
2220446049250313000 2220446049250313000 '2220446049250313000'
22204460492503130000 22204460492503130000 'NaN'
222044604925031300000 222044604925031320000 'NaN'
2.2204460492503127e+21 2.2204460492503127e+21 'NaN'
2.2204460492503126e+22 2.2204460492503126e+22 'NaN'
2.2204460492503126e+23 2.2204460492503126e+23 'NaN'
2.2204460492503126e+24 2.2204460492503126e+24 'NaN'
2.2204460492503127e+25 2.2204460492503127e+25 'NaN'
2.220446049250313e+26 2.220446049250313e+26 'NaN'
2.2204460492503128e+27 2.2204460492503128e+27 'NaN'
2.220446049250313e+28 2.220446049250313e+28 'NaN'
2.220446049250313e+29 2.2204460492503126e+29 'NaN'
2.220446049250313e+30 2.220446049250313e+30 'NaN'
2.220446049250313e+31 2.220446049250313e+31 'NaN'
2.220446049250313e+32 2.220446049250313e+32 'NaN'
2.2204460492503133e+33 2.220446049250313e+33 'NaN'
2.2204460492503133e+34 2.2204460492503133e+34 'NaN'
2.220446049250313e+35 2.220446049250313e+35 'NaN'
2.220446049250313e+36 2.2204460492503134e+36 'NaN'
2.220446049250313e+37 2.220446049250313e+37 'NaN'
2.2204460492503133e+38 2.2204460492503133e+38 'NaN'
2.2204460492503134e+39 2.2204460492503137e+39 'NaN'
2.2204460492503133e+40 2.2204460492503133e+40 'NaN'
2.220446049250313e+41 2.220446049250313e+41 'NaN'
2.2204460492503133e+42 2.2204460492503133e+42 'NaN'
2.2204460492503133e+43 2.2204460492503133e+43 'NaN'
2.2204460492503134e+44 2.2204460492503134e+44 'NaN'
2.2204460492503133e+45 2.2204460492503133e+45 'NaN'
2.2204460492503134e+46 2.2204460492503134e+46 'NaN'
2.2204460492503136e+47 2.2204460492503136e+47 'NaN'
2.2204460492503135e+48 2.2204460492503135e+48 'NaN'
2.2204460492503136e+49 2.2204460492503136e+49 'NaN'
2.2204460492503134e+50 2.2204460492503134e+50 'NaN'
2.2204460492503134e+51 2.2204460492503134e+51 'NaN'
2.2204460492503132e+52 2.2204460492503132e+52 'NaN'
2.2204460492503132e+53 2.2204460492503132e+53 'NaN'
2.2204460492503132e+54 2.2204460492503132e+54 'NaN'
2.2204460492503132e+55 2.2204460492503132e+55 'NaN'
2.2204460492503134e+56 2.2204460492503134e+56 'NaN'
2.2204460492503134e+57 2.2204460492503134e+57 'NaN'
2.2204460492503133e+58 2.2204460492503133e+58 'NaN'
2.2204460492503134e+59 2.2204460492503134e+59 'NaN'
2.2204460492503132e+60 2.2204460492503132e+60 'NaN'
2.2204460492503133e+61 2.2204460492503133e+61 'NaN'
2.2204460492503136e+62 2.2204460492503136e+62 'NaN'
2.2204460492503134e+63 2.2204460492503134e+63 'NaN'
2.2204460492503134e+64 2.2204460492503134e+64 'NaN'
2.2204460492503134e+65 2.2204460492503134e+65 'NaN'
2.2204460492503134e+66 2.2204460492503134e+66 'NaN'
2.2204460492503134e+67 2.2204460492503134e+67 'NaN'
2.2204460492503137e+68 2.2204460492503137e+68 'NaN'
2.220446049250314e+69 2.220446049250314e+69 'NaN'
2.2204460492503137e+70 2.2204460492503137e+70 'NaN'
2.220446049250314e+71 2.220446049250314e+71 'NaN'
2.220446049250314e+72 2.220446049250314e+72 'NaN'
2.220446049250314e+73 2.220446049250314e+73 'NaN'
2.220446049250314e+74 2.2204460492503143e+74 'NaN'
2.220446049250314e+75 2.220446049250314e+75 'NaN'
2.220446049250314e+76 2.2204460492503138e+76 'NaN'
2.2204460492503142e+77 2.2204460492503142e+77 'NaN'
2.2204460492503144e+78 2.2204460492503144e+78 'NaN'
2.2204460492503144e+79 2.2204460492503144e+79 'NaN'
2.2204460492503145e+80 2.2204460492503145e+80 'NaN'
2.2204460492503145e+81 2.2204460492503145e+81 'NaN'
2.2204460492503144e+82 2.2204460492503144e+82 'NaN'
2.2204460492503144e+83 2.2204460492503144e+83 'NaN'
2.2204460492503144e+84 2.2204460492503144e+84 'NaN'
2.2204460492503144e+85 2.2204460492503144e+85 'NaN'
2.2204460492503144e+86 2.2204460492503144e+86 'NaN'
2.2204460492503144e+87 2.2204460492503144e+87 'NaN'
2.2204460492503145e+88 2.2204460492503145e+88 'NaN'
2.2204460492503146e+89 2.2204460492503144e+89 'NaN'
2.2204460492503146e+90 2.2204460492503146e+90 'NaN'
2.2204460492503145e+91 2.2204460492503145e+91 'NaN'
2.2204460492503143e+92 2.2204460492503143e+92 'NaN'
2.2204460492503144e+93 2.2204460492503144e+93 'NaN'
2.2204460492503146e+94 2.220446049250315e+94 'NaN'
2.2204460492503145e+95 2.2204460492503148e+95 'NaN'
2.2204460492503143e+96 2.2204460492503143e+96 'NaN'
2.2204460492503143e+97 2.2204460492503143e+97 'NaN'
2.2204460492503143e+98 2.2204460492503143e+98 'NaN'
2.220446049250314e+99 2.220446049250314e+99 'NaN'
2.2204460492503143e+100 2.2204460492503143e+100 'NaN'
2.220446049250314e+101 2.220446049250314e+101 'NaN'
2.2204460492503142e+102 2.2204460492503142e+102 'NaN'
2.220446049250314e+103 2.220446049250314e+103 'NaN'
2.2204460492503143e+104 2.2204460492503143e+104 'NaN'
2.2204460492503144e+105 2.2204460492503144e+105 'NaN'
2.2204460492503143e+106 2.2204460492503143e+106 'NaN'
2.2204460492503143e+107 2.220446049250314e+107 'NaN'
2.2204460492503143e+108 2.2204460492503143e+108 'NaN'
2.220446049250314e+109 2.220446049250314e+109 'NaN'
2.2204460492503142e+110 2.2204460492503142e+110 'NaN'
2.2204460492503143e+111 2.2204460492503143e+111 'NaN'
2.220446049250314e+112 2.220446049250314e+112 'NaN'
2.220446049250314e+113 2.220446049250314e+113 'NaN'
2.220446049250314e+114 2.220446049250314e+114 'NaN'
2.220446049250314e+115 2.220446049250314e+115 'NaN'
2.220446049250314e+116 2.220446049250314e+116 'NaN'
2.2204460492503143e+117 2.220446049250314e+117 'NaN'
2.2204460492503143e+118 2.2204460492503143e+118 'NaN'
2.220446049250314e+119 2.220446049250314e+119 'NaN'
2.220446049250314e+120 2.2204460492503138e+120 'NaN'
2.220446049250314e+121 2.220446049250314e+121 'NaN'
2.220446049250314e+122 2.2204460492503135e+122 'NaN'
2.220446049250314e+123 2.2204460492503137e+123 'NaN'
2.220446049250314e+124 2.220446049250314e+124 'NaN'
2.2204460492503142e+125 2.2204460492503142e+125 'NaN'
2.220446049250314e+126 2.2204460492503144e+126 'NaN'
2.220446049250314e+127 2.220446049250314e+127 'NaN'
2.2204460492503143e+128 2.2204460492503143e+128 'NaN'
2.2204460492503143e+129 2.2204460492503143e+129 'NaN'
2.2204460492503144e+130 2.2204460492503144e+130 'NaN'
2.2204460492503146e+131 2.2204460492503146e+131 'NaN'
2.2204460492503145e+132 2.220446049250314e+132 'NaN'
2.2204460492503145e+133 2.2204460492503145e+133 'NaN'
2.2204460492503142e+134 2.2204460492503142e+134 'NaN'
2.220446049250314e+135 2.2204460492503138e+135 'NaN'
2.2204460492503142e+136 2.2204460492503142e+136 'NaN'
2.2204460492503143e+137 2.2204460492503143e+137 'NaN'
2.220446049250314e+138 2.220446049250314e+138 'NaN'
2.2204460492503142e+139 2.2204460492503142e+139 'NaN'
2.2204460492503143e+140 2.2204460492503143e+140 'NaN'
2.2204460492503144e+141 2.220446049250314e+141 'NaN'
2.2204460492503143e+142 2.2204460492503146e+142 'NaN'
2.2204460492503142e+143 2.2204460492503142e+143 'NaN'
2.220446049250314e+144 2.220446049250314e+144 'NaN'
2.2204460492503143e+145 2.2204460492503143e+145 'NaN'
2.2204460492503143e+146 2.2204460492503143e+146 'NaN'
2.2204460492503144e+147 2.2204460492503144e+147 'NaN'
2.2204460492503142e+148 2.220446049250314e+148 'NaN'
2.220446049250314e+149 2.220446049250314e+149 'NaN'
2.220446049250314e+150 2.220446049250314e+150 'NaN'
2.220446049250314e+151 2.220446049250314e+151 'NaN'
2.220446049250314e+152 2.220446049250314e+152 'NaN'
2.220446049250314e+153 2.220446049250314e+153 'NaN'
2.220446049250314e+154 2.220446049250314e+154 'NaN'
2.220446049250314e+155 2.220446049250314e+155 'NaN'
2.220446049250314e+156 2.220446049250314e+156 'NaN'
2.220446049250314e+157 2.220446049250314e+157 'NaN'
2.220446049250314e+158 2.220446049250314e+158 'NaN'
2.220446049250314e+159 2.220446049250314e+159 'NaN'
2.220446049250314e+160 2.220446049250314e+160 'NaN'
2.220446049250314e+161 2.220446049250314e+161 'NaN'
2.220446049250314e+162 2.220446049250314e+162 'NaN'
2.220446049250314e+163 2.220446049250314e+163 'NaN'
2.220446049250314e+164 2.220446049250314e+164 'NaN'
2.220446049250314e+165 2.220446049250314e+165 'NaN'
2.2204460492503142e+166 2.2204460492503145e+166 'NaN'
2.2204460492503143e+167 2.2204460492503143e+167 'NaN'
2.2204460492503144e+168 2.2204460492503144e+168 'NaN'
2.2204460492503144e+169 2.220446049250314e+169 'NaN'
2.2204460492503144e+170 2.2204460492503144e+170 'NaN'
2.220446049250314e+171 2.220446049250314e+171 'NaN'
2.220446049250314e+172 2.220446049250314e+172 'NaN'
2.2204460492503142e+173 2.2204460492503142e+173 'NaN'
2.220446049250314e+174 2.220446049250314e+174 'NaN'
2.2204460492503143e+175 2.2204460492503143e+175 'NaN'
2.2204460492503143e+176 2.2204460492503143e+176 'NaN'
2.220446049250314e+177 2.220446049250314e+177 'NaN'
2.220446049250314e+178 2.220446049250314e+178 'NaN'
2.220446049250314e+179 2.2204460492503136e+179 'NaN'
2.220446049250314e+180 2.220446049250314e+180 'NaN'
2.2204460492503137e+181 2.2204460492503137e+181 'NaN'
2.2204460492503137e+182 2.2204460492503137e+182 'NaN'
2.220446049250314e+183 2.220446049250314e+183 'NaN'
2.2204460492503138e+184 2.2204460492503138e+184 'NaN'
2.2204460492503138e+185 2.2204460492503138e+185 'NaN'
2.220446049250314e+186 2.220446049250314e+186 'NaN'
2.220446049250314e+187 2.220446049250314e+187 'NaN'
2.220446049250314e+188 2.220446049250314e+188 'NaN'
2.2204460492503139e+189 2.2204460492503139e+189 'NaN'
2.220446049250314e+190 2.220446049250314e+190 'NaN'
2.2204460492503138e+191 2.2204460492503138e+191 'NaN'
2.2204460492503138e+192 2.2204460492503138e+192 'NaN'
2.2204460492503136e+193 2.2204460492503136e+193 'NaN'
2.2204460492503136e+194 2.2204460492503136e+194 'NaN'
2.2204460492503135e+195 2.2204460492503135e+195 'NaN'
2.2204460492503137e+196 2.2204460492503137e+196 'NaN'
2.2204460492503137e+197 2.2204460492503137e+197 'NaN'
2.2204460492503138e+198 2.2204460492503138e+198 'NaN'
2.2204460492503137e+199 2.2204460492503137e+199 'NaN'
2.2204460492503138e+200 2.2204460492503138e+200 'NaN'
2.2204460492503138e+201 2.2204460492503138e+201 'NaN'
2.2204460492503137e+202 2.2204460492503137e+202 'NaN'
2.2204460492503137e+203 2.2204460492503137e+203 'NaN'
2.2204460492503136e+204 2.2204460492503136e+204 'NaN'
2.2204460492503136e+205 2.2204460492503136e+205 'NaN'
2.2204460492503135e+206 2.2204460492503135e+206 'NaN'
2.2204460492503135e+207 2.2204460492503132e+207 'NaN'
2.2204460492503135e+208 2.2204460492503135e+208 'NaN'
2.2204460492503133e+209 2.2204460492503133e+209 'NaN'
2.2204460492503133e+210 2.2204460492503133e+210 'NaN'
2.2204460492503133e+211 2.2204460492503133e+211 'NaN'
2.2204460492503133e+212 2.2204460492503133e+212 'NaN'
2.2204460492503134e+213 2.220446049250313e+213 'NaN'
2.2204460492503133e+214 2.2204460492503133e+214 'NaN'
2.2204460492503132e+215 2.2204460492503132e+215 'NaN'
2.2204460492503132e+216 2.2204460492503132e+216 'NaN'
2.220446049250313e+217 2.220446049250313e+217 'NaN'
2.220446049250313e+218 2.220446049250313e+218 'NaN'
2.2204460492503132e+219 2.2204460492503132e+219 'NaN'
2.2204460492503132e+220 2.2204460492503132e+220 'NaN'
2.2204460492503133e+221 2.2204460492503133e+221 'NaN'
2.2204460492503132e+222 2.2204460492503132e+222 'NaN'
2.2204460492503133e+223 2.2204460492503133e+223 'NaN'
2.2204460492503135e+224 2.2204460492503135e+224 'NaN'
2.2204460492503135e+225 2.2204460492503135e+225 'NaN'
2.2204460492503135e+226 2.2204460492503135e+226 'NaN'
2.2204460492503132e+227 2.2204460492503132e+227 'NaN'
2.2204460492503132e+228 2.2204460492503132e+228 'NaN'
2.2204460492503132e+229 2.2204460492503134e+229 'NaN'
2.2204460492503132e+230 2.2204460492503132e+230 'NaN'
2.2204460492503133e+231 2.2204460492503133e+231 'NaN'
2.2204460492503135e+232 2.2204460492503137e+232 'NaN'
2.2204460492503133e+233 2.2204460492503133e+233 'NaN'
2.2204460492503133e+234 2.2204460492503133e+234 'NaN'
2.2204460492503132e+235 2.2204460492503132e+235 'NaN'
2.2204460492503133e+236 2.2204460492503133e+236 'NaN'
2.2204460492503133e+237 2.2204460492503137e+237 'NaN'
2.2204460492503132e+238 2.2204460492503132e+238 'NaN'
2.220446049250313e+239 2.220446049250313e+239 'NaN'
2.220446049250313e+240 2.220446049250313e+240 'NaN'
2.220446049250313e+241 2.2204460492503134e+241 'NaN'
2.220446049250313e+242 2.220446049250313e+242 'NaN'
2.2204460492503133e+243 2.2204460492503133e+243 'NaN'
2.2204460492503132e+244 2.2204460492503132e+244 'NaN'
2.2204460492503134e+245 2.2204460492503134e+245 'NaN'
2.2204460492503135e+246 2.2204460492503135e+246 'NaN'
2.2204460492503134e+247 2.2204460492503137e+247 'NaN'
2.2204460492503134e+248 2.2204460492503134e+248 'NaN'
2.2204460492503136e+249 2.2204460492503136e+249 'NaN'
2.2204460492503138e+250 2.2204460492503138e+250 'NaN'
2.2204460492503138e+251 2.2204460492503138e+251 'NaN'
2.220446049250314e+252 2.220446049250314e+252 'NaN'
2.2204460492503138e+253 2.2204460492503138e+253 'NaN'
2.2204460492503137e+254 2.2204460492503134e+254 'NaN'
2.220446049250314e+255 2.220446049250314e+255 'NaN'
2.220446049250314e+256 2.220446049250314e+256 'NaN'
2.2204460492503138e+257 2.2204460492503138e+257 'NaN'
2.2204460492503138e+258 2.2204460492503138e+258 'NaN'
2.2204460492503137e+259 2.2204460492503137e+259 'NaN'
2.2204460492503137e+260 2.2204460492503137e+260 'NaN'
2.2204460492503135e+261 2.2204460492503135e+261 'NaN'
2.2204460492503133e+262 2.220446049250313e+262 'NaN'
2.2204460492503135e+263 2.2204460492503135e+263 'NaN'
2.2204460492503136e+264 2.2204460492503136e+264 'NaN'
2.2204460492503135e+265 2.2204460492503135e+265 'NaN'
2.2204460492503134e+266 2.2204460492503137e+266 'NaN'
2.2204460492503134e+267 2.2204460492503134e+267 'NaN'
2.2204460492503136e+268 2.2204460492503136e+268 'NaN'
2.2204460492503136e+269 2.2204460492503136e+269 'NaN'
2.2204460492503136e+270 2.2204460492503136e+270 'NaN'
2.2204460492503136e+271 2.2204460492503136e+271 'NaN'
2.2204460492503137e+272 2.2204460492503137e+272 'NaN'
2.2204460492503137e+273 2.2204460492503137e+273 'NaN'
2.2204460492503136e+274 2.2204460492503136e+274 'NaN'
2.2204460492503136e+275 2.2204460492503136e+275 'NaN'
2.2204460492503136e+276 2.2204460492503136e+276 'NaN'
2.2204460492503137e+277 2.2204460492503137e+277 'NaN'
2.2204460492503135e+278 2.2204460492503135e+278 'NaN'
2.2204460492503135e+279 2.2204460492503135e+279 'NaN'
2.2204460492503134e+280 2.2204460492503134e+280 'NaN'
2.2204460492503134e+281 2.2204460492503138e+281 'NaN'
2.2204460492503135e+282 2.2204460492503135e+282 'NaN'
2.2204460492503136e+283 2.2204460492503136e+283 'NaN'
2.2204460492503135e+284 2.2204460492503135e+284 'NaN'
2.2204460492503135e+285 2.2204460492503135e+285 'NaN'
2.2204460492503133e+286 2.2204460492503133e+286 'NaN'
2.220446049250313e+287 2.220446049250313e+287 'NaN'
2.220446049250313e+288 2.2204460492503127e+288 'NaN'
2.220446049250313e+289 2.220446049250313e+289 'NaN'
2.2204460492503128e+290 2.2204460492503128e+290 'NaN'
2.2204460492503128e+291 2.2204460492503128e+291 'NaN'
2.2204460492503127e+292 2.2204460492503127e+292 'NaN'
2.2204460492503128e+293 2.2204460492503124e+293 'NaN'
2.2204460492503128e+294 2.220446049250313e+294 'NaN'
2.220446049250313e+295 2.220446049250313e+295 'NaN'
2.220446049250313e+296 2.2204460492503126e+296 'NaN'
2.220446049250313e+297 2.220446049250313e+297 'NaN'
2.2204460492503132e+298 2.2204460492503132e+298 'NaN'
2.220446049250313e+299 2.2204460492503135e+299 'NaN'
2.220446049250313e+300 2.220446049250313e+300 'NaN'
2.220446049250313e+301 2.220446049250313e+301 'NaN'
2.220446049250313e+302 2.2204460492503127e+302 'NaN'
2.2204460492503132e+303 2.2204460492503132e+303 'NaN'
2.2204460492503134e+304 2.2204460492503134e+304 'NaN'
2.2204460492503133e+305 2.2204460492503133e+305 'NaN'
2.2204460492503133e+306 Infinity 'NaN'
2.2204460492503132e+307 Infinity 'NaN' This happens when the stringification in Would something as simple as starting from |
@SheetJSDev i've pushed a new commit that reverts to the native Math.round if the result is Did you have any luck at your end? |
@@ -14,3 +14,5 @@ value #.0000,,, #.0000,, #.0000, #,##0.0 ###,##0 ###,### #,###.00 | |||
123456789012 123.4568 123456.7890 123456789.0120 123,456,789,012.0 123,456,789,012 123,456,789,012 123,456,789,012.00 | |||
4321 .0000 .0043 4.3210 4,321.0 4,321 4,321 4,321.00 | |||
4321234 .0043 4.3212 4321.2340 4,321,234.0 4,321,234 4,321,234 4,321,234.00 | |||
1234652.68499999999995 .0012 1.2347 1234.6527 1,234,652.7 1,234,653 1,234,653 1,234,652.69 | |||
-4321234 -0.0043 -4.3212 -4321.2340 -4,321,234.0 -4,321,234 -4,321,234 -4,321,234.00 |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The first result should be -.0043 (no leading '0')
Suggested fix for SheetJS/sheetjs#1105