Skip to content

ENH: Improve numerical stability for window functions skew and kurt #37557

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 20 commits into from
Nov 9, 2020
Merged

ENH: Improve numerical stability for window functions skew and kurt #37557

Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
d4b1a9b
Improve numerical stability for window functions skew and kurt
phofl Nov 1, 2020
c634539
Fix typing
phofl Nov 1, 2020
bf88a63
Fix pattern
phofl Nov 1, 2020
e57f97c
Improve numerical stability for mean calculation
phofl Nov 1, 2020
a71f645
Change dtype assignment
phofl Nov 1, 2020
a1874d8
Improve performance
phofl Nov 1, 2020
288ac22
Merge branch 'master' of https://github.com/pandas-dev/pandas into 6929
phofl Nov 1, 2020
eb35925
Implement kahan summation
phofl Nov 1, 2020
480ba26
Change whatsnew
phofl Nov 1, 2020
c58be86
RUn black
phofl Nov 1, 2020
fa460b1
Fix pattern
phofl Nov 1, 2020
ce0a9a2
Fix failing test
phofl Nov 1, 2020
e643a45
Remove unnecessary division
phofl Nov 1, 2020
5998b7b
Merge branch 'master' of https://github.com/pandas-dev/pandas into 6929
phofl Nov 2, 2020
7109729
Merge branch 'master' of https://github.com/pandas-dev/pandas into 6929
phofl Nov 8, 2020
bfedac0
Hold no gil permanently
phofl Nov 8, 2020
b03d5c1
Merge branch 'master' of https://github.com/pandas-dev/pandas into 6929
phofl Nov 8, 2020
852f3ca
Use round from math
phofl Nov 8, 2020
4688648
Fix pattern
phofl Nov 8, 2020
50ea50a
Merge branch 'master' of https://github.com/pandas-dev/pandas into 6929
phofl Nov 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Use round from math
  • Loading branch information
@phofl
phofl committed Nov 8, 2020
commit 852f3ca89380de5f2f487a84e16674a6a48f488b
13 changes: 7 additions & 6 deletions pandas/_libs/window/aggregations.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import cython

from libc.math cimport round
from libcpp.deque cimport deque

import numpy as np
Expand Down Expand Up @@ -520,7 +521,7 @@ def roll_skew(ndarray[float64_t] values, ndarray[int64_t] start,
float64_t compensation_xx_add = 0, compensation_xx_remove = 0
float64_t compensation_x_add = 0, compensation_x_remove = 0
float64_t x = 0, xx = 0, xxx = 0
int64_t nobs = 0, i, j, N = len(values), nobs_mean = 0, mean_int = 0
int64_t nobs = 0, i, j, N = len(values), nobs_mean = 0
int64_t s, e
ndarray[float64_t] output, mean_array
bint is_monotonic_increasing_bounds
Expand All @@ -541,9 +542,9 @@ def roll_skew(ndarray[float64_t] values, ndarray[int64_t] start,
mean_val = sum_val / nobs_mean
# Other cases would lead to imprecision for smallest values
if min_val - mean_val > -1e5:
mean_int = <int64_t>mean_val
mean_val = round(mean_val)
for i in range(0, N):
values[i] = values[i] - mean_int
values[i] = values[i] - mean_val

for i in range(0, N):

Expand Down Expand Up @@ -701,7 +702,7 @@ def roll_kurt(ndarray[float64_t] values, ndarray[int64_t] start,
float64_t compensation_xx_remove = 0, compensation_xx_add = 0
float64_t compensation_x_remove = 0, compensation_x_add = 0
float64_t x = 0, xx = 0, xxx = 0, xxxx = 0
int64_t nobs = 0, i, j, s, e, N = len(values), nobs_mean = 0, mean_int = 0
int64_t nobs = 0, i, j, s, e, N = len(values), nobs_mean = 0
ndarray[float64_t] output
bint is_monotonic_increasing_bounds

Expand All @@ -721,9 +722,9 @@ def roll_kurt(ndarray[float64_t] values, ndarray[int64_t] start,
mean_val = sum_val / nobs_mean
# Other cases would lead to imprecision for smallest values
if min_val - mean_val > -1e4:
mean_int = <int64_t>mean_val
mean_val = round(mean_val)
for i in range(0, N):
values[i] = values[i] - mean_int
values[i] = values[i] - mean_val

for i in range(0, N):

Expand Down