Skip to content

Pandas and Numpy datetime serialization fixes #3022

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jan 14, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Support coercing pandas datetime DataFrames
  • Loading branch information
jonmmease committed Jan 13, 2021
commit 5fae4326ec83ece0fdae3dd39f09b4468dce6899
7 changes: 7 additions & 0 deletions packages/python/plotly/_plotly_utils/basevalidators.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,13 @@ def copy_to_readonly_numpy_array(v, kind=None, force_numeric=False):
else:
# DatetimeIndex
v = v.to_pydatetime()
elif pd and isinstance(v, pd.DataFrame) and len(set(v.dtypes)) == 1:
dtype = v.dtypes[0]
if dtype.kind in numeric_kinds:
v = v.values
elif dtype.kind == "M":
v = [row.dt.to_pydatetime().tolist() for i, row in v.iterrows()]

if not isinstance(v, np.ndarray):
# v has its own logic on how to convert itself into a numpy array
if is_numpy_convertable(v):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ def test_color_validator_categorical(color_validator, color_categorical_pandas):
np.testing.assert_array_equal(res, np.array(color_categorical_pandas))


def test_data_array_validator_dates(data_array_validator, datetime_pandas, dates_array):
def test_data_array_validator_dates_series(data_array_validator, datetime_pandas, dates_array):

res = data_array_validator.validate_coerce(datetime_pandas)

Expand All @@ -185,3 +185,18 @@ def test_data_array_validator_dates(data_array_validator, datetime_pandas, dates

# Check values
np.testing.assert_array_equal(res, dates_array)


def test_data_array_validator_dates_dataframe(data_array_validator, datetime_pandas, dates_array):

df = pd.DataFrame({"d": datetime_pandas})
res = data_array_validator.validate_coerce(df)

# Check type
assert isinstance(res, np.ndarray)

# Check dtype
assert res.dtype == "object"

# Check values
np.testing.assert_array_equal(res, dates_array.reshape(len(dates_array), 1))
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,61 @@ def test_pandas_json_encoding(self):
j6 = _json.dumps(ts.index, cls=utils.PlotlyJSONEncoder)
assert j6 == '["2011-01-01T00:00:00", "2011-01-01T01:00:00"]'

def test_encode_customdata_datetime_series(self):
df = pd.DataFrame(dict(t=pd.to_datetime(["2010-01-01", "2010-01-02"])))

# 1D customdata
fig = Figure(
Scatter(x=df["t"], customdata=df["t"]), layout=dict(template="none")
)
fig_json = _json.dumps(
fig, cls=utils.PlotlyJSONEncoder, separators=(",", ":"), sort_keys=True
)
self.assertTrue(
fig_json.startswith(
'{"data":[{"customdata":["2010-01-01T00:00:00","2010-01-02T00:00:00"]'
)
)

def test_encode_customdata_datetime_homogenous_dataframe(self):
df = pd.DataFrame(dict(
t1=pd.to_datetime(["2010-01-01", "2010-01-02"]),
t2=pd.to_datetime(["2011-01-01", "2011-01-02"]),
))
# 2D customdata
fig = Figure(
Scatter(x=df["t1"], customdata=df[["t1", "t2"]]), layout=dict(template="none")
)
fig_json = _json.dumps(
fig, cls=utils.PlotlyJSONEncoder, separators=(",", ":"), sort_keys=True
)
self.assertTrue(
fig_json.startswith(
'{"data":[{"customdata":'
'[["2010-01-01T00:00:00","2011-01-01T00:00:00"],'
'["2010-01-02T00:00:00","2011-01-02T00:00:00"]'
)
)

def test_encode_customdata_datetime_inhomogenous_dataframe(self):
df = pd.DataFrame(dict(
t=pd.to_datetime(["2010-01-01", "2010-01-02"]),
v=np.arange(2),
))
# 2D customdata
fig = Figure(
Scatter(x=df["t"], customdata=df[["t", "v"]]), layout=dict(template="none")
)
fig_json = _json.dumps(
fig, cls=utils.PlotlyJSONEncoder, separators=(",", ":"), sort_keys=True
)
self.assertTrue(
fig_json.startswith(
'{"data":[{"customdata":'
'[["2010-01-01T00:00:00",0],["2010-01-02T00:00:00",1]]'
)
)

def test_numpy_masked_json_encoding(self):
l = [1, 2, np.ma.core.masked]
j1 = _json.dumps(l, cls=utils.PlotlyJSONEncoder)
Expand Down