Skip to content

Commit

Permalink
TST: Make read_csv tests pyarrow 13 compatable on 2.1.x (#55855)
Browse files Browse the repository at this point in the history
* TST: Make read_csv tests pyarrow 13 compatable on 2.1.x

* Skip on windows for ARROW_TIMEZONE_DATABASE
  • Loading branch information
mroeschke authored Nov 7, 2023
1 parent 7702f41 commit c9854d9
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 4 deletions.
16 changes: 15 additions & 1 deletion pandas/tests/interchange/test_impl.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@
import pytest

from pandas._libs.tslibs import iNaT
from pandas.compat import (
is_ci_environment,
is_platform_windows,
)
import pandas.util._test_decorators as td

import pandas as pd
Expand Down Expand Up @@ -309,11 +313,21 @@ def test_datetimetzdtype(tz, unit):
tm.assert_frame_equal(df, from_dataframe(df.__dataframe__()))


def test_interchange_from_non_pandas_tz_aware():
def test_interchange_from_non_pandas_tz_aware(request):
# GH 54239, 54287
pa = pytest.importorskip("pyarrow", "11.0.0")
import pyarrow.compute as pc

if is_platform_windows() and is_ci_environment():
mark = pytest.mark.xfail(
raises=pa.ArrowInvalid,
reason=(
"TODO: Set ARROW_TIMEZONE_DATABASE environment variable "
"on CI to path to the tzdata for pyarrow."
),
)
request.node.add_marker(mark)

arr = pa.array([datetime(2020, 1, 1), None, datetime(2020, 1, 2)])
arr = pc.assume_timezone(arr, "Asia/Kathmandu")
table = pa.table({"arr": arr})
Expand Down
8 changes: 6 additions & 2 deletions pandas/tests/io/parser/common/test_common_basic.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,9 @@ def test_read_csv_local(all_parsers, csv1):

fname = prefix + str(os.path.abspath(csv1))
result = parser.read_csv(fname, index_col=0, parse_dates=True)

# TODO: make unit check more specific
if parser.engine == "pyarrow":
result.index = result.index.as_unit("ns")
expected = DataFrame(
[
[0.980269, 3.685731, -0.364216805298, -1.159738],
Expand Down Expand Up @@ -177,7 +179,9 @@ def test_read_csv_low_memory_no_rows_with_index(all_parsers):
def test_read_csv_dataframe(all_parsers, csv1):
parser = all_parsers
result = parser.read_csv(csv1, index_col=0, parse_dates=True)

# TODO: make unit check more specific
if parser.engine == "pyarrow":
result.index = result.index.as_unit("ns")
expected = DataFrame(
[
[0.980269, 3.685731, -0.364216805298, -1.159738],
Expand Down
8 changes: 7 additions & 1 deletion pandas/tests/io/parser/test_parse_dates.py
Original file line number Diff line number Diff line change
Expand Up @@ -979,12 +979,15 @@ def test_parse_dates_custom_euro_format(all_parsers, kwargs):
)


def test_parse_tz_aware(all_parsers, request):
def test_parse_tz_aware(all_parsers):
# See gh-1693
parser = all_parsers
data = "Date,x\n2012-06-13T01:39:00Z,0.5"

result = parser.read_csv(StringIO(data), index_col=0, parse_dates=True)
# TODO: make unit check more specific
if parser.engine == "pyarrow":
result.index = result.index.as_unit("ns")
expected = DataFrame(
{"x": [0.5]}, index=Index([Timestamp("2012-06-13 01:39:00+00:00")], name="Date")
)
Expand Down Expand Up @@ -2231,6 +2234,9 @@ def test_parse_dates_arrow_engine(all_parsers):
2000-01-01 00:00:01,1"""

result = parser.read_csv(StringIO(data), parse_dates=["a"])
# TODO: make unit check more specific
if parser.engine == "pyarrow":
result["a"] = result["a"].dt.as_unit("ns")
expected = DataFrame(
{
"a": [
Expand Down

0 comments on commit c9854d9

Please sign in to comment.