Skip to content

Commit

Permalink
Fix date parsing issue in arrow_parser_wrapper
Browse files Browse the repository at this point in the history
  • Loading branch information
Lavishgangwani committed Oct 16, 2024
1 parent 881f523 commit 75d6018
Showing 1 changed file with 12 additions and 0 deletions.
12 changes: 12 additions & 0 deletions pandas/io/parsers/arrow_parser_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,18 @@ def read(self) -> DataFrame:

dtype_backend = self.kwds["dtype_backend"]

# Handle missing date values by checking for timestamp columns
for i, field in enumerate(table.schema):
if pa.types.is_timestamp(field.type): # Check if the column is a timestamp
# Convert to a Pandas Series
column = table.column(i).to_pandas()

# Replace missing values with NaT
column.fillna(pd.NaT, inplace=True)

# Update the column back to the table
table = table.set_column(i, field.name, pa.array(column))

# Convert all pa.null() cols -> float64 (non nullable)
# else Int64 (nullable case, see below)
if dtype_backend is lib.no_default:
Expand Down

0 comments on commit 75d6018

Please sign in to comment.