From 75d601881c960e45dd27491783f1c64d0b28f428 Mon Sep 17 00:00:00 2001 From: Lavishgangwani Date: Wed, 16 Oct 2024 10:51:32 +0530 Subject: [PATCH] Fix date parsing issue in arrow_parser_wrapper --- pandas/io/parsers/arrow_parser_wrapper.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pandas/io/parsers/arrow_parser_wrapper.py b/pandas/io/parsers/arrow_parser_wrapper.py index 86bb5f190e403..d408c4ce8a0d5 100644 --- a/pandas/io/parsers/arrow_parser_wrapper.py +++ b/pandas/io/parsers/arrow_parser_wrapper.py @@ -274,6 +274,18 @@ def read(self) -> DataFrame: dtype_backend = self.kwds["dtype_backend"] + # Handle missing date values by checking for timestamp columns + for i, field in enumerate(table.schema): + if pa.types.is_timestamp(field.type): # Check if the column is a timestamp + # Convert to a Pandas Series + column = table.column(i).to_pandas() + + # Replace missing values with NaT + column.fillna(pd.NaT, inplace=True) + + # Update the column back to the table + table = table.set_column(i, field.name, pa.array(column)) + # Convert all pa.null() cols -> float64 (non nullable) # else Int64 (nullable case, see below) if dtype_backend is lib.no_default: