Merge pull request #10249 from cmeeren/datetime_res_coercion

Datetime resolution coercion

Author: jreback

doc/source/whatsnew/v0.16.2.txt

@@ -161,3 +161,4 @@ Bug Fixes
 - Bug in ``DataFrame.to_hdf()`` where table format would raise a seemingly unrelated error for invalid (non-string) column names. This is now explicitly forbidden. (:issue:`9057`)
 - Bug to handle masking empty ``DataFrame``(:issue:`10126`)
 - Bug where MySQL interface could not handle numeric table/column names (:issue:`10255`)
+- Bug where ``read_csv`` and similar failed if making ``MultiIndex`` and ``date_parser`` returned ``datetime64`` array of other time resolution than ``[ns]``.

pandas/io/parsers.py

@@ -2059,18 +2059,20 @@ def converter(*date_cols):
                     infer_datetime_format=infer_datetime_format
                 )
             except:
-                return lib.try_parse_dates(strs, dayfirst=dayfirst)
+                return tools.to_datetime(
+                    lib.try_parse_dates(strs, dayfirst=dayfirst))
         else:
             try:
-                result = date_parser(*date_cols)
+                result = tools.to_datetime(date_parser(*date_cols))
                 if isinstance(result, datetime.datetime):
                     raise Exception('scalar parser')
                 return result
             except Exception:
                 try:
-                    return lib.try_parse_dates(_concat_date_cols(date_cols),
-                                               parser=date_parser,
-                                               dayfirst=dayfirst)
+                    return tools.to_datetime(
+                        lib.try_parse_dates(_concat_date_cols(date_cols),
+                                            parser=date_parser,
+                                            dayfirst=dayfirst))
                 except Exception:
                     return generic_parser(date_parser, *date_cols)
 

pandas/io/tests/test_date_converters.py

@@ -11,7 +11,7 @@
 import numpy as np
 from numpy.testing.decorators import slow
 
-from pandas import DataFrame, Series, Index, isnull
+from pandas import DataFrame, Series, Index, MultiIndex, isnull
 import pandas.io.parsers as parsers
 from pandas.io.parsers import (read_csv, read_table, read_fwf,
                                TextParser)
@@ -119,6 +119,31 @@ def test_generic(self):
         self.assertIn('ym', df)
         self.assertEqual(df.ym.ix[0], date(2001, 1, 1))
 
+    def test_dateparser_resolution_if_not_ns(self):
+        # issue 10245
+        data = """\
+date,time,prn,rxstatus
+2013-11-03,19:00:00,126,00E80000
+2013-11-03,19:00:00,23,00E80000
+2013-11-03,19:00:00,13,00E80000
+"""
+
+        def date_parser(date, time):
+            datetime = np.array(date + 'T' + time + 'Z', dtype='datetime64[s]')
+            return datetime
+
+        df = read_csv(StringIO(data), date_parser=date_parser,
+                      parse_dates={'datetime': ['date', 'time']},
+                      index_col=['datetime', 'prn'])
+
+        datetimes = np.array(['2013-11-03T19:00:00Z']*3, dtype='datetime64[s]')
+        df_correct = DataFrame(data={'rxstatus': ['00E80000']*3},
+                               index=MultiIndex.from_tuples(
+                                   [(datetimes[0], 126),
+                                    (datetimes[1], 23),
+                                    (datetimes[2], 13)],
+                               names=['datetime', 'prn']))
+        assert_frame_equal(df, df_correct)
 
 if __name__ == '__main__':
     import nose