PERF: improve perf of writing csv's with datetimes

Author: jreback

doc/source/whatsnew/v0.16.1.txt

@@ -89,7 +89,7 @@ API changes
 Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-
+- Improved csv write performance with mixed dtypes, including datetimes (:issue:`9940`)
 
 
 

pandas/core/format.py

@@ -14,15 +14,14 @@
 from pandas.core.config import get_option, set_option
 import pandas.core.common as com
 import pandas.lib as lib
-from pandas.tslib import iNaT, Timestamp, Timedelta
-
+from pandas.tslib import iNaT, Timestamp, Timedelta, format_array_from_datetime
+from pandas.tseries.index import DatetimeIndex
+from pandas.tseries.period import PeriodIndex
 import numpy as np
 
 import itertools
 import csv
 
-from pandas.tseries.period import PeriodIndex, DatetimeIndex
-
 docstring_to_string = """
      Parameters
      ----------
@@ -2030,16 +2029,50 @@ def __init__(self, values, nat_rep='NaT', date_format=None, **kwargs):
         self.date_format = date_format
 
     def _format_strings(self):
-        formatter = (self.formatter or
-                     _get_format_datetime64_from_values(self.values,
-                                                        nat_rep=self.nat_rep,
-                                                        date_format=self.date_format))
 
-        fmt_values = [formatter(x) for x in self.values]
+        # we may have a tz, if so, then need to process element-by-element
+        # when DatetimeBlockWithTimezones is a reality this could be fixed
+        values = self.values
+        if not isinstance(values, DatetimeIndex):
+            values = DatetimeIndex(values)
+
+        if values.tz is None:
+
+            is_dates_only = _is_dates_only(values)
+            if is_dates_only:
+                formatter = self.date_format or "%Y-%m-%d"
+            else:
+                formatter = None
+
+            fmt_values = format_array_from_datetime(values.asi8.ravel(),
+                                                    format=formatter,
+                                                    na_rep=self.nat_rep).reshape(values.shape)
+            fmt_values = fmt_values.tolist()
+
+        else:
+
+            values = values.asobject
+            is_dates_only = _is_dates_only(values)
+            formatter = (self.formatter or _get_format_datetime64(is_dates_only, values, date_format=self.date_format))
+            fmt_values = [ formatter(x) for x in self.values ]
 
         return fmt_values
 
 
+def _is_dates_only(values):
+    # return a boolean if we are only dates (and don't have a timezone)
+    values = DatetimeIndex(values)
+    if values.tz is not None:
+        return False
+
+    values_int = values.asi8
+    consider_values = values_int != iNaT
+    one_day_nanos = (86400 * 1e9)
+    even_days = np.logical_and(consider_values, values_int % one_day_nanos != 0).sum() == 0
+    if even_days:
+        return True
+    return False
+
 def _format_datetime64(x, tz=None, nat_rep='NaT'):
     if x is None or lib.checknull(x):
         return nat_rep
@@ -2062,22 +2095,6 @@ def _format_datetime64_dateonly(x, nat_rep='NaT', date_format=None):
     else:
         return x._date_repr
 
-
-def _is_dates_only(values):
-    # return a boolean if we are only dates (and don't have a timezone)
-    from pandas import DatetimeIndex
-    values = DatetimeIndex(values)
-    if values.tz is not None:
-        return False
-
-    values_int = values.asi8
-    consider_values = values_int != iNaT
-    one_day_nanos = (86400 * 1e9)
-    even_days = np.logical_and(consider_values, values_int % one_day_nanos != 0).sum() == 0
-    if even_days:
-        return True
-    return False
-
 def _get_format_datetime64(is_dates_only, nat_rep='NaT', date_format=None):
 
     if is_dates_only:
@@ -2088,15 +2105,6 @@ def _get_format_datetime64(is_dates_only, nat_rep='NaT', date_format=None):
         return lambda x, tz=None: _format_datetime64(x, tz=tz, nat_rep=nat_rep)
 
 
-def _get_format_datetime64_from_values(values,
-                                       nat_rep='NaT',
-                                       date_format=None):
-    is_dates_only = _is_dates_only(values)
-    return _get_format_datetime64(is_dates_only=is_dates_only,
-                                  nat_rep=nat_rep,
-                                  date_format=date_format)
-
-
 class Timedelta64Formatter(GenericArrayFormatter):
 
     def __init__(self, values, nat_rep='NaT', box=False, **kwargs):

pandas/core/internals.py

@@ -1870,23 +1870,12 @@ def to_native_types(self, slicer=None, na_rep=None, date_format=None,
         values = self.values
         if slicer is not None:
             values = values[:, slicer]
-        mask = isnull(values)
 
-        rvalues = np.empty(values.shape, dtype=object)
-        if na_rep is None:
-            na_rep = 'NaT'
-        rvalues[mask] = na_rep
-        imask = (~mask).ravel()
-
-        if date_format is None:
-            date_formatter = lambda x: Timestamp(x)._repr_base
-        else:
-            date_formatter = lambda x: Timestamp(x).strftime(date_format)
-
-        rvalues.flat[imask] = np.array([date_formatter(val) for val in
-                                        values.ravel()[imask]], dtype=object)
-
-        return rvalues.tolist()
+        result = tslib.format_array_from_datetime(values.view('i8').ravel(),
+                                                  tz=None,
+                                                  format=date_format,
+                                                  na_rep=na_rep).reshape(values.shape)
+        return result.tolist()
 
     def should_store(self, value):
         return issubclass(value.dtype.type, np.datetime64)

pandas/tests/test_format.py

@@ -3010,12 +3010,12 @@ def test_format(self):
 
     def test_output_significant_digits(self):
         # Issue #9764
-        
+
         # In case default display precision changes:
         with pd.option_context('display.precision', 7):
             # DataFrame example from issue #9764
             d=pd.DataFrame({'col1':[9.999e-8, 1e-7, 1.0001e-7, 2e-7, 4.999e-7, 5e-7, 5.0001e-7, 6e-7, 9.999e-7, 1e-6, 1.0001e-6, 2e-6, 4.999e-6, 5e-6, 5.0001e-6, 6e-6]})
-            
+
             expected_output={
                 (0,6):'           col1\n0  9.999000e-08\n1  1.000000e-07\n2  1.000100e-07\n3  2.000000e-07\n4  4.999000e-07\n5  5.000000e-07',
                 (1,6):'           col1\n1  1.000000e-07\n2  1.000100e-07\n3  2.000000e-07\n4  4.999000e-07\n5  5.000000e-07',

pandas/tseries/index.py

@@ -673,9 +673,8 @@ def _add_delta(self, delta):
 
     def _format_native_types(self, na_rep=u('NaT'),
                              date_format=None, **kwargs):
-        data = self.asobject
         from pandas.core.format import Datetime64Formatter
-        return Datetime64Formatter(values=data,
+        return Datetime64Formatter(values=self,
                                    nat_rep=na_rep,
                                    date_format=date_format,
                                    justify='all').get_result()

pandas/tslib.pyx

@@ -1398,6 +1398,60 @@ def parse_datetime_string(date_string, **kwargs):
     dt = parse_date(date_string, **kwargs)
     return dt
 
+def format_array_from_datetime(ndarray[int64_t] values, object tz=None, object format=None, object na_rep=None):
+    """
+    return a np object array of the string formatted values
+
+    Parameters
+    ----------
+    values : a 1-d i8 array
+    tz : the timezone (or None)
+    format : optional, default is None
+          a strftime capable string
+    na_rep : optional, default is None
+          a nat format
+
+    """
+    cdef:
+        int64_t val, ns, N = len(values)
+        ndarray[object] result = np.empty(N, dtype=object)
+        object ts, res
+        pandas_datetimestruct dts
+
+    if na_rep is None:
+       na_rep = 'NaT'
+
+    for i in range(N):
+       val = values[i]
+
+       if val == iNaT:
+          result[i] = na_rep
+       else:
+          if format is None and tz is None:
+
+            pandas_datetime_to_datetimestruct(val, PANDAS_FR_ns, &dts)
+            res = '%d-%.2d-%.2d %.2d:%.2d:%.2d' % (dts.year,
+                                                   dts.month,
+                                                   dts.day,
+                                                   dts.hour,
+                                                   dts.min,
+                                                   dts.sec)
+
+            ns = dts.ps / 1000
+
+            if ns != 0:
+               res += '.%.9d' % (ns + 1000 * dts.us)
+            elif dts.us != 0:
+               res += '.%.6d' % dts.us
+
+            result[i] = res
+
+          else:
+             ts = Timestamp(val, tz=tz)
+             result[i] = ts.strftime(format)
+
+    return result
+
 def array_to_datetime(ndarray[object] values, raise_=False, dayfirst=False,
                       format=None, utc=None, coerce=False, unit=None):
     cdef: