TST: use xdist for multiple cpu testing

closes pandas-dev#15369

.travis.yml

@@ -320,7 +320,8 @@ before_script:
 script:
   - echo "script start"
   - ci/run_build_docs.sh
-  - ci/script.sh
+  - ci/script_single.sh
+  - ci/script_multi.sh
   - ci/lint.sh
   - echo "script done"
 

ci/script_multi.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+echo "[script multi]"
+
+source activate pandas
+
+# don't run the tests for the doc build
+if [ x"$DOC_BUILD" != x"" ]; then
+    exit 0
+fi
+
+if [ -n "$LOCALE_OVERRIDE" ]; then
+    export LC_ALL="$LOCALE_OVERRIDE";
+    echo "Setting LC_ALL to $LOCALE_OVERRIDE"
+
+    pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))'
+    python -c "$pycmd"
+fi
+
+if [ "$BUILD_TEST" ]; then
+    echo "We are not running pytest as this is simply a build test."
+elif [ "$COVERAGE" ]; then
+    echo pytest -s -n 2 -m "not single" --cov=pandas --cov-append --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
+    pytest -s -n 2 -m "not single" --cov=pandas --cov-append --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
+else
+    echo pytest -n 2 -m "not single" $TEST_ARGS pandas
+    pytest -n 2 -m "not single" $TEST_ARGS pandas # TODO: doctest
+fi
+
+RET="$?"
+
+exit "$RET"

ci/script.sh → ci/script_single.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-echo "inside $0"
+echo "[script_single]"
 
 source activate pandas
 
@@ -20,11 +20,11 @@ fi
 if [ "$BUILD_TEST" ]; then
     echo "We are not running pytest as this is simply a build test."
 elif [ "$COVERAGE" ]; then
-    echo pytest -s --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
-    pytest -s --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
+    echo pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
+    pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
 else
-    echo pytest $TEST_ARGS pandas
-    pytest $TEST_ARGS pandas # TODO: doctest
+    echo pytest -m "single" $TEST_ARGS pandas
+    pytest -m "single" $TEST_ARGS pandas # TODO: doctest
 fi
 
 RET="$?"

pandas/tests/indexes/datetimes/test_ops.py

@@ -1,7 +1,9 @@
+import pytest
 import warnings
 import numpy as np
 from datetime import timedelta
 
+from itertools import product
 import pandas as pd
 import pandas.tslib as tslib
 import pandas.util.testing as tm
@@ -958,134 +960,134 @@ def test_second(self):
         tm.assert_index_equal(r1, r2)
 
 
-class TestDatetimeIndex(tm.TestCase):
-
-    # GH 10699
-    def test_datetime64_with_DateOffset(self):
-        for klass, assert_func in zip([Series, DatetimeIndex],
-                                      [self.assert_series_equal,
-                                       tm.assert_index_equal]):
-            s = klass(date_range('2000-01-01', '2000-01-31'), name='a')
-            result = s + pd.DateOffset(years=1)
-            result2 = pd.DateOffset(years=1) + s
-            exp = klass(date_range('2001-01-01', '2001-01-31'), name='a')
+# GH 10699
+@pytest.mark.parametrize('klass,assert_func', zip([Series, DatetimeIndex],
+                                                  [tm.assert_series_equal,
+                                                   tm.assert_index_equal]))
+def test_datetime64_with_DateOffset(klass, assert_func):
+    s = klass(date_range('2000-01-01', '2000-01-31'), name='a')
+    result = s + pd.DateOffset(years=1)
+    result2 = pd.DateOffset(years=1) + s
+    exp = klass(date_range('2001-01-01', '2001-01-31'), name='a')
+    assert_func(result, exp)
+    assert_func(result2, exp)
+
+    result = s - pd.DateOffset(years=1)
+    exp = klass(date_range('1999-01-01', '1999-01-31'), name='a')
+    assert_func(result, exp)
+
+    s = klass([Timestamp('2000-01-15 00:15:00', tz='US/Central'),
+               pd.Timestamp('2000-02-15', tz='US/Central')], name='a')
+    result = s + pd.offsets.Day()
+    result2 = pd.offsets.Day() + s
+    exp = klass([Timestamp('2000-01-16 00:15:00', tz='US/Central'),
+                 Timestamp('2000-02-16', tz='US/Central')], name='a')
+    assert_func(result, exp)
+    assert_func(result2, exp)
+
+    s = klass([Timestamp('2000-01-15 00:15:00', tz='US/Central'),
+               pd.Timestamp('2000-02-15', tz='US/Central')], name='a')
+    result = s + pd.offsets.MonthEnd()
+    result2 = pd.offsets.MonthEnd() + s
+    exp = klass([Timestamp('2000-01-31 00:15:00', tz='US/Central'),
+                 Timestamp('2000-02-29', tz='US/Central')], name='a')
+    assert_func(result, exp)
+    assert_func(result2, exp)
+
+    # array of offsets - valid for Series only
+    if klass is Series:
+        with tm.assert_produces_warning(PerformanceWarning):
+            s = klass([Timestamp('2000-1-1'), Timestamp('2000-2-1')])
+            result = s + Series([pd.offsets.DateOffset(years=1),
+                                 pd.offsets.MonthEnd()])
+            exp = klass([Timestamp('2001-1-1'), Timestamp('2000-2-29')
+                         ])
             assert_func(result, exp)
-            assert_func(result2, exp)
 
-            result = s - pd.DateOffset(years=1)
-            exp = klass(date_range('1999-01-01', '1999-01-31'), name='a')
+            # same offset
+            result = s + Series([pd.offsets.DateOffset(years=1),
+                                 pd.offsets.DateOffset(years=1)])
+            exp = klass([Timestamp('2001-1-1'), Timestamp('2001-2-1')])
             assert_func(result, exp)
 
-            s = klass([Timestamp('2000-01-15 00:15:00', tz='US/Central'),
-                       pd.Timestamp('2000-02-15', tz='US/Central')], name='a')
-            result = s + pd.offsets.Day()
-            result2 = pd.offsets.Day() + s
-            exp = klass([Timestamp('2000-01-16 00:15:00', tz='US/Central'),
-                         Timestamp('2000-02-16', tz='US/Central')], name='a')
-            assert_func(result, exp)
-            assert_func(result2, exp)
-
-            s = klass([Timestamp('2000-01-15 00:15:00', tz='US/Central'),
-                       pd.Timestamp('2000-02-15', tz='US/Central')], name='a')
-            result = s + pd.offsets.MonthEnd()
-            result2 = pd.offsets.MonthEnd() + s
-            exp = klass([Timestamp('2000-01-31 00:15:00', tz='US/Central'),
-                         Timestamp('2000-02-29', tz='US/Central')], name='a')
-            assert_func(result, exp)
-            assert_func(result2, exp)
-
-            # array of offsets - valid for Series only
-            if klass is Series:
-                with tm.assert_produces_warning(PerformanceWarning):
-                    s = klass([Timestamp('2000-1-1'), Timestamp('2000-2-1')])
-                    result = s + Series([pd.offsets.DateOffset(years=1),
-                                         pd.offsets.MonthEnd()])
-                    exp = klass([Timestamp('2001-1-1'), Timestamp('2000-2-29')
-                                 ])
-                    assert_func(result, exp)
-
-                    # same offset
-                    result = s + Series([pd.offsets.DateOffset(years=1),
-                                         pd.offsets.DateOffset(years=1)])
-                    exp = klass([Timestamp('2001-1-1'), Timestamp('2001-2-1')])
-                    assert_func(result, exp)
-
-            s = klass([Timestamp('2000-01-05 00:15:00'),
+    s = klass([Timestamp('2000-01-05 00:15:00'),
+               Timestamp('2000-01-31 00:23:00'),
+               Timestamp('2000-01-01'),
+               Timestamp('2000-03-31'),
+               Timestamp('2000-02-29'),
+               Timestamp('2000-12-31'),
+               Timestamp('2000-05-15'),
+               Timestamp('2001-06-15')])
+
+    # DateOffset relativedelta fastpath
+    relative_kwargs = [('years', 2), ('months', 5), ('days', 3),
+                       ('hours', 5), ('minutes', 10), ('seconds', 2),
+                       ('microseconds', 5)]
+    for i, kwd in enumerate(relative_kwargs):
+        op = pd.DateOffset(**dict([kwd]))
+        assert_func(klass([x + op for x in s]), s + op)
+        assert_func(klass([x - op for x in s]), s - op)
+        op = pd.DateOffset(**dict(relative_kwargs[:i + 1]))
+        assert_func(klass([x + op for x in s]), s + op)
+        assert_func(klass([x - op for x in s]), s - op)
+
+    # assert these are equal on a piecewise basis
+    offsets = ['YearBegin', ('YearBegin', {'month': 5}),
+               'YearEnd', ('YearEnd', {'month': 5}),
+               'MonthBegin', 'MonthEnd',
+               'SemiMonthEnd', 'SemiMonthBegin',
+               'Week', ('Week', {'weekday': 3}),
+               'BusinessDay', 'BDay', 'QuarterEnd', 'QuarterBegin',
+               'CustomBusinessDay', 'CDay', 'CBMonthEnd',
+               'CBMonthBegin', 'BMonthBegin', 'BMonthEnd',
+               'BusinessHour', 'BYearBegin', 'BYearEnd',
+               'BQuarterBegin', ('LastWeekOfMonth', {'weekday': 2}),
+               ('FY5253Quarter', {'qtr_with_extra_week': 1,
+                                  'startingMonth': 1,
+                                  'weekday': 2,
+                                  'variation': 'nearest'}),
+               ('FY5253', {'weekday': 0,
+                           'startingMonth': 2,
+                           'variation':
+                           'nearest'}),
+               ('WeekOfMonth', {'weekday': 2,
+                                'week': 2}),
+               'Easter', ('DateOffset', {'day': 4}),
+               ('DateOffset', {'month': 5})]
+
+    with warnings.catch_warnings(record=True):
+        for normalize in (True, False):
+            for do in offsets:
+                if isinstance(do, tuple):
+                    do, kwargs = do
+                else:
+                    do = do
+                    kwargs = {}
+
+                    for n in [0, 5]:
+                        if (do in ['WeekOfMonth', 'LastWeekOfMonth',
+                                   'FY5253Quarter', 'FY5253'] and n == 0):
+                            continue
+                    op = getattr(pd.offsets, do)(n,
+                                                 normalize=normalize,
+                                                 **kwargs)
+                    assert_func(klass([x + op for x in s]), s + op)
+                    assert_func(klass([x - op for x in s]), s - op)
+                    assert_func(klass([op + x for x in s]), op + s)
+
+
+@pytest.mark.parametrize('years,months', product([-1, 0, 1], [-2, 0, 2]))
+def test_shift_months(years, months):
+    s = DatetimeIndex([Timestamp('2000-01-05 00:15:00'),
                        Timestamp('2000-01-31 00:23:00'),
                        Timestamp('2000-01-01'),
-                       Timestamp('2000-03-31'),
                        Timestamp('2000-02-29'),
-                       Timestamp('2000-12-31'),
-                       Timestamp('2000-05-15'),
-                       Timestamp('2001-06-15')])
-
-            # DateOffset relativedelta fastpath
-            relative_kwargs = [('years', 2), ('months', 5), ('days', 3),
-                               ('hours', 5), ('minutes', 10), ('seconds', 2),
-                               ('microseconds', 5)]
-            for i, kwd in enumerate(relative_kwargs):
-                op = pd.DateOffset(**dict([kwd]))
-                assert_func(klass([x + op for x in s]), s + op)
-                assert_func(klass([x - op for x in s]), s - op)
-                op = pd.DateOffset(**dict(relative_kwargs[:i + 1]))
-                assert_func(klass([x + op for x in s]), s + op)
-                assert_func(klass([x - op for x in s]), s - op)
-
-            # assert these are equal on a piecewise basis
-            offsets = ['YearBegin', ('YearBegin', {'month': 5}), 'YearEnd',
-                       ('YearEnd', {'month': 5}), 'MonthBegin', 'MonthEnd',
-                       'SemiMonthEnd', 'SemiMonthBegin',
-                       'Week', ('Week', {
-                           'weekday': 3
-                       }), 'BusinessDay', 'BDay', 'QuarterEnd', 'QuarterBegin',
-                       'CustomBusinessDay', 'CDay', 'CBMonthEnd',
-                       'CBMonthBegin', 'BMonthBegin', 'BMonthEnd',
-                       'BusinessHour', 'BYearBegin', 'BYearEnd',
-                       'BQuarterBegin', ('LastWeekOfMonth', {
-                           'weekday': 2
-                       }), ('FY5253Quarter', {'qtr_with_extra_week': 1,
-                                              'startingMonth': 1,
-                                              'weekday': 2,
-                                              'variation': 'nearest'}),
-                       ('FY5253', {'weekday': 0,
-                                   'startingMonth': 2,
-                                   'variation':
-                                   'nearest'}), ('WeekOfMonth', {'weekday': 2,
-                                                                 'week': 2}),
-                       'Easter', ('DateOffset', {'day': 4}),
-                       ('DateOffset', {'month': 5})]
-
-            with warnings.catch_warnings(record=True):
-                for normalize in (True, False):
-                    for do in offsets:
-                        if isinstance(do, tuple):
-                            do, kwargs = do
-                        else:
-                            do = do
-                            kwargs = {}
-
-                        for n in [0, 5]:
-                            if (do in ['WeekOfMonth', 'LastWeekOfMonth',
-                                       'FY5253Quarter', 'FY5253'] and n == 0):
-                                continue
-                            op = getattr(pd.offsets, do)(n,
-                                                         normalize=normalize,
-                                                         **kwargs)
-                            assert_func(klass([x + op for x in s]), s + op)
-                            assert_func(klass([x - op for x in s]), s - op)
-                            assert_func(klass([op + x for x in s]), op + s)
-
-    def test_shift_months(self):
-        s = DatetimeIndex([Timestamp('2000-01-05 00:15:00'), Timestamp(
-            '2000-01-31 00:23:00'), Timestamp('2000-01-01'), Timestamp(
-                '2000-02-29'), Timestamp('2000-12-31')])
-        for years in [-1, 0, 1]:
-            for months in [-2, 0, 2]:
-                actual = DatetimeIndex(tslib.shift_months(s.asi8, years * 12 +
-                                                          months))
-                expected = DatetimeIndex([x + offsets.DateOffset(
-                    years=years, months=months) for x in s])
-                tm.assert_index_equal(actual, expected)
+                       Timestamp('2000-12-31')])
+    actual = DatetimeIndex(tslib.shift_months(s.asi8, years * 12 +
+                                              months))
+    expected = DatetimeIndex([x + offsets.DateOffset(
+        years=years, months=months) for x in s])
+    tm.assert_index_equal(actual, expected)
 
 
 class TestBusinessDatetimeIndex(tm.TestCase):

pandas/tests/io/test_clipboard.py

@@ -20,6 +20,7 @@
     _DEPS_INSTALLED = 0
 
 
+@pytest.mark.single
 @pytest.mark.skipif(not _DEPS_INSTALLED,
                     reason="clipboard primitives not installed")
 class TestClipboard(tm.TestCase):

pandas/tests/io/test_pytables.py

@@ -36,12 +36,6 @@
 from pandas import concat, Timestamp
 from pandas import compat
 from pandas.compat import range, lrange, u
-
-try:
-    import tables
-except ImportError:
-    pytest.skip('no pytables')
-
 from distutils.version import LooseVersion
 
 _default_compressor = ('blosc' if LooseVersion(tables.__version__) >= '2.2'
@@ -165,6 +159,7 @@ def tearDown(self):
         pass
 
 
+@pytest.mark.single
 class TestHDFStore(Base, tm.TestCase):
 
     def test_factory_fun(self):