Merge remote-tracking branch 'upstream/master' into pd.Series.map_performance

Author: Rohith295

.devcontainer.json

@@ -17,7 +17,9 @@
 		"python.linting.pylintEnabled": false,
 		"python.linting.mypyEnabled": true,
 		"python.testing.pytestEnabled": true,
-		"python.testing.cwd": "pandas/tests"
+		"python.testing.pytestArgs": [
+			"pandas"
+		]
 	},
 
 	// Add the IDs of extensions you want installed when the container is created in the array below.

asv_bench/benchmarks/categoricals.py

@@ -34,6 +34,7 @@ def setup(self):
         self.values_all_int8 = np.ones(N, "int8")
         self.categorical = pd.Categorical(self.values, self.categories)
         self.series = pd.Series(self.categorical)
+        self.intervals = pd.interval_range(0, 1, periods=N // 10)
 
     def time_regular(self):
         pd.Categorical(self.values, self.categories)
@@ -44,6 +45,9 @@ def time_fastpath(self):
     def time_datetimes(self):
         pd.Categorical(self.datetimes)
 
+    def time_interval(self):
+        pd.Categorical(self.datetimes, categories=self.datetimes)
+
     def time_datetimes_with_nat(self):
         pd.Categorical(self.datetimes_with_nat)
 

asv_bench/benchmarks/groupby.py

@@ -16,7 +16,7 @@
 
 from .pandas_vb_common import tm
 
-method_blacklist = {
+method_blocklist = {
     "object": {
         "median",
         "prod",
@@ -403,7 +403,7 @@ class GroupByMethods:
     ]
 
     def setup(self, dtype, method, application):
-        if method in method_blacklist.get(dtype, {}):
+        if method in method_blocklist.get(dtype, {}):
             raise NotImplementedError  # skip benchmark
         ngroups = 1000
         size = ngroups * 2

asv_bench/benchmarks/io/json.py

@@ -1,3 +1,5 @@
+import sys
+
 import numpy as np
 
 from pandas import DataFrame, concat, date_range, read_json, timedelta_range
@@ -82,6 +84,7 @@ def setup(self, orient, frame):
         timedeltas = timedelta_range(start=1, periods=N, freq="s")
         datetimes = date_range(start=1, periods=N, freq="s")
         ints = np.random.randint(100000000, size=N)
+        longints = sys.maxsize * np.random.randint(100000000, size=N)
         floats = np.random.randn(N)
         strings = tm.makeStringIndex(N)
         self.df = DataFrame(np.random.randn(N, ncols), index=np.arange(N))
@@ -120,6 +123,18 @@ def setup(self, orient, frame):
             index=index,
         )
 
+        self.df_longint_float_str = DataFrame(
+            {
+                "longint_1": longints,
+                "longint_2": longints,
+                "float_1": floats,
+                "float_2": floats,
+                "str_1": strings,
+                "str_2": strings,
+            },
+            index=index,
+        )
+
     def time_to_json(self, orient, frame):
         getattr(self, frame).to_json(self.fname, orient=orient)
 
@@ -172,6 +187,7 @@ def setup(self):
         timedeltas = timedelta_range(start=1, periods=N, freq="s")
         datetimes = date_range(start=1, periods=N, freq="s")
         ints = np.random.randint(100000000, size=N)
+        longints = sys.maxsize * np.random.randint(100000000, size=N)
         floats = np.random.randn(N)
         strings = tm.makeStringIndex(N)
         self.df = DataFrame(np.random.randn(N, ncols), index=np.arange(N))
@@ -209,6 +225,17 @@ def setup(self):
             },
             index=index,
         )
+        self.df_longint_float_str = DataFrame(
+            {
+                "longint_1": longints,
+                "longint_2": longints,
+                "float_1": floats,
+                "float_2": floats,
+                "str_1": strings,
+                "str_2": strings,
+            },
+            index=index,
+        )
 
     def time_floats_with_int_idex_lines(self):
         self.df.to_json(self.fname, orient="records", lines=True)
@@ -225,6 +252,9 @@ def time_float_int_lines(self):
     def time_float_int_str_lines(self):
         self.df_int_float_str.to_json(self.fname, orient="records", lines=True)
 
+    def time_float_longint_str_lines(self):
+        self.df_longint_float_str.to_json(self.fname, orient="records", lines=True)
+
 
 class ToJSONMem:
     def setup_cache(self):

ci/code_checks.sh

@@ -248,19 +248,19 @@ fi
 ### CODE ###
 if [[ -z "$CHECK" || "$CHECK" == "code" ]]; then
 
-    MSG='Check import. No warnings, and blacklist some optional dependencies' ; echo $MSG
+    MSG='Check import. No warnings, and blocklist some optional dependencies' ; echo $MSG
     python -W error -c "
 import sys
 import pandas
 
-blacklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
+blocklist = {'bs4', 'gcsfs', 'html5lib', 'http', 'ipython', 'jinja2', 'hypothesis',
              'lxml', 'matplotlib', 'numexpr', 'openpyxl', 'py', 'pytest', 's3fs', 'scipy',
              'tables', 'urllib.request', 'xlrd', 'xlsxwriter', 'xlwt'}
 
 # GH#28227 for some of these check for top-level modules, while others are
 #  more specific (e.g. urllib.request)
 import_mods = set(m.split('.')[0] for m in sys.modules) | set(sys.modules)
-mods = blacklist & import_mods
+mods = blocklist & import_mods
 if mods:
     sys.stderr.write('err: pandas should not import: {}\n'.format(', '.join(mods)))
     sys.exit(len(mods))

doc/source/user_guide/advanced.rst

@@ -260,7 +260,9 @@ You don't have to specify all levels of the ``MultiIndex`` by passing only the
 first elements of the tuple. For example, you can use "partial" indexing to
 get all elements with ``bar`` in the first level as follows:
 
-df.loc['bar']
+.. ipython:: python
+
+   df.loc['bar']
 
 This is a shortcut for the slightly more verbose notation ``df.loc[('bar',),]`` (equivalent
 to ``df.loc['bar',]`` in this example).

doc/source/user_guide/computation.rst

@@ -561,7 +561,7 @@ For example, if we have the following ``DataFrame``:
    df
 
 and we want to use an expanding window where ``use_expanding`` is ``True`` otherwise a window of size
-1, we can create the following ``BaseIndexer``:
+1, we can create the following ``BaseIndexer`` subclass:
 
 .. code-block:: ipython
 
@@ -593,6 +593,8 @@ and we want to use an expanding window where ``use_expanding`` is ``True`` other
    3     3.0
    4    10.0
 
+You can view other examples of ``BaseIndexer`` subclasses `here <https://github.com/pandas-dev/pandas/blob/master/pandas/core/window/indexers.py>`__
+
 .. versionadded:: 1.1
 
 For some problems knowledge of the future is available for analysis. For example, this occurs when

doc/source/user_guide/visualization.rst

@@ -1108,6 +1108,34 @@ shown by default.
 
    plt.close('all')
 
+
+Controlling the labels
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.1.0
+
+You may set the ``xlabel`` and ``ylabel`` arguments to give the plot custom labels
+for x and y axis. By default, pandas will pick up index name as xlabel, while leaving
+it empty for ylabel.
+
+.. ipython:: python
+   :suppress:
+
+   plt.figure()
+
+.. ipython:: python
+
+   df.plot()
+
+   @savefig plot_xlabel_ylabel.png
+   df.plot(xlabel="new x", ylabel="new y")
+
+.. ipython:: python
+   :suppress:
+
+   plt.close('all')
+
+
 Scales
 ~~~~~~
 

doc/source/whatsnew/v0.14.1.rst

@@ -131,7 +131,7 @@ Enhancements
 
 - Implemented ``sem`` (standard error of the mean) operation for ``Series``,
   ``DataFrame``, ``Panel``, and ``Groupby`` (:issue:`6897`)
-- Add ``nlargest`` and ``nsmallest`` to the ``Series`` ``groupby`` whitelist,
+- Add ``nlargest`` and ``nsmallest`` to the ``Series`` ``groupby`` allowlist,
   which means you can now use these methods on a ``SeriesGroupBy`` object
   (:issue:`7053`).
 - All offsets ``apply``, ``rollforward`` and ``rollback`` can now handle ``np.datetime64``, previously results in ``ApplyTypeError`` (:issue:`7452`)

doc/source/whatsnew/v1.1.0.rst

@@ -13,6 +13,15 @@ including other versions of pandas.
 Enhancements
 ~~~~~~~~~~~~
 
+.. _whatsnew_110.specify_missing_labels:
+
+KeyErrors raised by loc specify missing labels
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Previously, if labels were missing for a loc call, a KeyError was raised stating that this was no longer supported.
+
+Now the error message also includes a list of the missing labels (max 10 items, display width 80 characters). See :issue:`34272`.
+
+
 .. _whatsnew_110.astype_string:
 
 All dtypes can now be converted to ``StringDtype``
@@ -303,6 +312,7 @@ Other enhancements
   :class:`~pandas.io.stata.StataWriter`, :class:`~pandas.io.stata.StataWriter117`,
   and :class:`~pandas.io.stata.StataWriterUTF8` (:issue:`26599`).
 - :meth:`HDFStore.put` now accepts `track_times` parameter. Parameter is passed to ``create_table`` method of ``PyTables`` (:issue:`32682`).
+- :meth:`Series.plot` and :meth:`DataFrame.plot` now accepts `xlabel` and `ylabel` parameters to present labels on x and y axis (:issue:`9093`).
 - Make :class:`pandas.core.window.Rolling` and :class:`pandas.core.window.Expanding` iterable（:issue:`11704`)
 - Make ``option_context`` a :class:`contextlib.ContextDecorator`, which allows it to be used as a decorator over an entire function (:issue:`34253`).
 - :meth:`DataFrame.to_csv` and :meth:`Series.to_csv` now accept an ``errors`` argument (:issue:`22610`)
@@ -314,7 +324,10 @@ Other enhancements
   result in object dtype but preserve the integer dtype (:issue:`33607`, :issue:`34339`).
 - :meth:`~pandas.io.gbq.read_gbq` now allows to disable progress bar (:issue:`33360`).
 - :meth:`~pandas.io.gbq.read_gbq` now supports the ``max_results`` kwarg from ``pandas-gbq`` (:issue:`34639`).
-- :meth:`DataFrame.to_html` and :meth:`DataFrame.to_string`'s ``col_space`` parameter now accepts a list of dict to change only some specific columns' width (:issue:`28917`).
+- :meth:`DataFrame.cov` and :meth:`Series.cov` now support a new parameter ddof to support delta degrees of freedom as in the corresponding numpy methods (:issue:`34611`).
+- :meth:`DataFrame.to_html` and :meth:`DataFrame.to_string`'s ``col_space`` parameter now accepts a list or dict to change only some specific columns' width (:issue:`28917`).
+- :meth:`DataFrame.to_excel` can now also write OpenOffice spreadsheet (.ods) files (:issue:`27222`)
+- :meth:`~Series.explode` now accepts ``ignore_index`` to reset the index, similarly to :meth:`pd.concat` or :meth:`DataFrame.sort_values` (:issue:`34932`).
 
 .. ---------------------------------------------------------------------------
 
@@ -800,6 +813,9 @@ Deprecations
 - The ``squeeze`` keyword in the ``groupby`` function is deprecated and will be removed in a future version (:issue:`32380`)
 - The ``tz`` keyword in :meth:`Period.to_timestamp` is deprecated and will be removed in a future version; use `per.to_timestamp(...).tz_localize(tz)`` instead (:issue:`34522`)
 - :meth:`DatetimeIndex.to_perioddelta` is deprecated and will be removed in a future version.  Use ``index - index.to_period(freq).to_timestamp()`` instead (:issue:`34853`)
+- :meth:`util.testing.assert_almost_equal` now accepts both relative and absolute
+  precision through the ``rtol``, and ``atol`` parameters, thus deprecating the
+  ``check_less_precise`` parameter. (:issue:`13357`).
 
 .. ---------------------------------------------------------------------------
 
@@ -823,6 +839,8 @@ Performance improvements
 - Performance improvement for groupby methods :meth:`~pandas.core.groupby.groupby.Groupby.first`
   and :meth:`~pandas.core.groupby.groupby.Groupby.last` (:issue:`34178`)
 - Performance improvement in :func:`factorize` for nullable (integer and boolean) dtypes (:issue:`33064`).
+- Performance improvement when constructing :class:`Categorical` objects (:issue:`33921`)
+- Fixed performance regression in :func:`pandas.qcut` and :func:`pandas.cut` (:issue:`33921`)
 - Performance improvement in reductions (sum, prod, min, max) for nullable (integer and boolean) dtypes (:issue:`30982`, :issue:`33261`, :issue:`33442`).
 - Performance improvement in arithmetic operations between two :class:`DataFrame` objects (:issue:`32779`)
 - Performance improvement in :class:`pandas.core.groupby.RollingGroupby` (:issue:`34052`)
@@ -844,6 +862,7 @@ Categorical
 - Bug when passing categorical data to :class:`Index` constructor along with ``dtype=object`` incorrectly returning a :class:`CategoricalIndex` instead of object-dtype :class:`Index` (:issue:`32167`)
 - Bug where :class:`Categorical` comparison operator ``__ne__`` would incorrectly evaluate to ``False`` when either element was missing (:issue:`32276`)
 - :meth:`Categorical.fillna` now accepts :class:`Categorical` ``other`` argument (:issue:`32420`)
+- Repr of :class:`Categorical` was not distinguishing between int and str (:issue:`33676`)
 
 Datetimelike
 ^^^^^^^^^^^^
@@ -952,6 +971,7 @@ Indexing
 - Bug in :meth:`Series.at` when used with a :class:`MultiIndex` would raise an exception on valid inputs (:issue:`26989`)
 - Bug in :meth:`DataFrame.loc` with dictionary of values changes columns with dtype of ``int`` to ``float`` (:issue:`34573`)
 - Bug in :meth:`Series.loc` when used with a :class:`MultiIndex` would raise an IndexingError when accessing a None value (:issue:`34318`)
+- Bug in :meth:`DataFrame.reset_index` and :meth:`Series.reset_index` would not preserve data types on an empty :class:`DataFrame` or :class:`Series` with a :class:`MultiIndex` (:issue:`19602`)
 
 Missing
 ^^^^^^^
@@ -1016,6 +1036,10 @@ I/O
 - Bug in :meth:`~SQLDatabase.execute` was raising a ``ProgrammingError`` for some DB-API drivers when the SQL statement contained the `%` character and no parameters were present (:issue:`34211`)
 - Bug in :meth:`~pandas.io.stata.StataReader` which resulted in categorical variables with difference dtypes when reading data using an iterator. (:issue:`31544`)
 - :meth:`HDFStore.keys` has now an optional `include` parameter that allows the retrieval of all native HDF5 table names (:issue:`29916`)
+- `TypeError` exceptions raised by :meth:`read_csv` and :meth:`read_table` were showing as ``parser_f`` when an unexpected keyword argument was passed (:issue:`25648`)
+- Bug in :meth:`read_excel` for ODS files removes 0.0 values (:issue:`27222`)
+- Bug in :meth:`ujson.encode` was raising an `OverflowError` with numbers larger than sys.maxsize (:issue: `34395`)
+- Bug in :meth:`HDFStore.append_to_multiple` was raising a ``ValueError`` when the min_itemsize parameter is set (:issue:`11238`)
 
 Plotting
 ^^^^^^^^
@@ -1026,6 +1050,7 @@ Plotting
 - Bug in :meth:`DataFrame.hist` where the order of ``column`` argument was ignored (:issue:`29235`)
 - Bug in :meth:`DataFrame.plot.scatter` that when adding multiple plots with different ``cmap``, colorbars alway use the first ``cmap`` (:issue:`33389`)
 - Bug in :meth:`DataFrame.plot.scatter` was adding a colorbar to the plot even if the argument `c` was assigned to a column containing color names (:issue:`34316`)
+- Bug in :meth:`pandas.plotting.bootstrap_plot` was causing cluttered axes and overlapping labels (:issue:`34905`)
 
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -1047,6 +1072,7 @@ Groupby/resample/rolling
 - Bug in :meth:`SeriesGroupBy.agg` where any column name was accepted in the named aggregation of ``SeriesGroupBy`` previously. The behaviour now allows only ``str`` and callables else would raise ``TypeError``. (:issue:`34422`)
 - Bug in :meth:`DataFrame.groupby` lost index, when one of the ``agg`` keys referenced an empty list (:issue:`32580`)
 - Bug in :meth:`Rolling.apply` where ``center=True`` was ignored when ``engine='numba'`` was specified (:issue:`34784`)
+- Bug in :meth:`DataFrame.ewm.cov` was throwing ``AssertionError`` for :class:`MultiIndex` inputs (:issue:`34440`)
 
 Reshaping
 ^^^^^^^^^
@@ -1087,6 +1113,7 @@ Sparse
 - Bug in :meth:`Series.sum` with ``SparseArray`` raises ``TypeError`` (:issue:`25777`)
 - Bug where :class:`DataFrame` containing :class:`SparseArray` filled with ``NaN`` when indexed by a list-like (:issue:`27781`, :issue:`29563`)
 - The repr of :class:`SparseDtype` now includes the repr of its ``fill_value`` attribute. Previously it used ``fill_value``'s  string representation (:issue:`34352`)
+- Bug where empty :class:`DataFrame` could not be cast to :class:`SparseDtype` (:issue:`33113`)
 
 ExtensionArray
 ^^^^^^^^^^^^^^
@@ -1118,6 +1145,7 @@ Other
 - :class:`IntegerArray` now implements the ``sum`` operation (:issue:`33172`)
 - Bug in :class:`Tick` comparisons raising ``TypeError`` when comparing against timedelta-like objects (:issue:`34088`)
 - Bug in :class:`Tick` multiplication raising ``TypeError`` when multiplying by a float (:issue:`34486`)
+- Passing a `set` as `names` argument to :func:`pandas.read_csv`, :func:`pandas.read_table`, or :func:`pandas.read_fwf` will raise ``ValueError: Names should be an ordered collection.`` (:issue:`34946`)
 
 .. ---------------------------------------------------------------------------
 

environment.yml

@@ -37,7 +37,6 @@ dependencies:
   # Dask and its dependencies (that dont install with dask)
   - dask-core
   - toolz>=0.7.3
-  - fsspec>=0.5.1
   - partd>=0.3.10
   - cloudpickle>=0.2.1
 

pandas/_libs/internals.pyx

@@ -16,6 +16,7 @@ cnp.import_array()
 from pandas._libs.algos import ensure_int64
 
 
+@cython.final
 cdef class BlockPlacement:
     # __slots__ = '_as_slice', '_as_array', '_len'
     cdef: