Merge remote-tracking branch 'upstream/master' into numpy-types

.github/ISSUE_TEMPLATE/documentation_improvement.md

@@ -9,7 +9,7 @@ labels: "Docs, Needs Triage"
 
 #### Location of the documentation
 
-[this should provide the location of the documentation, e.g. "pandas.read_csv" or the URL of the documentation, e.g. "https://dev.pandas.io/docs/reference/api/pandas.read_csv.html"]
+[this should provide the location of the documentation, e.g. "pandas.read_csv" or the URL of the documentation, e.g. "https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html"]
 
 **Note**: You can check the latest versions of the docs on `master` [here](https://pandas.pydata.org/docs/dev/).
 

.github/workflows/comment_bot.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Install-pre-commit
         run: python -m pip install --upgrade pre-commit
       - name: Run pre-commit
-        run: pre-commit run --all-files || (exit 0)
+        run: pre-commit run --from-ref=origin/master --to-ref=HEAD --all-files || (exit 0)
       - name: Commit results
         run: |
           git config user.name "$(git log -1 --pretty=format:%an)"

.pre-commit-config.yaml

@@ -135,7 +135,7 @@ repos:
         files: ^pandas/tests/
     -   id: FrameOrSeriesUnion
         name: Check for use of Union[Series, DataFrame] instead of FrameOrSeriesUnion alias
-        entry: Union\[.*(Series.*DataFrame|DataFrame.*Series).*\]
+        entry: Union\[.*(Series,.*DataFrame|DataFrame,.*Series).*\]
         language: pygrep
         types: [python]
         exclude: ^pandas/_typing\.py$
@@ -189,8 +189,9 @@ repos:
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown]
-        files: ^pandas/core/
+        files: ^pandas/
+        exclude: ^pandas/tests/
 -   repo: https://github.com/MarcoGorelli/no-string-hints
-    rev: v0.1.5
+    rev: v0.1.6
     hooks:
     -   id: no-string-hints

README.md

@@ -1,5 +1,5 @@
 <div align="center">
-  <img src="https://dev.pandas.io/static/img/pandas.svg"><br>
+  <img src="https://pandas.pydata.org/static/img/pandas.svg"><br>
 </div>
 
 -----------------

asv_bench/benchmarks/indexing.py

@@ -243,6 +243,20 @@ def time_loc_list(self, monotonic):
         monotonic.loc[80000:]
 
 
+class DatetimeIndexIndexing:
+    def setup(self):
+        dti = date_range("2016-01-01", periods=10000, tz="US/Pacific")
+        dti2 = dti.tz_convert("UTC")
+        self.dti = dti
+        self.dti2 = dti2
+
+    def time_get_indexer_mismatched_tz(self):
+        # reached via e.g.
+        #  ser = Series(range(len(dti)), index=dti)
+        #  ser[dti2]
+        self.dti.get_indexer(self.dti2)
+
+
 class CategoricalIndexIndexing:
 
     params = ["monotonic_incr", "monotonic_decr", "non_monotonic"]

asv_bench/benchmarks/io/csv.py

@@ -76,6 +76,54 @@ def time_frame(self, obs):
         self.data.to_csv(self.fname)
 
 
+class ToCSVIndexes(BaseIO):
+
+    fname = "__test__.csv"
+
+    @staticmethod
+    def _create_df(rows, cols):
+        index_cols = {
+            "index1": np.random.randint(0, rows, rows),
+            "index2": np.full(rows, 1, dtype=np.int),
+            "index3": np.full(rows, 1, dtype=np.int),
+        }
+        data_cols = {
+            f"col{i}": np.random.uniform(0, 100000.0, rows) for i in range(cols)
+        }
+        df = DataFrame({**index_cols, **data_cols})
+        return df
+
+    def setup(self):
+        ROWS = 100000
+        COLS = 5
+        # For tests using .head(), create an initial dataframe with this many times
+        # more rows
+        HEAD_ROW_MULTIPLIER = 10
+
+        self.df_standard_index = self._create_df(ROWS, COLS)
+
+        self.df_custom_index_then_head = (
+            self._create_df(ROWS * HEAD_ROW_MULTIPLIER, COLS)
+            .set_index(["index1", "index2", "index3"])
+            .head(ROWS)
+        )
+
+        self.df_head_then_custom_index = (
+            self._create_df(ROWS * HEAD_ROW_MULTIPLIER, COLS)
+            .head(ROWS)
+            .set_index(["index1", "index2", "index3"])
+        )
+
+    def time_standard_index(self):
+        self.df_standard_index.to_csv(self.fname)
+
+    def time_multiindex(self):
+        self.df_head_then_custom_index.to_csv(self.fname)
+
+    def time_head_of_multiindex(self):
+        self.df_custom_index_then_head.to_csv(self.fname)
+
+
 class StringIORewind:
     def data(self, stringio_object):
         stringio_object.seek(0)

asv_bench/benchmarks/series_methods.py

@@ -284,16 +284,29 @@ def time_clip(self, n):
 
 class ValueCounts:
 
-    params = ["int", "uint", "float", "object"]
-    param_names = ["dtype"]
+    params = [[10 ** 3, 10 ** 4, 10 ** 5], ["int", "uint", "float", "object"]]
+    param_names = ["N", "dtype"]
 
-    def setup(self, dtype):
-        self.s = Series(np.random.randint(0, 1000, size=100000)).astype(dtype)
+    def setup(self, N, dtype):
+        self.s = Series(np.random.randint(0, N, size=10 * N)).astype(dtype)
 
-    def time_value_counts(self, dtype):
+    def time_value_counts(self, N, dtype):
         self.s.value_counts()
 
 
+class Mode:
+
+    params = [[10 ** 3, 10 ** 4, 10 ** 5], ["int", "uint", "float", "object"]]
+    param_names = ["N", "dtype"]
+
+    def setup(self, N, dtype):
+        np.random.seed(42)
+        self.s = Series(np.random.randint(0, N, size=10 * N)).astype(dtype)
+
+    def time_mode(self, N, dtype):
+        self.s.mode()
+
+
 class Dir:
     def setup(self):
         self.s = Series(index=tm.makeStringIndex(10000))

ci/code_checks.sh

@@ -64,7 +64,7 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
     # this particular codebase (e.g. src/headers, src/klib). However,
     # we can lint all header files since they aren't "generated" like C files are.
     MSG='Linting .c and .h' ; echo $MSG
-    cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/ujson pandas/_libs/tslibs/src/datetime pandas/_libs/*.cpp
+    cpplint --quiet --extensions=c,h --headers=h --recursive --filter=-readability/casting,-runtime/int,-build/include_subdir pandas/_libs/src/*.h pandas/_libs/src/parser pandas/_libs/src/ujson pandas/_libs/tslibs/src/datetime pandas/_libs/*.cpp
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/azure-37.yaml

@@ -18,7 +18,7 @@ dependencies:
   - numpy
   - python-dateutil
   - nomkl
-  - pyarrow
+  - pyarrow=0.15.1
   - pytz
   - s3fs>=0.4.0
   - moto>=1.3.14

ci/deps/azure-macos-37.yaml

@@ -21,7 +21,7 @@ dependencies:
   - numexpr
   - numpy=1.16.5
   - openpyxl
-  - pyarrow>=0.15.0
+  - pyarrow=0.15.1
   - pytables
   - python-dateutil==2.7.3
   - pytz

doc/source/getting_started/comparison/comparison_with_spreadsheets.rst

@@ -89,13 +89,6 @@ Both `Excel <https://support.microsoft.com/en-us/office/import-data-from-externa
 and :ref:`pandas <10min_tut_02_read_write>` can import data from various sources in various
 formats.
 
-Excel files
-'''''''''''
-
-Excel opens `various Excel file formats <https://support.microsoft.com/en-us/office/file-formats-that-are-supported-in-excel-0943ff2c-6014-4e8d-aaea-b83d51d46247>`_
-by double-clicking them, or using `the Open menu <https://support.microsoft.com/en-us/office/open-files-from-the-file-menu-97f087d8-3136-4485-8e86-c5b12a8c4176>`_.
-In pandas, you use :ref:`special methods for reading and writing from/to Excel files <io.excel>`.
-
 CSV
 '''
 
@@ -125,6 +118,27 @@ would be:
    # alternatively, read_table is an alias to read_csv with tab delimiter
    tips = pd.read_table("tips.csv", header=None)
 
+Excel files
+'''''''''''
+
+Excel opens `various Excel file formats <https://support.microsoft.com/en-us/office/file-formats-that-are-supported-in-excel-0943ff2c-6014-4e8d-aaea-b83d51d46247>`_
+by double-clicking them, or using `the Open menu <https://support.microsoft.com/en-us/office/open-files-from-the-file-menu-97f087d8-3136-4485-8e86-c5b12a8c4176>`_.
+In pandas, you use :ref:`special methods for reading and writing from/to Excel files <io.excel>`.
+
+Let's first :ref:`create a new Excel file <io.excel_writer>` based on the ``tips`` dataframe in the above example:
+
+.. code-block:: python
+
+    tips.to_excel("./tips.xlsx")
+
+Should you wish to subsequently access the data in the ``tips.xlsx`` file, you can read it into your module using
+
+.. code-block:: python
+
+    tips_df = pd.read_excel("./tips.xlsx", index_col=0)
+
+You have just read in an Excel file using pandas!
+
 
 Limiting output
 ~~~~~~~~~~~~~~~