Merge remote-tracking branch 'upstream/main' into 2022-06-29-FIX-46671

Author: jorisvandenbossche

.github/workflows/32-bit-linux.yml

@@ -39,8 +39,9 @@ jobs:
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
           pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
-          python setup.py build_ext -q -j2 && \
+          python setup.py build_ext -q -j1 && \
           python -m pip install --no-build-isolation --no-use-pep517 -e . && \
+          python -m pip list && \
           export PANDAS_CI=1 && \
           pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml"
 

.github/workflows/codeql.yml

@@ -0,0 +1,31 @@
+name: CodeQL
+on:
+  schedule:
+    # every day at midnight
+    - cron: "0 0 * * *"
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language:
+          - python
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: github/codeql-action/init@v2
+        with:
+          languages: ${{ matrix.language }}
+      - uses: github/codeql-action/autobuild@v2
+      - uses: github/codeql-action/analyze@v2

.github/workflows/python-dev.yml

@@ -1,9 +1,21 @@
-# This file is purposely frozen(does not run). DO NOT DELETE IT
-# Unfreeze(by commentingthe if: false() condition) once the
-# next Python Dev version has released beta 1 and both Cython and numpy support it
-# After that Python has released, migrate the workflows to the
-# posix GHA workflows and "freeze" this file by
-# uncommenting the if: false() condition
+# This workflow may or may not run depending on the state of the next
+# unreleased Python version. DO NOT DELETE IT.
+#
+# In general, this file will remain frozen(present, but not running) until:
+#    - The next unreleased Python version has released beta 1
+#      - This version should be available on Github Actions.
+#    - Our required build/runtime dependencies(numpy, pytz, Cython, python-dateutil)
+#      support that unreleased Python version.
+#    To unfreeze, comment out the ``if: false`` condition, and make sure you update
+#    the name of the workflow and Python version in actions/setup-python to: '3.12-dev'
+#
+# After it has been unfrozen, this file should remain unfrozen(present, and running) until:
+#    - The next Python version has been officially released.
+#    OR
+#    - Most/All of our optional dependencies support Python 3.11 AND
+#    - The next Python version has released a rc(we are guaranteed a stable ABI).
+#    To freeze this file, uncomment out the ``if: false`` condition, and migrate the jobs
+#    to the corresponding posix/windows-macos/sdist etc. workflows.
 # Feel free to modify this comment as necessary.
 
 name: Python Dev
@@ -32,7 +44,7 @@ permissions:
 
 jobs:
   build:
-    if: false # Comment this line out to "unfreeze"
+    # if: false # Uncomment this to freeze the workflow, comment it to unfreeze
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -53,27 +65,27 @@ jobs:
         fetch-depth: 0
 
     - name: Set up Python Dev Version
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: '3.11-dev'
 
     - name: Install dependencies
-      shell: bash -el {0}
       run: |
-        python3 -m pip install --upgrade pip setuptools wheel
-        python3 -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
-        python3 -m pip install git+https://github.com/nedbat/coveragepy.git
-        python3 -m pip install cython python-dateutil pytz hypothesis pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
-        python3 -m pip list
+        python --version
+        python -m pip install --upgrade pip setuptools wheel
+        python -m pip install git+https://github.com/numpy/numpy.git
+        python -m pip install git+https://github.com/nedbat/coveragepy.git
+        python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
+        python -m pip list
 
     - name: Build Pandas
       run: |
-        python3 setup.py build_ext -q -j2
-        python3 -m pip install -e . --no-build-isolation --no-use-pep517
+        python setup.py build_ext -q -j2
+        python -m pip install -e . --no-build-isolation --no-use-pep517
 
     - name: Build Version
       run: |
-        python3 -c "import pandas; pandas.show_versions();"
+        python -c "import pandas; pandas.show_versions();"
 
     - name: Test
       uses: ./.github/actions/run-tests

.github/workflows/sdist.yml

@@ -79,9 +79,9 @@ jobs:
       run: |
         case "${{matrix.python-version}}" in
         3.8)
-          pip install numpy==1.19.5 ;;
+          pip install numpy==1.20.3 ;;
         3.9)
-          pip install numpy==1.19.5 ;;
+          pip install numpy==1.20.3 ;;
         3.10)
           pip install numpy==1.21.2 ;;
         esac

.github/workflows/ubuntu.yml

@@ -64,7 +64,7 @@ jobs:
             env_file: actions-310-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_testing_mode: "deprecate"
-            test_args: "-W error::DeprecationWarning:numpy"
+            test_args: "-W error::DeprecationWarning:numpy -W error::FutureWarning:numpy"
         exclude:
           - env_file: actions-39.yaml
             pyarrow_version: "6"

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/jendrikseipp/vulture
-    rev: 'v2.4'
+    rev: 'v2.5'
     hooks:
       - id: vulture
         entry: python scripts/run_vulture.py
@@ -46,20 +46,19 @@ repos:
         exclude: ^pandas/_libs/src/(klib|headers)/
         args: [--quiet, '--extensions=c,h', '--headers=h', --recursive, '--filter=-readability/casting,-runtime/int,-build/include_subdir']
 -   repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
+    rev: 5.0.4
     hooks:
     -   id: flake8
         additional_dependencies: &flake8_dependencies
-        - flake8==4.0.1
-        - flake8-comprehensions==3.7.0
-        - flake8-bugbear==21.3.2
+        - flake8==5.0.4
+        - flake8-bugbear==22.7.1
         - pandas-dev-flaker==0.5.0
 -   repo: https://github.com/PyCQA/isort
     rev: 5.10.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.34.0
+    rev: v2.37.3
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -85,38 +84,42 @@ repos:
 -   repo: local
     hooks:
     -   id: pyright
+        # note: assumes python env is setup and activated
         name: pyright
         entry: pyright
-        # note: assumes python env is setup and activated
         language: node
         pass_filenames: false
         types: [python]
         stages: [manual]
         additional_dependencies: &pyright_dependencies
-        - pyright@1.1.258
--   repo: local
-    hooks:
+        - pyright@1.1.264
     -   id: pyright_reportGeneralTypeIssues
+        # note: assumes python env is setup and activated
         name: pyright reportGeneralTypeIssues
         entry: pyright --skipunannotated -p pyright_reportGeneralTypeIssues.json
-        # note: assumes python env is setup and activated
         language: node
         pass_filenames: false
         types: [python]
         stages: [manual]
         additional_dependencies: *pyright_dependencies
--   repo: local
-    hooks:
     -   id: mypy
+        # note: assumes python env is setup and activated
         name: mypy
         entry: mypy
-        # note: assumes python env is setup and activated
         language: system
         pass_filenames: false
         types: [python]
         stages: [manual]
--   repo: local
-    hooks:
+    -   id: stubtest
+        # note: assumes python env is setup and activated
+        # note: requires pandas dev to be installed
+        name: mypy (stubtest)
+        entry: python
+        language: system
+        pass_filenames: false
+        types: [pyi]
+        args: [scripts/run_stubtest.py]
+        stages: [manual]
     -   id: flake8-rst
         name: flake8-rst
         description: Run flake8 on code snippets in docstrings or RST files
@@ -235,5 +238,17 @@ repos:
         types: [pyi]
         language: python
         additional_dependencies:
-        - flake8==4.0.1
-        - flake8-pyi==22.5.1
+        - flake8==5.0.4
+        - flake8-pyi==22.8.1
+    -   id: future-annotations
+        name: import annotations from __future__
+        entry: 'from __future__ import annotations'
+        language: pygrep
+        args: [--negate]
+        files: ^pandas/
+        types: [python]
+        exclude: |
+            (?x)
+            /(__init__\.py)|(api\.py)|(_version\.py)|(testing\.py)|(conftest\.py)$
+            |/tests/
+            |/_testing/

Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/condaforge/miniforge3
+FROM quay.io/condaforge/mambaforge
 
 # if you forked pandas, you can pass in your own GitHub username to use your fork
 # i.e. gh_username=myname
@@ -40,7 +40,6 @@ RUN mkdir "$pandas_home" \
 # we just update the base/root one from the 'environment.yml' file instead of creating a new one.
 #
 # Set up environment
-RUN conda install -y mamba
 RUN mamba env update -n base -f "$pandas_home/environment.yml"
 
 # Build C extensions and pandas

asv_bench/asv.conf.json

@@ -42,7 +42,7 @@
     // followed by the pip installed packages).
     "matrix": {
         "numpy": [],
-        "Cython": ["0.29.30"],
+        "Cython": ["0.29.32"],
         "matplotlib": [],
         "sqlalchemy": [],
         "scipy": [],

asv_bench/benchmarks/hash_functions.py

@@ -39,6 +39,21 @@ def time_unique(self, exponent):
         pd.unique(self.a2)
 
 
+class Unique:
+    params = ["Int64", "Float64"]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        self.ser = pd.Series(([1, pd.NA, 2] + list(range(100_000))) * 3, dtype=dtype)
+        self.ser_unique = pd.Series(list(range(300_000)) + [pd.NA], dtype=dtype)
+
+    def time_unique_with_duplicates(self, exponent):
+        pd.unique(self.ser)
+
+    def time_unique(self, exponent):
+        pd.unique(self.ser_unique)
+
+
 class NumericSeriesIndexing:
 
     params = [

asv_bench/benchmarks/reshape.py

@@ -268,9 +268,7 @@ def setup(self, bins):
         self.datetime_series = pd.Series(
             np.random.randint(N, size=N), dtype="datetime64[ns]"
         )
-        self.interval_bins = pd.IntervalIndex.from_breaks(
-            np.linspace(0, N, bins), "right"
-        )
+        self.interval_bins = pd.IntervalIndex.from_breaks(np.linspace(0, N, bins))
 
     def time_cut_int(self, bins):
         pd.cut(self.int_series, bins)