Merge remote-tracking branch 'upstream/main' into cow_series_copy_false_everywhere

Author: phofl

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 jobs:
   test-arm:
     machine:
-      image: ubuntu-2004:202101-01
+      image: ubuntu-2004:2022.04.1
     resource_class: arm.large
     environment:
       ENV_FILE: ci/deps/circle-38-arm64.yaml
@@ -23,7 +23,7 @@ jobs:
       cibw-build:
         type: string
     machine:
-      image: ubuntu-2004:202101-01
+      image: ubuntu-2004:2022.04.1
     resource_class: arm.large
     environment:
       ENV_FILE: ci/deps/circle-38-arm64.yaml
@@ -48,7 +48,7 @@ jobs:
       - run:
           name: Build aarch64 wheels
           command: |
-            pip3 install cibuildwheel==2.9.0
+            pip3 install cibuildwheel==2.12.1
             cibuildwheel --output-dir wheelhouse
           environment:
             CIBW_BUILD: << parameters.cibw-build >>
@@ -57,7 +57,7 @@ jobs:
           name: Install Anaconda Client & Upload Wheels
           command: |
             echo "Install Mambaforge"
-            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/4.14.0-0/Mambaforge-4.14.0-0-Linux-aarch64.sh"
+            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/23.1.0-0/Mambaforge-23.1.0-0-Linux-aarch64.sh"
             echo "Downloading $MAMBA_URL"
             wget -q $MAMBA_URL -O minimamba.sh
             chmod +x minimamba.sh

.circleci/setup_env.sh

@@ -1,7 +1,7 @@
 #!/bin/bash -e
 
 echo "Install Mambaforge"
-MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/4.14.0-0/Mambaforge-4.14.0-0-Linux-aarch64.sh"
+MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/23.1.0-0/Mambaforge-23.1.0-0-Linux-aarch64.sh"
 echo "Downloading $MAMBA_URL"
 wget -q $MAMBA_URL -O minimamba.sh
 chmod +x minimamba.sh

.github/actions/setup-conda/action.yml

@@ -9,20 +9,9 @@ inputs:
   extra-specs:
     description: Extra packages to install
     required: false
-  pyarrow-version:
-    description: If set, overrides the PyArrow version in the Conda environment to the given string.
-    required: false
 runs:
   using: composite
   steps:
-    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
-      run: |
-        grep -q '  - pyarrow' ${{ inputs.environment-file }}
-        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
-        cat ${{ inputs.environment-file }}
-      shell: bash
-      if: ${{ inputs.pyarrow-version }}
-
     - name: Install ${{ inputs.environment-file }}
       uses: mamba-org/provision-with-micromamba@v12
       with:

.github/dependabot.yml

@@ -0,0 +1,9 @@
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      interval: weekly
+    labels:
+      - "CI"
+      - "Dependencies"

.github/workflows/code-checks.yml

@@ -35,7 +35,7 @@ jobs:
         python-version: '3.9'
 
     - name: Run pre-commit
-      uses: pre-commit/action@v2.0.3
+      uses: pre-commit/action@v3.0.0
       with:
         extra_args: --verbose --all-files
 
@@ -93,7 +93,7 @@ jobs:
       if: ${{ steps.build.outcome == 'success' && always() }}
 
     - name: Typing + pylint
-      uses: pre-commit/action@v2.0.3
+      uses: pre-commit/action@v3.0.0
       with:
         extra_args: --verbose --hook-stage manual --all-files
       if: ${{ steps.build.outcome == 'success' && always() }}

.github/workflows/macos-windows.yml

@@ -52,7 +52,6 @@ jobs:
       uses: ./.github/actions/setup-conda
       with:
         environment-file: ci/deps/${{ matrix.env_file }}
-        pyarrow-version: ${{ matrix.os == 'macos-latest' && '9' || '' }}
 
     - name: Build Pandas
       uses: ./.github/actions/build_pandas

.github/workflows/stale-pr.yml

@@ -13,7 +13,7 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/stale@v4
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity. Please [update](https://pandas.pydata.org/pandas-docs/stable/development/contributing.html#updating-your-pull-request) and respond to this comment if you're still interested in working on this."

.github/workflows/ubuntu.yml

@@ -26,9 +26,8 @@ jobs:
     strategy:
       matrix:
         env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml, actions-311.yaml]
-        pattern: ["not single_cpu", "single_cpu"]
-        pyarrow_version: ["8", "9", "10"]
-        pandas_ci: [1]
+        # Prevent the include jobs from overriding other jobs
+        pattern: [""]
         include:
           - name: "Downstream Compat"
             env_file: actions-38-downstream_compat.yaml
@@ -75,22 +74,12 @@ jobs:
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
             # TODO(cython3): Re-enable once next-beta(after beta 1) comes out
             # There are some warnings failing the build with -werror
-            pandas_ci: 0
-        exclude:
-          - env_file: actions-38.yaml
-            pyarrow_version: "8"
-          - env_file: actions-38.yaml
-            pyarrow_version: "9"
-          - env_file: actions-39.yaml
-            pyarrow_version: "8"
-          - env_file: actions-39.yaml
-            pyarrow_version: "9"
-          - env_file: actions-310.yaml
-            pyarrow_version: "8"
-          - env_file: actions-310.yaml
-            pyarrow_version: "9"
+            pandas_ci: "0"
+          - name: "Pyarrow Nightly"
+            env_file: actions-311-pyarrownightly.yaml
+            pattern: "not slow and not network and not single_cpu"
       fail-fast: false
-    name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }}
+    name: ${{ matrix.name || matrix.env_file }}
     env:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
@@ -99,16 +88,16 @@ jobs:
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_DATA_MANAGER: ${{ matrix.pandas_data_manager || 'block' }}
       PANDAS_COPY_ON_WRITE: ${{ matrix.pandas_copy_on_write || '0' }}
-      PANDAS_CI: ${{ matrix.pandas_ci }}
+      PANDAS_CI: ${{ matrix.pandas_ci || '1' }}
       TEST_ARGS: ${{ matrix.test_args || '' }}
-      PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }}
+      PYTEST_WORKERS: 'auto'
       PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
       IS_PYPY: ${{ contains(matrix.env_file, 'pypy') }}
       # TODO: re-enable coverage on pypy, its slow
       COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }}
       cancel-in-progress: true
 
     services:
@@ -167,12 +156,24 @@ jobs:
       uses: ./.github/actions/setup-conda
       with:
         environment-file: ${{ env.ENV_FILE }}
-        pyarrow-version: ${{ matrix.pyarrow_version }}
 
     - name: Build Pandas
+      id: build
       uses: ./.github/actions/build_pandas
 
-    - name: Test
+    - name: Test (not single_cpu)
+      uses: ./.github/actions/run-tests
+      # TODO: Don't continue on error for PyPy
+      continue-on-error: ${{ env.IS_PYPY == 'true' }}
+      env:
+        # Set pattern to not single_cpu if not already set
+        PATTERN: ${{ env.PATTERN == '' && 'not single_cpu' || matrix.pattern }}
+
+    - name: Test (single_cpu)
       uses: ./.github/actions/run-tests
       # TODO: Don't continue on error for PyPy
       continue-on-error: ${{ env.IS_PYPY == 'true' }}
+      env:
+        PATTERN: 'single_cpu'
+        PYTEST_WORKERS: 1
+      if: ${{ matrix.pattern == '' && (always() && steps.build.outcome == 'success')}}

.github/workflows/wheels.yml

@@ -71,7 +71,7 @@ jobs:
           fetch-depth: 0
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.9.0
+        uses: pypa/cibuildwheel@v2.12.1
         env:
           CIBW_BUILD: ${{ matrix.python[0] }}-${{ matrix.buildplat[1] }}
 

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
         types_or: [python, pyi]
         additional_dependencies: [black==23.1.0]
 -   repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.255
+    rev: v0.0.259
     hooks:
     -   id: ruff
         args: [--exit-non-zero-on-fix]
@@ -392,14 +392,6 @@ repos:
         files: ^pandas/
         exclude: ^(pandas/_libs/|pandas/tests/|pandas/errors/__init__.py$|pandas/_version.py)
         types: [python]
-    -   id: flake8-pyi
-        name: flake8-pyi
-        entry: flake8 --extend-ignore=E301,E302,E305,E701,E704
-        types: [pyi]
-        language: python
-        additional_dependencies:
-        - flake8==5.0.4
-        - flake8-pyi==22.8.1
     -   id: future-annotations
         name: import annotations from __future__
         entry: 'from __future__ import annotations'
@@ -421,8 +413,8 @@ repos:
         language: python
         stages: [manual]
         additional_dependencies:
-        - autotyping==22.9.0
-        - libcst==0.4.7
+        - autotyping==23.3.0
+        - libcst==0.4.9
     -   id: check-test-naming
         name: check that test names start with 'test'
         entry: python -m scripts.check_test_naming

asv_bench/benchmarks/arithmetic.py

@@ -266,10 +266,14 @@ def setup(self, tz):
         self.ts = self.s[halfway]
 
         self.s2 = Series(date_range("20010101", periods=N, freq="s", tz=tz))
+        self.ts_different_reso = Timestamp("2001-01-02", tz=tz)
 
     def time_series_timestamp_compare(self, tz):
         self.s <= self.ts
 
+    def time_series_timestamp_different_reso_compare(self, tz):
+        self.s <= self.ts_different_reso
+
     def time_timestamp_series_compare(self, tz):
         self.ts >= self.s
 

asv_bench/benchmarks/strings.py

@@ -34,7 +34,6 @@ def setup(self, dtype):
 
         # GH37371. Testing construction of string series/frames from ExtensionArrays
         self.series_cat_arr = Categorical(self.series_arr)
-        self.frame_cat_arr = Categorical(self.frame_arr)
 
     def time_series_construction(self, dtype):
         Series(self.series_arr, dtype=dtype)
@@ -54,12 +53,6 @@ def time_cat_series_construction(self, dtype):
     def peakmem_cat_series_construction(self, dtype):
         Series(self.series_cat_arr, dtype=dtype)
 
-    def time_cat_frame_construction(self, dtype):
-        DataFrame(self.frame_cat_arr, dtype=dtype)
-
-    def peakmem_cat_frame_construction(self, dtype):
-        DataFrame(self.frame_cat_arr, dtype=dtype)
-
 
 class Methods(Dtypes):
     def time_center(self, dtype):

ci/code_checks.sh

@@ -86,8 +86,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
     MSG='Partially validate docstrings (EX01)' ;  echo $MSG
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX01 --ignore_functions \
         pandas.Series.index \
-        pandas.Series.hasnans \
-        pandas.Series.to_list \
         pandas.Series.__iter__ \
         pandas.Series.keys \
         pandas.Series.item \
@@ -309,7 +307,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas_object \
         pandas.api.interchange.from_dataframe \
         pandas.Index.values \
-        pandas.Index.hasnans \
         pandas.Index.dtype \
         pandas.Index.inferred_type \
         pandas.Index.shape \

ci/deps/actions-310.yaml

@@ -41,7 +41,7 @@ dependencies:
   - psycopg2>=2.8.6
   - pymysql>=1.0.2
   - pytables>=3.6.1
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat>=1.1.2
   - python-snappy>=0.6.0
   - pyxlsb>=1.0.8

ci/deps/actions-311-pyarrownightly.yaml

@@ -0,0 +1,29 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11
+
+  # build dependencies
+  - versioneer[toml]
+  - cython>=0.29.33
+
+  # test dependencies
+  - pytest>=7.0.0
+  - pytest-cov
+  - pytest-xdist>=2.2.0
+  - hypothesis>=6.34.2
+  - pytest-asyncio>=0.17.0
+
+  # required dependencies
+  - python-dateutil
+  - numpy
+  - pytz
+  - pip
+
+  - pip:
+    - "tzdata>=2022.1"
+    - "--extra-index-url https://pypi.fury.io/arrow-nightlies/"
+    - "--prefer-binary"
+    - "--pre"
+    - "pyarrow"

ci/deps/actions-311.yaml

@@ -41,7 +41,7 @@ dependencies:
   - psycopg2>=2.8.6
   - pymysql>=1.0.2
   # - pytables>=3.8.0  # first version that supports 3.11
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat>=1.1.2
   - python-snappy>=0.6.0
   - pyxlsb>=1.0.8

ci/deps/actions-38-downstream_compat.yaml

@@ -39,7 +39,7 @@ dependencies:
   - openpyxl<3.1.1, >=3.0.7
   - odfpy>=1.4.1
   - psycopg2>=2.8.6
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql>=1.0.2
   - pyreadstat>=1.1.2
   - pytables>=3.6.1

ci/deps/actions-38.yaml

@@ -39,7 +39,7 @@ dependencies:
   - odfpy>=1.4.1
   - pandas-gbq>=0.15.0
   - psycopg2>=2.8.6
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql>=1.0.2
   - pyreadstat>=1.1.2
   - pytables>=3.6.1

ci/deps/actions-39.yaml

@@ -40,7 +40,7 @@ dependencies:
   - pandas-gbq>=0.15.0
   - psycopg2>=2.8.6
   - pymysql>=1.0.2
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat>=1.1.2
   - pytables>=3.6.1
   - python-snappy>=0.6.0

ci/deps/circle-38-arm64.yaml

@@ -39,7 +39,7 @@ dependencies:
   - odfpy>=1.4.1
   - pandas-gbq>=0.15.0
   - psycopg2>=2.8.6
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql>=1.0.2
   # Not provided on ARM
   #- pyreadstat