Merge branch 'main' of https://github.com/pandas-dev/pandas into feature/44764_perf_issue_new_period

Author: Sylvain MARIE

.github/workflows/32-bit-linux.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     paths-ignore:
       - "doc/**"
@@ -52,3 +54,7 @@ jobs:
           name: Test results
           path: test-data.xml
         if: failure()
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-32bit
+      cancel-in-progress: true

.github/workflows/code-checks.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
 
 env:

.github/workflows/docbuild-and-upload.yml

@@ -4,12 +4,14 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
 
 env:

.github/workflows/macos-windows.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     paths-ignore:
       - "doc/**"
@@ -16,8 +18,6 @@ env:
   PANDAS_CI: 1
   PYTEST_TARGET: pandas
   PATTERN: "not slow and not db and not network and not single_cpu"
-  ERROR_ON_WARNINGS: "1"
-
 
 permissions:
   contents: read

.github/workflows/package-checks.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     types: [ labeled, opened, synchronize, reopened ]
 

.github/workflows/sdist.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     types: [labeled, opened, synchronize, reopened]
     paths-ignore:

.github/workflows/ubuntu.yml

@@ -4,10 +4,12 @@ on:
   push:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
   pull_request:
     branches:
       - main
+      - 2.0.x
       - 1.5.x
     paths-ignore:
       - "doc/**"
@@ -38,7 +40,6 @@ jobs:
           - name: "Minimum Versions"
             env_file: actions-38-minimum_versions.yaml
             pattern: "not slow and not network and not single_cpu"
-            error_on_warnings: "0"
           - name: "Locale: it_IT"
             env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -63,12 +64,10 @@ jobs:
             env_file: actions-310.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_copy_on_write: "1"
-            error_on_warnings: "0"
           - name: "Data Manager"
             env_file: actions-38.yaml
             pattern: "not slow and not network and not single_cpu"
             pandas_data_manager: "array"
-            error_on_warnings: "0"
           - name: "Pypy"
             env_file: actions-pypy-38.yaml
             pattern: "not slow and not network and not single_cpu"
@@ -77,7 +76,6 @@ jobs:
             env_file: actions-310-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
-            error_on_warnings: "0"
         exclude:
           - env_file: actions-38.yaml
             pyarrow_version: "8"
@@ -97,7 +95,6 @@ jobs:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
       EXTRA_APT: ${{ matrix.extra_apt || '' }}
-      ERROR_ON_WARNINGS: ${{ matrix.error_on_warnings || '1' }}
       LANG: ${{ matrix.lang || '' }}
       LC_ALL: ${{ matrix.lc_all || '' }}
       PANDAS_DATA_MANAGER: ${{ matrix.pandas_data_manager || 'block' }}

.github/workflows/wheels.yml

@@ -173,8 +173,8 @@ jobs:
           pip install hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17
           cd .. # Not a good idea to test within the src tree
           python -c "import pandas; print(pandas.__version__);
-          pandas.test(extra_args=['-m not clipboard and not single_cpu', '--skip-slow', '--skip-network', '--skip-db', '-n=2']);
-          pandas.test(extra_args=['-m not clipboard and single_cpu', '--skip-slow', '--skip-network', '--skip-db'])"
+          pandas.test(extra_args=['-m not clipboard and not single_cpu and not slow and not network and not db', '-n 2']);
+          pandas.test(extra_args=['-m not clipboard and single_cpu and not slow and not network and not db'])"
       - uses: actions/upload-artifact@v3
         with:
           name: sdist

.pre-commit-config.yaml

@@ -313,7 +313,7 @@ repos:
         entry: python scripts/generate_pip_deps_from_conda.py
         files: ^(environment.yml|requirements-dev.txt)$
         pass_filenames: false
-        additional_dependencies: [pyyaml, toml]
+        additional_dependencies: [tomli, pyyaml]
     -   id: title-capitalization
         name: Validate correct capitalization among titles in documentation
         entry: python scripts/validate_rst_title_capitalization.py
@@ -391,10 +391,11 @@ repos:
         types: [yaml]
     -   id: validate-min-versions-in-sync
         name: Check minimum version of dependencies are aligned
-        entry: python scripts/validate_min_versions_in_sync.py
+        entry: python -m scripts.validate_min_versions_in_sync
         language: python
         files: ^(ci/deps/actions-.*-minimum_versions\.yaml|pandas/compat/_optional\.py)$
-        additional_dependencies: [tomli]
+        additional_dependencies: [tomli, pyyaml]
+        pass_filenames: false
     -   id: validate-errors-locations
         name: Validate errors locations
         description: Validate errors are in appropriate locations.

asv_bench/benchmarks/frame_methods.py

@@ -17,6 +17,21 @@
 from .pandas_vb_common import tm
 
 
+class Clip:
+    params = [
+        ["float64", "Float64", "float64[pyarrow]"],
+    ]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        data = np.random.randn(100_000, 10)
+        df = DataFrame(data, dtype=dtype)
+        self.df = df
+
+    def time_clip(self, dtype):
+        self.df.clip(-1.0, 1.0)
+
+
 class GetNumericData:
     def setup(self):
         self.df = DataFrame(np.random.randn(10000, 25))
@@ -739,4 +754,29 @@ def time_memory_usage_object_dtype(self):
         self.df2.memory_usage(deep=True)
 
 
+class FindValidIndex:
+    param_names = ["dtype"]
+    params = [
+        ["float", "Float64", "float64[pyarrow]"],
+    ]
+
+    def setup(self, dtype):
+        df = DataFrame(
+            np.random.randn(100000, 2),
+            columns=list("AB"),
+            dtype=dtype,
+        )
+        df.iloc[:100, 0] = None
+        df.iloc[:200, 1] = None
+        df.iloc[-100:, 0] = None
+        df.iloc[-200:, 1] = None
+        self.df = df
+
+    def time_first_valid_index(self, dtype):
+        self.df.first_valid_index()
+
+    def time_last_valid_index(self, dtype):
+        self.df.last_valid_index()
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

ci/code_checks.sh

@@ -86,24 +86,15 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
     MSG='Partially validate docstrings (EX01)' ;  echo $MSG
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX01 --ignore_functions \
         pandas.Series.index \
-        pandas.Series.nbytes \
-        pandas.Series.ndim \
-        pandas.Series.size \
-        pandas.Series.T \
         pandas.Series.hasnans \
-        pandas.Series.to_timestamp \
         pandas.Series.to_list \
         pandas.Series.__iter__ \
         pandas.Series.keys \
         pandas.Series.item \
         pandas.Series.pipe \
-        pandas.Series.kurt \
-        pandas.Series.mean \
-        pandas.Series.median \
         pandas.Series.mode \
         pandas.Series.sem \
         pandas.Series.skew \
-        pandas.Series.kurtosis \
         pandas.Series.is_unique \
         pandas.Series.is_monotonic_increasing \
         pandas.Series.is_monotonic_decreasing \
@@ -218,7 +209,6 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.Period.year \
         pandas.Period.asfreq \
         pandas.Period.now \
-        pandas.Period.to_timestamp \
         pandas.arrays.PeriodArray \
         pandas.Interval.closed \
         pandas.Interval.left \
@@ -551,18 +541,13 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
         pandas.DataFrame.keys \
         pandas.DataFrame.iterrows \
         pandas.DataFrame.pipe \
-        pandas.DataFrame.kurt \
-        pandas.DataFrame.kurtosis \
-        pandas.DataFrame.mean \
-        pandas.DataFrame.median \
         pandas.DataFrame.sem \
         pandas.DataFrame.skew \
         pandas.DataFrame.backfill \
         pandas.DataFrame.pad \
         pandas.DataFrame.swapaxes \
         pandas.DataFrame.first_valid_index \
         pandas.DataFrame.last_valid_index \
-        pandas.DataFrame.to_timestamp \
         pandas.DataFrame.attrs \
         pandas.DataFrame.plot \
         pandas.DataFrame.sparse.density \
@@ -576,24 +561,9 @@ if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
     $BASE_DIR/scripts/validate_docstrings.py --format=actions --errors=EX02 --ignore_functions \
         pandas.DataFrame.plot.line \
         pandas.Series.plot.line \
-        pandas.api.types.infer_dtype \
         pandas.api.types.is_datetime64_any_dtype \
         pandas.api.types.is_datetime64_ns_dtype \
         pandas.api.types.is_datetime64tz_dtype \
-        pandas.api.types.is_integer_dtype \
-        pandas.api.types.is_interval_dtype \
-        pandas.api.types.is_period_dtype \
-        pandas.api.types.is_signed_integer_dtype \
-        pandas.api.types.is_sparse \
-        pandas.api.types.is_string_dtype \
-        pandas.api.types.is_timedelta64_dtype \
-        pandas.api.types.is_timedelta64_ns_dtype \
-        pandas.api.types.is_unsigned_integer_dtype \
-        pandas.io.formats.style.Styler.concat \
-        pandas.io.formats.style.Styler.export \
-        pandas.io.formats.style.Styler.set_td_classes \
-        pandas.io.formats.style.Styler.use \
-        pandas.io.json.build_table_schema \
         pandas.plotting.andrews_curves \
         pandas.plotting.autocorrelation_plot \
         pandas.plotting.lag_plot \

ci/deps/actions-310-numpydev.yaml

@@ -12,7 +12,7 @@ dependencies:
   - pytest-cov
   - pytest-xdist>=2.2.0
   - hypothesis>=6.34.2
-  - pytest-asyncio>=0.17
+  - pytest-asyncio>=0.17.0
 
   # pandas dependencies
   - python-dateutil

ci/deps/actions-310.yaml

@@ -12,8 +12,7 @@ dependencies:
   - pytest>=7.0.0
   - pytest-cov
   - pytest-xdist>=2.2.0
-  - psutil
-  - pytest-asyncio>=0.17
+  - pytest-asyncio>=0.17.0
   - boto3
 
   # required dependencies
@@ -22,36 +21,36 @@ dependencies:
   - pytz
 
   # optional dependencies
-  - beautifulsoup4
+  - beautifulsoup4>=4.9.3
   - blosc
-  - bottleneck
-  - brotlipy
-  - fastparquet
-  - fsspec
-  - html5lib
-  - hypothesis
-  - gcsfs
-  - jinja2
-  - lxml
+  - bottleneck>=1.3.2
+  - brotlipy>=0.7.0
+  - fastparquet>=0.6.3
+  - fsspec>=2021.07.0
+  - html5lib>=1.1
+  - hypothesis>=6.34.2
+  - gcsfs>=2021.07.0
+  - jinja2>=3.0.0
+  - lxml>=4.6.3
   - matplotlib>=3.6.1, <3.7.0
-  - numba
-  - numexpr
-  - openpyxl<3.1.1
-  - odfpy
-  - pandas-gbq
-  - psycopg2
-  - pymysql
-  - pytables
+  - numba>=0.53.1
+  - numexpr>=2.7.3
+  - openpyxl<3.1.1, >=3.0.7
+  - odfpy>=1.4.1
+  - pandas-gbq>=0.15.0
+  - psycopg2>=2.8.6
+  - pymysql>=1.0.2
+  - pytables>=3.6.1
   - pyarrow
-  - pyreadstat
-  - python-snappy
-  - pyxlsb
+  - pyreadstat>=1.1.2
+  - python-snappy>=0.6.0
+  - pyxlsb>=1.0.8
   - s3fs>=2021.08.0
-  - scipy
-  - sqlalchemy
-  - tabulate
+  - scipy>=1.7.1
+  - sqlalchemy>=1.4.16
+  - tabulate>=0.8.9
   - tzdata>=2022a
-  - xarray
-  - xlrd
-  - xlsxwriter
-  - zstandard
+  - xarray>=0.21.0
+  - xlrd>=2.0.1
+  - xlsxwriter>=1.4.3
+  - zstandard>=0.15.2