Shift

.circleci/config.yml

@@ -46,6 +46,7 @@ jobs:
             fi
       - run:
           name: Build aarch64 wheels
+          no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that
           command: |
             pip3 install cibuildwheel==2.14.1
             cibuildwheel --prerelease-pythons --output-dir wheelhouse

.circleci/setup_env.sh

@@ -48,10 +48,10 @@ source activate pandas-dev
 # downstream CI jobs that may also build pandas from source.
 export PANDAS_CI=1
 
-if pip list | grep -q ^pandas; then
+if pip show pandas 1>/dev/null; then
     echo
     echo "remove any installed pandas package w/o removing anything else"
-    pip uninstall -y pandas || true
+    pip uninstall -y pandas
 fi
 
 echo "Install pandas"

.gitattributes

@@ -14,3 +14,71 @@
 *.xls binary
 *.xlsx binary
 pandas/_version.py export-subst
+
+
+*.bz2 export-ignore
+*.csv export-ignore
+*.data export-ignore
+*.dta export-ignore
+*.feather export-ignore
+*.tar export-ignore
+*.gz export-ignore
+*.h5 export-ignore
+*.html export-ignore
+*.json export-ignore
+*.jsonl export-ignore
+*.kml export-ignore
+*.msgpack export-ignore
+*.pdf export-ignore
+*.parquet export-ignore
+*.pickle export-ignore
+*.pkl export-ignore
+*.png export-ignore
+*.pptx export-ignore
+*.ods export-ignore
+*.odt export-ignore
+*.orc export-ignore
+*.sas7bdat export-ignore
+*.sav export-ignore
+*.so export-ignore
+*.txt export-ignore
+*.xls export-ignore
+*.xlsb export-ignore
+*.xlsm export-ignore
+*.xlsx export-ignore
+*.xpt export-ignore
+*.cpt export-ignore
+*.xml export-ignore
+*.xsl export-ignore
+*.xz export-ignore
+*.zip export-ignore
+*.zst export-ignore
+*~ export-ignore
+.DS_Store export-ignore
+.git* export-ignore
+
+*.py[ocd] export-ignore
+*.pxi export-ignore
+
+# Ignoring stuff from the top level
+.circleci export-ignore
+.github export-ignore
+asv_bench export-ignore
+ci export-ignore
+doc export-ignore
+gitpod export-ignore
+MANIFEST.in export-ignore
+scripts export-ignore
+typings export-ignore
+web export-ignore
+CITATION.cff export-ignore
+codecov.yml export-ignore
+Dockerfile export-ignore
+environment.yml export-ignore
+setup.py export-ignore
+
+
+# GH 39321
+# csv_dir_path fixture checks the existence of the directory
+# exclude the whole directory to avoid running related tests in sdist
+pandas/tests/io/parser/data export-ignore

.github/workflows/code-checks.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.0.x
+      - 2.1.x
   pull_request:
     branches:
       - main
-      - 2.0.x
+      - 2.1.x
 
 env:
   ENV_FILE: environment.yml
@@ -17,28 +17,8 @@ env:
 permissions:
   contents: read
 
+# pre-commit run by https://pre-commit.ci/
 jobs:
-  pre_commit:
-    name: pre-commit
-    runs-on: ubuntu-22.04
-    concurrency:
-      # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit
-      cancel-in-progress: true
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-
-    - name: Install Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: '3.9'
-
-    - name: Run pre-commit
-      uses: pre-commit/action@v3.0.0
-      with:
-        extra_args: --verbose --all-files
-
   docstring_typing_manual_hooks:
     name: Docstring validation, typing, and other manual pre-commit hooks
     runs-on: ubuntu-22.04
@@ -144,7 +124,7 @@ jobs:
       run: |
         cd asv_bench
         asv machine --yes
-        asv run --quick --dry-run --strict --durations=30 --python=same
+        asv run --quick --dry-run --durations=30 --python=same
 
   build_docker_dev_environment:
     name: Build Docker Dev Environment

.github/workflows/docbuild-and-upload.yml

@@ -4,13 +4,13 @@ on:
   push:
     branches:
       - main
-      - 2.0.x
+      - 2.1.x
     tags:
       - '*'
   pull_request:
     branches:
       - main
-      - 2.0.x
+      - 2.1.x
 
 env:
   ENV_FILE: environment.yml

.github/workflows/package-checks.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.0.x
+      - 2.1.x
   pull_request:
     branches:
       - main
-      - 2.0.x
+      - 2.1.x
     types: [ labeled, opened, synchronize, reopened ]
 
 permissions:
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "all"]
+        extra: ["test", "performance", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "consortium-standard", "all"]
       fail-fast: false
     name: Install Extras - ${{ matrix.extra }}
     concurrency:

.github/workflows/unit-tests.yml

@@ -4,11 +4,11 @@ on:
   push:
     branches:
       - main
-      - 2.0.x
+      - 2.1.x
   pull_request:
     branches:
       - main
-      - 2.0.x
+      - 2.1.x
     paths-ignore:
       - "doc/**"
       - "web/**"
@@ -77,16 +77,12 @@ jobs:
             env_file: actions-311-numpydev.yaml
             pattern: "not slow and not network and not single_cpu"
             test_args: "-W error::DeprecationWarning -W error::FutureWarning"
-            # TODO(cython3): Re-enable once next-beta(after beta 1) comes out
-            # There are some warnings failing the build with -werror
-            pandas_ci: "0"
           - name: "Pyarrow Nightly"
             env_file: actions-311-pyarrownightly.yaml
             pattern: "not slow and not network and not single_cpu"
       fail-fast: false
     name: ${{ matrix.name || format('ubuntu-latest {0}', matrix.env_file) }}
     env:
-      ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
       EXTRA_APT: ${{ matrix.extra_apt || '' }}
       LANG: ${{ matrix.lang || 'C.UTF-8' }}
@@ -150,14 +146,13 @@ jobs:
 
     - name: Generate extra locales
       # These extra locales will be available for locale.setlocale() calls in tests
-      run: |
-        sudo locale-gen ${{ matrix.extra_loc }}
+      run: sudo locale-gen ${{ matrix.extra_loc }}
       if: ${{ matrix.extra_loc }}
 
     - name: Set up Conda
       uses: ./.github/actions/setup-conda
       with:
-        environment-file: ${{ env.ENV_FILE }}
+        environment-file: ci/deps/${{ matrix.env_file }}
 
     - name: Build Pandas
       id: build
@@ -239,7 +234,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson[ninja]==1.0.1 meson-python==0.13.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] "cython<3.0.1" numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
           export PANDAS_CI=1
@@ -277,7 +272,7 @@ jobs:
           /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
           . ~/virtualenvs/pandas-dev/bin/activate
           python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.0.1
-          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir versioneer[toml] "cython<3.0.1" numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.46.1
           python -m pip install --no-cache-dir --no-build-isolation -e .
           python -m pip list --no-cache-dir
 
@@ -312,15 +307,14 @@ jobs:
     #    to the corresponding posix/windows-macos/sdist etc. workflows.
     # Feel free to modify this comment as necessary.
     #if: false # Uncomment this to freeze the workflow, comment it to unfreeze
+    defaults:
+      run:
+        shell: bash -eou pipefail {0}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        # TODO: Disable macOS for now, Github Actions bug where python is not
-        # symlinked correctly to 3.12
-        # xref https://github.com/actions/setup-python/issues/701
-        #os: [ubuntu-22.04, macOS-latest, windows-latest]
-        os: [ubuntu-22.04, windows-latest]
+        os: [ubuntu-22.04, macOS-latest, windows-latest]
 
     timeout-minutes: 180
 
@@ -333,7 +327,6 @@ jobs:
       PYTEST_WORKERS: "auto"
       PANDAS_CI: 1
       PATTERN: "not slow and not network and not clipboard and not single_cpu"
-      COVERAGE: true
       PYTEST_TARGET: pandas
 
     steps:
@@ -346,23 +339,15 @@ jobs:
         with:
           python-version: '3.12-dev'
 
-      - name: Install dependencies
+      - name: Build Environment
         run: |
           python --version
           python -m pip install --upgrade pip setuptools wheel meson[ninja]==1.0.1 meson-python==0.13.1
           python -m pip install --pre --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple numpy
-          python -m pip install git+https://github.com/nedbat/coveragepy.git
           python -m pip install versioneer[toml]
-          python -m pip install python-dateutil pytz tzdata cython hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
-          python -m pip list
-
-      - name: Build Pandas
-        run: |
+          python -m pip install python-dateutil pytz tzdata "cython<3.0.1" hypothesis>=6.46.1 pytest>=7.3.2 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17
           python -m pip install -ve . --no-build-isolation --no-index
+          python -m pip list
 
-      - name: Build Version
-        run: |
-          python -c "import pandas; pandas.show_versions();"
-
-      - name: Test
+      - name: Run Tests
         uses: ./.github/actions/run-tests