Merge remote-tracking branch 'upstream/main' into GH45857

.circleci/config.yml

@@ -3,20 +3,135 @@ version: 2.1
 jobs:
   test-arm:
     machine:
-      image: ubuntu-2004:202101-01
-    resource_class: arm.medium
+      image: ubuntu-2004:2022.04.1
+    resource_class: arm.large
     environment:
-      ENV_FILE: ci/deps/circle-38-arm64.yaml
+      ENV_FILE: ci/deps/circle-310-arm64.yaml
       PYTEST_WORKERS: auto
       PATTERN: "not single_cpu and not slow and not network and not clipboard and not arm_slow and not db"
       PYTEST_TARGET: "pandas"
       PANDAS_CI: "1"
     steps:
       - checkout
       - run: .circleci/setup_env.sh
-      - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh
+      - run: >
+          PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH
+          LD_PRELOAD=$HOME/miniconda3/envs/pandas-dev/lib/libgomp.so.1:$LD_PRELOAD
+          ci/run_tests.sh
+  linux-musl:
+    docker:
+      - image: quay.io/pypa/musllinux_1_1_aarch64
+    resource_class: arm.large
+    steps:
+      # Install pkgs first to have git in the image
+      # (needed for checkout)
+      - run: |
+          apk update
+          apk add git
+          apk add musl-locales
+      - checkout
+      - run: |
+          /opt/python/cp311-cp311/bin/python -m venv ~/virtualenvs/pandas-dev
+          . ~/virtualenvs/pandas-dev/bin/activate
+          python -m pip install --no-cache-dir -U pip wheel setuptools meson-python==0.13.1 meson[ninja]==1.2.1
+          python -m pip install --no-cache-dir versioneer[toml] cython numpy python-dateutil pytz pytest>=7.3.2 pytest-xdist>=2.2.0 hypothesis>=6.46.1
+          python -m pip install --no-cache-dir --no-build-isolation -e . --config-settings=setup-args="--werror"
+          python -m pip list --no-cache-dir
+      - run: |
+          . ~/virtualenvs/pandas-dev/bin/activate
+          export PANDAS_CI=1
+          python -m pytest -m 'not slow and not network and not clipboard and not single_cpu' pandas --junitxml=test-data.xml
+  build-aarch64:
+    parameters:
+      cibw-build:
+        type: string
+    machine:
+      image: ubuntu-2004:2022.04.1
+    resource_class: arm.large
+    environment:
+      TRIGGER_SOURCE: << pipeline.trigger_source >>
+    steps:
+      - checkout
+      - run:
+          name: Check if build is necessary
+          command: |
+            # Check if tag is defined or TRIGGER_SOURCE is scheduled
+            if [[ -n "$CIRCLE_TAG" ]]; then
+                echo 'export IS_PUSH="true"' >> "$BASH_ENV"
+            elif [[ $TRIGGER_SOURCE == "scheduled_pipeline" ]]; then
+                echo 'export IS_SCHEDULE_DISPATCH="true"' >> "$BASH_ENV"
+            # Look for the build label/[wheel build] in commit
+            # grep takes a regex, so need to escape brackets
+            elif (git log --format=oneline -n 1 $CIRCLE_SHA1) | grep -q '\[wheel build\]'; then
+                : # Do nothing
+            elif ! (curl https://api.github.com/repos/pandas-dev/pandas/issues/$CIRCLE_PR_NUMBER | jq '.labels' | grep -q 'Build'); then
+                circleci-agent step halt
+            fi
+      - run:
+          name: Build aarch64 wheels
+          no_output_timeout: 30m # Sometimes the tests won't generate any output, make sure the job doesn't get killed by that
+          command: |
+            pip3 install cibuildwheel==2.15.0
+            # When this is a nightly wheel build, allow picking up NumPy 2.0 dev wheels:
+            if [[ "$IS_SCHEDULE_DISPATCH" == "true" || "$IS_PUSH" != 'true' ]]; then
+                export CIBW_ENVIRONMENT="PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
+            fi
+            cibuildwheel --prerelease-pythons --output-dir wheelhouse
+
+          environment:
+            CIBW_BUILD: << parameters.cibw-build >>
+
+      - run:
+          name: Install Anaconda Client & Upload Wheels
+          command: |
+            echo "Install Mambaforge"
+            MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/23.1.0-0/Mambaforge-23.1.0-0-Linux-aarch64.sh"
+            echo "Downloading $MAMBA_URL"
+            wget -q $MAMBA_URL -O minimamba.sh
+            chmod +x minimamba.sh
+
+            MAMBA_DIR="$HOME/miniconda3"
+            rm -rf $MAMBA_DIR
+            ./minimamba.sh -b -p $MAMBA_DIR
+
+            export PATH=$MAMBA_DIR/bin:$PATH
+
+            mamba install -y -c conda-forge anaconda-client
+
+            source ci/upload_wheels.sh
+            set_upload_vars
+            upload_wheels
+      - store_artifacts:
+          path: wheelhouse/
 
 workflows:
   test:
+    # Don't run trigger this one when scheduled pipeline runs
+    when:
+      not:
+        equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
     jobs:
       - test-arm
+  test-musl:
+    # Don't run trigger this one when scheduled pipeline runs
+    when:
+      not:
+        equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
+    jobs:
+      - linux-musl
+  build-wheels:
+    jobs:
+      - build-aarch64:
+          filters:
+            tags:
+              only: /^v.*/
+          matrix:
+            parameters:
+              cibw-build: ["cp39-manylinux_aarch64",
+                           "cp310-manylinux_aarch64",
+                           "cp311-manylinux_aarch64",
+                           "cp312-manylinux_aarch64",
+                           "cp39-musllinux_aarch64",
+                           "cp310-musllinux_aarch64",
+                           "cp311-musllinux_aarch64",
+                           "cp312-musllinux_aarch64",]

.circleci/setup_env.sh

@@ -1,47 +1,16 @@
 #!/bin/bash -e
 
-# edit the locale file if needed
-if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
-    echo "Adding locale to the first line of pandas/__init__.py"
-    rm -f pandas/__init__.pyc
-    SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n"
-    sed -i "$SEDC" pandas/__init__.py
-
-    echo "[head -4 pandas/__init__.py]"
-    head -4 pandas/__init__.py
-    echo
-fi
+echo "Install Mambaforge"
+MAMBA_URL="https://github.com/conda-forge/miniforge/releases/download/23.1.0-0/Mambaforge-23.1.0-0-Linux-aarch64.sh"
+echo "Downloading $MAMBA_URL"
+wget -q $MAMBA_URL -O minimamba.sh
+chmod +x minimamba.sh
 
+MAMBA_DIR="$HOME/miniconda3"
+rm -rf $MAMBA_DIR
+./minimamba.sh -b -p $MAMBA_DIR
 
-MINICONDA_DIR=/usr/local/miniconda
-if [ -e $MINICONDA_DIR ] && [ "$BITS32" != yes ]; then
-    echo "Found Miniconda installation at $MINICONDA_DIR"
-else
-    echo "Install Miniconda"
-    DEFAULT_CONDA_URL="https://repo.continuum.io/miniconda/Miniconda3-latest"
-    if [[ "$(uname -m)" == 'aarch64' ]]; then
-        CONDA_URL="https://github.com/conda-forge/miniforge/releases/download/4.10.1-4/Miniforge3-4.10.1-4-Linux-aarch64.sh"
-    elif [[ "$(uname)" == 'Linux' ]]; then
-        if [[ "$BITS32" == "yes" ]]; then
-            CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86.sh"
-        else
-            CONDA_URL="$DEFAULT_CONDA_URL-Linux-x86_64.sh"
-        fi
-    elif [[ "$(uname)" == 'Darwin' ]]; then
-        CONDA_URL="$DEFAULT_CONDA_URL-MacOSX-x86_64.sh"
-    else
-      echo "OS $(uname) not supported"
-      exit 1
-    fi
-    echo "Downloading $CONDA_URL"
-    wget -q $CONDA_URL -O miniconda.sh
-    chmod +x miniconda.sh
-
-    MINICONDA_DIR="$HOME/miniconda3"
-    rm -rf $MINICONDA_DIR
-    ./miniconda.sh -b -p $MINICONDA_DIR
-fi
-export PATH=$MINICONDA_DIR/bin:$PATH
+export PATH=$MAMBA_DIR/bin:$PATH
 
 echo
 echo "which conda"
@@ -51,7 +20,7 @@ echo
 echo "update conda"
 conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
-conda install -y -c conda-forge -n base 'mamba>=0.21.2' pip setuptools
+mamba install -y -c conda-forge -n base pip setuptools
 
 echo "conda info -a"
 conda info -a
@@ -70,11 +39,6 @@ time mamba env update -n pandas-dev --file="${ENV_FILE}"
 echo "conda list -n pandas-dev"
 conda list -n pandas-dev
 
-if [[ "$BITS32" == "yes" ]]; then
-    # activate 32-bit compiler
-    export CONDA_BUILD=1
-fi
-
 echo "activate pandas-dev"
 source activate pandas-dev
 
@@ -84,23 +48,13 @@ source activate pandas-dev
 # downstream CI jobs that may also build pandas from source.
 export PANDAS_CI=1
 
-if pip list | grep -q ^pandas; then
+if pip show pandas 1>/dev/null; then
     echo
     echo "remove any installed pandas package w/o removing anything else"
-    pip uninstall -y pandas || true
+    pip uninstall -y pandas
 fi
 
-if [ "$(conda list -f qt --json)" != [] ]; then
-    echo
-    echo "remove qt"
-    echo "causes problems with the clipboard, we use xsel for that"
-    conda remove qt -y --force || true
-fi
-
-echo "Build extensions"
-python setup.py build_ext -q -j3
-
 echo "Install pandas"
-python -m pip install --no-build-isolation --no-use-pep517 -e .
+python -m pip install --no-build-isolation -ve . --config-settings=setup-args="--werror"
 
 echo "done"

.devcontainer.json

@@ -9,8 +9,7 @@
 	// You can edit these settings after create using File > Preferences > Settings > Remote.
 	"settings": {
 		"terminal.integrated.shell.linux": "/bin/bash",
-		"python.condaPath": "/opt/conda/bin/conda",
-		"python.pythonPath": "/opt/conda/bin/python",
+		"python.pythonPath": "/usr/local/bin/python",
 		"python.formatting.provider": "black",
 		"python.linting.enabled": true,
 		"python.linting.flake8Enabled": true,

.gitattributes

@@ -14,3 +14,71 @@
 *.xls binary
 *.xlsx binary
 pandas/_version.py export-subst
+
+
+*.bz2 export-ignore
+*.csv export-ignore
+*.data export-ignore
+*.dta export-ignore
+*.feather export-ignore
+*.tar export-ignore
+*.gz export-ignore
+*.h5 export-ignore
+*.html export-ignore
+*.json export-ignore
+*.jsonl export-ignore
+*.kml export-ignore
+*.msgpack export-ignore
+*.pdf export-ignore
+*.parquet export-ignore
+*.pickle export-ignore
+*.pkl export-ignore
+*.png export-ignore
+*.pptx export-ignore
+*.ods export-ignore
+*.odt export-ignore
+*.orc export-ignore
+*.sas7bdat export-ignore
+*.sav export-ignore
+*.so export-ignore
+*.txt export-ignore
+*.xls export-ignore
+*.xlsb export-ignore
+*.xlsm export-ignore
+*.xlsx export-ignore
+*.xpt export-ignore
+*.cpt export-ignore
+*.xml export-ignore
+*.xsl export-ignore
+*.xz export-ignore
+*.zip export-ignore
+*.zst export-ignore
+*~ export-ignore
+.DS_Store export-ignore
+.git* export-ignore
+
+*.py[ocd] export-ignore
+*.pxi export-ignore
+
+# Ignoring stuff from the top level
+.circleci export-ignore
+.github export-ignore
+asv_bench export-ignore
+ci export-ignore
+doc export-ignore
+gitpod export-ignore
+MANIFEST.in export-ignore
+scripts export-ignore
+typings export-ignore
+web export-ignore
+CITATION.cff export-ignore
+codecov.yml export-ignore
+Dockerfile export-ignore
+environment.yml export-ignore
+setup.py export-ignore
+
+
+# GH 39321
+# csv_dir_path fixture checks the existence of the directory
+# exclude the whole directory to avoid running related tests in sdist
+pandas/tests/io/parser/data export-ignore

.github/CODEOWNERS

@@ -0,0 +1,23 @@
+# github
+.github/                          @mroeschke
+
+# ci
+ci/                               @mroeschke
+
+# web
+web/                              @datapythonista
+
+# docs
+doc/cheatsheet                    @Dr-Irv
+doc/source/development            @noatamir
+
+# pandas
+pandas/_libs/                     @WillAyd
+pandas/_libs/tslibs/*             @MarcoGorelli
+pandas/_typing.py                 @Dr-Irv
+pandas/core/groupby/*             @rhshadrach
+pandas/core/tools/datetimes.py    @MarcoGorelli
+pandas/io/excel/*                 @rhshadrach
+pandas/io/formats/style.py        @attack68
+pandas/io/formats/style_render.py @attack68
+pandas/io/formats/templates       @attack68

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -17,7 +17,9 @@ body:
             [latest version](https://pandas.pydata.org/docs/whatsnew/index.html) of pandas.
           required: true
         - label: >
-            I have confirmed this bug exists on the main branch of pandas.
+            I have confirmed this bug exists on the
+            [main branch](https://pandas.pydata.org/docs/dev/getting_started/install.html#installing-the-development-version-of-pandas)
+            of pandas.
   - type: textarea
     id: example
     attributes: