Merge branch 'master' of https://github.com/pandas-dev/pandas into cln-gb

Author: jbrockmendel

.github/workflows/assign.yml

@@ -0,0 +1,15 @@
+name: Assign
+on:
+  issue_comment:
+    types: created
+
+jobs:
+  one:
+    runs-on: ubuntu-latest
+    steps:
+      - name:
+        run: |
+            if [[ "${{ github.event.comment.body }}" == "take" ]]; then
+                echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"
+                curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
+            fi

.travis.yml

@@ -85,15 +85,6 @@ install:
   - ci/submit_cython_cache.sh
   - echo "install done"
 
-
-before_script:
-  # display server (for clipboard functionality) needs to be started here,
-  # does not work if done in install:setup_env.sh (GH-26103)
-  - export DISPLAY=":99.0"
-  - echo "sh -e /etc/init.d/xvfb start"
-  - if [ "$JOB" != "3.8-dev" ]; then sh -e /etc/init.d/xvfb start; fi
-  - sleep 3
-
 script:
   - echo "script start"
   - echo "$JOB"

ci/azure/posix.yml

@@ -73,33 +73,16 @@ jobs:
 
     - task: PublishTestResults@2
       inputs:
-        testResultsFiles: 'test-data-*.xml'
+        testResultsFiles: 'test-data.xml'
         testRunTitle: ${{ format('{0}-$(CONDA_PY)', parameters.name) }}
       displayName: 'Publish test results'
 
     - powershell: |
-        $junitXml = "test-data-single.xml"
-        $(Get-Content $junitXml | Out-String) -match 'failures="(.*?)"'
-        if ($matches[1] -eq 0)
-        {
-          Write-Host "No test failures in test-data-single"
-        }
-        else
-        {
-          # note that this will produce $LASTEXITCODE=1
-          Write-Error "$($matches[1]) tests failed"
-        }
-
-        $junitXmlMulti = "test-data-multiple.xml"
-        $(Get-Content $junitXmlMulti | Out-String) -match 'failures="(.*?)"'
-        if ($matches[1] -eq 0)
-        {
-          Write-Host "No test failures in test-data-multi"
-        }
-        else
-        {
-          # note that this will produce $LASTEXITCODE=1
-          Write-Error "$($matches[1]) tests failed"
+        $(Get-Content "test-data.xml" | Out-String) -match 'failures="(.*?)"'
+        if ($matches[1] -eq 0) {
+          Write-Host "No test failures in test-data"
+        } else {
+          Write-Error "$($matches[1]) tests failed"  # will produce $LASTEXITCODE=1
         }
       displayName: 'Check for test failures'
 

ci/print_skipped.py

@@ -27,14 +27,13 @@ def main(filename):
 if __name__ == "__main__":
     print("SKIPPED TESTS:")
     i = 1
-    for file_type in ("-single", "-multiple", ""):
-        for test_data in main("test-data{}.xml".format(file_type)):
-            if test_data is None:
-                print("-" * 80)
-            else:
-                print(
-                    "#{i} {class_name}.{test_name}: {message}".format(
-                        **dict(test_data, i=i)
-                    )
+    for test_data in main("test-data.xml"):
+        if test_data is None:
+            print("-" * 80)
+        else:
+            print(
+                "#{i} {class_name}.{test_name}: {message}".format(
+                    **dict(test_data, i=i)
                 )
-                i += 1
+            )
+            i += 1

ci/run_tests.sh

@@ -15,37 +15,29 @@ if [ -n "$LOCALE_OVERRIDE" ]; then
         # exit 1
     fi
 fi
+
 if [[ "not network" == *"$PATTERN"* ]]; then
     export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
 fi
 
-
-if [ -n "$PATTERN" ]; then
-    PATTERN=" and $PATTERN"
+if [ "$COVERAGE" ]; then
+    COVERAGE_FNAME="/tmp/test_coverage.xml"
+    COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME"
 fi
 
-for TYPE in single multiple
-do
-    if [ "$COVERAGE" ]; then
-        COVERAGE_FNAME="/tmp/coc-$TYPE.xml"
-        COVERAGE="-s --cov=pandas --cov-report=xml:$COVERAGE_FNAME"
-    fi
+PYTEST_CMD="pytest -m \"$PATTERN\" -n auto --dist=loadfile -s --strict --durations=10 --junitxml=test-data.xml $TEST_ARGS $COVERAGE pandas"
 
-    TYPE_PATTERN=$TYPE
-    NUM_JOBS=1
-    if [[ "$TYPE_PATTERN" == "multiple" ]]; then
-        TYPE_PATTERN="not single"
-        NUM_JOBS=2
-    fi
+# Travis does not have have an X server
+if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+    DISPLAY=DISPLAY=:99.0
+    PYTEST_CMD="xvfb-run -e /dev/stdout $PYTEST_CMD"
+fi
 
-    PYTEST_CMD="pytest -m \"$TYPE_PATTERN$PATTERN\" -n $NUM_JOBS -s --strict --durations=10 --junitxml=test-data-$TYPE.xml $TEST_ARGS $COVERAGE pandas"
-    echo $PYTEST_CMD
-    # if no tests are found (the case of "single and slow"), pytest exits with code 5, and would make the script fail, if not for the below code
-    sh -c "$PYTEST_CMD; ret=\$?; [ \$ret = 5 ] && exit 0 || exit \$ret"
+echo $PYTEST_CMD
+sh -c "$PYTEST_CMD"
 
-    if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then
-        echo "uploading coverage for $TYPE tests"
-        echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
-              bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
-    fi
-done
+if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then
+    echo "uploading coverage"
+    echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
+          bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
+fi

doc/source/development/contributing.rst

@@ -24,6 +24,27 @@ and `good first issue
 where you could start out. Once you've found an interesting issue, you can
 return here to get your development environment setup.
 
+When you start working on an issue, it's a good idea to assign the issue to yourself,
+so nobody else duplicates the work on it. GitHub restricts assigning issues to maintainers
+of the project only. In most projects, and until recently in pandas, contributors added a
+comment letting others know they are working on an issue. While this is ok, you need to
+check each issue individually, and it's not possible to find the unassigned ones.
+
+For this reason, we implemented a workaround consisting of adding a comment with the exact
+text `take`. When you do it, a GitHub action will automatically assign you the issue
+(this will take seconds, and may require refreshint the page to see it).
+By doing this, it's possible to filter the list of issues and find only the unassigned ones.
+
+So, a good way to find an issue to start contributing to pandas is to check the list of
+`unassigned good first issues <https://github.com/pandas-dev/pandas/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22+no%3Aassignee>`_
+and assign yourself one you like by writing a comment with the exact text `take`.
+
+If for whatever reason you are not able to continue working with the issue, please try to
+unassign it, so other people know it's available again. You can check the list of
+assigned issues, since people may not be working in them anymore. If you want to work on one
+that is assigned, feel free to kindly ask the current assignee if you can take it
+(please allow at least a week of inactivity before considering work in the issue discontinued).
+
 Feel free to ask questions on the `mailing list
 <https://groups.google.com/forum/?fromgroups#!forum/pydata>`_ or on `Gitter`_.
 

doc/source/development/index.rst

@@ -13,6 +13,7 @@ Development
     :maxdepth: 2
 
     contributing
+    maintaining
     internals
     extending
     developer

doc/source/development/maintaining.rst

@@ -0,0 +1,193 @@
+.. _maintaining:
+
+******************
+Pandas Maintenance
+******************
+
+This guide is for pandas' maintainers. It may also be interesting to contributors
+looking to understand the pandas development process and what steps are necessary
+to become a maintainer.
+
+The main contributing guide is available at :ref:`contributing`.
+
+Roles
+-----
+
+Pandas uses two levels of permissions: **triage** and **core** team members.
+
+Triage members can label and close issues and pull requests.
+
+Core team members can label and close issues and pull request, and can merge
+pull requests.
+
+GitHub publishes the full `list of permissions`_.
+
+Tasks
+-----
+
+Pandas is largely a volunteer project, so these tasks shouldn't be read as
+"expectations" of triage and maintainers. Rather, they're general descriptions
+of what it means to be a maintainer.
+
+* Triage newly filed issues (see :ref:`maintaining.triage`)
+* Review newly opened pull requests
+* Respond to updates on existing issues and pull requests
+* Drive discussion and decisions on stalled issues and pull requests
+* Provide experience / wisdom on API design questions to ensure consistency and maintainability
+* Project organization (run / attend developer meetings, represent pandas)
+
+http://matthewrocklin.com/blog/2019/05/18/maintainer may be interesting background
+reading.
+
+.. _maintaining.triage:
+
+Issue Triage
+------------
+
+
+Here's a typical workflow for triaging a newly opened issue.
+
+1. **Thank the reporter for opening an issue**
+
+   The issue tracker is many people's first interaction with the pandas project itself,
+   beyond just using the library. As such, we want it to be a welcoming, pleasant
+   experience.
+
+2. **Is the necessary information provided?**
+
+   Ideally reporters would fill out the issue template, but many don't.
+   If crucial information (like the version of pandas they used), is missing
+   feel free to ask for that and label the issue with "Needs info". The
+   report should follow the guidelines in :ref:`contributing.bug_reports`.
+   You may want to link to that if they didn't follow the template.
+
+   Make sure that the title accurately reflects the issue. Edit it yourself
+   if it's not clear.
+
+3. **Is this a duplicate issue?**
+
+   We have many open issues. If a new issue is clearly a duplicate, label the
+   new issue as "Duplicate" assign the milestone "No Action", and close the issue
+   with a link to the original issue. Make sure to still thank the reporter, and
+   encourage them to chime in on the original issue, and perhaps try to fix it.
+
+   If the new issue provides relevant information, such as a better or slightly
+   different example, add it to the original issue as a comment or an edit to
+   the original post.
+
+4. **Is the issue minimal and reproducible**?
+
+   For bug reports, we ask that the reporter provide a minimal reproducible
+   example. See http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports
+   for a good explanation. If the example is not reproducible, or if it's
+   *clearly* not minimal, feel free to ask the reporter if they can provide
+   and example or simplify the provided one. Do acknowledge that writing
+   minimal reproducible examples is hard work. If the reporter is struggling,
+   you can try to write one yourself and we'll edit the original post to include it.
+
+   If a reproducible example can't be provided, add the "Needs info" label.
+
+   If a reproducible example is provided, but you see a simplification,
+   edit the original post with your simpler reproducible example.
+
+5. **Is this a clearly defined feature request?**
+
+   Generally, pandas prefers to discuss and design new features in issues, before
+   a pull request is made. Encourage the submitter to include a proposed API
+   for the new feature. Having them write a full docstring is a good way to
+   pin down specifics.
+
+   We'll need a discussion from several pandas maintainers before deciding whether
+   the proposal is in scope for pandas.
+
+6. **Is this a usage question?**
+
+   We prefer that usage questions are asked on StackOverflow with the pandas
+   tag. https://stackoverflow.com/questions/tagged/pandas
+
+   If it's easy to answer, feel free to link to the relevant documentation section,
+   let them know that in the future this kind of question should be on
+   StackOverflow, and close the issue.
+
+7. **What labels and milestones should I add?**
+
+   Apply the relevant labels. This is a bit of an art, and comes with experience.
+   Look at similar issues to get a feel for how things are labeled.
+
+   If the issue is clearly defined and the fix seems relatively straightforward,
+   label the issue as "Good first issue".
+
+   Typically, new issues will be assigned the "Contributions welcome" milestone,
+   unless it's know that this issue should be addressed in a specific release (say
+   because it's a large regression).
+
+.. _maintaining.closing:
+
+Closing Issues
+--------------
+
+Be delicate here: many people interpret closing an issue as us saying that the
+conversation is over. It's typically best to give the reporter some time to
+respond or self-close their issue if it's determined that the behavior is not a bug,
+or the feature is out of scope. Sometimes reporters just go away though, and
+we'll close the issue after the conversation has died.
+
+Reviewing Pull Requests
+-----------------------
+
+Anybody can review a pull request: regular contributors, triagers, or core-team
+members. Here are some guidelines to check.
+
+* Tests should be in a sensible location.
+* New public APIs should be included somewhere in ``doc/source/reference/``.
+* New / changed API should use the ``versionadded`` or ``versionchanged`` directives in the docstring.
+* User-facing changes should have a whatsnew in the appropriate file.
+* Regression tests should reference the original GitHub issue number like ``# GH-1234``.
+
+Cleaning up old Issues
+----------------------
+
+Every open issue in pandas has a cost. Open issues make finding duplicates harder,
+and can make it harder to know what needs to be done in pandas. That said, closing
+issues isn't a goal on its own. Our goal is to make pandas the best it can be,
+and that's best done by ensuring that the quality of our open issues is high.
+
+Occasionally, bugs are fixed but the issue isn't linked to in the Pull Request.
+In these cases, comment that "This has been fixed, but could use a test." and
+label the issue as "Good First Issue" and "Needs Test".
+
+If an older issue doesn't follow our issue template, edit the original post to
+include a minimal example, the actual output, and the expected output. Uniformity
+in issue reports is valuable.
+
+If an older issue lacks a reproducible example, label it as "Needs Info" and
+ask them to provide one (or write one yourself if possible). If one isn't
+provide reasonably soon, close it according to the policies in :ref:`maintaining.closing`.
+
+Cleaning up old Pull Requests
+-----------------------------
+
+Occasionally, contributors are unable to finish off a pull request.
+If some time has passed (two weeks, say) since the last review requesting changes,
+gently ask if they're still interested in working on this. If another two weeks or
+so passes with no response, thank them for their work and close the pull request.
+Comment on the original issue that "There's a stalled PR at #1234 that may be
+helpful.", and perhaps label the issue as "Good first issue" if the PR was relatively
+close to being accepted.
+
+Additionally, core-team members can push to contributors branches. This can be
+helpful for pushing an important PR across the line, or for fixing a small
+merge conflict.
+
+Becoming a pandas maintainer
+----------------------------
+
+The full process is outlined in our `governance documents`_. In summary,
+we're happy to give triage permissions to anyone who shows interest by
+being helpful on the issue tracker.
+
+The current list of core-team members is at
+https://github.com/pandas-dev/pandas-governance/blob/master/people.md
+
+.. _governance documents: https://github.com/pandas-dev/pandas-governance
+.. _list of permissions: https://help.github.com/en/github/setting-up-and-managing-organizations-and-teams/repository-permission-levels-for-an-organization

environment.yml

@@ -21,7 +21,7 @@ dependencies:
   - flake8-comprehensions>=3.1.0  # used by flake8, linting of unnecessary comprehensions
   - flake8-rst>=0.6.0,<=0.7.0  # linting of code blocks in rst files
   - isort  # check that imports are in the right order
-  - mypy=0.720
+  - mypy=0.730
   - pycodestyle  # used by flake8
 
   # documentation
@@ -53,7 +53,7 @@ dependencies:
   - moto  # mock S3
   - pytest>=4.0.2
   - pytest-cov
-  - pytest-xdist
+  - pytest-xdist>=1.21
   - seaborn
   - statsmodels