FIX-modin-project#1867: add test-all, test-windows, test-pyarrow with…

… conda usage Signed-off-by: Anatoly Myachev <anatoly.myachev@intel.com>
anmyachev · Jul 31, 2020 · 52e4ba5 · 52e4ba5
1 parent 8d7ce4a
commit 52e4ba5
Showing 1 changed file with 190 additions and 1 deletion.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -19,6 +19,7 @@ jobs:
       - run: git remote add upstream https://github.com/modin-project/modin.git
       - run: git fetch upstream
       - run: npx commitlint --from upstream/master --to HEAD --verbose
+
   lint-black:
     name: lint (black)
     runs-on: ubuntu-latest
@@ -32,6 +33,7 @@ jobs:
           architecture: "x64"
       - run: pip install black
       - run: black --check --diff modin/
+
   lint-flake8:
     name: lint (flake8)
     runs-on: ubuntu-latest
@@ -45,6 +47,7 @@ jobs:
           architecture: "x64"
       - run: pip install flake8 flake8-print
       - run: flake8 --enable=T modin
+
   test-api:
     runs-on: ubuntu-latest
     name: test api
@@ -64,6 +67,192 @@ jobs:
           conda info
           conda list
       - run: sudo apt update && sudo apt install -y libhdf5-dev
-      - name: api tests
+      - name: Api tests
         shell: bash -l {0}
         run: python -m pytest modin/pandas/test/test_api.py
+
+  test-headers:
+    runs-on: ubuntu-latest
+    name: test-headers
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: goanpeca/setup-miniconda@v1.6.0
+        with:
+          activate-environment: modin
+          environment-file: requirements.yml
+          python-version: "3.7"
+          auto-activate-base: false
+      - name: Conda environment
+        shell: bash -l {0}
+        run: |
+          conda info
+          conda list
+      - name: Headers tests
+        shell: bash -l {0}
+        run: python -m pytest modin/test/test_headers.py
+
+  test-internals:
+    runs-on: ubuntu-latest
+    name: test-internals
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: goanpeca/setup-miniconda@v1.6.0
+        with:
+          activate-environment: modin
+          environment-file: requirements.yml
+          python-version: "3.6"
+          auto-activate-base: false
+      - name: Conda environment
+        shell: bash -l {0}
+        run: |
+          conda info
+          conda list
+      - name: Internals tests
+        shell: bash -l {0}
+        run: python -m pytest modin/test/test_publisher.py modin/data_management/test/test_dispatcher.py
+
+  test-all:
+    needs: [lint-commit, lint-flake8, lint-black, test-api, test-headers]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7"]
+        engine: ["python", "ray", "dask"]
+        part: ["Reduction_A", "Reduction_B", "Binary", "MapMetadata", "UDF", "Default", "Window", "Indexing", "Iter", "JoinSort", 3]
+    env:
+      MODIN_ENGINE: ${{matrix.engine}}
+      MODIN_MEMORY: 1000000000
+    name: test (${{matrix.engine}}, part ${{matrix.part}}, python ${{matrix.python-version}})
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: goanpeca/setup-miniconda@v1.6.0
+        with:
+          activate-environment: modin
+          environment-file: requirements.yml
+          python-version: ${{matrix.python-version}}
+          auto-activate-base: false
+      - name: Conda environment
+        shell: bash -l {0}
+        run: |
+          conda info
+          conda list
+      - name: Install HDF5
+        if: matrix.part == 3
+        run: sudo apt update && sudo apt install -y libhdf5-dev
+      - shell: bash -l {0}
+        run: bash run-tests.sh ${{matrix.engine}} -k "TestDataFrame${{matrix.part}}"
+        if: matrix.part != 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_series.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_rolling.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_concat.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_groupby.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_reshape.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_general.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_io.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/experimental/pandas/test/test_io_exp.py
+        if: matrix.part == 3
+      - run: bash <(curl -s https://codecov.io/bash)
+
+  test-windows:
+    needs: [lint-commit, lint-flake8, lint-black, test-api, test-headers]
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7"]
+        engine: ["ray", "dask"]
+        part: ["Reduction_A", "Reduction_B", "Binary", "MapMetadata", "UDF", "Default", "Window", "Indexing", "Iter", "JoinSort", 3]
+    env:
+      MODIN_ENGINE: ${{matrix.engine}}
+      MODIN_MEMORY: 1000000000
+    name: test-windows
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: goanpeca/setup-miniconda@v1.6.0
+        with:
+          activate-environment: modin
+          environment-file: requirements.yml
+          python-version: ${{matrix.python-version}}
+          auto-activate-base: false
+      - name: Conda environment
+        shell: bash -l {0}
+        run: |
+          conda info
+          conda list
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_dataframe.py::TestDataFrame${{matrix.part}}
+        if: matrix.part != 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_series.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_rolling.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_concat.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_groupby.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_reshape.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_general.py
+        if: matrix.part == 3
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_io.py
+        if: matrix.part == 3
+      - run: choco install codecov
+      - run: codecov -f .\coverage.xml -t ${{secrets.CODECOV_TOKEN}}
+
+  test-pyarrow:
+    needs: [lint-commit, lint-flake8, lint-black, test-api, test-headers]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7"]
+    env:
+      MODIN_BACKEND: pyarrow
+      MODIN_EXPERIMENTAL: "True"
+    name: test (pyarrow, python ${{matrix.python-version}})
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+      - uses: goanpeca/setup-miniconda@v1.6.0
+        with:
+          activate-environment: modin
+          environment-file: requirements.yml
+          python-version: ${{matrix.python-version}}
+          auto-activate-base: false
+      - name: Conda environment
+        shell: bash -l {0}
+        run: |
+          conda info
+          conda list
+      - run: sudo apt update && sudo apt install -y libhdf5-dev
+      - shell: bash -l {0}
+        run: python -m pytest modin/pandas/test/test_io.py::test_from_csv