Merge pull request #262 from FluxML/bc/testitems

Switch to (Re)TestItems
FluxML · Dec 23, 2023 · b2ec1d6 · b2ec1d6
2 parents 3bd48ad + 929aca0
commit b2ec1d6
Show file tree

Hide file tree

Showing 11 changed files with 582 additions and 486 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -12,8 +12,17 @@ steps:
     agents:
       queue: "juliagpu"
       cuda: "*"
+    commands: |
+      julia --project=test -e '
+          if VERSION < v"1.7"
+            using Pkg
+            Pkg.rm("ReTestItems") # not compatible with 1.6
+          end'
     env:
-      GROUP: "CUDA"
+      GROUP: "All"
+      TEST_FAST: true
+      RETESTITEMS_NWORKERS: 2
+      # JULIA_CUDA_HARD_MEMORY_LIMIT: "50%"
     if: build.message !~ /\[skip tests\]/
     timeout_in_minutes: 180
     matrix:

diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -24,11 +24,15 @@ jobs:
   test:
     name: ${{ matrix.suite }} - Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
+    env:
+      GROUP: ${{ matrix.suite }}
+      TEST_FAST: ${{ matrix.version != '1' || matrix.os != 'ubuntu-latest' }}
     strategy:
       fail-fast: false
       matrix:
         version:
-          - '1' # Replace this with the minimum Julia version that your package supports.
+          - '1.6' # Replace this with the minimum Julia version that your package supports.
+          - '1'
           - 'nightly'
         os:
           - ubuntu-latest
@@ -37,18 +41,19 @@ jobs:
         arch:
           - x64
         suite:
-          - '["AlexNet", "VGG"]'
-          - '["GoogLeNet", "SqueezeNet", "MobileNets"]'
-          - '"EfficientNet"'
-          - 'r"/*/ResNet*"'
-          - 'r"/*/SEResNet*"'
-          - '[r"Res2Net", r"Res2NeXt"]'
-          - '"Inception"'
-          - '"DenseNet"'
-          - '"UNet"'
-          - '["ConvNeXt", "ConvMixer"]'
-          - 'r"Mixers"'
-          - 'r"ViTs"'
+          - 'AlexNet|VGG'
+          - 'GoogLeNet|SqueezeNet|MobileNet|MNASNet'
+          - 'EfficientNet'
+          - '^ResNet|WideResNet'
+          - '^ResNeXt' # split off from ResNet to reduce overall runtime
+          - 'SEResNet|SEResNeXt'
+          - 'Res2Net|Res2NeXt'
+          - 'Inception'
+          - 'DenseNet'
+          - 'UNet'
+          - 'ConvNeXt|ConvMixer'
+          - 'MLP-Mixer|ResMLP|gMLP'
+          - 'ViT'
     steps:
       - uses: actions/checkout@v3
       - uses: julia-actions/setup-julia@v1
@@ -57,28 +62,24 @@ jobs:
           arch: ${{ matrix.arch }}
       - uses: julia-actions/cache@v1
       - uses: julia-actions/julia-buildpkg@v1
-      - name: "Setup environment"
-        run: |
-          julia --project=./test -e 'using Pkg; Pkg.develop(path = ".")'
-      - name: "Run tests + coverage"
-        if: matrix.version == '1' && matrix.os == 'ubuntu-latest'
-        run: |
-          julia --code-coverage=user --color=yes --depwarn=yes --project=./test -e 'include("test/retest.jl"); retest(${{ matrix.suite }})'
-        shell: bash
-      - name: "Run tests only"
-        if: ${{ !(matrix.version == '1' && matrix.os == 'ubuntu-latest') }}
+
+      - name: Setup test env for 1.6
+        if: ${{ matrix.version == '1.6' }}
         run: |
-          julia --color=yes --depwarn=yes --project=./test -e 'include("test/retest.jl"); retest(${{ matrix.suite }})'
-        continue-on-error: ${{ matrix.version == 'nightly' }}
-        shell: bash
+          julia --color=yes --depwarn=yes --project=./test -e 'using Pkg; Pkg.rm("ReTestItems")'
+      - name: Run tests
+        uses: julia-actions/julia-runtest@v1
+        continue-on-error: ${{ !(matrix.version == '1' && matrix.os == 'ubuntu-latest') && matrix.version == 'nightly' }}
+        with:
+          coverage: ${{ matrix.version == '1' && matrix.os == 'ubuntu-latest' }}
       - uses: actions/upload-artifact@v3
         with:
           name: coverage-${{ hashFiles('**/*.cov') }}
           path: '**/*.cov'
         if: matrix.version == '1' && matrix.os == 'ubuntu-latest'
 
   coverage:
-    name: "Coverage"
+    name: Coverage
     runs-on: ubuntu-latest
     needs: test
     steps:
@@ -94,7 +95,7 @@ jobs:
           cp -r coverage-*/* .
           rm -rf coverage-*
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v2
+      - uses: codecov/codecov-action@v3
         with:
           file: lcov.info
 

diff --git a/test/Project.toml b/test/Project.toml
@@ -2,5 +2,18 @@
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0"
-ReTest = "e0db7c4e-2690-44b9-bad6-7687da720f89"
+ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+TestImages = "5e47fb64-e119-507b-a336-dd2b206d9990"
+TestItemRunner = "f8b46487-2199-4994-9208-9a1283c18c0a"
+cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd"
+
+[compat]
+CUDA = "4, 5"
+Flux = "0.13.16, 0.14"
+Images = "0.26"
+ReTestItems = "1"
+Test = "1"
+TestImages = "1.8"
+TestItemRunner = "0.2"
+cuDNN = "1"