Merge pull request #2350 from github/angelapwen/add-exclude-pr-check-…

…param

PR check generator: add `excludeOsAndVersionCombination`

.github/workflows/debug-artifacts.yml

@@ -22,20 +22,18 @@ jobs:
       fail-fast: false
       matrix:
         version:
-        # TODO: Once CLI v2.17.4 is available and the platform is switched back to ubuntu,
-        #       stable-20230403, stable-v2.13.5, and stable-v2.14.6 can be added back to this matrix,
-        #       and the VERSIONS variable in the bash script below. 
-        #       Prior to CLI v2.15.1, ARM runners were not supported by the build tracer.
+        - stable-20230403
+        - stable-v2.13.5
+        - stable-v2.14.6
         - stable-v2.15.5
-        - stable-v2.16.6
         - default
         - linked
         - nightly-latest
     name: Upload debug artifacts
     env:
       CODEQL_ACTION_TEST_MODE: true
     timeout-minutes: 45
-    runs-on: macos-latest # TODO: Switch back to ubuntu for `nightly-latest` and `linked` once CLI v2.17.4 is available.
+    runs-on: ubuntu-latest
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -73,7 +71,7 @@ jobs:
       - name: Check expected artifacts exist
         shell: bash
         run: |
-          VERSIONS="stable-v2.15.5 stable-v2.16.6 default linked nightly-latest"
+          VERSIONS="stable-20230403 stable-v2.13.5 stable-v2.14.6 stable-v2.15.5 default linked nightly-latest"
           LANGUAGES="cpp csharp go java javascript python"
           for version in $VERSIONS; do
             pushd "./my-debug-artifacts-${version//./}"

pr-checks/checks/multi-language-autodetect.yml

@@ -1,7 +1,14 @@
 name: "Multi-language repository"
 description: "An end-to-end integration test of a multi-language repository using automatic language detection"
-# TODO: Add ubuntu back for `nightly-latest` and `latest` once CLI v2.17.4 is available.
-operatingSystems: ["macos"] 
+operatingSystems: ["macos", "ubuntu"]
+excludeOsAndVersionCombination: [ 
+  # Known failure for Swift on Linux before CLI v2.17.4.
+  [ "ubuntu", "stable-20230403" ], 
+  [ "ubuntu", "stable-v2.13.5" ], 
+  [ "ubuntu", "stable-v2.14.6" ], 
+  [ "ubuntu", "stable-v2.15.5" ],
+  [ "ubuntu", "stable-v2.16.6" ],
+]
 steps:
   - uses: actions/setup-go@v5
     with:

pr-checks/checks/scaling-reserved-ram.yml

@@ -1,7 +1,14 @@
 name: "Scaling reserved RAM"
 description: "An end-to-end integration test of a multi-language repository with the scaling_reserved_ram feature flag enabled"
-# TODO: Add ubuntu back for `nightly-latest` and `latest` once CLI v2.17.4 is available.
-operatingSystems: ["macos"]
+operatingSystems: ["macos", "ubuntu"]
+excludeOsAndVersionCombination: [ 
+  # Known failure for Swift on Linux before CLI v2.17.4.
+  [ "ubuntu", "stable-20230403" ], 
+  [ "ubuntu", "stable-v2.13.5" ], 
+  [ "ubuntu", "stable-v2.14.6" ], 
+  [ "ubuntu", "stable-v2.15.5" ],
+  [ "ubuntu", "stable-v2.16.6" ],
+]
 env:
   CODEQL_ACTION_SCALING_RESERVED_RAM: true
 steps:

pr-checks/checks/swift-custom-build.yml

@@ -1,8 +1,7 @@
 name: "Swift analysis using a custom build command"
 description: "Tests creation of a Swift database using custom build"
 versions: ["linked", "default", "nightly-latest"]
-# TODO: Add ubuntu back for `nightly-latest` and `latest` once CLI v2.17.4 is available.
-operatingSystems: ["macos"] 
+operatingSystems: ["macos", "ubuntu"]
 env:
   DOTNET_GENERATE_ASPNET_CERTIFICATE: "false"
 steps:

pr-checks/checks/unset-environment.yml

@@ -1,8 +1,15 @@
 name: "Test unsetting environment variables"
 description: "An end-to-end integration test that unsets some environment variables"
-# TODO: Switch back to all versions once CLI v2.17.4 is available and running on ubuntu again.
-versions: ["stable-v2.14.6", "stable-v2.15.5", "stable-v2.16.6", "linked", "default", "nightly-latest"] 
-operatingSystems: ["macos"] # TODO: Switch back to ubuntu for `nightly-latest` and `latest` once CLI v2.17.4 is available.
+operatingSystems: ["ubuntu"]
+excludeOsAndVersionCombination: [ 
+  # Known failure for Swift on Linux before CLI v2.17.4.
+  [ "ubuntu", "stable-20230403" ], 
+  [ "ubuntu", "stable-v2.13.5" ], 
+  [ "ubuntu", "stable-v2.14.6" ], 
+  [ "ubuntu", "stable-v2.15.5" ],
+  [ "ubuntu", "stable-v2.16.6" ],
+]
+
 steps:
   - uses: ./../action/init
     id: init

pr-checks/sync.py

@@ -27,6 +27,12 @@
     "nightly-latest"
 ]
 
+def is_os_and_version_excluded(os, version, exclude_params):
+    for exclude_param in exclude_params:
+        if exclude_param[0] == os and exclude_param[1] == version:
+            return True
+    return False
+
 # When updating the ruamel.yaml version here, update the PR check in
 # `.github/workflows/pr-checks.yml` too.
 header = """# Warning: This file is generated automatically, and should not be modified.
@@ -56,27 +62,32 @@ def writeHeader(checkStream):
 for file in (this_dir / 'checks').glob('*.yml'):
     with open(file, 'r') as checkStream:
         checkSpecification = yaml.load(checkStream)
-
     matrix = []
+    excludedOsesAndVersions = checkSpecification.get('excludeOsAndVersionCombination', [])
     for version in checkSpecification.get('versions', defaultTestVersions):
-        runnerImages = ["ubuntu-latest", "macos-latest", "windows-latest"]
-        if checkSpecification.get('operatingSystems', None):
-            runnerImages = [image for image in runnerImages for operatingSystem in checkSpecification['operatingSystems']
-                            if image.startswith(operatingSystem)]
-
-        for runnerImage in runnerImages:
-            # Prior to CLI v2.15.1, ARM runners were not supported by the build tracer.
-            # "macos-latest" is now an ARM runner, so we run tests on the old CLIs on Intel runners instead.
-            if version in ["stable-20230403", "stable-v2.13.4", "stable-v2.13.5", "stable-v2.14.6"] and runnerImage == "macos-latest":
-                matrix.append({
-                    'os': "macos-12",
-                    'version': version
-                })
-            else:
-                matrix.append({
-                    'os': runnerImage,
-                    'version': version
-                })
+        runnerImages = ["ubuntu-latest", "macos-latest", "windows-latest"]            
+        operatingSystems = checkSpecification.get('operatingSystems', ["ubuntu", "macos", "windows"])
+
+        for operatingSystem in operatingSystems:
+            runnerImagesForOs = [image for image in runnerImages if image.startswith(operatingSystem)]
+
+            for runnerImage in runnerImagesForOs:
+                # Skip appending this combination to the matrix if it is explicitly excluded.
+                if is_os_and_version_excluded(operatingSystem, version, excludedOsesAndVersions):
+                    continue
+
+                # Prior to CLI v2.15.1, ARM runners were not supported by the build tracer.
+                # "macos-latest" is now an ARM runner, so we run tests on the old CLIs on Intel runners instead.
+                if version in ["stable-20230403", "stable-v2.13.4", "stable-v2.13.5", "stable-v2.14.6"] and runnerImage == "macos-latest":
+                    matrix.append({
+                        'os': "macos-12",
+                        'version': version
+                    })
+                else:
+                    matrix.append({
+                        'os': runnerImage,
+                        'version': version
+                    })
 
         useAllPlatformBundle = "false" # Default to false
         if checkSpecification.get('useAllPlatformBundle'):