[CI] Regression test jobs now include the C++ standard version and (if applicable) the standard library to link with

.github/workflows/regression-tests.yml

@@ -11,7 +11,7 @@ on:
 
 jobs:
   regression-tests:
-    name: Run on ${{ matrix.os }} using ${{ matrix.compiler }}
+    name: ${{ matrix.shortosname }} | ${{ matrix.compiler }} | ${{ matrix.cxx_std }} | ${{ matrix.stdlib }} | ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     env:
       CXX: ${{ matrix.compiler }}
@@ -20,22 +20,61 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-24.04]
-        compiler: [g++-13, g++-14, clang++-18]
+        shortosname: [ubu-24]
+        compiler: [g++-14, g++-13]
+        cxx_std: [c++2b]
+        stdlib: [libstdc++]
         include:
+          - os: ubuntu-20.04
+            shortosname: ubu-20
+            compiler: g++-10
+            cxx_std: c++20
+            stdlib: libstdc++
+          - os: ubuntu-24.04
+            shortosname: ubu-24
+            compiler: clang++-18
+            cxx_std: c++20
+            stdlib: libstdc++
+          - os: ubuntu-22.04
+            shortosname: ubu-22
+            compiler: clang++-15
+            cxx_std: c++20
+            stdlib: libstdc++
           - os: ubuntu-22.04
+            shortosname: ubu-22
             compiler: clang++-15
+            cxx_std: c++20
+            stdlib: libc++-15-dev
           - os: ubuntu-20.04
+            shortosname: ubu-20
             compiler: clang++-12
-          - os: ubuntu-20.04
-            compiler: g++-10
+            cxx_std: c++20
+            stdlib: libstdc++
           - os: macos-14
+            shortosname: mac-14
             compiler: clang++
+            cxx_std: c++2b
+            stdlib: default
           - os: macos-13
+            shortosname: mac-13
             compiler: clang++
+            cxx_std: c++2b
+            stdlib: default
           - os: macos-13
+            shortosname: mac-13
             compiler: clang++-15
-          - os: windows-latest
+            cxx_std: c++2b
+            stdlib: default
+          - os: windows-2022
+            shortosname: win-22
+            compiler: cl.exe
+            cxx_std: c++latest
+            stdlib: default
+          - os: windows-2022
+            shortosname: win-22
             compiler: cl.exe
+            cxx_std: c++20
+            stdlib: default
     steps:
     - name: Checkout repo
       uses: actions/checkout@v4
@@ -50,21 +89,21 @@ jobs:
       if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
       run: |
         cd regression-tests
-        bash run-tests.sh -c ${{ matrix.compiler }} -l ${{ matrix.os }}
+        bash run-tests.sh -c ${{ matrix.compiler }} -s ${{ matrix.cxx_std }} -d ${{ matrix.stdlib }} -l ${{ matrix.os }}
 
     - name: Run regression tests - Windows version
       if: matrix.os == 'windows-latest'
       run: |
         "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
         git config --local core.autocrlf false && ^
         cd regression-tests && ^
-        bash run-tests.sh -c ${{ matrix.compiler }} -l ${{ matrix.os }}
+        bash run-tests.sh -c ${{ matrix.compiler }} -s ${{ matrix.cxx_std }} -d ${{ matrix.stdlib }} -l ${{ matrix.os }}
       shell: cmd
 
     - name: Upload patch
       if: ${{ !cancelled() }}
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ matrix.os }}-${{ matrix.compiler }}.patch
-        path: regression-tests/${{ matrix.os }}-${{ matrix.compiler }}.patch
+        name: ${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.cxx_std }}-${{ matrix.stdlib }}.patch
+        path: regression-tests/${{ matrix.os }}-${{ matrix.compiler }}-${{ matrix.cxx_std }}-${{ matrix.stdlib }}.patch
         if-no-files-found: ignore

regression-tests/run-tests.sh

@@ -4,6 +4,8 @@
 usage() {
     echo "Usage: $0 -c <compiler> [-l <run label>] [-t <tests to run>]"
     echo "    -c <compiler>     The compiler to use for the test"
+    echo "    -s <cxx_std>      The C++ standard to compile with (e.g. 'c++20', 'c++2b', 'c++latest' depending on the compiler)"
+    echo "    -d <stdlib>       Clang-only: the C++ Standard Library to link with ('libstdc++', 'libc++', or 'default' for platform default)"
     echo "    -l <run label>    The label to use in output patch file name"
     echo "    -t <tests to run> Runs only the provided, comma-separated tests (filenames including .cpp2)"
     echo "                      If the argument is not used all tests are run"
@@ -46,7 +48,7 @@ check_file () {
     git ls-files --error-unmatch "$file" > /dev/null 2>&1
     untracked=$?
 
-    patch_file="$label$cxx_compiler.patch"
+    patch_file="${label}-${cxx_compiler}-${cxx_std}-${cxx_stdlib}.patch"
 
     if [[ $untracked -eq 1 ]]; then
         # Add the file to the index to be able to diff it...
@@ -67,14 +69,20 @@ check_file () {
     fi
 }
 
-optstring="c:l:t:"
+optstring="c:s:d:l:t:"
 while getopts ${optstring} arg; do
   case "${arg}" in
     c)
         cxx_compiler="${OPTARG}"
         ;;
+    s)
+        cxx_std="${OPTARG}"
+        ;;
+    d)
+        cxx_stdlib="${OPTARG}"
+        ;;
     l)
-        label="${OPTARG}-"
+        label="${OPTARG}"
         ;;
     t)
         # Replace commas with spaces
@@ -125,8 +133,8 @@ expected_results_dir="test-results"
 ################
 # Get the directory with the exec outputs and compilation command
 if [[ "$cxx_compiler" == *"cl.exe"* ]]; then
-    compiler_cmd='cl.exe -nologo -std:c++latest -MD -EHsc -I ..\..\..\include -Fe:'
-    exec_out_dir="$expected_results_dir/msvc-2022"
+    compiler_cmd="cl.exe -nologo -std:${cxx_std} -MD -EHsc -I ..\..\..\include -Fe:"
+    exec_out_dir="$expected_results_dir/msvc-2022-${cxx_std}"
     compiler_version=$(cl.exe)
 else
     # Verify the compiler command
@@ -136,7 +144,6 @@ else
         exit 2
     fi
 
-    cpp_std=c++2b
     compiler_version=$("$cxx_compiler" --version)
 
     if [[ "$compiler_version" == *"Apple clang version 14.0"* ||
@@ -148,16 +155,10 @@ else
         exec_out_dir="$expected_results_dir/clang-12"
     elif [[ "$compiler_version" == *"clang version 15.0"* ]]; then 
         exec_out_dir="$expected_results_dir/clang-15"
-        # c++2b causes starge issues on GitHub ubuntu runners
-        cpp_std="c++20"
     elif [[ "$compiler_version" == *"clang version 18.1"* ]]; then 
         exec_out_dir="$expected_results_dir/clang-18"
-        # c++2b causes starge issues on GitHub ubuntu runners
-        cpp_std="c++20"
     elif [[ "$compiler_version" == *"g++-10"* ]]; then
         exec_out_dir="$expected_results_dir/gcc-10"
-        # GCC 10 does not support c++2b
-        cpp_std=c++20
     elif [[ "$compiler_version" == *"g++-12"* ||
             "$compiler_version" == *"g++-13"*
          ]]; then
@@ -169,15 +170,44 @@ else
         exit 2
     fi
 
-    compiler_cmd="$cxx_compiler -I../../../include -std=$cpp_std -pthread -o "
+    # Append the C++ standard (e.g. 'c++20') to the expected output dir name
+    exec_out_dir="${exec_out_dir}-${cxx_std}"
+
+    # Clang can link with either libstdc++ or libc++
+    # By default clang on ubuntu links with libstdc++ and on macOS links with libc++.
+    if [[ "$compiler_version" == *"clang"* ]]; then 
+        if [[ "$cxx_stdlib" == "default" || "$cxx_stdlib" == "" ]]; then
+            cxx_stdlib_link_arg=""  # Use compiler/platform default
+        elif [[ "$cxx_stdlib" == "libstdc++" ]]; then
+            cxx_stdlib_link_arg="-stdlib=libstdc++"
+        elif [[ "$cxx_stdlib" == *"libc++"* ]]; then
+
+            # Need to install the correct libc++ packages, e.g. `libc++-15-dev` and `libc++abi-15-dev` for clang 15.
+            # Our `cxx_stdlib` variable contains the `libc++-XX-dev` package name so we need to create the abi version.
+            cxx_stdlib_abi_package="${cxx_stdlib/libc++/libc++abi}"
+            printf "Installing packages: $cxx_stdlib $cxx_stdlib_abi_package\n\n"
+            sudo apt-get install -y $cxx_stdlib $cxx_stdlib_abi_package
+
+            cxx_stdlib_link_arg="-stdlib=libc++"
+            exec_out_dir="${exec_out_dir}-libcpp"
+        else
+            printf "Unhandled C++ Standard Library option:\n$cxx_stdlib\n\n"
+            exit 2
+        fi
+    else
+        cxx_stdlib_link_arg=""  # Use compiler/platform default
+    fi
+
+    compiler_cmd="$cxx_compiler -I../../../include -std=$cxx_std $cxx_stdlib_link_arg -pthread -o "
+    printf "\ncompiler_cmd: $compiler_cmd\n\n"
 fi
 
 if [[ -d "$exec_out_dir" ]]; then
     printf "Full compiler version for '$cxx_compiler':\n$compiler_version\n\n"
 
     printf "Directory with reference compilation/execution files to use:\n$exec_out_dir\n\n"
 else
-    printf "Directory with reference compilation/execution files not found for compiler: '$cxx_compiler'\n\n"
+    printf "Directory with reference compilation/execution files not found for compiler: '$cxx_compiler' at $exec_out_dir\n\n"
     exit 2
 fi
 
@@ -197,8 +227,8 @@ regression_test_link_obj=""
 if [[ "$cxx_compiler" == *"cl.exe"* ]]; then
     echo "Building std and std.compat modules"
     (cd $exec_out_dir; \
-     cl.exe -nologo -std:c++latest -MD -EHsc -c "${VCToolsInstallDir}/modules/std.ixx";
-     cl.exe -nologo -std:c++latest -MD -EHsc -c "${VCToolsInstallDir}/modules/std.compat.ixx")
+     cl.exe -nologo -std:${cxx_std} -MD -EHsc -c "${VCToolsInstallDir}/modules/std.ixx";
+     cl.exe -nologo -std:${cxx_std} -MD -EHsc -c "${VCToolsInstallDir}/modules/std.compat.ixx")
     regression_test_link_obj="std.obj std.compat.obj"
 fi