Merge pull request #1644 from LLNL/long58/ci_expansion

GitLab CI testing expansion

.gitlab-ci.yml

@@ -91,13 +91,12 @@ stages:
 trigger-rajaperf:
   stage: multi-project
   rules:
-    - if: '$CI_COMMIT_BRANCH == "${MP_BRANCH}" || $MULTI_PROJECT == "ON"' #run only if ...
+    - if: $CI_COMMIT_BRANCH == $MP_BRANCH || $MULTI_PROJECT == "ON" #run only if ...
   variables:
     UPDATE_RAJA: ${MP_BRANCH}
   trigger:
     project: radiuss/rajaperf
     branch: develop
-    strategy: depend
 
 include:
   # Sets ID tokens for every job using `default:`

.gitlab/custom-jobs-and-variables.yml

@@ -21,7 +21,7 @@ variables:
 # Project specific variants for ruby
   PROJECT_RUBY_VARIANTS: "~shared +openmp +vectorization +tests"
 # Project specific deps for ruby
-  PROJECT_RUBY_DEPS: ""
+  PROJECT_RUBY_DEPS: "^blt@develop "
 
 # Poodle
 # Arguments for top level allocation
@@ -31,7 +31,7 @@ variables:
 # Project specific variants for poodle
   PROJECT_POODLE_VARIANTS: "~shared +openmp +vectorization +tests"
 # Project specific deps for poodle
-  PROJECT_POODLE_DEPS: ""
+  PROJECT_POODLE_DEPS: "^blt@develop "
 
 # Corona
 # Arguments for top level allocation

.gitlab/jobs/corona.yml

@@ -31,3 +31,9 @@ rocmcc_5_7_0_hip_desul_atomics:
     SPEC: " ~shared +rocm ~openmp +tests +desul amdgpu_target=gfx906 %rocmcc@=5.7.0 ^hip@5.7.0 ^blt@develop"
   extends: .job_on_corona
 
+clang_19_0_0_sycl_gcc_10_3_1_rocmcc_5_7_1_hip:
+  variables:
+    SPEC: " ~shared +sycl ~openmp +tests %clang@=19.0.0 cxxflags==\"-w -fsycl -fsycl-unnamed-lambda -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend --offload-arch=gfx906\" ^blt@develop"
+    MODULE_LIST: "rocm/5.7.1"
+  extends: .job_on_corona
+

.gitlab/jobs/lassen.yml

@@ -66,3 +66,13 @@ gcc_8_3_1_cuda_10_1_243_desul_atomics:
   variables:
     SPEC: " ~shared +openmp +tests +cuda +desul %gcc@=8.3.1 cuda_arch=70 ^cuda@10.1.243+allow-unsupported-compilers ^blt@develop"
   extends: .job_on_lassen
+
+# Warning: Allowed to fail temporarily
+# Deactivated due to issues with OpenMP Target and various tests and compilers.
+clang_16_0_6_ibm_omptarget:
+  variables:
+    SPEC: " ~shared +openmp +omptarget +tests %clang@=16.0.6.ibm.gcc.8.3.1 ^blt@develop"
+    ON_LASSEN: "OFF"
+  extends: .job_on_lassen
+  allow_failure: true
+

.gitlab/jobs/poodle.yml

@@ -15,26 +15,32 @@
 ########################
 # We duplicate the shared jobs description and add necessary changes for this
 # project. We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS}
-# So that the comparison with the original job is easier.
+# when possible so that the comparison with the original job is easier.
 
+# Identical to shared job, but use OpenMP tasks and no vectorization
 clang_14_0_6:
   variables:
-    SPEC: " ~shared +openmp +omptask +tests %clang@=14.0.6 ^blt@develop"
+    SPEC: " ~shared +openmp +omptask +tests %clang@=14.0.6 ${PROJECT_POODLE_DEPS}"
   extends: .job_on_poodle
 
+# Identical to shared job, but use OpenMP tasks and no vectorization
 gcc_10_3_1:
   variables:
-    SPEC: " ~shared +openmp +omptask +tests %gcc@=10.3.1 ^blt@develop"
+    SPEC: " ~shared +openmp +omptask +tests %gcc@=10.3.1 ${PROJECT_POODLE_DEPS}"
   extends: .job_on_poodle
 
+# Identical to shared job, but use OpenMP tasks and no vectorization
+# Deactivated (too long on poodle)
 intel_19_1_2_gcc_10_3_1:
   variables:
-    SPEC: " ~shared +openmp +omptask +tests %intel@=19.1.2.gcc.10.3.1 ^blt@develop"
+    ON_POODLE: "OFF"
+    SPEC: " ~shared +openmp +omptask +tests %intel@=19.1.2.gcc.10.3.1 ${PROJECT_POODLE_DEPS}"
   extends: .job_on_poodle
 
+# Allowed to fail
 intel_2022_1_0:
   variables:
-    SPEC: "${PROJECT_POODLE_VARIANTS} %intel@=2022.1.0 ${PROJECT_POODLE_DEPS} ^blt@develop"
+    SPEC: "${PROJECT_POODLE_VARIANTS} %intel@=2022.1.0 ${PROJECT_POODLE_DEPS}"
   allow_failure: true
   extends: .job_on_poodle
 

.gitlab/jobs/ruby.yml

@@ -15,26 +15,30 @@
 ########################
 # We duplicate the shared jobs description and add necessary changes for this
 # project. We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS}
-# So that the comparison with the original job is easier.
+# when possible so that the comparison with the original job is easier.
 
+# Identical to shared job, but use OpenMP tasks and no vectorization
 clang_14_0_6:
   variables:
-    SPEC: " ~shared +openmp +omptask +tests %clang@=14.0.6 ^blt@develop"
+    SPEC: " ~shared +openmp +omptask +tests %clang@=14.0.6 ${PROJECT_RUBY_DEPS}"
   extends: .job_on_ruby
 
+# Identical to shared job, but use OpenMP tasks and no vectorization
 gcc_10_3_1:
   variables:
-    SPEC: " ~shared +openmp +omptask +tests %gcc@=10.3.1 ^blt@develop"
+    SPEC: " ~shared +openmp +omptask +tests %gcc@=10.3.1 ${PROJECT_RUBY_DEPS}"
   extends: .job_on_ruby
 
+# Identical to shared job, but use OpenMP tasks and no vectorization
 intel_19_1_2_gcc_10_3_1:
   variables:
-    SPEC: " ~shared +openmp +omptask +tests %intel@=19.1.2.gcc.10.3.1 ^blt@develop"
+    SPEC: " ~shared +openmp +omptask +tests %intel@=19.1.2.gcc.10.3.1 ${PROJECT_RUBY_DEPS}"
   extends: .job_on_ruby
 
+# Allowed to fail
 intel_2022_1_0:
   variables:
-    SPEC: "${PROJECT_RUBY_VARIANTS} %intel@=2022.1.0 ${PROJECT_RUBY_DEPS} ^blt@develop"
+    SPEC: "${PROJECT_RUBY_VARIANTS} %intel@=2022.1.0 ${PROJECT_RUBY_DEPS}"
   allow_failure: true
   extends: .job_on_ruby
 

docs/sphinx/dev_guide/ci_tasks.rst

@@ -157,6 +157,114 @@ annotate the job for this. For example:
    describe the change in the ``RAJA/.gitlab/jobs/<MACHINE>.yml`` file where
    the job is overridden.
 
+
+Building the Intel clang + SYCL HIP compiler for use in CI
+----------------------------------------------------------
+
+The SYCL CI tests on corona rely on a custom Intel Clang SYCL compiler that we 
+build ourselves. This compiler lives in the ``/usr/workspace/raja-dev/`` folder so 
+that it can be accessed by the gitlab CI system. Since the intel compiler does
+not do releases in the typical sense (they simply update their repo *every night*), 
+it may become necessary to periodically build a new version of the compiler to 
+ensure that we are using the most up-to-date version available. The steps for 
+building, installing, and running are shown here.
+
+Building the Compiler
+^^^^^^^^^^^^^^^^^^^^^
+
+.. important:: Because intel updates their compiler repo daily, there is a nonzero possibility that the head of the sycl branch will fail to build. 
+  In the event that it does not build, try checking out a different commit. On the intel/llvm GitHub page, one can see which of their 
+  commits builds by checking the status badge next to each commit. Look for a commit that passes. 
+
+
+#. Load the version of GCC that you want to use. In this case, we are using LC's gcc/10.3.1-magic installation::
+
+    module load gcc/10.3.1-magic
+
+#. Load the version of rocm that you want to use. In this case, we are using 5.7.1::
+
+    module load rocm/5.7.1 
+
+#. Clone the "sycl" branch of intel's llvm compiler fork::
+
+    git clone https://github.com/intel/llvm -b sycl
+
+#. cd into that folder:: 
+
+    cd llvm
+
+   In the event that the head of the sycl branch does not build, run ``git checkout <git sha>`` to checkout a version that does build.
+
+#. Build the compiler. 
+
+   Note that in this example, we are using rocm5.7.1, but one can change the version they wish to use simply by changing the paths in the configure step
+
+   a. Configure
+
+     .. code-block:: bash 
+
+        srun -n1 /usr/bin/python3 buildbot/configure.py --hip -o buildrocm5.7.1 \
+        --cmake-gen "Unix Makefiles" \
+        --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_DIR=/opt/rocm-5.7.1 \
+        --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_INCLUDE_DIR=/opt/rocm-5.7.1/include \
+        --cmake-opt=-DSYCL_BUILD_PI_HIP_ROCM_LIB_DIR=/opt/rocm-5.7.1/lib \
+        --cmake-opt=-DSYCL_BUILD_PI_HIP_INCLUDE_DIR=/opt/rocm-5.7.1/include \
+        --cmake-opt=-DSYCL_BUILD_PI_HIP_HSA_INCLUDE_DIR=/opt/rocm-5.7.1/hsa/include/hsa \
+        --cmake-opt=-DSYCL_BUILD_PI_HIP_LIB_DIR=/opt/rocm-5.7.1/lib \
+        --cmake-opt=-DUR_HIP_ROCM_DIR=/opt/rocm-5.7.1 \
+        --cmake-opt=-DUR_HIP_INCLUDE_DIR=/opt/rocm-5.7.1/include \
+        --cmake-opt=-DUR_HIP_HSA_INCLUDE_DIR=/opt/rocm-5.7.1/hsa/include/hsa \
+        --cmake-opt=-DUR_HIP_LIB_DIR=/opt/rocm-5.7.1/lib
+
+   b. Build
+
+     .. code-block:: bash
+
+      srun -n1 /usr/bin/python3 buildbot/compile.py -o buildrocm5.7.1
+
+#. Test the compiler
+
+   Follow the steps in the `Using the compiler`_ section to test this installation
+
+#. Install
+
+  a. The build step will install the compiler to the folder ``buildrocm<version>/install``. Simply copy this folder to the ``/usr/workspace/raja-dev/`` directory using the naming scheme ``clang_sycl_<git sha>_hip_gcc<version>_rocm<version>``
+
+  #. Set the permissions of the folder, and everything in it to 750::
+
+      chmod 750 /usr/workspace/raja-dev/<foldername>/ -R  
+
+  #. Change the group of the folder and everything in it to raja-dev::
+
+      chgrp raja-dev /usr/workspace/raja-dev/<foldername>/ -R  
+
+
+Using the compiler
+^^^^^^^^^^^^^^^^^^
+
+#. Load the version of rocm that you used when building the compiler::
+
+    module load rocm/5.7.1
+
+#. Navigate to the root of your local checkout space of the RAJA repo::
+
+    cd /path/to/raja
+
+#. Run the test config script::
+
+    ./scripts/lc-builds/corona_sycl.sh /usr/workspace/raja-dev/clang_sycl_2f03ef85fee5_hip_gcc10.3.1_rocm5.7.1
+
+   Note that at the time of writing, the newest compiler we had built was at ``clang_sycl_2f03ef85fee5_hip_gcc10.3.1_rocm5.7.1``
+
+#. cd into the auto generated build directory::
+
+    cd {build directory}
+
+#. Run the tests::
+
+    make -j
+
+
 ==============
 Azure CI Tasks
 ==============

scripts/gitlab/build_and_test.sh

@@ -200,7 +200,7 @@ then
     mkdir -p ${build_dir} && cd ${build_dir}
 
     timed_message "Building RAJA"
-    if [[ "${truehostname}" == "corona" || "${truehostname}" == "tioga" ]]
+    if [[ "${truehostname}" == "tioga" ]]
     then
         module unload rocm
     fi

test/functional/kernel/hyperplane/CMakeLists.txt

@@ -13,7 +13,7 @@ set(TESTTYPES 2D 3D)
 foreach( BACKEND ${KERNEL_BACKENDS} )
   foreach( TEST_TYPE ${TESTTYPES} )
     # Removing Sycl backend, implementation of Hyperplane does not exist
-    if( NOT ((BACKEND STREQUAL "Sycl")) )
+    if( NOT ((BACKEND STREQUAL "Sycl")) AND NOT ((BACKEND STREQUAL "OpenMPTarget")) )
       configure_file( test-kernel-hyperplane-${TEST_TYPE}.cpp.in
                       test-kernel-hyperplane-${TEST_TYPE}-${BACKEND}.cpp )
       raja_add_test( NAME test-kernel-hyperplane-${TEST_TYPE}-${BACKEND}

test/functional/kernel/single-loop-tile-icount-tcount/CMakeLists.txt

@@ -17,16 +17,19 @@ set(TILESIZES 8 32)
 # Note: KERNEL_BACKENDS is defined in ../CMakeLists.txt
 #
 foreach( BACKEND ${KERNEL_BACKENDS} )
-  foreach( TESTTYPE ${TESTTYPES} )
-    foreach( TILESIZE ${TILESIZES} )
-      configure_file( test-kernel-single-loop-tile-count.cpp.in
-                      test-kernel-single-loop-${TESTTYPE}-${TILESIZE}-${BACKEND}.cpp )
-      raja_add_test( NAME test-kernel-single-loop-${TESTTYPE}-${TILESIZE}-${BACKEND}
-                     SOURCES ${CMAKE_CURRENT_BINARY_DIR}/test-kernel-single-loop-${TESTTYPE}-${TILESIZE}-${BACKEND}.cpp )
-      target_include_directories(test-kernel-single-loop-${TESTTYPE}-${TILESIZE}-${BACKEND}.exe
-                                 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests)
+  # using omp target crashes the compiler with this one 
+  if( NOT ((BACKEND STREQUAL "OpenMPTarget")) )
+    foreach( TESTTYPE ${TESTTYPES} )
+      foreach( TILESIZE ${TILESIZES} )
+        configure_file( test-kernel-single-loop-tile-count.cpp.in
+                        test-kernel-single-loop-${TESTTYPE}-${TILESIZE}-${BACKEND}.cpp )
+        raja_add_test( NAME test-kernel-single-loop-${TESTTYPE}-${TILESIZE}-${BACKEND}
+                      SOURCES ${CMAKE_CURRENT_BINARY_DIR}/test-kernel-single-loop-${TESTTYPE}-${TILESIZE}-${BACKEND}.cpp )
+        target_include_directories(test-kernel-single-loop-${TESTTYPE}-${TILESIZE}-${BACKEND}.exe
+                                  PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests)
+      endforeach()
     endforeach()
-  endforeach()
+  endif()
 endforeach()
 
 unset( TILESIZES )

test/functional/kernel/tile-variants/CMakeLists.txt

@@ -12,13 +12,16 @@ set(TILETYPES Fixed2D Fixed2DSum Fixed2DMinMax)
 
 foreach( TILE_BACKEND ${KERNEL_BACKENDS} )
   foreach( TILE_TYPE ${TILETYPES} )
-    configure_file( test-kernel-tilefixed.cpp.in
-                    test-kernel-tile-${TILE_TYPE}-${TILE_BACKEND}.cpp )
-    raja_add_test( NAME test-kernel-tile-${TILE_TYPE}-${TILE_BACKEND}
-                   SOURCES ${CMAKE_CURRENT_BINARY_DIR}/test-kernel-tile-${TILE_TYPE}-${TILE_BACKEND}.cpp )
+    # OpenMPTarget crashes the xl compiler when building this test... 
+    if( NOT((TILE_BACKEND STREQUAL "OpenMPTarget")) )
+      configure_file( test-kernel-tilefixed.cpp.in
+                      test-kernel-tile-${TILE_TYPE}-${TILE_BACKEND}.cpp )
+      raja_add_test( NAME test-kernel-tile-${TILE_TYPE}-${TILE_BACKEND}
+                    SOURCES ${CMAKE_CURRENT_BINARY_DIR}/test-kernel-tile-${TILE_TYPE}-${TILE_BACKEND}.cpp )
 
-    target_include_directories(test-kernel-tile-${TILE_TYPE}-${TILE_BACKEND}.exe
-                               PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests)
+      target_include_directories(test-kernel-tile-${TILE_TYPE}-${TILE_BACKEND}.exe
+                                PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests)
+    endif()
   endforeach()
 endforeach()
 

test/unit/workgroup/CMakeLists.txt

@@ -62,6 +62,11 @@ if(RAJA_TEST_EXHAUSTIVE OR NOT RAJA_COMPILER MATCHES "RAJA_COMPILER_Intel")
   set(Constructor_SUBTESTS Single)
   buildunitworkgrouptest(Constructor "${Constructor_SUBTESTS}" "${DISPATCHERS}" "${BACKENDS}")
 
+  if(RAJA_ENABLE_TARGET_OPENMP)
+    # WorkGroup dispatcher for OpenMPTarget not implemented yet
+    list(REMOVE_ITEM BACKENDS OpenMPTarget)
+  endif()
+
   set(Enqueue_SUBTESTS Single Multiple)
   buildunitworkgrouptest(Enqueue     "${Enqueue_SUBTESTS}"     "${DISPATCHERS}" "${BACKENDS}")
 
@@ -70,10 +75,12 @@ if(RAJA_TEST_EXHAUSTIVE OR NOT RAJA_COMPILER MATCHES "RAJA_COMPILER_Intel")
 endif()
 
 set(Dispatcher_SUBTESTS Single)
+
 if(RAJA_ENABLE_TARGET_OPENMP)
   # WorkGroup dispatcher for OpenMPTarget not implemented yet
   list(REMOVE_ITEM BACKENDS OpenMPTarget)
 endif()
+
 buildunitworkgrouptest(Dispatcher    "${Dispatcher_SUBTESTS}"  "${DISPATCHERS}" "${BACKENDS}")
 
 set(WorkStorage_SUBTESTS Constructor Iterator InsertCall Multiple)