Merge pull request #1502 from LLNL/v2023.06.0-RC

v2023.06.0 RC

Author: rhornung67

.gitlab-ci.yml

@@ -24,9 +24,11 @@
 # We define the following GitLab pipeline variables:
 variables:
   MP_BRANCH: "develop"
+  GITHUB_PROJECT_NAME: "RAJA"
+  GITHUB_PROJECT_ORG: "LLNL"
 # Use a service user to run CI. This prevents from running pipelines as an
 # actual user.
-  LLNL_SERVICE_USER: ""
+  LLNL_SERVICE_USER: rajasa
 # Use a service user workspace. Solves permission issues, stores everything
 # at the same location whoever triggers a pipeline.
 #  CUSTOM_CI_BUILDS_DIR: ""
@@ -35,12 +37,18 @@ variables:
 # We build the projects in the CI clone directory.
 # TODO: add a clean-up mechanism
   BUILD_ROOT: ${CI_PROJECT_DIR}
+# Set the build-and-test command.
+  BUILD_AND_TEST_CMD: "./scripts/gitlab/build_and_test.sh"
+# Override the list of branch that will skip the "draft PR test".
+# Add protected branches here. Defaults to "develop main master".
+  ALWAYS_RUN_LIST: "develop main"
 
 # We organize the build-and-test stage in sub-pipelines. Each sub-pipeline
 # corresponds to a test batch on a given machine.
 
 # High level stages
 stages:
+  - machine-checks
   - build-and-test
   - multi_project
 
@@ -51,7 +59,7 @@ stages:
     include:
       - local: '.gitlab/custom-jobs-and-variables.yml'
       - project: 'radiuss/radiuss-shared-ci'
-        ref: v2022.09.0
+        ref: v2023.03.1
         file: '${CI_MACHINE}-build-and-test.yml'
       - local: '.gitlab/${CI_MACHINE}-build-and-test-extra.yml'
     strategy: depend
@@ -74,6 +82,10 @@ trigger-rajaperf:
     branch: develop
     strategy: depend
 
-# pipelines subscribed by the project
 include:
-  - local: .gitlab/subscribed-pipelines.yml
+  # checks preliminary to running the actual CI test (optional)
+  - project: 'radiuss/radiuss-shared-ci'
+    ref: v2023.03.1
+    file: 'preliminary-ignore-draft-pr.yml'
+  # pipelines subscribed by the project
+  - local: '.gitlab/subscribed-pipelines.yml'

.gitlab/corona-build-and-test-extra.yml

@@ -1,5 +1,5 @@
 #############################################################################
-# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC
+# Copyright (c) 2016-23, Lawrence Livermore National Security, LLC
 # and RAJA project contributors. See the RAJA/LICENSE file for details.
 #
 # SPDX-License-Identifier: (BSD-3-Clause)
@@ -21,8 +21,8 @@
 # ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is no reason not to fully
 # describe the spec here.
 
-rocm_5_1_1_clang_13_0_0_desul_atomics:
+rocmcc_5_4_1_hip_desul_atomics:
   variables:
-    SPEC: " ~shared +rocm ~openmp +tests +desul amdgpu_target=gfx906 %clang@13.0.0 ^hip@5.1.1 ^blt@develop"
+    SPEC: " ~shared +rocm ~openmp +tests +desul amdgpu_target=gfx906 %rocmcc@5.4.1 ^hip@5.4.1 ^blt@develop"
   extends: .build_and_test_on_corona
 

.gitlab/custom-jobs-and-variables.yml

@@ -1,6 +1,6 @@
 ###############################################################################
-# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC and RADIUSS
-# project contributors. See the COPYRIGHT file for details.
+# Copyright (c) 2016-23, Lawrence Livermore National Security, LLC and RADIUSS
+# project contributors. See the RAJA/LICENSE file for details.
 #
 # SPDX-License-Identifier: (MIT)
 ###############################################################################
@@ -14,34 +14,43 @@ variables:
 
 # Ruby
 # Arguments for top level allocation
-  RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --partition=pdebug --time=60 --nodes=1"
+  RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --reservation=ci --qos=ci_ruby --time=60 --nodes=2"
 # Arguments for job level allocation
-  RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=45 --nodes=1"
+  RUBY_BUILD_AND_TEST_JOB_ALLOC: "--reservation=ci --qos=ci_ruby --time=45 --nodes=1"
 # Project specific variants for ruby
   PROJECT_RUBY_VARIANTS: "~shared +openmp +tests"
 # Project specific deps for ruby
   PROJECT_RUBY_DEPS: ""
 
 # Corona
 # Arguments for top level allocation
-  CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--time-limit=60m --nodes=1"
+  CORONA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=60m --nodes=1"
 # Arguments for job level allocation
-  CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=45m --nodes=1"
+  CORONA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=45m --nodes=1 --begin-time=+5s"
 # Project specific variants for corona
   PROJECT_CORONA_VARIANTS: "~shared ~openmp +tests"
 # Project specific deps for corona
   PROJECT_CORONA_DEPS: "^blt@develop "
 
+# Tioga
+# Arguments for top level allocation
+  TIOGA_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --time-limit=60m --nodes=1"
+# Arguments for job level allocation
+  TIOGA_BUILD_AND_TEST_JOB_ALLOC: "--time-limit=45m --nodes=1 --begin-time=+5s"
+# Project specific variants for corona
+  PROJECT_TIOGA_VARIANTS: "~shared ~openmp +tests"
+# Project specific deps for corona
+  PROJECT_TIOGA_DEPS: "^blt@develop "
+
 # Lassen and Butte use a different job scheduler (spectrum lsf) that does not
 # allow pre-allocation the same way slurm does.
 # Arguments for job level allocation
   LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 60"
 # Project specific variants for lassen
-  PROJECT_LASSEN_VARIANTS: "~shared +openmp +tests"
+  PROJECT_LASSEN_VARIANTS: "~shared +openmp +tests cuda_arch=70"
 # Project specific deps for lassen
   PROJECT_LASSEN_DEPS: ""
 
-
 # Configuration shared by build and test jobs specific to this project.
 # Not all configuration can be shared. Here projects can fine tune the
 # CI behavior.

.gitlab/lassen-build-and-test-extra.yml

@@ -1,5 +1,5 @@
 ##############################################################################
-# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC
+# Copyright (c) 2016-23, Lawrence Livermore National Security, LLC
 # and RAJA project contributors. See the RAJA/LICENSE file for details.
 #
 # SPDX-License-Identifier: (BSD-3-Clause)
@@ -12,80 +12,20 @@
 # We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
 # the comparison with the original job is easier.
 
-# Overriding shared spec: Allow failures
-ibm_clang_9_0_0:
+# Overriding shared spec: Longer allocation + extra flags
+xl_2022_08_19_gcc_8_3_1_cuda_11_2_0:
   variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %clang@ibm.9.0.0 ${PROJECT_LASSEN_DEPS}"
-  extends: .build_and_test_on_lassen
-  allow_failure: true
-
-# Overriding shared spec: Allow failures
-ibm_clang_9_0_0_gcc_8_3_1:
-  variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %clang@ibm.9.0.0 cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ${PROJECT_LASSEN_DEPS}"
-  extends: .build_and_test_on_lassen
-  allow_failure: true
-
-# Overriding shared spec: Extra flags
-gcc_8_3_1:
-  variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %gcc@8.3.1 cxxflags=\"-finline-functions -finline-limit=20000\" cflags=\"-finline-functions -finline-limit=20000\" ${PROJECT_LASSEN_DEPS}"
-  extends: .build_and_test_on_lassen
-
-# Overriding shared spec: Longer allocation + Allow failures
-pgi_20_4_gcc_8_3_1:
-  extends: .build_and_test_on_lassen
-  variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %pgi@20.4 cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ${PROJECT_LASSEN_DEPS}"
-    LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 80"
-  allow_failure: true
-
-# Overriding shared spec: Extra flags
-xl_16_1_1_12:
-  variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %xl@16.1.1.12 cxxflags=\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" ${PROJECT_LASSEN_DEPS}"
-  extends: .build_and_test_on_lassen
-
-# Overriding shared spec: Extra flags
-xl_16_1_1_12_gcc_8_3_1:
-  variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %xl@16.1.1.12 cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 -qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 ${PROJECT_LASSEN_DEPS}"
-  extends: .build_and_test_on_lassen
-
-# Overriding shared spec: Allow failures
-ibm_clang_9_0_0_gcc_8_3_1_cuda_10_1_168:
-  extends: .build_and_test_on_lassen
-  variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} +cuda %clang@ibm.9.0.0 cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^cuda@10.1.168 ${PROJECT_LASSEN_DEPS}"
-  allow_failure: true
-
-# Overriding shared spec: Longer allocation + Extra flags + Allow failure + Updated cuda
-xl_16_1_1_12_cuda_11_1_1:
-  variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} +cuda %xl@16.1.1.12 cxxflags=\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" cuda_arch=70 ^cuda@11.1.0 ^cmake@3.14.5 ${PROJECT_LASSEN_DEPS}"
+    SPEC: "${PROJECT_LASSEN_VARIANTS} +cuda cxxflags==\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" %xl@16.1.1.12.gcc.8.3.1 ^cuda@11.2.0+allow-unsupported-compilers ${PROJECT_LASSEN_DEPS}"
+    MODULE_LIST: "cuda/11.2.0"
     LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 120"
-  allow_failure: true
   extends: .build_and_test_on_lassen
 
-# Overriding shared spec: Deactivated spec. This spec will be removed soon.
-xl_16_1_1_12_gcc_8_3_1_cuda_11_0_2:
+# Overriding shared spec: Longer allocation + extra flags
+xl_2022_08_19_gcc_8_3_1_cuda_11_7_0:
   variables:
-    SPEC: "${PROJECT_LASSEN_VARIANTS} +cuda %xl@16.1.1.12 cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^cuda@11.0.2 ^cmake@3.14.5 ${PROJECT_LASSEN_DEPS}"
-  extends: .build_and_test_on_lassen
-  script:
-    - |
-      echo -e "\e[31mDeactivated spec !\e[0m"
-      echo -e "\e[31m${SPEC}\e[0m"
-      echo -e "\e[31mRAJA won’t build with Cuda 11.0.2 due to a known issue.\e[0m"
-    - exit 1
-  allow_failure: true
-
-# Overriding shared spec: Longer allocation + Extra flags + Allow failure + Updated cuda
-xl_16_1_1_12_gcc_8_3_1_cuda_11_1_0:
-  variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} +cuda %xl@16.1.1.12 cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 -qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cuda_arch=70 ^cuda@11.1.0 ^cmake@3.14.5 ${PROJECT_LASSEN_DEPS}"
+    SPEC: "${PROJECT_LASSEN_VARIANTS} +cuda cxxflags==\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" %xl@16.1.1.12.gcc.8.3.1 ^cuda@11.7.0+allow-unsupported-compilers ${PROJECT_LASSEN_DEPS}"
+    MODULE_LIST: "cuda/11.7.0"
     LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 120"
-  allow_failure: true
   extends: .build_and_test_on_lassen
 
 
@@ -96,33 +36,11 @@ xl_16_1_1_12_gcc_8_3_1_cuda_11_1_0:
 # ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is no reason not to fully
 # describe the spec here.
 
-##########
-# CPU ONLY
-##########
-
-clang_14_0_5:
-  variables:
-    SPEC: " ~shared +openmp +tests %clang@14.0.5"
-  extends: .build_and_test_on_lassen
-
-##########
-# CUDA
-##########
-
-clang_12_0_1_cuda_11_5_0:
-  variables:
-    SPEC: " ~shared +openmp +tests +cuda cuda_arch=70 %clang@12.0.1 ^cuda@11.5.0"
-  extends: .build_and_test_on_lassen
-
-gcc_8_3_1_cuda_11_1_0:
-  variables:
-    SPEC: " ~shared +openmp +tests +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.1.0"
-  extends: .build_and_test_on_lassen
-
 gcc_8_3_1_cuda_11_5_0_ats_disabled:
   extends: .build_and_test_on_lassen
   variables:
-    SPEC: " ~shared +openmp +tests +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.5.0"
+    SPEC: " ~shared +openmp +tests +cuda %gcc@8.3.1 cuda_arch=70 ^cuda@11.5.0+allow-unsupported-compilers"
+    MODULE_LIST: "cuda/11.5.0"
     LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 --atsdisable -W 60"
 
 ##########
@@ -131,16 +49,17 @@ gcc_8_3_1_cuda_11_5_0_ats_disabled:
 
 clang_13_0_1_libcpp:
   variables:
-    SPEC: " ~shared +openmp +tests %clang@13.0.1+libcpp"
+    SPEC: " ~shared +openmp +tests %clang@13.0.1 cflags==\"-DGTEST_HAS_CXXABI_H_=0\" cxxflags==\"-stdlib=libc++ -DGTEST_HAS_CXXABI_H_=0\""
   extends: .build_and_test_on_lassen
 
-clang_14_0_5_asan:
-  variables:
-    SPEC: " ~shared +openmp +tests %clang@14.0.5 cxxflags=-fsanitize=address"
-    ASAN_OPTIONS: "detect_leaks=1"
-  extends: .build_and_test_on_lassen
+#clang_14_0_5_asan:
+#  variables:
+#    SPEC: " ~shared +openmp +tests %clang@14.0.5 cxxflags==\"-fsanitize=address\" "
+#    ASAN_OPTIONS: "detect_leaks=1"
+#    LSAN_OPTIONS: "suppressions=${CI_PROJECT_DIR}/suppressions.asan"
+#  extends: .build_and_test_on_lassen
 
-gcc_8_3_1_cuda_10_1_168_desul_atomics:
+gcc_8_3_1_cuda_10_1_243_desul_atomics:
   variables:
-    SPEC: " ~shared +openmp +tests +cuda +desul %gcc@8.3.1 cuda_arch=70 ^cuda@10.1.168"
+    SPEC: " ~shared +openmp +tests +cuda +desul %gcc@8.3.1 cuda_arch=70 ^cuda@10.1.243+allow-unsupported-compilers"
   extends: .build_and_test_on_lassen

.gitlab/ruby-build-and-test-extra.yml

@@ -1,5 +1,5 @@
 ##############################################################################
-# Copyright (c) 2016-22, Lawrence Livermore National Security, LLC
+# Copyright (c) 2016-23, Lawrence Livermore National Security, LLC
 # and RAJA project contributors. See the RAJA/LICENSE file for details.
 #
 # SPDX-License-Identifier: (BSD-3-Clause)
@@ -12,19 +12,7 @@
 # We keep ${PROJECT_<MACHINE>_VARIANTS} and ${PROJECT_<MACHINE>_DEPS} So that
 # the comparison with the original job is easier.
 
-# Overriding shared config for longer run
-gcc_8_1_0:
-  variables:
-    SPEC: " ${PROJECT_RUBY_VARIANTS} %gcc@8.1.0 ${PROJECT_RUBY_DEPS}"
-    RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=60 --nodes=1"
-  extends: .build_and_test_on_ruby
-
-# Overriding shared spec: Allow failures
-pgi_20_1_gcc_local_8_3_1:
-  variables:
-    SPEC: " ${PROJECT_RUBY_VARIANTS} %pgi@20.1 cxxflags\"=-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" cflags\"=-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" fflags=\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" ${PROJECT_RUBY_DEPS}"
-  extends: .build_and_test_on_ruby
-  allow_failure: true
+# No overridden jobs so far.
 
 ############
 # Extra jobs
@@ -33,26 +21,26 @@ pgi_20_1_gcc_local_8_3_1:
 # ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is no reason not to fully
 # describe the spec here.
 
-clang_9_0_0_openmp_off:
+clang_14_0_6_openmp_off:
   variables:
-    SPEC: " ~shared ~openmp +tests %clang@9.0.0"
+    SPEC: " ~shared ~openmp +tests %clang@14.0.6"
   extends: .build_and_test_on_ruby
 
-gcc_8_1_0_openmp_default:
+gcc_10_3_1_openmp_default:
   variables:
-    SPEC: " ~shared +tests %gcc@8.1.0"
+    SPEC: " ~shared +tests %gcc@10.3.1"
     RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=60 --nodes=1"
   extends: .build_and_test_on_ruby
 
-icpc_19_1_0:
-  variables:
-    SPEC: " ~shared +openmp +tests %intel@19.1.0"
-    RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=40 --nodes=1"
-  extends: .build_and_test_on_ruby
-
 # OTHERS
-clang_10_0_1_gcc_8_3_1_desul_atomics:
+clang_14_0_6_gcc_10_3_1_desul_atomics:
   variables:
-    SPEC: " ~shared +openmp +tests +desul %clang@10.0.1 cxxflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1"
+    SPEC: " ~shared +openmp +tests +desul %clang@14.0.6.gcc.10.3.1"
   extends: .build_and_test_on_ruby
 
+# Ideally, we want to use this spec, but the build takes too much time...
+#intel_oneapi_2022_2_gcc_8_3_1:
+#  variables:
+#    SPEC: "~shared +openmp +tests %oneapi@2022.2.gcc.8.3.1"
+#    RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=90 --nodes=1"
+#  extends: .build_and_test_on_ruby