ExaWorks · mtitov · Sep 11, 2023 · Aug 29, 2023 · Aug 29, 2023 · Aug 29, 2023
diff --git a/.gitlab/README.md b/.gitlab/README.md
@@ -1,5 +1,17 @@
 # Troubleshooting
 
+## Environment variables
+
+The following environment variables should be set within CI/CD settings:
+- `SDK_DASHBOARD_URL` - used to publish results from the executed tests;
+- `SDK_DASHBOARD_TOKEN` - unique token per facility (randomly generated UUID);
+- `SDK_WORK_DIR` - working directory in a shared space;
+   - In case of OLCF: `SDK_WORK_DIR_OPEN`, `SDK_WORK_DIR_ALPINE`, 
+     `SDK_WORK_DIR_ORION`;
+- `RADICAL_PILOT_DBURL` - MongoDB URL for the RADICAL-Pilot component;
+   - In case of LLNL: `MONGODB_CONNECTION_STRING`, which will be used within 
+     configuration scripts for `RADICAL_PILOT_DBURL`.
+
 ## GitLab working directory
 
 ```shell
@@ -12,6 +24,7 @@ chmod 700 <scratch_space>/gitlab-runner
 `<scratch_space>` per facility
 * LLNL: `/usr/workspace/${USER}`
 * OLCF: `$MEMBERWORK/csc449`
+* ALCF: TBD...
 
 ## SPACK-related issues
 

diff --git a/.gitlab/llnl-ci-conda.yml b/.gitlab/llnl-ci-conda.yml
@@ -14,13 +14,13 @@ stages:
 ### MACHINES
 .on_quartz:
   variables:
-    HOST: "quartz"
+    SITE: "quartz"
     CONDA_ENV_NAME: "quartz-env"
     CONDA_ARCH: "x86_64"
 
 .on_ruby:
   variables:
-    HOST: "ruby"
+    SITE: "ruby"
     CONDA_ENV_NAME: "ruby-env"
     CONDA_ARCH: "x86_64"
 
@@ -33,53 +33,51 @@ stages:
 
 .job_tags:
   tags:
-    - $HOST
+    - $SITE
     - $RUNNER_TYPE
 
 .conda_setup:
   stage: setup
   script:
     - mkdir -p ${CONDA_WORK_DIR}
-    - test -d ${CONDA_WORK_DIR}/miniconda3-${HOST} && exit 0
-    - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-${CONDA_ARCH}.sh -O ${CONDA_WORK_DIR}/miniconda-${HOST}.sh
-    - chmod +x ${CONDA_WORK_DIR}/miniconda-${HOST}.sh
-    - ${CONDA_WORK_DIR}/miniconda-${HOST}.sh -b -p ${CONDA_WORK_DIR}/miniconda3-${HOST}
+    - test -d ${CONDA_WORK_DIR}/miniconda3-${SITE} && exit 0
+    - wget -q https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-${CONDA_ARCH}.sh -O ${CONDA_WORK_DIR}/miniconda-${SITE}.sh
+    - chmod +x ${CONDA_WORK_DIR}/miniconda-${SITE}.sh
+    - ${CONDA_WORK_DIR}/miniconda-${SITE}.sh -b -p ${CONDA_WORK_DIR}/miniconda3-${SITE}
     - *finalize
 
 .conda_env_setup:
   stage: setup
   script:
     - test -d ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} && exit 0
-    - source ${CONDA_WORK_DIR}/miniconda3-${HOST}/etc/profile.d/conda.sh
+    - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh
     - eval "$(conda shell.posix hook)"
     - conda create --prefix ${CONDA_WORK_DIR}/${CONDA_ENV_NAME} -y
     - *finalize
 
 .conda_build:
   stage: build
   script:
-    - source ${CONDA_WORK_DIR}/miniconda3-${HOST}/etc/profile.d/conda.sh
+    - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh
     - eval "$(conda shell.posix hook)"
     - conda activate ${CONDA_WORK_DIR}/${CONDA_ENV_NAME}
     - conda env update --file ${CONDA_REQUIREMENTS}
     - *finalize
 
 .conda_test:
   stage: test
+  variables:
+    MAINTAINER: "arambula2@llnl.gov"
+    IM_NUMBER: "LLNL-MI-834241"
+    TESTS_GROUP: "conda"
+    SITE_ID: "llnl-$SITE"
   script:
-    - source ${CONDA_WORK_DIR}/miniconda3-${HOST}/etc/profile.d/conda.sh
+    - source ${CONDA_WORK_DIR}/miniconda3-${SITE}/etc/profile.d/conda.sh
     - eval "$(conda shell.posix hook)"
     - conda activate ${CONDA_WORK_DIR}/${CONDA_ENV_NAME}
-    - export run_id=${CI_PIPELINE_ID}
-    - export branch=${CI_COMMIT_BRANCH}
-    - export url=${TESTING_HOST}
-    - export test="Conda"
-    - export location="llnl-$HOST"
-    - export contact="arambula2@llnl.gov"
-    - export imnumber="LLNL-MI-834241"
     - python3 ci/tests/test.py -s
     - for TEST in flux parsl rp swift-t parsl-flux rp-flux; do
-        python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh";
+        python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout;
       done
     - python3 ci/tests/test.py -e
     - *finalize
@@ -107,7 +105,7 @@ conda_build_quartz:
 conda_test_quartz:
   variables:
     RUNNER_TYPE: "batch"
-    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 60"
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45"
   needs: [ conda_build_quartz ]
   extends: [ .on_quartz, .job_tags, .conda_test ]
 
@@ -133,7 +131,7 @@ conda_build_ruby:
 conda_test_ruby:
   variables:
     RUNNER_TYPE: "batch"
-    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 60"
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45"
   needs: [ conda_build_ruby ]
   extends: [ .on_ruby, .job_tags, .conda_test ]
 

diff --git a/.gitlab/llnl-ci-pip.yml b/.gitlab/llnl-ci-pip.yml
@@ -14,17 +14,17 @@ stages:
 ### MACHINES
 .on_quartz:
   variables:
-    HOST: "quartz"
+    SITE: "quartz"
     VENV_ENV_NAME: "quartz-env"
 
 .on_ruby:
   variables:
-    HOST: "ruby"
+    SITE: "ruby"
     VENV_ENV_NAME: "ruby-env"
 
 .on_lassen:
   variables:
-    HOST: "lassen"
+    SITE: "lassen"
     VENV_ENV_NAME: "lassen-env"
 
 
@@ -36,7 +36,7 @@ stages:
 
 .job_tags:
   tags:
-    - $HOST
+    - $SITE
     - $RUNNER_TYPE
 
 .pip_env_setup:
@@ -59,18 +59,16 @@ stages:
 
 .pip_test:
   stage: test
+  variables:
+    MAINTAINER: "arambula2@llnl.gov"
+    IM_NUMBER: "LLNL-MI-834241"
+    TESTS_GROUP: "pip"
+    SITE_ID: "llnl-$SITE"
   script:
     - source ${PIP_WORK_DIR}/${VENV_ENV_NAME}/bin/activate
-    - export run_id=${CI_PIPELINE_ID}
-    - export branch=${CI_COMMIT_BRANCH}
-    - export url=${TESTING_HOST}
-    - export test="Pip"
-    - export location="llnl-$HOST"
-    - export contact="arambula2@llnl.gov"
-    - export imnumber="LLNL-MI-834241"
     - python3 ci/tests/test.py -s
-    - for TEST in flux parsl rp swift-t parsl-flux rp-flux maestro; do
-        python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh";
+    - for TEST in flux parsl rp parsl-flux rp-flux maestro; do
+        python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout;
       done
     - python3 ci/tests/test.py -e
     - deactivate
@@ -92,7 +90,7 @@ pip_build_quartz:
 pip_test_quartz:
   variables:
     RUNNER_TYPE: "batch"
-    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 30"
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45"
   needs: [ pip_build_quartz ]
   extends: [ .on_quartz, .job_tags, .pip_test ]
 
@@ -112,7 +110,7 @@ pip_build_ruby:
 pip_test_ruby:
   variables:
     RUNNER_TYPE: "batch"
-    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 30"
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45"
   needs: [ pip_build_ruby ]
   extends: [ .on_ruby, .job_tags, .pip_test ]
 
@@ -134,7 +132,7 @@ pip_build_lassen:
 pip_test_lassen:
   variables:
     RUNNER_TYPE: "batch"
-    LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -W 30"
+    LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -W 45"
     LSB_JOB_STARTER: "ENVIRONMENT=BATCH /usr/tcetmp/bin/bsub_job_starter %USRCMD"
     EVIRONMENT: "BATCH"
   needs: [ pip_build_lassen ]

diff --git a/.gitlab/llnl-ci-spack.yml b/.gitlab/llnl-ci-spack.yml
@@ -13,21 +13,21 @@ stages:
 ### MACHINES
 .on_quartz:
   variables:
-    HOST: "quartz"
+    SITE: "quartz"
     SPACK_ENV_NAME: "rhel7-broadwell"
     COMPILER: "gcc@8.1.0"
     TARGET_ARCH_OPT: "target=x86_64"
 
 .on_ruby:
   variables:
-    HOST: "ruby"
+    SITE: "ruby"
     SPACK_ENV_NAME: "rhel7-cascadelake"
     COMPILER: "gcc@8.2.0"
     TARGET_ARCH_OPT: ""
 
 .on_lassen:
   variables:
-    HOST: "lassen"
+    SITE: "lassen"
     SPACK_ENV_NAME: "rhel7-ppc64le"
     COMPILER: "gcc@9.4.0"
     TARGET_ARCH_OPT: "target=ppc64le"
@@ -41,7 +41,7 @@ stages:
 
 .job_tags:
   tags:
-    - $HOST
+    - $SITE
     - $RUNNER_TYPE
 
 .spack_env_setup:
@@ -77,22 +77,20 @@ stages:
 
 .spack_test:
   stage: test
+  variables:
+    MAINTAINER: "arambula2@llnl.gov"
+    IM_NUMBER: "LLNL-MI-834241"
+    TESTS_GROUP: "spack"
+    SITE_ID: "llnl-$SITE"
   script:
     - . ${SPACK_WORK_DIR}/share/spack/setup-env.sh
     - for i in {1..3}; do
         spack env activate ${SPACK_ENV_NAME} && break || sleep 5;
       done
     - spack load exaworks py-pytest
-    - export run_id=${CI_PIPELINE_ID}
-    - export branch=${CI_COMMIT_BRANCH}
-    - export url=${TESTING_HOST}
-    - export test="Spack"
-    - export location="llnl-$HOST"
-    - export contact="arambula2@llnl.gov"
-    - export imnumber="LLNL-MI-834241"
     - python3 ci/tests/test.py -s
     - for TEST in flux parsl rp swift-t parsl-flux rp-flux; do
-        python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh";
+        python3 ci/tests/test.py -n $TEST -c "bash ci/tests/${TEST}/test.sh" --stdout;
       done
     - python3 ci/tests/test.py -e
 
@@ -137,7 +135,7 @@ spack_build_quartz:
 spack_test_quartz:
   variables:
     RUNNER_TYPE: "batch"
-    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 30"
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45"
   needs: [ spack_build_quartz ]
   extends: [ .on_quartz, .job_tags, .spack_test ]
 
@@ -158,7 +156,7 @@ spack_build_ruby:
 spack_test_ruby:
   variables:
     RUNNER_TYPE: "batch"
-    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 30"
+    LLNL_SLURM_SCHEDULER_PARAMETERS: "--nodes=1 -t 45"
   needs: [ spack_build_ruby ]
   extends: [ .on_ruby, .job_tags, .spack_test ]
 
@@ -179,7 +177,7 @@ spack_build_lassen:
 spack_test_lassen:
   variables:
     RUNNER_TYPE: "batch"
-    LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -W 30"
+    LLNL_LSF_SCHEDULER_PARAMETERS: "-nnodes 1 -W 45"
     LSB_JOB_STARTER: "ENVIRONMENT=BATCH /usr/tcetmp/bin/bsub_job_starter %USRCMD"
     ENVIRONMENT: "BATCH"
   needs: [ spack_build_lassen ]

diff --git a/ci/tests/rp-flux/test.sh b/ci/tests/rp-flux/test.sh
@@ -15,20 +15,21 @@ fi
 mkdir -p .radical/pilot/configs
 cp ci/tests/rp-flux/resource_flux.json .radical/pilot/configs/resource_flux.json
 export RADICAL_CONFIG_USER_DIR=$(pwd)
+export RADICAL_REPORT_ANIME=FALSE
 
-if [[ ! -d "./radical.pilot" ]]; then
-    git clone -b v$rp_version --single-branch \
-    https://github.com/radical-cybertools/radical.pilot.git
-fi
-cd radical.pilot
+wget -q "https://raw.githubusercontent.com/radical-cybertools/radical.pilot/v$rp_version/examples/config.json"
+wget -q "https://raw.githubusercontent.com/radical-cybertools/radical.pilot/v$rp_version/examples/00_getting_started.py"
+chmod +x 00_getting_started.py
 
-echo '--- smoke test'
-./examples/00_getting_started.py 'flux.localhost_flux'
+radical-stack
+./00_getting_started.py 'flux.localhost_flux'
 ret=$?
 echo "--- smoke test $ret"
 
-rm -rf $HOME/radical.pilot.sandbox/*
+SID=$(ls -rt | grep rp.session)
+test -z "$SID" || rm -rf "$HOME/radical.pilot.sandbox/$SID"
 echo '--- cleaned pilot sandbox'
 
 test "$ret" = 0 && echo "Success!"
 exit $ret
+
diff --git a/ci/tests/rp/test.sh b/ci/tests/rp/test.sh
@@ -21,32 +21,19 @@ cat > .radical/pilot/configs/resource_local.json <<EOF
 }
 EOF
 export RADICAL_CONFIG_USER_DIR=$(pwd)
+export RADICAL_REPORT_ANIME=FALSE
 
-if [[ ! -d "./radical.pilot" ]]; then
-    git clone -b v$rp_version --single-branch \
-    https://github.com/radical-cybertools/radical.pilot.git
-fi
-cd radical.pilot
+wget -q "https://raw.githubusercontent.com/radical-cybertools/radical.pilot/v$rp_version/examples/config.json"
+wget -q "https://raw.githubusercontent.com/radical-cybertools/radical.pilot/v$rp_version/examples/00_getting_started.py"
+chmod +x 00_getting_started.py
 
-echo '--- smoke test'
-./examples/00_getting_started.py local.localhost
+radical-stack
+./00_getting_started.py local.localhost
 ret=$?
 echo "--- smoke test $ret"
 
 SID=$(ls -rt | grep rp.session)
 test -z "$SID" || rm -rf "$HOME/radical.pilot.sandbox/$SID"
-
-echo '--- unit test'
-pytest -vvv tests/unit_tests
-test "$ret" = 0 && ret=$?
-echo "--- unit test $ret"
-
-echo '--- component test'
-pytest -vvv tests/component_tests
-test "$ret" = 0 && ret=$?
-echo "--- component test $ret"
-
-rm -rf $HOME/radical.pilot.sandbox/*
 echo '--- cleaned pilot sandbox'
 
 test "$ret" = 0 && echo "Success!"