Update integration test scripts to run on WashU Compute1 cluster

CHANGELOG.md

@@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 `SatDiagnEdge` collection to all GEOS-Chem Classic `HISTORY.rc` templates
 - Added new GCHP config file ESMF.rc for configuring ESMF logging
 - Added several new run directory files for use with GEOS-Chem in GEOS
+- GCClassic integration tests now display proper commit info in `results.compile.log`
 - Stopped OCEAN_CONC from needlessly being pushed through vertical regridding for Hg simulations
 - Added warning in GCHP HISTORY.rc about outputting area-dependent variables on custom grids
 
@@ -30,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update `GeosCore/carbon_gases_mod.F90` for consistency with config file updates in PR #1916
 - Update MPI usage in CESM-only code to match new conventions in CAM
 - Updated GEPA inventory to GHGI v2 for CH4 and carbon simulations
+- Updated integration tests scripts to run on the WashU Compute1 cluster
 
 ### Fixed
 - Add missing mol wt for HgBrO in `run/shared/species_database_hg.yml`

test/integration/GCClassic/integrationTest.sh

@@ -60,19 +60,20 @@ if [[ $? -ne 0 ]]; then
 fi
 
 # Parse arguments and set variables accordingly
+# TODO: replace "scheduler" with "site" argument; set scheduler from site.
 eval set -- "${validArgs}"
 while [ : ]; do
     case "${1}" in
 
-	# -d or --directory specifies the root folder for tests
-	-d | --directory)
-	    itRoot="${2}"
+        # -d or --directory specifies the root folder for tests
+        -d | --directory)
+            itRoot="${2}"
             shift 2
             ;;
 
 	# -e or --env-file specifies the environment file
 	-e | --env-file)
-	    envFile="${2}"
+            envFile="${2}"
             shift 2
             ;;
 
@@ -84,19 +85,19 @@ while [ : ]; do
 
 	# -p or --partition replaces REQUESTED_PARTITION with the user's choice
 	-p | --partition)
-	    sedCmd="s/REQUESTED_PARTITION/${2}/"
-	    shift 2
-	    ;;
+            sedCmd="s/REQUESTED_PARTITION/${2}/"
+            shift 2
+            ;;
 
 	# -q or --quick runs a quick set of integration tests (for testing)
 	-q | --quick)
-	    quick="yes"
+            quick="yes"
             shift
-	    ;;
+            ;;
 
 	# -s or --scheduler selects the scheduler (case-insensitive)
 	-s | --scheduler)
-	    scheduler="${2^^}"
+            scheduler="${2^^}"
             shift 2
             ;;
 
@@ -114,10 +115,13 @@ if [[ "x${itRoot}" == "xnone" ]]; then
 fi
 
 # Error check environment file
-if [[ "x${envFile}" == "xnone" ]]; then
-    echo "ERROR: The enviroment file (module loads) has not been specified!"
-    echo "${usage}"
-    exit 1
+# TODO: Add a test on site name rather than scheduler
+if [[ "x${scheduler}" != "xLSF" ]]; then
+    if [[ "x${envFile}" == "xnone" ]]; then
+        echo "ERROR: The enviroment file (module loads) has not been specified!"
+        echo "${usage}"
+        exit 1
+    fi
 fi
 
 # Exit if no partition has been selected for SLURM
@@ -192,21 +196,21 @@ if [[ "x${scheduler}" == "xSLURM" ]]; then
     sed_ie '/#BSUB -W 0:30/d'                "${scriptsDir}/integrationTestCompile.sh"
     sed_ie '/#BSUB -o lsf-%J.txt/d'          "${scriptsDir}/integrationTestCompile.sh"
     sed_ie \
-	'/#BSUB -R "rusage\[mem=8GB\] span\[ptile=1\] select\[mem < 1TB\]"/d' \
-	"${scriptsDir}/integrationTestCompile.sh"
+      '/#BSUB -R "rusage\[mem=8GB\] span\[ptile=1\] select\[mem < 1TB\]"/d' \
+      "${scriptsDir}/integrationTestCompile.sh"
     sed_ie \
-	"/#BSUB -a 'docker(registry\.gsc\.wustl\.edu\/sleong\/esm\:intel\-2021\.1\.2)'/d" \
-	"${scriptsDir}/integrationTestCompile.sh"
+      "/#BSUB -a 'docker(registry\.gsc\.wustl\.edu\/sleong\/esm\:intel\-2021\.1\.2)'/d" \
+      "${scriptsDir}/integrationTestCompile.sh"
     sed_ie '/#BSUB -q REQUESTED_PARTITION/d' "${scriptsDir}/integrationTestExecute.sh"
     sed_ie '/#BSUB -n 24/d'                  "${scriptsDir}/integrationTestExecute.sh"
     sed_ie '/#BSUB -W 6:00/d'                "${scriptsDir}/integrationTestExecute.sh"
     sed_ie '/#BSUB -o lsf-%J.txt/d'          "${scriptsDir}/integrationTestExecute.sh"
     sed_ie \
-	'/#BSUB -R "rusage\[mem=90GB\] span\[ptile=1\] select\[mem < 2TB\]"/d' \
-	"${scriptsDir}/integrationTestExecute.sh"
+      '/#BSUB -R "rusage\[mem=8GB\] span\[ptile=1\] select\[mem < 1TB\]"/d' \
+      "${scriptsDir}/integrationTestCompile.sh"
     sed_ie \
-	"/#BSUB -a 'docker(registry\.gsc\.wustl\.edu\/sleong\/esm\:intel\-2021\.1\.2)'/d" \
-	"${scriptsDir}/integrationTestExecute.sh"
+      "/#BSUB -a 'docker(registry\.gsc\.wustl\.edu\/sleong\/esm\:intel\-2021\.1\.2)'/d" \
+      "${scriptsDir}/integrationTestCompile.sh"
 
     # Replace "REQUESTED_PARTITION" with the partition name
     sed_ie "${sedCmd}" "${scriptsDir}/integrationTestCompile.sh"

test/integration/GCClassic/integrationTestCompile.sh

@@ -61,7 +61,7 @@ baseOptions="-DCMAKE_BUILD_TYPE=Debug -DRUNDIR='' -DINSTALLCOPY=${binDir}"
 head_gcc=$(export GIT_DISCOVERY_ACROSS_FILESYSTEM=1; \
            git -C "${codeDir}" log --oneline --no-decorate -1)
 head_gc=$(export GIT_DISCOVERY_ACROSS_FILESYSTEM=1; \
-          git -C "${codeDir}" log --oneline --no-decorate -1)
+          git -C "${codeDir}/src/GEOS-Chem" log --oneline --no-decorate -1)
 head_hco=$(export GIT_DISCOVERY_ACROSS_FILESYSTEM=1; \
            git -C "${codeDir}/src/HEMCO" log --oneline --no-decorate -1)
 
@@ -86,7 +86,7 @@ elif [[ "x${scheduler}" == "xLSF" ]]; then
     #-----------------------
 
     # Set OMP_NUM_THREADS to the same # of cores requested with #BSUB -n
-    export OMP_NUM_THREADS=${$LSB_DJOB_NUMPROC}
+    export OMP_NUM_THREADS=${LSB_DJOB_NUMPROC}
 
 else
 

test/integration/GCClassic/integrationTestCreate.sh

@@ -145,6 +145,9 @@ cp -f ${commonFuncs}                 ${scriptsDir}
 cp -f ${thisDir}/README.md           ${scriptsDir}
 cp -f ${thisDir}/README.testroot.md  ${itRoot}/README.md
 
+# This is necessary on Compute1 to make all scripts executable
+chmod 755 -R ${scriptsDir}
+
 # Log file with echoback from rundir creation
 log="${logsDir}/createIntegrationTests.log"
 

test/integration/GCClassic/integrationTestExecute.sh

@@ -185,26 +185,34 @@ for runDir in *; do
             # Copy the executable file here
             cp -f "${binDir}/${exeFile}" .
 
+            # Update to make sure the run directory is executable
+            # on Compute1.  We will later replace this test with
+            # a test on the site name instead of on the scheduler.
+            # TODO: Test on name rather than scheduler.
+            if [[ "x${scheduler}" == "xLSF" ]]; then
+                chmod 755 -R "${runAbsPath}"
+            fi
+
             # Remove any leftover files in the run dir
             ./cleanRunDir.sh --no-interactive >> "${log}" 2>&1
 
-	    # Change time cycle flags in HEMCO_Config.rc from EFYO to CYS,
-	    # to allow missing species to be set a default value.
-	    sed_ie "s/EFYO/CYS/"            HEMCO_Config.rc  # GC_RESTART
-	    sed_ie "s/EFY xyz 1/CYS xyz 1/" HEMCO_Config.rc  # GC_BCs
+            # Change time cycle flags in HEMCO_Config.rc from EFYO to CYS,
+            # to allow missing species to be set a default value.
+            sed_ie "s/EFYO/CYS/"            HEMCO_Config.rc  # GC_RESTART
+            sed_ie "s/EFY xyz 1/CYS xyz 1/" HEMCO_Config.rc  # GC_BCs
 
             # Run the code if the executable is present.  Then update the
             # pass/fail counters and write a message to the results log file.
             if [[ "x${scheduler}" == "xSLURM" ]]; then
                 srun -c ${OMP_NUM_THREADS} ./${exeFile} >> "${log}" 2>&1
             else
-		./${exeFile} >> "${log}" 2>&1
+                ./${exeFile} >> "${log}" 2>&1
             fi
 
             # Determine if the job succeeded or failed
             if [[ $? -eq 0 ]]; then
                 let passed++
-	        print_to_log "${passMsg}" "${results}"
+                print_to_log "${passMsg}" "${results}"
             else
                 let failed++
                 print_to_log "${failMsg}" "${results}"

test/integration/GCHP/integrationTest.sh

@@ -60,6 +60,7 @@ if [[ $? -ne 0 ]]; then
 fi
 
 # Parse arguments and set variables accordingly
+# TODO: replace "scheduler" with "site" argument; set scheduler from site.
 eval set -- "${validArgs}"
 while [ : ]; do
     case "${1}" in
@@ -114,10 +115,13 @@ if [[ "x${itRoot}" == "xnone" ]]; then
 fi
 
 # Error check environment file
-if [[ "x${envFile}" == "xnone" ]]; then
-    echo "ERROR: The enviroment file (module loads) has not been specified!"
-    echo "${usage}"
-    exit 1
+# TODO: Add a test on site name rather than scheduler
+if [[ "x${scheduler}" != "xLSF" ]]; then
+    if [[ "x${envFile}" == "xnone" ]]; then
+        echo "ERROR: The enviroment file (module loads) has not been specified!"
+        echo "${usage}"
+        exit 1
+    fi
 fi
 
 # Exit if no partition has been selected for SLURM

test/integration/GCHP/integrationTestCreate.sh

@@ -145,6 +145,9 @@ cp -f ${commonFuncs}                 ${scriptsDir}
 cp -f ${thisDir}/README.md           ${scriptsDir}
 cp -f ${thisDir}/README.testroot.md  ${itRoot}/README.md
 
+# This is necessary on Compute1 to make all scripts executable
+chmod 755 -R ${scriptsDir}
+
 # Log file with echoback from rundir creation
 log="${logsDir}/createIntegrationTests.log"
 

test/integration/GCHP/integrationTestExecute.sh

@@ -179,37 +179,45 @@ for runDir in *; do
         # Test if the executable exists
         if [[ -f "${binDir}/${exeFile}" ]]; then
 
-	    #----------------------------------------------------------------
-	    # If the executable file exists, we can do the test
             #----------------------------------------------------------------
+            # If the executable file exists, we can do the test
+            #----------------------------------------------------------------
+
+            # Change to the run directory
+            cd "${runAbsPath}"
 
-	    # Change to the run directory
-	    cd "${runAbsPath}"
+            # Copy the executable file here
+            cp "${binDir}/${exeFile}" .
 
-	    # Copy the executable file here
-	    cp "${binDir}/${exeFile}" .
+            # Update to make sure the run directory is executable
+            # on Compute1.  We will later replace this test with
+            # a test on the site name instead of on the scheduler.
+            # TODO: Test on name rather than scheduler
+            if [[ "x${scheduler}" == "xLSF" ]]; then
+		chmod 755 -R "${runAbsPath}"
+            fi
 
-	    # Remove any leftover files in the run dir
-	    ./cleanRunDir.sh --no-interactive >> "${log}" 2>&1
+            # Remove any leftover files in the run dir
+            ./cleanRunDir.sh --no-interactive >> "${log}" 2>&1
 
-	    # Link to the environment file
-	    ./setEnvironmentLink.sh "${envDir}/gchp.env"
+            # Link to the environment file
+            ./setEnvironmentLink.sh "${envDir}/gchp.env"
 
-	    # Update config files, set links, load environment, sanity checks
-	    . setCommonRunSettings.sh >> "${log}" 2>&1
-	    . setRestartLink.sh       >> "${log}" 2>&1
-	    . gchp.env                >> "${log}" 2>&1
-	    . checkRunSettings.sh     >> "${log}" 2>&1
+            # Update config files, set links, load environment, sanity checks
+            . setCommonRunSettings.sh >> "${log}" 2>&1
+            . setRestartLink.sh       >> "${log}" 2>&1
+            . gchp.env                >> "${log}" 2>&1
+            . checkRunSettings.sh     >> "${log}" 2>&1
 
-	    # For safety's sake, remove restarts that weren't renamed
-	    for rst in Restarts; do
+            # For safety's sake, remove restarts that weren't renamed
+            for rst in Restarts; do
 		if [[ "${rst}" =~ "gcchem_internal_checkpoint" ]]; then
 		    rm -f "${rst}"
 		fi
-	    done
+            done
 
-	    # Run GCHP and evenly distribute tasks across nodes
-	    if [[ "x${scheduler}" == "xSLURM" ]]; then
+            # Run GCHP and evenly distribute tasks across nodes
+            if [[ "x${scheduler}" == "xSLURM" ]]; then
 
 		#---------------------------------------------
 		# Executing GCHP on SLURM (Harvard Cannon)
@@ -228,25 +236,25 @@ for runDir in *; do
 
 		# Execute GCHP with srun
 		srun -n ${coreCt} -N ${SLURM_NNODES} -m plane=${planeCt} \
-		    --mpi=pmix ./${exeFile} >> "${log}" 2>&1
+		     --mpi=pmix ./${exeFile} >> "${log}" 2>&1
 
-	    elif [[ "x${scheduler}" == "xLSF" ]]; then
+            elif [[ "x${scheduler}" == "xLSF" ]]; then
 
 		#---------------------------------------------
 		# Executing GCHP on LSF (WashU Compute1)
 		#---------------------------------------------
 		mpiexec -n 24 ./${exeFile} > "${log}" 2>&1
 
-	    else
+            else
 
 		#---------------------------------------------
 		# Executing GCHP interactively
 		#---------------------------------------------
 		mpirun -n 24 ./${exeFile} >> "${log}" 2>&1
-	    fi
+            fi
 
-	    # Update pass/failed counts and write to results.log
-	    if [[ $? -eq 0 ]]; then
+            # Update pass/failed counts and write to results.log
+            if [[ $? -eq 0 ]]; then
 
 		# The run passed ...
                 let passed++
@@ -279,7 +287,7 @@ for runDir in *; do
         fi
 
         # Decrement the count of remaining tests
-	let remain--
+        let remain--
     fi
 done