develop: Fix to run workflow tasks on WCOSS (#390)

Fix to run workflow tasks on the WCOSS dell and cray
ufs-community · Jan 15, 2021 · 16cfd14 · 16cfd14
1 parent 4faf50d
commit 16cfd14
Show file tree

Hide file tree

Showing 22 changed files with 164 additions and 141 deletions.
diff --git a/modulefiles/tasks/wcoss_cray/get_extrn_ics.local b/modulefiles/tasks/wcoss_cray/get_extrn_ics.local
@@ -4,5 +4,8 @@
 
 module purge
 
+module use /usrx/local/prod/modulefiles
 module load hpss/4.1.0.3
 
+module use /usrx/local/nceplibs/modulefiles 
+module load srw-app-python/1.0.0
diff --git a/modulefiles/tasks/wcoss_cray/get_extrn_lbcs.local b/modulefiles/tasks/wcoss_cray/get_extrn_lbcs.local
@@ -4,5 +4,8 @@
 
 module purge
 
+module use /usrx/local/prod/modulefiles
 module load hpss/4.1.0.3
 
+module use /usrx/local/nceplibs/modulefiles 
+module load srw-app-python/1.0.0
diff --git a/modulefiles/tasks/wcoss_cray/make_grid.local b/modulefiles/tasks/wcoss_cray/make_grid.local
@@ -1,34 +1,15 @@
 #%Module
-module load modules
 
-module load xt-lsfhpc
-module load ncep
+module use /opt/cray/ari/modulefiles
 module load alps
-module load dvs
-module load xpmem
-module load ugni
-module load craype-network-aries
-module load switch
-module load rca
 module load gni-headers
+module load pmi/5.0.11
+module load rca
 module load udreg
-module load hpss
-
-module load prod_util
-module load g2tmpl-intel/1.4.0
-module load crtm-intel/2.2.6
-module load iobuf/2.0.7
-module load gempak/7.3.0
-
-module load nco-gnu-sandybridge/4.4.4
-module load NetCDF-intel-sandybridge/4.2
-module load cfp-intel-sandybridge/1.1.0
-export USE_CFP=YES
-
-module load grib_util/1.1.0
-
-module use -a /gpfs/hps3/emc/nems/noscrub/emc.nemspara/soft/modulefiles
-module load esmf/8.0.0
+module load ugni
+module load xpmem
 
-module load python/3.6.3
+module use /usrx/local/nceplibs/modulefiles
+module load srw-app-python/1.0.0
 
+module list
diff --git a/modulefiles/tasks/wcoss_cray/make_ics.local b/modulefiles/tasks/wcoss_cray/make_ics.local
@@ -1,34 +1,17 @@
 #%Module
-module load modules
 
-module load xt-lsfhpc
-module load ncep
+module use /opt/cray/ari/modulefiles
 module load alps
-module load dvs
-module load xpmem
-module load ugni
-module load craype-network-aries
-module load switch
-module load rca
 module load gni-headers
+module load pmi/5.0.11
+module load rca
 module load udreg
-module load hpss
-
-module load prod_util
-module load g2tmpl-intel/1.4.0
-module load crtm-intel/2.2.6
-module load iobuf/2.0.7
-module load gempak/7.3.0
-
-module load nco-gnu-sandybridge/4.4.4
-module load NetCDF-intel-sandybridge/4.2
-module load cfp-intel-sandybridge/1.1.0
-export USE_CFP=YES
-
-module load grib_util/1.1.0
+module load ugni
+module load xpmem
 
-module use -a /gpfs/hps3/emc/nems/noscrub/emc.nemspara/soft/modulefiles
-module load esmf/8.0.0
+module use /usrx/local/prod/modulefiles
+module load hpss/4.1.0.3
 
-module load python/3.6.3
+module use /usrx/local/nceplibs/modulefiles
+module load srw-app-python/1.0.0
 
diff --git a/modulefiles/tasks/wcoss_cray/make_lbcs.local b/modulefiles/tasks/wcoss_cray/make_lbcs.local
@@ -1,34 +1,17 @@
 #%Module
-module load modules
 
-module load xt-lsfhpc
-module load ncep
+module use /opt/cray/ari/modulefiles
 module load alps
-module load dvs
-module load xpmem
-module load ugni
-module load craype-network-aries
-module load switch
-module load rca
 module load gni-headers
+module load pmi/5.0.11
+module load rca
 module load udreg
-module load hpss
-
-module load prod_util
-module load g2tmpl-intel/1.4.0
-module load crtm-intel/2.2.6
-module load iobuf/2.0.7
-module load gempak/7.3.0
-
-module load nco-gnu-sandybridge/4.4.4
-module load NetCDF-intel-sandybridge/4.2
-module load cfp-intel-sandybridge/1.1.0
-export USE_CFP=YES
-
-module load grib_util/1.1.0
+module load ugni
+module load xpmem
 
-module use -a /gpfs/hps3/emc/nems/noscrub/emc.nemspara/soft/modulefiles
-module load esmf/8.0.0
+module use /usrx/local/prod/modulefiles
+module load hpss/4.1.0.3
 
-module load python/3.6.3
+module use /usrx/local/nceplibs/modulefiles
+module load srw-app-python/1.0.0
 
diff --git a/modulefiles/tasks/wcoss_cray/make_orog.local b/modulefiles/tasks/wcoss_cray/make_orog.local
@@ -0,0 +1,12 @@
+#%Module
+
+module use /opt/cray/ari/modulefiles
+module load alps
+module load gni-headers
+module load pmi/5.0.11
+module load rca
+module load udreg
+module load ugni
+module load xpmem
+
+module list
diff --git a/modulefiles/tasks/wcoss_cray/make_sfc_climo.local b/modulefiles/tasks/wcoss_cray/make_sfc_climo.local
@@ -0,0 +1,12 @@
+#%Module
+
+module use /opt/cray/ari/modulefiles
+module load alps
+module load gni-headers
+module load pmi/5.0.11
+module load rca
+module load udreg
+module load ugni
+module load xpmem
+
+module list
diff --git a/modulefiles/tasks/wcoss_cray/run_fcst.local b/modulefiles/tasks/wcoss_cray/run_fcst.local
@@ -1,3 +1,15 @@
 #%Module
-module unload python/2.7.14
-module load python/3.6.3
+
+module use /opt/cray/ari/modulefiles
+module load alps
+module load gni-headers
+module load pmi/5.0.11
+module load rca
+module load udreg
+module load ugni
+module load xpmem
+
+module use /usrx/local/nceplibs/modulefiles
+module load srw-app-python/1.0.0
+
+module list
diff --git a/modulefiles/tasks/wcoss_cray/run_post.local b/modulefiles/tasks/wcoss_cray/run_post.local
@@ -1,14 +1,18 @@
 #%Module
-module load modules
 
-module load xt-lsfhpc
-module load ncep
+module use /opt/cray/ari/modulefiles
 module load alps
-module load dvs
-module load xpmem
-module load ugni
-module load craype-network-aries
-module load switch
-module load rca
 module load gni-headers
+module load pmi/5.0.11
+module load rca
 module load udreg
+module load ugni
+module load xpmem
+
+module use /usrx/local/prod/modulefiles
+module load hpss/4.1.0.3
+
+module use /usrx/local/nceplibs/modulefiles
+module load srw-app-python/1.0.0
+
+
diff --git a/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local b/modulefiles/tasks/wcoss_dell_p3/get_extrn_ics.local
@@ -3,6 +3,7 @@
 #############################################################
 
 module purge
-
 module load HPSS/5.0.2.5
 
+module use /usrx/local/nceplibs/dev/modulefiles
+module load srw-app-python/1.0.0
diff --git a/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local b/modulefiles/tasks/wcoss_dell_p3/get_extrn_lbcs.local
@@ -3,6 +3,7 @@
 #############################################################
 
 module purge
-
 module load HPSS/5.0.2.5
 
+module use /usrx/local/nceplibs/dev/modulefiles
+module load srw-app-python/1.0.0
diff --git a/modulefiles/tasks/wcoss_dell_p3/make_grid.local b/modulefiles/tasks/wcoss_dell_p3/make_grid.local
@@ -1,3 +1,7 @@
 #%Module
+
 module load lsf/10.1
 module load python/3.6.3
+
+module use /usrx/local/nceplibs/dev/modulefiles
+module load srw-app-python/1.0.0
diff --git a/modulefiles/tasks/wcoss_dell_p3/make_ics.local b/modulefiles/tasks/wcoss_dell_p3/make_ics.local
@@ -1,3 +1,7 @@
 #%Module
+
 module load lsf/10.1
 module load python/3.6.3
+
+module use /usrx/local/nceplibs/dev/modulefiles
+module load srw-app-python/1.0.0
diff --git a/modulefiles/tasks/wcoss_dell_p3/make_lbcs.local b/modulefiles/tasks/wcoss_dell_p3/make_lbcs.local
@@ -1,3 +1,7 @@
 #%Module
+
 module load lsf/10.1
 module load python/3.6.3
+
+module use /usrx/local/nceplibs/dev/modulefiles
+module load srw-app-python/1.0.0
diff --git a/modulefiles/tasks/wcoss_dell_p3/run_fcst.local b/modulefiles/tasks/wcoss_dell_p3/run_fcst.local
@@ -1,3 +1,7 @@
 #%Module
+
 module unload python/2.7.14
 module load python/3.6.3
+
+module use /usrx/local/nceplibs/dev/modulefiles
+module load srw-app-python/1.0.0
diff --git a/modulefiles/tasks/wcoss_dell_p3/run_post.local b/modulefiles/tasks/wcoss_dell_p3/run_post.local
@@ -1,2 +1,7 @@
 #%Module
+
 module load lsf/10.1
+
+module use /usrx/local/nceplibs/dev/modulefiles
+module load srw-app-python/1.0.0
+
diff --git a/scripts/exregional_make_orog.sh b/scripts/exregional_make_orog.sh
@@ -287,50 +287,12 @@ cat "${input_redirect_fn}"
 print_info_msg "$VERBOSE" "
 Starting orography file generation..."
 
-case $MACHINE in
-
-  "WCOSS_CRAY")
-#
-# On WCOSS and WCOSS_C, use cfp to run multiple tiles simulatneously for
-# the orography.  For now, we have only one tile in the regional case,
-# but in the future we will have more.  First, create an input file for
-# cfp.
-#
-    ufs_utils_ushdir="${UFS_UTILS_DIR}/ush"
-    res="0"  # What should this be set to???
-    printf "%s\n" "\
-${ufs_utils_ushdir}/${orog_gen_scr} \
-$res \
-${TILE_RGNL} \
-${FIXLAM} \
-${raw_dir} \
-${UFS_UTILS_DIR} \
-${TOPO_DIR} \
-${tmp_dir}" \
-    >> ${tmp_dir}/orog.file1
-    aprun -j 1 -n 4 -N 4 -d 6 -cc depth cfp ${tmp_dir}/orog.file1
-    rm_vrfy ${tmp_dir}/orog.file1
-    ;;
-
-  "WCOSS_DELL_P3")
-    ufs_utils_ushdir="${UFS_UTILS_DIR}/ush"
-    res="0"  # What should this be set to???
-    "${exec_fp}" < "${input_redirect_fn}" || \
-      print_err_msg_exit "\
-Call to executable (exec_fp) that generates the raw orography file returned
-with nonzero exit code:
-  exec_fp = \"${exec_fp}\""
-    ;;
-
-  "CHEYENNE" | "HERA" | "ORION" | "JET" | "ODIN" | "STAMPEDE")
-    $APRUN "${exec_fp}" < "${input_redirect_fn}" || \
+$APRUN "${exec_fp}" < "${input_redirect_fn}" || \
       print_err_msg_exit "\
 Call to executable (exec_fp) that generates the raw orography file returned
 with nonzero exit code:
   exec_fp = \"${exec_fp}\""
-    ;;
 
-esac
 #
 # Change location to the original directory.
 #
@@ -457,7 +419,13 @@ cp_vrfy "${raw_orog_fp}" "${filtered_orog_fp}"
 # filtering executable will run) with the same name as the grid file and
 # point it to the actual grid file specified by grid_fp.
 #
-ln_vrfy -fs --relative "${grid_fp}" "${filter_dir}/${grid_fn}"
+
+if [ "${MACHINE}" = "WCOSS_CRAY" ]; then
+  ln_vrfy -fs "${grid_fp}" "${filter_dir}/${grid_fn}"
+else
+  ln_vrfy -fs --relative "${grid_fp}" "${filter_dir}/${grid_fn}"
+fi
+
 #
 # Create the namelist file (in the filter_dir directory) that the orography
 # filtering executable will read in.