run_gfsmos_master.sh.dell

#!/bin/sh
#######################################################################
# run_gfsmos.sh
#
#  History:             
#     03/29/13   Scallion      Created (Adapted from Eric Engle's 
#                              "master" scripts)
#     12/03/13   Scallion      Added ptype/wxgrid
#
#  Purpose:
#      To run the GFS-MOS operational suite
#
#######################################################################
set -x

if (( $# > 1 )); then
   echo "Incorrect number of arguments."
   echo "Syntax: $0 [PDYCYC (optional)]"
   echo "Exiting..."
   exit 1
fi

##export PATH=./:$PATH

#######################################################################
#  Only run on the dev machine
#######################################################################
#/u/Scott.Scallion/bin/run-on-dev.sh
#[[ $? -eq 1 ]] && exit 1

#######################################################################
#  Source bash_profile to run with proper modules on cron
#######################################################################
#elim . ~/.bash_profile 1> /dev/null 2>&1
#. ~/.bash_profile 

#######################################################################
#  Check the host to determine whether tide or gyre is prod
#######################################################################
#chkhost=`hostname | cut -c1`
if [[ $SITE == "GYRE" ]] || [[ $SITE == "SURGE" ]] || [[ $SITE == "VENUS" ]]; then
   gort="g"
elif [[ $SITE == "TIDE" ]] || [[ $SITE == "LUNA" ]] || [[ $SITE == "MARS" ]]; then
   gort="t"
fi

#######################################################################
#  Set global variables neede in the run script and/or each individual
#  job script.
#######################################################################
. $MODULESHOME/init/bash 
#module purge                  2>/dev/null
module load EnvVars/1.0.2     2>/dev/null
module load ips/18.0.1.163    2>/dev/null
module load impi/18.0.1       2>/dev/null
module load lsf/10.1          2>/dev/null
module load prod_envir/1.0.3  2>/dev/null
module load prod_util/1.1.4   2>/dev/null
module load CFP/2.0.1         2>/dev/null
 
module use -a /gpfs/dell1/nco/ops/nwpara/modulefiles/compiler_prod/ips/18.0.1
module load grib_util/1.1.1   2>/dev/null

export FORT_BUFFERED=TRUE
export KMP_AFFINITY=disabled
export envir=prod
export RUN_ENVIR=${RUN_ENVIR:-""}
#export QUEUE=dev

#------------------
export SENDCOM=YES
export SENDCOM_SAVE=$SENDCOM
#------------------

#--------------------------------
# COMDATEROOT defined by module prod_util
##export PTMPROOT=/gpfs/dell2/ptmp/$USER
##export STMPROOT=/gpfs/dell2/stmp/$USER
##export MODELROOT=/gpfs/dell2/mdl/mdlstat/noscrub/usr/$USER/nwprod
##export MODELDIR=$MODELROOT/gfsmos.v5.0.6
##export CODEDIR=$MODELROOT/mos_shared.v2.6.1

export PTMPROOT=$ROTDIR/gfsmos.$PDY
export STMPROOT=$RUNDIR/gfsmos.$PDY
export DATAROOT=$STMPROOT
##export MODELROOT=/gpfs/dell2/emc/modeling/noscrub/emc.glopara/git/gfsmos
export MODELROOT=/gpfs/dell2/mdl/mdlstat/noscrub/usr/Scott.Scallion/gfsv16
#export MODELDIR=$MODELROOT/gfsmos.v5.2.0.1
export MODELDIR=$MODELROOT/gfsmos.v5.2.0.1-restructure
export CODEDIR=/gpfs/dell1/nco/ops/nwpara/mos_shared.v2.6.5
#--------------------------------

#--------------------------------
##export PDY=20180419
##export PDY=`date -u +%Y%m%d`
##export prevday=`date -u --date="${PDY} 0000 UTC 24 hours ago" +%Y%m%d`

dateIn=$1
#if [ $REALTIME = "YES" ]; then
#    GDATE=`$NDATE -24 $dateIn`
#    dateIn=$GDATE
#fi
export PDY=`echo $dateIn | cut -c 1-8`
export cyc=`echo $dateIn | cut -c 9-10`
export prevday=`$NDATE -24 ${PDY}00 | cut -c1-8`
#--------------------------------


#--------------------------------
##let hour=`date -u +"%-H"`
##if [[ $hour -ge 4 && $hour -lt 10 ]]; then
##   export cyc=00
##   export range=both
##elif [[ $hour -ge 10 && $hour -lt 16 ]]; then
##   export cyc=06
##   export range=both
##elif [[ $hour -ge 16 && $hour -lt 22 ]]; then
##   export cyc=12
##   export range=both
##elif [[ $hour -ge 22 && $hour -le 23 ]]; then
##   export cyc=18
##   export range=both
##else
##   export cyc=18
##   export PDY=$prevday
##   export range=both
##fi
##
##cyc_list="00 06 12 18"
##if [[ $# == 1 ]] && [[ $cyc_list =~ $1 ]]; then
##   export cyc=$1
##   if [ "$cyc" == "00" -o "$cyc" == "12" ]; then
##      export range=both
##   else
##      export range=both
##   fi
##elif [[ $# == 1 ]]; then
##   echo "$1 is not a valid cycle (choose 00 or 12)" 
##   exit 1
##fi

# ERIC ENGLE 05/30/2018 - PATCH FOR FV3GFS WINTER 17/18 PARALLEL
export range=${range:-"both"}

#if [ $cyc -eq 00 -o $cyc -eq 12 ]; then
# if [ $cyc -eq 00 ]; then
#    export range=both
# else
#    export range=short
# fi
# ERIC ENGLE 05/30/2018 - PATCH FOR FV3GFS WINTER 17/18 PARALLEL
export stnonly='Y'
export skipmodel=n
export skipprep=n
export cycle="t${cyc}z"
export pid="gfs_qprod.$$"
export dailylog=$PTMPROOT/dailylog/log.$PDY
export jlogfile=$dailylog/jlogfile_gfsmos
mkdir -p $dailylog

export SENDDBN=NO
export SENDDBN_NTC=NO
export GET_IOPROFILE=NO

# Specify Execution Areas
export HOMEmdl=$MODELDIR
export HOMEcode=$CODEDIR
#export utilscript=/gpfs/hps/mdl/mdlstat/noscrub/usr/Scott.Scallion/ush

if [ ! -d $HOMEmdl ]; then
   echo  "$HOMEmdl does not exist"
   exit 1
fi

if [ ! -d $HOMEcode ]; then
   echo  "$HOMEcode does not exist"
   exit 1
fi


#-------------------------------------
# Define COMOUT (COMIN will vary by job)
#export GFSDIR=$COMROOThps/gfs/prod/gfs.$PDY
##export GFSDIR=$COMROOThps/gfs/prod/gfs.$PDY
##export COMINgfs=$GFSDIR
##export COMOUT=$PTMPROOT/qprod/gfsmos.$PDY

export GFSDIR=$COMROOT/gfs/prod/gfs.${PDY}
if [[ "$RUN_ENVIR" = "emc" ]] ; then
    export GFSDIR=$ROTDIR/gfs.${PDY}
fi
export COMINgfs=$GFSDIR
export COMOUT=$ROTDIR/gfsmos.$PDY

#export COMINm1=$PTMPROOT/gfsmos.$prevday
#export COMINm1=$PTMPROOT/qprod/gfsmos_gmos_pre-nbmv2.$prevday
#export COMINm1=$PTMPROOT/qprod/gfsmos.$prevday

if [[ ! -d $PTMPROOT/qprod ]]; then
   mkdir -p $PTMPROOT/qprod
fi

if [[ ! -d $COMOUT ]]; then
   mkdir -p $COMOUT
fi

export COMOUTwmo=$PTMPROOT/wmo

if [[ ! -d $COMOUTwmo ]]; then
   mkdir -p $COMOUTwmo
fi


# NOTE: On WCOSS_DELL_P3 the directory from which bsub
# is executed must exist when the submitted job
# begins.  Otherwise, the submitted job fails
# with TERM_CWD_NOTEXIST error.

mkdir -p $DATAROOT
cd $DATAROOT


if [ "$range" == "short" -o "$range" == "both" ]; then
########################################################################
########################################################################
#             SHORT-RANGE JOBS
########################################################################
########################################################################

######################################################################## 
#  Wait for 1 degree model data
######################################################################## 
##if [ "$skipmodel" != "y" ]; then
##let attempts=1
##while [[ $attempts -le 120 ]]
##do
##   if [[ -f $GFSDIR/gfs.$cycle.pgrb2.1p00.f096 ]]; then
##      echo "Model file found.  Proceeding..."
##      break
##   else
##      if [[ $attempts -le 60 ]]; then
##         sleep 60
##      else
##         sleep 180 
##      fi  
##      attempts=$((attempts+1))
##   fi  
##done
##
##if [[ $attempts -gt 120 ]]; then
##   echo "Waited 4 hours for model file, but it did not appear."
##   echo "Exiting..."
##   exit 1
##fi
##
##fi #endif for skipmodel

######################################################################## 
#  Wait for 1/4 degree model data
######################################################################## 
##if [ "$skipmodel" != "y" ]; then
##let attempts=1
##while [[ $attempts -le 120 ]]
##do
##   if [[ -f $GFSDIR/gfs.$cycle.pgrb2.0p25.f096 ]]; then
##      echo "Model file found.  Proceeding..."
##      break
##   else
##      if [[ $attempts -le 60 ]]; then
##         sleep 60
##      else
##         sleep 180
##      fi
##      attempts=$((attempts+1))
##   fi
##done

##if [[ $attempts -gt 120 ]]; then
##   echo "Waited 4 hours for model file, but it did not appear."
##   echo "Exiting..."
##   exit 1
##fi
##
##fi #endif for skipmodel

######################################################################## 
#  Wait for 1.0 degree GFS model files before running (Pacific MOS)
######################################################################## 
##if [ "$skipmodel" != "y" ]; then
##let attempts=1
##while [[ $attempts -le 120 ]]
##do
##   if [[ -f $GFSDIR/gfs.$cycle.pgrb2.1p00.f096 ]]; then
##      echo "1.0 degree model file found.  Proceeding..."
##      break
##   else
##      if [[ $attempts -le 60 ]]; then
##         sleep 60
##      else
##         sleep 180
##      fi
##      attempts=$((attempts+1))
##   fi
##done
##
##if [[ $attempts -gt 120 ]]; then
##   echo "Waited 4 hours for model file, but it did not appear."
##   echo "Exiting..."
##   exit 1
##fi
##
##fi #endif for skipmodel


######################################################################## 
#  JGFSMOS_PREP47
######################################################################## 
if [[ "$skipprep" != "y" ]]; then
   export job=gfsmos_prep_${cyc}_${pid}
   export COMIN=$GFSDIR
   jobCard=$HOMEmdl/jobs/JGFSMOS_STN_PREP
#    Define DATA and create directory
   export DATA=$STMPROOT/qprod/gfsmos_prep_${cyc}
   export logfile=$dailylog/$job.out
   export out_dir=$dailylog

   export NTASK=3
   export PTILE=3
   export OMP_NUM_THREADS=1

   bsub -J $job \
        -o $logfile \
        -q $QUEUE \
        -x \
        -n $NTASK \
        -R "span[ptile=$PTILE]" \
        -W 0:25 \
        -P $ACCOUNT \
        $jobCard

fi #end for skipprep

######################################################################## 
#  JGFSMOS_FORECAST
######################################################################## 
if [[ ! -d /gpfs/dell1/nco/ops/com/mos/prod/hry_mos.$PDY ]]; then
   export ROTATE=/gpfs/dell2/mdl/mdlstat/noscrub/rotate
   export COMINhry_mos=$PTMPROOT/hourly.$PDY
   if [[ ! -d $COMINhry_mos ]]; then
      mkdir -p $COMINhry_mos
   fi
   \cp $ROTATE/hry/${PDY}03 $COMINhry_mos/sfctbl.03
   \cp $ROTATE/hry/${PDY}09 $COMINhry_mos/sfctbl.09
   \cp $ROTATE/hry/${PDY}15 $COMINhry_mos/sfctbl.15
   \cp $ROTATE/hry/${PDY}21 $COMINhry_mos/sfctbl.21
fi

# Change COMIN to get files from user's PTMP "qprod" area 
export COMIN=$COMOUT

export job=gfsmos_fcst_${cyc}_${pid}
jobCard=$HOMEmdl/jobs/JGFSMOS_STN_FORECAST
export DATA=$STMPROOT/qprod/gfsmos_fcst_${cyc}
export logfile=$dailylog/$job.out
export out_dir=$dailylog
# Set dependencies
if [[ "$skipprep" != "y" ]]; then
   deps="done(gfsmos_prep_${cyc}_${pid})"
else
   deps=""
fi
if [[ $cyc == "00" || $cyc == "12" ]] && [[ "$stnonly" != "Y" ]]; then
   complist="metar pac cooprfcmeso goe higoe akgoe tstms"
   complist2="copopo3 coptype akpopo3 akptype"
else
   complist="metar cooprfcmeso tstms"
   complist2=""
fi

if [[ $cyc == "00" || $cyc == "12" ]] && [[ "$stnonly" != "Y" ]]; then
   export NTASK=11
   export PTILE=1
   export OMP_NUM_THREADS=1
elif [[ $cyc == "00" || $cyc == "12" ]] && [[ "$stnonly" == "Y" ]]; then
   export NTASK=5
   export PTILE=5
   export OMP_NUM_THREADS=1
else
   export NTASK=4
   export PTILE=4
   export OMP_NUM_THREADS=1
fi

bsub -J ${job} \
     -o $logfile \
     -q ${QUEUE}  \
     -x \
     -n $NTASK \
     -R "span[ptile=$PTILE]" \
     -W 0:20 \
     -P $ACCOUNT \
     -w "$deps" \
     $jobCard

######################################################################## 
#     JGFSMOS_PRDGEN
######################################################################## 
jobCard=$HOMEmdl/jobs/JGFSMOS_STN_PRDGEN
export job=gfsmos_prdgen_${cyc}_${pid}
# Change COMIN back to COMOUT
export COMIN=$COMOUT
# Define DATA and create directory
export DATA=$STMPROOT/qprod/gfsmos_prdgen_${cyc}
export logfile=$dailylog/$job.out
export out_dir=$dailylog
# Set dependencies
deps="done(gfsmos_fcst_${cyc}_${pid})"
# Set Nodes
if [ $cyc -eq 00 -o $cyc -eq 12 ] && [[ "$stnonly" != "Y" ]]; then
   #nodes='1*{select[craylinux && !vnode]} + 168*{select[craylinux && vnode]span[ptile=24]}'
   export NTASK=1
   export PTILE=1
   export OMP_NUM_THREADS=20
else
   #nodes='1*{select[craylinux && !vnode]} + 24*{select[craylinux && vnode]span[ptile=24]}'
   export NTASK=1
   export PTILE=1
   export OMP_NUM_THREADS=1
fi

bsub -J ${job} \
     -o $logfile \
     -q ${QUEUE}  \
     -x \
     -n $NTASK \
     -R "span[ptile=$PTILE]" \
     -W 0:30 \
     -P $ACCOUNT \
     -w "$deps" \
     $jobCard

########################################################################
#   JGFSMOS_WX_PRDGEN (00z and 12z only)
########################################################################
#if [ $cyc -eq 00 -o $cyc -eq 12 ]; then
#   jobCard=$HOMEmdl/jobs/JGFSMOS_WX_PRDGEN
#   export job=gfsmos_wx_prdgen_${cyc}_${pid}
#   # Change COMIN back to COMOUT
#   export COMIN=$COMOUT
#   # Define DATA and create directory
#   export DATA=$STMPROOT/qprod/gfsmos_wx_prdgen_${cyc}
#   export logfile=$dailylog/$job.out
#   export out_dir=$dailylog
#   # Set dependencies
#   deps="done(gfsmos_prdgen_${cyc}_${pid})"
#
#   export NTASK=2
#   export PTILE=1
#   export OMP_NUM_THREADS=20
#
#   #bsub -J ${job} -oo $logfile -q ${QUEUE} -P MDLST-T2O \
#   #     -W 1:00 -M 1000 \
#   #     -extsched 'CRAYLINUX[]' \
#   #     -R '1*{select[craylinux && !vnode]} + 48*{select[craylinux && vnode]span[ptile=24]}' \
#   #     -w "$deps" \
#   #     $jobCard
#
#   bsub -J ${job} \
#        -o $logfile \
#        -q ${QUEUE} -x \
#        -n $NTASK \
#        -R "span[ptile=$PTILE]" \
#        -W 1:00 \
#        -M 3000 \
#        -P $ACCOUNT \
#        -w "$deps" \
#        $jobCard
#fi

fi #endif short

########################################################################
########################################################################
#             EXTENDED-RANGE JOBS
########################################################################
########################################################################

if [ "$range" == "ext" -o "$range" == "both" ]; then

######################################################################## 
#  Wait for 1/4 degree model data
######################################################################## 
#if [ "$skipmodel" != "y" ]; then
#let attempts=1
#proj_list=`seq -f %03g 252 12 384`
#for tau in $proj_list
#do
#   while [[ $attempts -le 120 ]]
#   do
#      if [[ -f $GFSDIR/gfs.$cycle.pgrb2.0p25.f${tau} && -f $GFSDIR/gfs.$cycle.pgrb2.0p50.f${tau} ]]; then
#         echo "Model file found.  Proceeding to next..."
#         break
#      else
#         if [[ $attempts -le 60 ]]; then
#            sleep 60
#         else
#            sleep 180
#         fi
#         attempts=$((attempts+1))
#      fi
#   done
#done
#
#if [[ $attempts -gt 120 ]]; then
#   echo "Waited 4 hours for model file, but it did not appear."
#   echo "Exiting..."
#   exit 1
#fi

#fi #endif for skipmodel

######################################################################## 
#  Wait for 1.0/2.5 degree GFS model files before running (Pacific GFS)
######################################################################## 
#if [ "$skipmodel" != "y" ]; then
#let attempts1deg=1
#proj_list=`seq -f %03g 204 12 384`
#for tau in $proj_list
#do
#   while [[ $attempts1deg -le 120 ]]
#   do
## ERIC ENGLE 05/30/2018 - PATCH FOR FV3GFS WINTER 17/18 PARALLEL
#      #if [[ -f $GFSDIR/gfs.$cycle.pgrb2.1p00.f384 && -f $GFSDIR/gfs.$cycle.pgrb2.2p50.f240 ]]; then
#      if [ $cyc  -eq 00 ]; then
#         waitfile=$GFSDIR/gfs.$cycle.pgrb2.1p00.f384
#      else
#         waitfile=$GFSDIR/gfs.$cycle.pgrb2.1p00.f180
#      fi
#      if [[ -f $waitfile ]]; then
## ERIC ENGLE 05/30/2018 - PATCH FOR FV3GFS WINTER 17/18 PARALLEL
#         #echo "1.0/2.5 degree model files found.  Proceeding to next..."
#         echo "1.0 degree model files found.  Proceeding to next..."
#         break
#      else
#         if [[ $attempts1deg -le 60 ]]; then
#            sleep 60
#         else
#            sleep 180
#         fi
#         attempts1deg=$((aattempts1deg+1))
#      fi
#   done
#done
#
#if [[ $attempts1deg -gt 120 ]]; then
#   echo "Waited 4 hours for 1.0 degree model file, but it did not appear."
#   echo "Exiting..."
#   exit 1
#fi

#fi #endif for skipmodel

########################################################################
#  JGFSMOS_EXT_PREP47
########################################################################
if [[ "$skipprep" != "y" ]]; then
   export job=gfsmos_extprep_${cyc}_${pid}
   export COMIN=$GFSDIR
   jobCard=$HOMEmdl/jobs/JGFSMOS_EXT_STN_PREP
#    Define DATA and create directory
   export DATA=$STMPROOT/qprod/gfsmos_extprep_${cyc}
   export logfile=$dailylog/$job.out
   export out_dir=$dailylog

   export NTASK=2
   export PTILE=2
   export OMP_NUM_THREADS=1

#   bsub -J $job \
#        -o $logfile \
#        -q $QUEUE \
#        -W 2:00 \
#        -M 2500 \
#        -P MDLST-T2O \
#        -extsched 'CRAYLINUX[]' \
#        -R '1*{select[craylinux && !vnode]} + 24*{select[craylinux && vnode]span[ptile=24]}' \
#        $jobCard

bsub -J ${job} \
     -o $logfile \
     -q ${QUEUE}  \
     -n $NTASK \
     -R "span[ptile=$PTILE]" \
     -W 2:00 \
     -P $ACCOUNT \
     $jobCard

fi #end for skipprep

#  Skip EXT_FORECAST for 06/18
if [ $cyc -eq 00 -o $cyc -eq 12 ]; then
######################################################################## 
#  JGFSMOS_EXT_FORECAST
######################################################################## 
if [[ ! -d /gpfs/dell1/nco/ops/com/mos/prod/hry_mos.$PDY ]]; then
   export ROTATE=/gpfs/dell2/mdl/mdlstat/noscrub/rotate
   export COMINhry_mos=$PTMPROOT/hourly.$PDY
   if [[ ! -d $COMINhry_mos ]]; then
      mkdir -p $COMINhry_mos
   fi  
   \cp $ROTATE/hry/${PDY}03 $COMINhry_mos/sfctbl.03
   \cp $ROTATE/hry/${PDY}09 $COMINhry_mos/sfctbl.09
   \cp $ROTATE/hry/${PDY}15 $COMINhry_mos/sfctbl.15
   \cp $ROTATE/hry/${PDY}21 $COMINhry_mos/sfctbl.21
fi

# Change COMIN to get files from user's PTMP "qprod" area 
export COMIN=$COMOUT

export job=gfsmos_extfcst_${cyc}_${pid}
jobCard=$HOMEmdl/jobs/JGFSMOS_EXT_STN_FORECAST
export DATA=$STMPROOT/qprod/gfsmos_extfcst_${cyc}
export logfile=$dailylog/$job.out
export out_dir=$dailylog
# Set dependencies
if [[ "$skipprep" != "y" && "$range" == "both" ]]; then
   deps="done(gfsmos_extprep_${cyc}_${pid}) && done(gfsmos_fcst_${cyc}_${pid})"
elif [[ "$skipprep" != "y" && "$range" == "ext" ]]; then
   deps="done(gfsmos_extprep_${cyc}_${pid})"
elif [[ "$skipprep" == "y" && "$range" == "ext" ]]; then
   deps=""
else
   deps="done(gfsmos_fcst_${cyc}_${pid})"
fi

if [[ $stnonly != "Y" ]]; then
   export NTASK=10
   export PTILE=1
   export OMP_NUM_THREADS=1
else
   export NTASK=3
   export PTILE=3
   export OMP_NUM_THREADS=1
fi

#bsub -J ${job} -oo $logfile -q $QUEUE -P MDLST-T2O \
#     -W 1:00 -M 2000 \
#     -extsched 'CRAYLINUX[]' \
#     -R '1*{select[craylinux && !vnode]} + 24*{select[craylinux && vnode]span[ptile=24]}' \
#     -w "$deps" \
#     $jobCard

bsub -J ${job} \
     -o $logfile \
     -q ${QUEUE} \
     -x \
     -n $NTASK \
     -R "span[ptile=$PTILE]" \
     -W 1:00 \
     -P $ACCOUNT \
     -w "$deps" \
     $jobCard

fi #endif for skipping 06/18 ext_fcst
######################################################################## 
#  JGFSMOS_EXT_PRDGEN
######################################################################## 
jobCard=$HOMEmdl/jobs/JGFSMOS_EXT_STN_PRDGEN
export job=gfsmos_extprdgen_${cyc}_${pid}
# Change COMIN back to COMOUT
export COMIN=$COMOUT
# Define DATA and create directory
export DATA=$STMPROOT/qprod/gfsmos_extprdgen_${cyc}
export logfile=$dailylog/$job.out
export out_dir=$dailylog
# Set dependencies
if [[ "$cyc" == "06" || "$cyc" == "18" ]]; then
   deps="done(gfsmos_prdgen_${cyc}_${pid})"
elif [[ "$range" == "both" ]]; then
   deps="done(gfsmos_extfcst_${cyc}_${pid}) && done(gfsmos_prdgen_${cyc}_${pid})"
else
   deps="done(gfsmos_extfcst_${cyc}_${pid})"
fi
# Set Nodes
if [ $cyc -eq 00 -o $cyc -eq 12 ] && [[ "$stnonly" != "Y" ]]; then
   #nodes='1*{select[craylinux && !vnode]} + 168*{select[craylinux && vnode]span[ptile=24]}'
   export NTASK=1
   export PTILE=1
   export OMP_NUM_THREADS=20
else
   #nodes='1*{select[craylinux && !vnode]} + 24*{select[craylinux && vnode]span[ptile=24]}'
   export NTASK=1
   export PTILE=1
   export OMP_NUM_THREADS=1
fi


#bsub -J ${job} -oo $logfile -q ${QUEUE} -P MDLST-T2O \
#     -W 1:00 -M 2000 \
#     -extsched 'CRAYLINUX[]' \
#     -R "$nodes" \
#     -w "$deps" \
#     $jobCard

bsub -J ${job} \
     -o $logfile \
     -q ${QUEUE} \
     -x \
     -n $NTASK \
     -R "span[ptile=$PTILE]" \
     -W 1:00 \
     -P $ACCOUNT \
     -w "$deps" \
     $jobCard

#  Exit here with $cyc is 06 or 18
if [ $cyc -eq 06 -o $cyc -eq 18 ]; then
   exit 0
fi
########################################################################
#   JGFSMOS_WX_EXT_PRDGEN
########################################################################
#jobCard=$HOMEmdl/jobs/JGFSMOS_WX_EXT_PRDGEN
#export job=gfsmos_wx_extprdgen_${cyc}_${pid}
## Change COMIN back to COMOUT
#export COMIN=$COMOUT
## Define DATA and create directory
#export DATA=$STMPROOT/qprod/gfsmos_wx_extprdgen_${cyc}
#export logfile=$dailylog/$job.out
#export out_dir=$dailylog
## Set dependencies
#if [[ "$range" == "both" ]]; then
#   deps="done(gfsmos_extprdgen_${cyc}_${pid}) && done(gfsmos_wx_prdgen_${cyc}_${pid})"
#else
#   deps="done(gfsmos_extprdgen_${cyc}_${pid})"
#fi
#
#export NTASK=1
#export PTILE=1
#export OMP_NUM_THREADS=20

##bsub -J ${job} -oo $logfile -q ${QUEUE} -P MDLST-T2O \
##     -W 1:00 -M 1000 \
##     -extsched 'CRAYLINUX[]' \
##     -R '1*{select[craylinux && !vnode]} + 48*{select[craylinux && vnode]span[ptile=24]}' \
##     -w "$deps" \
##     $jobCard

#bsub -J ${job} \
#     -o $logfile \
#     -q ${QUEUE} -x \
#     -n $NTASK \
#     -R "span[ptile=$PTILE]" \
#     -W 1:00 \
#     -M 3000 \
#     -P $ACCOUNT \
#     -w "$deps" \
#     $jobCard

fi #endif for ext/both

#---------------------
export SENDCOM=$SENDCOM_SAVE

exit 0