Add continuous aerosols support (#693)

Adds support for continuous aerosols when running with GOCART (using the forecast aerosol tracers from the previous cycle as initial tracer fields). This adds a new task to the workflow mesh in forecast-only mode. The new task does not run in the first cycle, though it will appear in rocotostat/rocotoviewer. In subsequent cycles, the task will launch as soon as the needed restart files from the previous cycle are available and gfs_init has completed. There is currently no mechanism to have aerosol_init run for the first cycle (for instance, continuing from a previous experiment); users would have to run the scripts off-line to add aerosols to the initial conditions.

To support the introduction of the necessary python scripts, a new python environment is added to provide necessary libraries. This virtual environment is temporarily being housed in personal space, but an issue has been opened with hpc-stack to add it to the standard stack installations. The introduction of miniconda also caused the conversion of all of the rocoto entry scripts (the ones rocoto calls to run the job) to bash. Most of them had been in korne.

To make sure the needed restart files are produced, config.fcst now makes sure the forecast cadence is included in the list of restart times. This may need more testing to ensure it works properly with other restart lists. As part of this, STEP_GFS was updated to properly determine the step from gfs_cyc instead of being a set value.

Closes #366
Fixes #516
Fixes #630

jobs/rocoto/aerosol_init.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+set -x
+
+###############################################################
+# Source FV3GFS workflow modules
+source $HOMEgfs/ush/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+# Source relevant configs
+configs="base aerosol_init"
+for config in $configs; do
+    source $EXPDIR/config.${config}
+    status=$?
+    [[ $status -ne 0 ]] && exit $status
+done
+
+###############################################################
+# Source machine runtime environment
+source $BASE_ENV/${machine}.env aerosol_init
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+$HOMEgfs/scripts/exgfs_aero_init_aerosol.py
+
+status=$?
+if [[ $status -ne 0 ]]; then
+    echo "FATAL ERROR: exgfs_chem_init_aerosol.py failed with error code $status"
+    exit $status
+fi
+
+##############################################################
+# Exit cleanly
+
+set +x
+exit 0

jobs/rocoto/anal.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/analcalc.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/analdiag.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/awips.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 ## Abstract:

jobs/rocoto/earc.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 ## Abstract:

jobs/rocoto/ecen.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/echgres.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/ediag.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/efcs.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/eobs.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/eomg.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/epos.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/esfc.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/eupd.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/fcst.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/gempak.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 ## Abstract:

jobs/rocoto/getic.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 ## Abstract:

jobs/rocoto/gldas.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/init.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 ## Abstract:

jobs/rocoto/metp.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 ## Abstract:

jobs/rocoto/ocnpost.sh

@@ -81,16 +81,8 @@ fi
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
-
-if [ $RUN_ENVIR = "nco" ]; then
-    export COMIN=${COMIN:-$ROTDIR/$RUN.$PDY/$cyc}
-    export COMOUT=${COMOUT:-$ROTDIR/$RUN.$PDY/$cyc}
-else
-    export COMIN="$ROTDIR/$CDUMP.$PDY/$cyc"
-    export COMOUT="$ROTDIR/$CDUMP.$PDY/$cyc"
-fi
-[[ ! -d $COMOUT/ocean ]] && mkdir -p $COMOUT/ocean
-[[ ! -d $COMOUT/ice ]] && mkdir -p $COMOUT/ice
+[[ ! -d $COMOUTocean ]] && mkdir -p $COMOUTocean
+[[ ! -d $COMOUTice ]] && mkdir -p $COMOUTice
 
 fhrlst=$(echo $FHRLST | sed -e 's/_/ /g; s/f/ /g; s/,/ /g')
 
@@ -121,31 +113,31 @@ for fhr in $fhrlst; do
 
 
     #break up ocn netcdf into multiple files:  
-    if [ -f $COMOUT/ocean/ocn_2D_$VDATE.$ENSMEM.$IDATE.nc ]; then 
-      echo "File $COMOUT/ocean/ocn_2D_$VDATE.$ENSMEM.$IDATE.nc already exists"
+    if [ -f $COMOUTocean/ocn_2D_$VDATE.$ENSMEM.$IDATE.nc ]; then 
+      echo "File $COMOUTocean/ocn_2D_$VDATE.$ENSMEM.$IDATE.nc already exists"
     else
-      ncks -x -v vo,uo,so,temp $COMOUT/ocean/ocn$VDATE.$ENSMEM.$IDATE.nc $COMOUT/ocean/ocn_2D_$VDATE.$ENSMEM.$IDATE.nc
+      ncks -x -v vo,uo,so,temp $COMOUTocean/ocn$VDATE.$ENSMEM.$IDATE.nc $COMOUTocean/ocn_2D_$VDATE.$ENSMEM.$IDATE.nc
       status=$?
       [[ $status -ne 0 ]] && exit $status
     fi 
-    if [ -f $COMOUT/ocean/ocn_3D_$VDATE.$ENSMEM.$IDATE.nc ]; then 
-       echo "File $COMOUT/ocean/ocn_3D_$VDATE.$ENSMEM.$IDATE.nc already exists" 
+    if [ -f $COMOUTocean/ocn_3D_$VDATE.$ENSMEM.$IDATE.nc ]; then 
+       echo "File $COMOUTocean/ocn_3D_$VDATE.$ENSMEM.$IDATE.nc already exists" 
     else 
-      ncks -x -v Heat_PmE,LW,LwLatSens,MLD_003,MLD_0125,SSH,SSS,SST,SSU,SSV,SW,cos_rot,ePBL,evap,fprec,frazil,latent,lprec,lrunoff,sensible,sin_rot,speed,taux,tauy,wet_c,wet_u,wet_v $COMOUT/ocean/ocn$VDATE.$ENSMEM.$IDATE.nc $COMOUT/ocean/ocn_3D_$VDATE.$ENSMEM.$IDATE.nc
+      ncks -x -v Heat_PmE,LW,LwLatSens,MLD_003,MLD_0125,SSH,SSS,SST,SSU,SSV,SW,cos_rot,ePBL,evap,fprec,frazil,latent,lprec,lrunoff,sensible,sin_rot,speed,taux,tauy,wet_c,wet_u,wet_v $COMOUTocean/ocn$VDATE.$ENSMEM.$IDATE.nc $COMOUTocean/ocn_3D_$VDATE.$ENSMEM.$IDATE.nc
       status=$?
       [[ $status -ne 0 ]] && exit $status
     fi 
-    if [ -f $COMOUT/ocean/ocn-temp-EQ_$VDATE.$ENSMEM.$IDATE.nc ]; then 
-       echo "File $COMOUT/ocean/ocn-temp-EQ_$VDATE.$ENSMEM.$IDATE.nc already exists" 
+    if [ -f $COMOUTocean/ocn-temp-EQ_$VDATE.$ENSMEM.$IDATE.nc ]; then 
+       echo "File $COMOUTocean/ocn-temp-EQ_$VDATE.$ENSMEM.$IDATE.nc already exists" 
     else 
-      ncks -v temp -d yh,503 -d xh,-299.92,60.03 $COMOUT/ocean/ocn_3D_$VDATE.$ENSMEM.$IDATE.nc $COMOUT/ocean/ocn-temp-EQ_$VDATE.$ENSMEM.$IDATE.nc
+      ncks -v temp -d yh,503 -d xh,-299.92,60.03 $COMOUTocean/ocn_3D_$VDATE.$ENSMEM.$IDATE.nc $COMOUTocean/ocn-temp-EQ_$VDATE.$ENSMEM.$IDATE.nc
       status=$?
       [[ $status -ne 0 ]] && exit $status
     fi 
-    if [ -f $COMOUT/ocean/ocn-uo-EQ_$VDATE.$ENSMEM.$IDATE.nc ]; then 
-       echo "File $COMOUT/ocean/ocn-uo-EQ_$VDATE.$ENSMEM.$IDATE.nc already exists" 
+    if [ -f $COMOUTocean/ocn-uo-EQ_$VDATE.$ENSMEM.$IDATE.nc ]; then 
+       echo "File $COMOUTocean/ocn-uo-EQ_$VDATE.$ENSMEM.$IDATE.nc already exists" 
     else 
-      ncks -v uo -d yh,503 -d xh,-299.92,60.03 $COMOUT/ocean/ocn_3D_$VDATE.$ENSMEM.$IDATE.nc $COMOUT/ocean/ocn-uo-EQ_$VDATE.$ENSMEM.$IDATE.nc
+      ncks -v uo -d yh,503 -d xh,-299.92,60.03 $COMOUTocean/ocn_3D_$VDATE.$ENSMEM.$IDATE.nc $COMOUTocean/ocn-uo-EQ_$VDATE.$ENSMEM.$IDATE.nc
       status=$?
       [[ $status -ne 0 ]] && exit $status
     fi
@@ -155,7 +147,7 @@ done
 
 # Restore CDATE to what is expected
 export CDATE=$IDATE
-$NMV ocn_ice*.grb2 $COMOUT/ocean/
+$NMV ocn_ice*.grb2 $COMOUTocean/
 status=$?
 [[ $status -ne 0 ]] && exit $status
 

jobs/rocoto/post.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 ## NCEP post driver script

jobs/rocoto/postsnd.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/prep.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 # Source FV3GFS workflow modules

jobs/rocoto/vrfy.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 ## Abstract:

jobs/rocoto/wafs.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/wafsblending.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/wafsblending0p25.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/wafsgcip.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/wafsgrib2.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/wafsgrib20p25.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/waveawipsbulls.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/waveawipsgridded.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/wavegempak.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/waveinit.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/wavepostbndpnt.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/wavepostbndpntbll.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/wavepostpnt.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/wavepostsbs.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

jobs/rocoto/waveprep.sh

@@ -1,4 +1,4 @@
-#!/bin/ksh -x
+#!/bin/bash -x
 
 ###############################################################
 echo

modulefiles/module_base.hera.lua

@@ -41,9 +41,17 @@ load(pathJoin("w3nco", "2.4.1"))
 load(pathJoin("wgrib2", "2.0.8"))
 setenv("WGRIB2","wgrib2")
 
-append_path("MODULEPATH", "/contrib/anaconda/modulefiles")
-load(pathJoin("anaconda", "2.3.0"))
-
 load(pathJoin("cdo", "1.9.5"))
 
+load(pathJoin("R", "3.5.0"))
+
+-- Temporary until official hpc-stack is updated
+prepend_path("MODULEPATH", "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/modulefiles/stack")
+load(pathJoin("hpc", "1.2.0"))
+load(pathJoin("hpc-intel", "18.0.5.274"))
+load(pathJoin("hpc-miniconda3", "4.6.14"))
+load(pathJoin("ufswm", "1.0.0"))
+load(pathJoin("met", "9.1"))
+load(pathJoin("metplus", "3.1"))
+
 whatis("Description: GFS run environment")

modulefiles/module_base.orion.lua

@@ -44,8 +44,15 @@ load("contrib")
 load(pathJoin("rocoto", "1.3.3"))
 load(pathJoin("slurm", "19.05.3-2"))
 
-load(pathJoin("python", "3.7.5"))
-
 load(pathJoin("cdo", "1.9.5"))
 
+-- Temporary until official hpc-stack is updated
+prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefiles/stack")
+load(pathJoin("hpc", "1.2.0"))
+load(pathJoin("hpc-intel", "2018.4"))
+load(pathJoin("hpc-miniconda3", "4.6.14"))
+load(pathJoin("ufswm", "1.0.0"))
+load(pathJoin("met", "9.1"))
+load(pathJoin("metplus", "3.1"))
+
 whatis("Description: GFS run environment")