Separate tracker and genesis jobs from vrfy job (NOAA-EMC#2058)

This PR creates new individual jobs for the tracker and genesis jobs that are being
pulled out of the `vrfy` job (issue NOAA-EMC#235). The `tracker` and `genesis` jobs will be on
by default while the `genesis_fsu` job will be off by default. This mimics the current
settings in the `vrfy` job. 

No enhancements or upgrades are made to the jobs themselves in this PR, only to
separate them from the `vrfy` job. However, one change compared to how they
currently run in the `vrfy` job is to no longer run the `tracker` in the gdas suite. This
mimics how the `ens_tracker` package is run in operations (gfs only).

Changes in this PR:

1. Create rocoto job scripts for tracker/genesis jobs.
2. Create configs for tracker/genesis jobs, as well as `config.tropcy` for shared
settings.
3. Add tracker/genesis jobs to `config.resources`; use resources from the ecf scripts
for `ens_tracker` jobs in operations
4. Add tracker/genesis jobs to rocoto setup for both cycled and forecast-only
modes.
5. Update arguments for `jjob_header.sh` script in tracker/genesis JJOB scripts to use
specific job names and configs instead of vrfy.
6. Rename `tracker_ver` to `ens_tracker_ver` in version files to mimic variable within
tracker package.
7. Rename and move tracker/genesis switches to `config.base`.
8. Remove tracker/genesis jobs from `vrfy` job (`jobs/rocoto/vrfy` and `config.vrfy`).
9. Reduce vrfy job resources to accommodate tasks having been removed.

Refs NOAA-EMC#235
Resolves NOAA-EMC#1988
Resolves NOAA-EMC#2048

jobs/JGFS_ATMOS_CYCLONE_GENESIS

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "genesis" -c "base genesis"
 
 
 ##############################################

jobs/JGFS_ATMOS_CYCLONE_TRACKER

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "tracker" -c "base tracker"
 
 
 export COMPONENT="atmos"

jobs/JGFS_ATMOS_FSU_GENESIS

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "genesis_fsu" -c "base genesis_fsu"
 
 export COMPONENT="atmos"
 

jobs/rocoto/genesis.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+export job="genesis"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+
+"${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_GENESIS"
+status=$?
+
+exit "${status}"

jobs/rocoto/genesis_fsu.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+export job="genesis_fsu"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+
+"${HOMEgfs}/jobs/JGFS_ATMOS_FSU_GENESIS"
+status=$?
+
+exit "${status}"

jobs/rocoto/tracker.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+export job="tracker"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+
+"${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_TRACKER"
+status=$?
+
+exit "${status}"

jobs/rocoto/vrfy.sh

@@ -52,34 +52,6 @@ if [[ "${RUNMOS}" == "YES" && "${CDUMP}" == "gfs" ]]; then
 fi
 
 
-################################################################################
-echo
-echo "=============== START TO RUN CYCLONE TRACK VERIFICATION ==============="
-if [[ ${VRFYTRAK} = "YES" ]]; then
-
-    COMINsyn=${COMINsyn:-$(compath.py "${envir}/com/gfs/${gfs_ver}")/syndat}
-    export COMINsyn
-
-    ${TRACKERSH}
-fi
-
-
-################################################################################
-echo
-echo "=============== START TO RUN CYCLONE GENESIS VERIFICATION ==============="
-if [[ ${VRFYGENESIS} = "YES" && "${CDUMP}" = "gfs" ]]; then
-    ${GENESISSH}
-fi
-
-
-################################################################################
-echo
-echo "=============== START TO RUN CYCLONE GENESIS VERIFICATION (FSU) ==============="
-if [[ ${VRFYFSU} = "YES" && "${CDUMP}" = "gfs" ]]; then
-    ${GENESISFSU}
-fi
-
-
 ###############################################################
 # Force Exit out cleanly
 cd "${DATAROOT}"

parm/config/gfs/config.base.emc.dyn

@@ -57,6 +57,9 @@ export DO_BUFRSND="NO"     # BUFR sounding products
 export DO_GEMPAK="NO"      # GEMPAK products
 export DO_AWIPS="NO"       # AWIPS products
 export DO_VRFY="YES"       # VRFY step
+export DO_TRACKER="YES"    # Hurricane track verification
+export DO_GENESIS="YES"    # Cyclone genesis verification
+export DO_GENESIS_FSU="NO" # Cyclone genesis verification (FSU)
 export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
 export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
 export DO_VMINMON="YES"    # GSI minimization monitoring

parm/config/gfs/config.genesis

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+########## config.genesis ##########
+echo "BEGIN: config.genesis"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" genesis
+
+# Get tropcy settings
+. "${EXPDIR}/config.tropcy"
+
+echo "END: config.genesis"

parm/config/gfs/config.genesis_fsu

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+########## config.genesis_fsu ##########
+echo "BEGIN: config.genesis_fsu"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" genesis_fsu
+
+# Get tropcy settings
+. "${EXPDIR}/config.tropcy"
+
+echo "END: config.genesis_fsu"

parm/config/gfs/config.resources

@@ -15,6 +15,7 @@ if [[ $# -ne 1 ]]; then
     echo "landanl"
     echo "aeroanlinit aeroanlrun aeroanlfinal"
     echo "anal sfcanl analcalc analdiag fcst post echgres"
+    echo "tracker genesis genesis_fsu"
     echo "verfozn verfrad vminmon vrfy fit2obs metp arch cleanup"
     echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
     echo "init_chem mom6ic ocnpost"
@@ -749,19 +750,39 @@ elif [[ ${step} = "vminmon" ]]; then
     export npe_node_vminmon_gfs=1
     export memory_vminmon="1G"
 
+elif [[ ${step} = "tracker" ]]; then
+
+    export wtime_tracker="00:10:00"
+    export npe_tracker=1
+    export nth_tracker=1
+    export npe_node_tracker=1
+    export memory_tracker="4G"
+
+elif [[ ${step} = "genesis" ]]; then
+
+    export wtime_genesis="00:25:00"
+    export npe_genesis=1
+    export nth_genesis=1
+    export npe_node_genesis=1
+    export memory_genesis="4G"
+
+elif [[ ${step} = "genesis_fsu" ]]; then
+
+    export wtime_genesis_fsu="00:10:00"
+    export npe_genesis_fsu=1
+    export nth_genesis_fsu=1
+    export npe_node_genesis_fsu=1
+    export memory_genesis_fsu="4G"
+
 elif [[ ${step} = "vrfy" ]]; then
 
-    export wtime_vrfy="03:00:00"
-    export wtime_vrfy_gfs="06:00:00"
-    export npe_vrfy=3
+    export wtime_vrfy="00:10:00"
+    export wtime_vrfy_gfs="00:10:00"
+    export npe_vrfy=1
     export nth_vrfy=1
     export npe_node_vrfy=1
     export npe_vrfy_gfs=1
     export npe_node_vrfy_gfs=1
-    if [[ ${machine} == "HERA" ]]; then
-      export memory_vrfy="16384M"
-    fi
-    export is_exclusive=True
 
 elif [[ "${step}" = "fit2obs" ]]; then
 

parm/config/gfs/config.tracker

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+########## config.tracker ##########
+echo "BEGIN: config.tracker"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" tracker
+
+# Get tropcy settings
+. "${EXPDIR}/config.tropcy"
+
+echo "END: config.tracker"

parm/config/gfs/config.tropcy

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+########## config.tropcy ##########
+echo "BEGIN: config.tropcy"
+
+# Tracker/genesis package location
+export HOMEens_tracker=${BASE_GIT}/TC_tracker/${ens_tracker_ver}
+
+export SENDCOM="YES" # Needed by tracker scripts still
+
+export FHOUT_CYCLONE=6
+FHMAX_CYCLONE=$(( FHMAX_GFS<240 ? FHMAX_GFS : 240 ))
+export FHMAX_CYCLONE
+
+echo "END: config.tropcy"

parm/config/gfs/config.vrfy

@@ -10,45 +10,8 @@ echo "BEGIN: config.vrfy"
 
 export CDFNL="gdas"               # Scores verification against GDAS/GFS analysis
 export MKPGB4PRCP="YES"           # Make 0.25-deg pgb files in ARCDIR for precip verification
-export VRFYTRAK="YES"             # Hurricane track verification
-export VRFYGENESIS="YES"          # Cyclone genesis verification
-export VRFYFSU="NO"               # Cyclone genesis verification (FSU)
 export RUNMOS="NO"                # whether to run entire MOS package
 
-#-------------------------------------------------
-# Cyclone genesis and cyclone track verification
-#-------------------------------------------------
-
-export SENDCOM="YES" # Needed by tracker/genesis scripts still
-
-export HOMEens_tracker=$BASE_GIT/TC_tracker/${tracker_ver}
-
-if [[ "${VRFYTRAK}" = "YES" ]]; then
-
-    export TRACKERSH="${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_TRACKER"
-    COMINsyn=${COMINsyn:-$(compath.py "${envir}"/com/gfs/"${gfs_ver}")/syndat}
-    export COMINsyn
-    if [[ "${RUN}" = "gdas" ]]; then
-        export FHOUT_CYCLONE=3
-        export FHMAX_CYCLONE=${FHMAX}
-    else
-        export FHOUT_CYCLONE=6
-        FHMAX_CYCLONE=$(( FHMAX_GFS<240 ? FHMAX_GFS : 240 ))
-        export FHMAX_CYCLONE
-    fi
-fi
-
-
-if [[ "${VRFYGENESIS}" == "YES" && "${RUN}" == "gfs" ]]; then
-
-    export GENESISSH="${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_GENESIS"
-fi
-
-if [[ "${VRFYFSU}" == "YES" && "${RUN}" == "gfs" ]]; then
-
-    export GENESISFSU="${HOMEgfs}/jobs/JGFS_ATMOS_FSU_GENESIS"
-fi
-
 if [[ "${RUNMOS}" == "YES" && "${RUN}" == "gfs" ]]; then
 
     if [[ "${machine}" = "HERA" ]] ; then

versions/run.hera.ver

@@ -24,5 +24,5 @@ export wgrib2_ver=2.0.8
 export obsproc_run_ver=1.1.2
 export prepobs_run_ver=1.0.1
 
-export tracker_ver=feature-GFSv17_com_reorg
+export ens_tracker_ver=feature-GFSv17_com_reorg
 export fit2obs_ver=1.0.0

versions/run.jet.ver

@@ -26,5 +26,5 @@ export anaconda_ver=5.3.1
 export obsproc_run_ver=1.1.2
 export prepobs_run_ver=1.0.1
 
-export tracker_ver=feature-GFSv17_com_reorg
+export ens_tracker_ver=feature-GFSv17_com_reorg
 export fit2obs_ver=1.0.0

versions/run.orion.ver

@@ -23,5 +23,5 @@ export wgrib2_ver=2.0.8
 export obsproc_run_ver=1.1.2
 export prepobs_run_ver=1.0.1
 
-export tracker_ver=feature-GFSv17_com_reorg
+export ens_tracker_ver=feature-GFSv17_com_reorg
 export fit2obs_ver=1.0.0

versions/run.s4.ver

@@ -23,5 +23,5 @@ export wgrib2_ver=2.0.8
 export obsproc_run_ver=1.1.2
 export prepobs_run_ver=1.0.1
 
-export tracker_ver=feature-GFSv17_com_reorg
+export ens_tracker_ver=feature-GFSv17_com_reorg
 export fit2obs_ver=1.0.0

versions/run.wcoss2.ver

@@ -45,5 +45,5 @@ export wgrib2_ver=2.0.8
 export obsproc_run_ver=1.1.2
 export prepobs_run_ver=1.0.1
 
-export tracker_ver=feature-GFSv17_com_reorg
+export ens_tracker_ver=feature-GFSv17_com_reorg
 export fit2obs_ver=1.0.0

workflow/applications/applications.py

@@ -58,6 +58,9 @@ def __init__(self, conf: Configuration) -> None:
         self.do_verfozn = _base.get('DO_VERFOZN', True)
         self.do_verfrad = _base.get('DO_VERFRAD', True)
         self.do_vminmon = _base.get('DO_VMINMON', True)
+        self.do_tracker = _base.get('DO_TRACKER', True)
+        self.do_genesis = _base.get('DO_GENESIS', True)
+        self.do_genesis_fsu = _base.get('DO_GENESIS_FSU', False)
         self.do_metp = _base.get('DO_METP', False)
 
         self.do_hpssarch = _base.get('HPSSARCH', False)

workflow/applications/gfs_cycled.py

@@ -68,6 +68,15 @@ def _get_app_configs(self):
         if self.do_vminmon:
             configs += ['vminmon']
 
+        if self.do_tracker:
+            configs += ['tracker']
+
+        if self.do_genesis:
+            configs += ['genesis']
+
+        if self.do_genesis_fsu:
+            configs += ['genesis_fsu']
+
         if self.do_metp:
             configs += ['metp']
 
@@ -195,6 +204,15 @@ def get_task_names(self):
         if self.do_vminmon:
             gfs_tasks += ['vminmon']
 
+        if self.do_tracker:
+            gfs_tasks += ['tracker']
+
+        if self.do_genesis:
+            gfs_tasks += ['genesis']
+
+        if self.do_genesis_fsu:
+            gfs_tasks += ['genesis_fsu']
+
         if self.do_metp:
             gfs_tasks += ['metp']
 

workflow/applications/gfs_forecast_only.py

@@ -26,6 +26,15 @@ def _get_app_configs(self):
         if self.do_ocean or self.do_ice:
             configs += ['ocnpost']
 
+        if self.do_atm and self.do_tracker:
+            configs += ['tracker']
+
+        if self.do_atm and self.do_genesis:
+            configs += ['genesis']
+
+        if self.do_atm and self.do_genesis_fsu:
+            configs += ['genesis_fsu']
+
         if self.do_atm and self.do_metp:
             configs += ['metp']
 
@@ -85,6 +94,15 @@ def get_task_names(self):
         if self.do_atm:
             tasks += ['vrfy']
 
+        if self.do_atm and self.do_tracker:
+            tasks += ['tracker']
+
+        if self.do_atm and self.do_genesis:
+            tasks += ['genesis']
+
+        if self.do_atm and self.do_genesis_fsu:
+            tasks += ['genesis_fsu']
+
         if self.do_atm and self.do_metp:
             tasks += ['metp']