Modify crontab management, use config_defaults.sh.

tests/WE2E/run_WE2E_tests.sh

@@ -1281,7 +1281,7 @@ exist or is not a directory:
 #
 #-----------------------------------------------------------------------
 #
-  $ushdir/generate_FV3LAM_wflow.sh || \
+  $ushdir/generate_FV3LAM_wflow.py || \
     print_err_msg_exit "\
 Could not generate an experiment for the test specified by test_name:
   test_name = \"${test_name}\""

ush/config_defaults.sh

@@ -4,7 +4,7 @@
 # This file sets the experiment's configuration variables (which are
 # global shell variables) to their default values.  For many of these
 # variables, the valid values that they may take on are defined in the
-# file $USHDIR/valid_param_vals.sh.
+# file $USHDIR/valid_param_vals.py.
 #
 #-----------------------------------------------------------------------
 #
@@ -45,7 +45,7 @@ RUN_ENVIR="nco"
 # supported platform, and you want to use the Rocoto workflow manager,
 # you will need set MACHINE="linux" and WORKFLOW_MANAGER="rocoto". This
 # combination will assume a Slurm batch manager when generating the XML.
-# Please see ush/valid_param_vals.sh for a full list of supported
+# Please see ush/valid_param_vals.py for a full list of supported
 # platforms.
 #
 # MACHINE_FILE:
@@ -623,7 +623,7 @@ NDAS_OBS_DIR=""
 # conditions from the GFS started 6 hours earlier, then
 # EXTRN_MDL_LBCS_OFFSET_HRS=6.
 # Note: the default value is model-dependent and set in
-# set_extrn_mdl_params.sh
+# set_extrn_mdl_params.py
 #
 # FV3GFS_FILE_FMT_ICS:
 # If using the FV3GFS model as the source of the ICs (i.e. if EXTRN_MDL_NAME_ICS
@@ -1016,7 +1016,7 @@ GFDLgrid_USE_NUM_CELLS_IN_FILENAMES=""
 # in generating the files with 0-cell-, 3-cell-, and 4-cell-wide halos;
 # they are not needed by the forecast model.  
 # NOTE: Probably don't need to make ESGgrid_WIDE_HALO_WIDTH a user-specified 
-#       variable.  Just set it in the function set_gridparams_ESGgrid.sh.
+#       variable.  Just set it in the function set_gridparams_ESGgrid.py.
 #
 # Note that:
 #
@@ -1176,7 +1176,7 @@ WRTCMP_dy=""
 # commonly used set of grid-dependent parameters.  The predefined grid 
 # parameters are specified in the script 
 #
-#   $HOMErrfs/ush/set_predef_grid_params.sh
+#   $HOMErrfs/ush/set_predef_grid_params.py
 #
 #-----------------------------------------------------------------------
 #

ush/generate_FV3LAM_wflow.py

@@ -15,7 +15,7 @@
 
 from setup import setup
 from set_FV3nml_sfc_climo_filenames import set_FV3nml_sfc_climo_filenames
-from get_crontab_contents import get_crontab_contents
+from get_crontab_contents import add_crontab_line
 from fill_jinja_template import fill_jinja_template
 from set_namelist import set_namelist
 
@@ -449,77 +449,7 @@ def generate_FV3LAM_wflow():
     #-----------------------------------------------------------------------
     #
     if USE_CRON_TO_RELAUNCH:
-    #
-    # Make a backup copy of the user's crontab file and save it in a file.
-    #
-      time_stamp = datetime.now().strftime("%F_%T")
-      crontab_backup_fp=os.path.join(EXPTDIR,f"crontab.bak.{time_stamp}")
-      print_info_msg(f'''
-        Copying contents of user cron table to backup file:
-          crontab_backup_fp = \"{crontab_backup_fp}\"''',verbose=VERBOSE)
-
-      global called_from_cron
-      try: called_from_cron
-      except: called_from_cron = False
-
-      crontab_cmd,crontab_contents = get_crontab_contents(called_from_cron=called_from_cron)
-      # To create the backup crontab file and add a new job to the user's 
-      # existing cron table, use the "printf" command, not "echo", to print 
-      # out variables.  This is because "echo" will add a newline at the end 
-      # of its output even if its input argument is a null string, resulting
-      # in extranous blank lines in the backup crontab file and/or the cron 
-      # table itself.  Using "printf" prevents the appearance of these blank
-      # lines.
-      run_command(f'''printf "%s" '{crontab_contents}' > "{crontab_backup_fp}"''')
-      #
-      # Below, we use "grep" to determine whether the crontab line that the
-      # variable CRONTAB_LINE contains is already present in the cron table.
-      # For that purpose, we need to escape the asterisks in the string in
-      # CRONTAB_LINE with backslashes.  Do this next.
-      #
-      (_,crontab_line_esc_astr,_) = run_command(f'''printf "%s" '{CRONTAB_LINE}' | \
-                               {SED} -r -e "s%[*]%\\\\*%g"''')
-      # In the grep command below, the "^" at the beginning of the string 
-      # passed to grep is a start-of-line anchor, and the "$" at the end is
-      # an end-of-line anchor.  Thus, in order for grep to find a match on 
-      # any given line of the cron table's contents, that line must contain 
-      # exactly the string in the variable crontab_line_esc_astr without any 
-      # leading or trailing characters.  This is to eliminate situations in 
-      # which a line in the cron table contains the string in crontab_line_esc_astr 
-      # but is precedeeded, for example, by the comment character "#" (in which
-      # case cron ignores that line) and/or is followed by further commands 
-      # that are not part of the string in crontab_line_esc_astr (in which 
-      # case it does something more than the command portion of the string in 
-      # crontab_line_esc_astr does).
-      #
-      if MACHINE == "WCOSS_DELL_P3":
-        (exit_status,grep_output,_)=run_command(f'''grep '^{crontab_line_esc_astr}$' "/u/{USER}/cron/mycrontab"''')
-      else:
-        (exit_status,grep_output,_)=run_command(f'''printf "%s" '{crontab_contents}' | grep "^{crontab_line_esc_astr}$"''')
-
-      if exit_status == 0:
-
-        print_info_msg(f'''
-            The following line already exists in the cron table and thus will not be
-            added:
-              CRONTAB_LINE = \"{CRONTAB_LINE}\"''')
-
-      else:
-
-        print_info_msg(f'''
-            Adding the following line to the user's cron table in order to automatically
-            resubmit SRW workflow:
-              CRONTAB_LINE = \"{CRONTAB_LINE}\"''',verbose=VERBOSE)
-
-        if MACHINE == "WCOSS_DELL_P3":
-          run_command(f'''printf "%s\n" '{CRONTAB_LINE}' >> "/u/{USER}/cron/mycrontab"''')
-        else:
-          # Add a newline to the end of crontab_contents only if it is not empty.
-          # This is needed so that when CRONTAB_LINE is printed out, it appears on
-          # a separate line.
-          if crontab_contents:
-            crontab_contents += "\n"
-          run_command(f'''( printf "%s" '{crontab_contents}'; printf "%s\n" '{CRONTAB_LINE}' ) | {crontab_cmd}''')
+       add_crontab_line()
     #
     #-----------------------------------------------------------------------
     #

ush/get_crontab_contents.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
 import os
+import sys
 import unittest
+import argparse
 from datetime import datetime
 
 from python_utils import import_vars, set_env_var, print_input_args, \
-                         run_command, define_macos_utilities, check_var_valid_value
+                         run_command, define_macos_utilities, print_info_msg
 from constants import valid_vals_BOOLEAN
 
 def get_crontab_contents(called_from_cron):
@@ -36,15 +38,10 @@ def get_crontab_contents(called_from_cron):
     """
 
     print_input_args(locals())
-  
-    #import all env vars
+
+    #import selected env vars
     IMPORTS = ["MACHINE", "USER"]
     import_vars(env_vars=IMPORTS)
-
-    #
-    # Make sure called_from_cron is set to a valid value.
-    #
-    check_var_valid_value(called_from_cron, valid_vals_BOOLEAN)
 
     if MACHINE == "WCOSS_DELL_P3":
       __crontab_cmd__=""
@@ -69,10 +66,116 @@ def get_crontab_contents(called_from_cron):
     # copy of this header to the user's cron table.
     #
     if MACHINE == "CHEYENNE":
-      (_,__crontab_contents__,_)=run_command(f'''printf "%s" "{__crontab_contents__}" | tail -n +4 ''')
+      (_,__crontab_contents__,_)=run_command(f'''printf "%s" '{__crontab_contents__}' | tail -n +4 ''')
 
     return __crontab_cmd__, __crontab_contents__
 
+def add_crontab_line():
+    """ Add crontab line to cron table """
+
+    #import selected env vars
+    IMPORTS = ["MACHINE", "USER", "CRONTAB_LINE", "VERBOSE", "EXPTDIR"]
+    import_vars(env_vars=IMPORTS)
+
+    #
+    # Make a backup copy of the user's crontab file and save it in a file.
+    #
+    time_stamp = datetime.now().strftime("%F_%T")
+    crontab_backup_fp=os.path.join(EXPTDIR,f"crontab.bak.{time_stamp}")
+    print_info_msg(f'''
+      Copying contents of user cron table to backup file:
+        crontab_backup_fp = \"{crontab_backup_fp}\"''',verbose=VERBOSE)
+
+    global called_from_cron
+    try: called_from_cron
+    except: called_from_cron = False
+
+    # Get crontab contents
+    crontab_cmd,crontab_contents = get_crontab_contents(called_from_cron=called_from_cron)
+
+    # Create backup
+    run_command(f'''printf "%s" '{crontab_contents}' > "{crontab_backup_fp}"''')
+
+    # Add crontab line
+    if CRONTAB_LINE in crontab_contents:
+
+      print_info_msg(f'''
+          The following line already exists in the cron table and thus will not be
+          added:
+            CRONTAB_LINE = \"{CRONTAB_LINE}\"''')
+
+    else:
+
+      print_info_msg(f'''
+          Adding the following line to the user's cron table in order to automatically
+          resubmit SRW workflow:
+            CRONTAB_LINE = \"{CRONTAB_LINE}\"''',verbose=VERBOSE)
+
+      #add new line to crontab contents if it doesn't have one
+      NEWLINE_CHAR=""
+      if crontab_contents and crontab_contents[-1] != "\n": 
+        NEWLINE_CHAR="\n"
+
+      #add the crontab line
+      if MACHINE == "WCOSS_DELL_P3":
+        run_command(f'''printf "%b%s\n" '{NEWLINE_CHAR}' '{CRONTAB_LINE}' >> "/u/{USER}/cron/mycrontab"''')
+      else:
+        run_command(f'''printf "%s%b%s\n" '{crontab_contents}' '{NEWLINE_CHAR}' '{CRONTAB_LINE}' | {crontab_cmd}''')
+
+def delete_crontab_line(called_from_cron):
+    """ Delete crontab line after job is complete i.e. either SUCCESS/FAILURE
+    but not IN PROGRESS status"""
+
+    print_input_args(locals())
+
+    #import selected env vars
+    IMPORTS = ["MACHINE", "USER", "CRONTAB_LINE"]
+    import_vars(env_vars=IMPORTS)
+
+    #
+    # Get the full contents of the user's cron table.  
+    #
+    (crontab_cmd,crontab_contents) = get_crontab_contents(called_from_cron)
+    #
+    # Remove the line in the contents of the cron table corresponding to the 
+    # current forecast experiment (if that line is part of the contents).
+    # Then record the results back into the user's cron table.
+    #
+    if (CRONTAB_LINE + '\n') in crontab_contents:
+        crontab_contents = crontab_contents.replace(CRONTAB_LINE+'\n','')
+    else:
+        crontab_contents = crontab_contents.replace(CRONTAB_LINE,'')
+
+    if MACHINE == "WCOSS_DELL_P3":
+        run_command(f'''echo '{crontab_contents}' > "/u/{USER}/cron/mycrontab"''')
+    else:
+        run_command(f'''echo '{crontab_contents}' | {crontab_cmd}''')
+
+def parse_args(argv):
+    """ Parse command line arguments for deleting crontab line.
+    This is needed because it is called from shell script
+    """
+    parser = argparse.ArgumentParser(
+        description='Crontab job manupilation program.'
+    )
+
+    parser.add_argument('-d', '--delete',
+                        dest='delete',
+                        action='store_true',
+                        help='Delete crontab line.')
+
+    parser.add_argument('-c', '--called-from-cron',
+                        dest='called_from_cron',
+                        action='store_true',
+                        help='Called from cron.')
+
+    return parser.parse_args(argv)
+
+if __name__ == '__main__':
+    args = parse_args(sys.argv[1:])
+    if args.delete:
+        delete_crontab_line(args.called_from_cron)	
+
 class Testing(unittest.TestCase):
     def test_get_crontab_contents(self):
         crontab_cmd,crontab_contents = get_crontab_contents(called_from_cron=True)

ush/launch_FV3LAM_wflow.sh

@@ -99,7 +99,6 @@ fi
 . $USHDIR/constants.sh
 . $USHDIR/source_util_funcs.sh
 . $USHDIR/init_env.sh
-. $USHDIR/get_crontab_contents.sh
 #
 #-----------------------------------------------------------------------
 #
@@ -393,35 +392,14 @@ script for this experiment:
   CRONTAB_LINE = \"${CRONTAB_LINE}\"
 "
 #
-# Below, we use "grep" to determine whether the crontab line that the
-# variable CRONTAB_LINE contains is already present in the cron table.
-# For that purpose, we need to escape the asterisks in the string in
-# CRONTAB_LINE with backslashes.  Do this next.
+# Remove CRONTAB_LINE from cron table
 #
-    crontab_line_esc_astr=$( printf "%s" "${CRONTAB_LINE}" | \
-                             $SED -r -e "s%[*]%\\\\*%g" )
-#
-# Get the full contents of the user's cron table.  
-#
-    get_crontab_contents called_from_cron=${called_from_cron} \
-                         outvarname_crontab_cmd="crontab_cmd" \
-                         outvarname_crontab_contents="crontab_contents"
-#
-# Remove the line in the contents of the cron table corresponding to the 
-# current forecast experiment (if that line is part of the contents).
-# Then record the results back into the user's cron table.
-#
-# In the string passed to the grep command below, we use the line start
-# and line end anchors ("^" and "$", respectively) to ensure that we
-# only find lines in the crontab that contain exactly the string in 
-# crontab_line_esc_astr without any leading or trailing characters.
-#
-    crontab_contents=$( echo "${crontab_contents}" | grep -v "^${crontab_line_esc_astr}$" )
-
-    if [ "$MACHINE" = "WCOSS_DELL_P3" ]; then
-      echo "${crontab_contents}" > "/u/$USER/cron/mycrontab"
+    if [ "${called_from_cron}" = "TRUE" ]; then
+       MACHINE=$MACHINE CRONTAB_LINE=$CRONTAB_LINE \
+           python3 $USHDIR/get_crontab_contents.py --delete --called-from-cron
     else
-      echo "${crontab_contents}" | ${crontab_cmd} 
+       MACHINE=$MACHINE CRONTAB_LINE=$CRONTAB_LINE \
+           python3 $USHDIR/get_crontab_contents.py --delete
     fi
 
   fi

ush/python_utils/__init__.py

@@ -1,22 +1,25 @@
-from .misc import uppercase, lowercase, find_pattern_in_str, find_pattern_in_file
-from .check_for_preexist_dir_file import check_for_preexist_dir_file
-from .check_var_valid_value import check_var_valid_value
-from .count_files import count_files
-from .create_symlink_to_file import create_symlink_to_file
-from .define_macos_utilities import define_macos_utilities
-from .environment import str_to_date, date_to_str, str_to_type, type_to_str, list_to_str, \
-      str_to_list, set_env_var, get_env_var, import_vars, export_vars
-from .filesys_cmds_vrfy import cmd_vrfy, cp_vrfy, mv_vrfy, rm_vrfy, ln_vrfy, mkdir_vrfy, cd_vrfy
-from .get_charvar_from_netcdf import get_charvar_from_netcdf
-from .get_elem_inds import get_elem_inds
-from .interpol_to_arbit_CRES import interpol_to_arbit_CRES
-from .print_input_args import print_input_args
-from .print_msg import print_info_msg, print_err_msg_exit
-from .process_args import process_args
-from .run_command import run_command
-from .config_parser import load_yaml_config, cfg_to_yaml_str, \
-                           load_json_config, cfg_to_json_str, \
-                           load_ini_config, cfg_to_ini_str, get_ini_value, \
-                           load_shell_config, cfg_to_shell_str, \
-                           load_config_file
-from .xml_parser import load_xml_file, has_tag_with_value
+try:
+  from .misc import uppercase, lowercase, find_pattern_in_str, find_pattern_in_file
+  from .check_for_preexist_dir_file import check_for_preexist_dir_file
+  from .check_var_valid_value import check_var_valid_value
+  from .count_files import count_files
+  from .create_symlink_to_file import create_symlink_to_file
+  from .define_macos_utilities import define_macos_utilities
+  from .environment import str_to_date, date_to_str, str_to_type, type_to_str, list_to_str, \
+        str_to_list, set_env_var, get_env_var, import_vars, export_vars
+  from .filesys_cmds_vrfy import cmd_vrfy, cp_vrfy, mv_vrfy, rm_vrfy, ln_vrfy, mkdir_vrfy, cd_vrfy
+  from .get_elem_inds import get_elem_inds
+  from .interpol_to_arbit_CRES import interpol_to_arbit_CRES
+  from .print_input_args import print_input_args
+  from .print_msg import print_info_msg, print_err_msg_exit
+  from .process_args import process_args
+  from .run_command import run_command
+  from .get_charvar_from_netcdf import get_charvar_from_netcdf
+  from .xml_parser import load_xml_file, has_tag_with_value
+  from .config_parser import load_yaml_config, cfg_to_yaml_str, \
+                             load_json_config, cfg_to_json_str, \
+                             load_ini_config, cfg_to_ini_str, get_ini_value, \
+                             load_shell_config, cfg_to_shell_str, \
+                             load_config_file
+except:
+  pass