From 7fe418f71044a782fdce41160f043434f2e0749b Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Mon, 13 May 2024 07:25:56 -0400 Subject: [PATCH] Clean up old scripts from homedir and add static to FIX dir (#93) * clean up scripts * revive necessary files * clean up prep_exp script * adjust indentation in ex-scripts * clean up scripts * remove unnecessary variables from scripts * change path to rundir * remove RSTRDIR from script * remove MEM_MODL_OUTDIR * remove fv3_run * add FIXlandda to j-job * change laddda_inputs to FIXlandda * increase wtime for forecast on orion * change rsync to ln --- do_submit_cycle.sh | 206 --------- fix/.gitignore | 4 + fv3_run | 48 -- incdate.sh | 8 - jobs/JLANDDA_ANALYSIS | 1 + jobs/JLANDDA_FORECAST | 1 + jobs/JLANDDA_PREP_BMAT | 1 + jobs/JLANDDA_PREP_EXP | 1 + jobs/JLANDDA_PREP_OBS | 1 + land_mods | 5 - module_check.sh | 44 -- .../datm_cdeps_lnd_gswp3_rst | 0 parm/land_analysis_era5_hera.yaml | 6 - parm/land_analysis_era5_orion.yaml | 6 - parm/land_analysis_gswp3_hera.yaml | 6 - parm/land_analysis_gswp3_orion.yaml | 8 +- .../run_container_executable.sh | 4 +- parm/run_without_rocoto.sh | 2 - parm/templates/template.jedi2ufs | 2 +- parm/templates/template.tile2vector | 2 +- .../template.ufs-noahMP.namelist.era5 | 6 +- parm/templates/template.ufs2jedi | 2 +- parm/templates/template.vector2tile | 2 +- release.environment | 43 -- scripts/exlandda_analysis.sh | 135 +++--- scripts/exlandda_forecast.sh | 199 +++++---- scripts/exlandda_prep_bmat.sh | 52 ++- scripts/exlandda_prep_exp.sh | 135 +++--- scripts/exlandda_prep_obs.sh | 13 +- settings_DA_cycle_era5 | 49 -- settings_DA_cycle_gswp3 | 53 --- settings_DA_test | 45 -- sorc/app_build.sh | 8 + sorc/test/CMakeLists.txt | 12 +- sorc/test/ci/Dockerfile | 2 +- sorc/test/runtime_vars.sh | 4 +- sorc/test/test_letkfoi_snowda.sh | 2 +- sorc/test/test_ufs_land_driver.sh | 2 +- sorc/test/test_vector2tile.sh | 4 +- sorc/test/testinput/template.vector2tile | 2 +- submit_cycle.sh | 418 ------------------ 41 files changed, 306 insertions(+), 1238 deletions(-) delete mode 100755 do_submit_cycle.sh create mode 100644 fix/.gitignore delete mode 100644 fv3_run delete mode 100755 incdate.sh delete mode 100644 land_mods delete mode 100755 module_check.sh rename datm_cdeps_lnd_gswp3_rst => parm/datm_cdeps_lnd_gswp3_rst (100%) rename run_container_executable.sh => parm/run_container_executable.sh (82%) delete mode 100644 release.environment delete mode 100644 settings_DA_cycle_era5 delete mode 100644 settings_DA_cycle_gswp3 delete mode 100644 settings_DA_test delete mode 100755 submit_cycle.sh diff --git a/do_submit_cycle.sh b/do_submit_cycle.sh deleted file mode 100755 index 76c04878..00000000 --- a/do_submit_cycle.sh +++ /dev/null @@ -1,206 +0,0 @@ -#!/bin/bash - -############################ -# load config file - -if [[ $# -gt 0 ]]; then - config_file=$1 -else - config_file=settings -fi - -if [[ ! -e $config_file ]]; then - echo 'Config file does not exist. Exiting. ' - echo $config_file - exit -fi - -echo "reading cycle settings from $config_file" -source $config_file - -export KEEPWORKDIR="YES" - -############################ -# ensure necessary envars are set -envars=("exp_name" "STARTDATE" "ENDDATE" "LANDDAROOT" "LANDDA_INPUTS" "HOMElandda" \ - "LANDDA_EXPTS" "BUILDDIR" "atmos_forc" "OBSDIR" "WORKDIR" \ - "OUTDIR" "TEST_BASEDIR" "JEDI_EXECDIR" "JEDI_STATICDIR" "ensemble_size" \ - "FCSTHR" "RES" "TPATH" "TSTUB" "cycles_per_job" "ICSDIR" "DA_config" \ - "DA_config00" "DA_config06" "DA_config12" "DA_config18" "BASELINE") -if [[ ! $BASELINE =~ 'hera.internal' ]]; then - envars=("PYTHON") -fi - -for var in "${envars[@]}"; do - if [ -z "${!var}" ]; then - unset_envars+=("$var") - fi -done - -if [ ${#unset_envars[@]} -ne 0 ]; then - echo "ERROR: the following environmental variables have not been set: ${unset_envars[@]}." - exit 1 -fi - -############################ -# check that modules are loaded in the environment - -if [[ ! $BASELINE =~ 'hera.internal' ]]; then - ${HOMElandda}/module_check.sh -fi - -if [[ $? -ne 0 ]]; then - exit 1 -fi - -############################ -# check that a valid account for job submission -# is set in submit_cycle.sh (only on Orion/Hera) - -if [[ ${HOSTNAME} == *"Orion"* || ${HOSTNAME} == *"hfe"* ]]; then - user_accounts=$(echo $(sacctmgr show assoc where user=$USER format=account) | sed 's|.*--- \(.*\)|\1|') - preset_account=$(grep '#SBATCH --account=' submit_cycle.sh | cut -d= -f2) - - # if the account set in submit_cycle matches any of ${user_accounts}, continue; - # if not, esuggest a compute account to which the user has access and exit. - if echo "$user_accounts" | grep -q -w "${preset_account}"; then - echo "Account for sbatch submission set to ${preset_account}." - else - echo "Warning: You don't have access to the ${preset_account} compute account. You might try setting the #SBATCH --account in submit_cycle.sh to one of the following accounts instead before re-submitting: ${user_acceunts}." - exit 1 - fi -fi - -############################ -# set executables - -if [[ -e ${HOMElandda}/exec/vector2tile_converter.exe ]]; then #prefer cmake-built executables - export vec2tileexec=${HOMElandda}/exec/vector2tile_converter.exe -else - export vec2tileexec=${HOMElandda}/sorc/vector2tile/vector2tile_converter.exe -fi -if [[ -e ${HOMElandda}/exec/tile2tile_converter.exe ]]; then #prefer cmake-built executables - export tile2tileexec=${HOMElandda}/exec/tile2tile_converter.exe -else - export tile2tileexec=${HOMElandda}/sorc/tile2tile/tile2tile_converter.exe -fi -if [[ -e ${HOMElandda}/exec/ufsLand.exe ]]; then - export LSMexec=${HOMElandda}/exec/ufsLand.exe -else - export LSMexec=${HOMElandda}/sorc/ufsLand.fd/run/ufsLand.exe -fi - -export DADIR=${HOMElandda}/sorc/DA_update/ -export DAscript=${DADIR}/do_landDA.sh -export MPIEXEC=`which mpiexec` -export LANDDADIR=${DADIR} - -export analdate=${HOMElandda}/analdates.sh -export incdate=${HOMElandda}/incdate.sh - -export BUILDDIR=${HOMElandda}/sorc/build -export INCR_EXECDIR=${DADIR}/add_jedi_incr/exec/ - -############################ -# read in dates - -export logfile=${HOMElandda}/cycle.log -touch $logfile -echo "***************************************" >> $logfile -echo "cycling from $STARTDATE to $ENDDATE" >> $logfile - -sYYYY=`echo $STARTDATE | cut -c1-4` -sMM=`echo $STARTDATE | cut -c5-6` -sDD=`echo $STARTDATE | cut -c7-8` -sHH=`echo $STARTDATE | cut -c9-10` - -# compute the restart frequency, run_days and run_hours -export FREQ=$(( 3600 * $FCSTHR )) -export RDD=$(( $FCSTHR / 24 )) -export RHH=$(( $FCSTHR % 24 )) - -############################ -# set up directories - -#workdir -if [[ -e ${WORKDIR} ]]; then - rm -rf ${WORKDIR} -fi -mkdir ${WORKDIR} - -#outdir for model -if [[ ! -e ${OUTDIR} ]]; then - mkdir -p ${OUTDIR} -fi - -############################### -# create dirs and copy in ICS if needed - -mem_ens="mem000" # single member, us ensemble 0 - -MEM_WORKDIR=${WORKDIR}/${mem_ens} -if [[ ! -e $MEM_WORKDIR ]]; then - mkdir $MEM_WORKDIR -fi - -# ensemble outdir (model only) -MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} -if [[ ! -e $MEM_MODL_OUTDIR ]]; then #ensemble outdir - mkdir -p $MEM_MODL_OUTDIR -fi - -# outdir subdirs -if [[ ! -e ${MEM_MODL_OUTDIR}/restarts/ ]]; then # subdirectories - mkdir -p ${MEM_MODL_OUTDIR}/restarts/vector/ - mkdir ${MEM_MODL_OUTDIR}/restarts/tile/ - mkdir -p ${MEM_MODL_OUTDIR}/noahmp/ -fi -ln -sf ${MEM_MODL_OUTDIR}/noahmp ${MEM_WORKDIR}/noahmp_output - -# copy ICS into restarts, if needed -if [[ $atmos_forc == "era5" ]]; then - rst_out=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc - rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.nc - # if restart not in experiment out directory, copy the restarts from the ICSDIR - if [[ ! -e ${rst_out} ]]; then - echo "Looking for ICS: ${rst_in}" - if [[ -e ${rst_in} ]]; then - echo "ICS found, copying" - cp ${rst_in} ${rst_out} - else - echo "ICS not found. Exiting" - exit 10 - fi - fi -fi - -if [[ $atmos_forc == "gswp3" ]]; then - - for tile in 1 2 3 4 5 6 - do - rst_out=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${sYYYY}-${sMM}-${sDD}_${sHH}-00-00.tile${tile}.nc - rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs.cpld.lnd.out.${sYYYY}-${sMM}-${sDD}-00000.tile${tile}.nc - # if restart not in experiment out directory, copy the restarts from the ICSDIR - if [[ ! -e ${rst_out} ]]; then - echo "Looking for ICS: ${rst_in}" - if [[ -e ${rst_in} ]]; then - echo "ICS found, copying" - cp ${rst_in} ${rst_out} - else - echo "ICS not found. Exiting" - exit 10 - fi - fi - done -fi - -# create dates file -touch analdates.sh -cat << EOF > analdates.sh -STARTDATE=$STARTDATE -ENDDATE=$ENDDATE -EOF - -# submit script -sbatch submit_cycle.sh - diff --git a/fix/.gitignore b/fix/.gitignore new file mode 100644 index 00000000..5e7d2734 --- /dev/null +++ b/fix/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/fv3_run b/fv3_run deleted file mode 100644 index 4b8a0b2f..00000000 --- a/fv3_run +++ /dev/null @@ -1,48 +0,0 @@ -mkdir -p RESTART INPUT -cd INPUT - -rsync -arv ${LANDDA_INPUTS}/UFS_WM/DATM_GSWP3_input_data/. . - -cd - - -SUFFIX=${RT_SUFFIX} -# restart -if [ $WARM_START = .true. ]; then - # NoahMP restart files - cp ${MEM_MODL_OUTDIR}/restarts/tile/ufs.cpld.lnd.out.${RESTART_FILE_SUFFIX_SECS}.tile*.nc RESTART/. - - # CMEPS restart and pointer files - RFILE1=ufs.cpld.cpl.r.${RESTART_FILE_SUFFIX_SECS}.nc - cp ${LANDDA_INPUTS}/restarts/gswp3/${RFILE1} RESTART/. - ls -1 "RESTART/${RFILE1}">rpointer.cpl - - # CDEPS restart and pointer files - RFILE2=ufs.cpld.datm.r.${RESTART_FILE_SUFFIX_SECS}.nc - cp ${LANDDA_INPUTS}/restarts/gswp3/${RFILE2} RESTART/. - ls -1 "RESTART/${RFILE2}">rpointer.atm -fi - -cd INPUT -rsync -arv ${LANDDA_INPUTS}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile1.nc C96.initial.tile1.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile2.nc C96.initial.tile2.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile3.nc C96.initial.tile3.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile4.nc C96.initial.tile4.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile5.nc C96.initial.tile5.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile6.nc C96.initial.tile6.nc - -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/C96.maximum_snow_albedo.tile*.nc . -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/C96.slope_type.tile*.nc . -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/C96.soil_type.tile*.nc . -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/C96.soil_color.tile*.nc . -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/C96.substrate_temperature.tile*.nc . -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/C96.vegetation_greenness.tile*.nc . -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/C96.vegetation_type.tile*.nc . -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile1.nc oro_data.tile1.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile2.nc oro_data.tile2.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile3.nc oro_data.tile3.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile4.nc oro_data.tile4.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile5.nc oro_data.tile5.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile6.nc oro_data.tile6.nc -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_input_data/INPUT/C96_grid.tile*.nc . -rsync -arv ${LANDDA_INPUTS}/UFS_WM/FV3_input_data/INPUT/grid_spec.nc C96_mosaic.nc -cd - diff --git a/incdate.sh b/incdate.sh deleted file mode 100755 index 1c69c1c8..00000000 --- a/incdate.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -if [ $# != 2 ]; then - echo "usage: incdate YYYYMMDDHH hrs" - echo "where YYYYMMDDHH is a 10 character date string (e.g. 2002050312)" - echo "and hrs is integer number of hours to increment YYYMMDDHH" - exit 1 -fi -date -u -d "${1:0:4}-${1:4:2}-${1:6:2} ${1:8:2}:00:00 UTC $2 hour" +%Y%m%d%H diff --git a/jobs/JLANDDA_ANALYSIS b/jobs/JLANDDA_ANALYSIS index 3318b221..9afa8043 100755 --- a/jobs/JLANDDA_ANALYSIS +++ b/jobs/JLANDDA_ANALYSIS @@ -13,6 +13,7 @@ set -xue export USHlandda="${HOMElandda}/ush" export EXEClandda="${HOMElandda}/exec" export PARMlandda="${HOMElandda}/parm" +export FIXlandda="${HOMElandda}/fix" export SCRIPTSlandda="${HOMElandda}/scripts" # #----------------------------------------------------------------------- diff --git a/jobs/JLANDDA_FORECAST b/jobs/JLANDDA_FORECAST index d5e65880..13f42b5a 100755 --- a/jobs/JLANDDA_FORECAST +++ b/jobs/JLANDDA_FORECAST @@ -13,6 +13,7 @@ set -xue export USHlandda="${HOMElandda}/ush" export EXEClandda="${HOMElandda}/exec" export PARMlandda="${HOMElandda}/parm" +export FIXlandda="${HOMElandda}/fix" export SCRIPTSlandda="${HOMElandda}/scripts" # #----------------------------------------------------------------------- diff --git a/jobs/JLANDDA_PREP_BMAT b/jobs/JLANDDA_PREP_BMAT index 3ae23e6a..b8c35834 100755 --- a/jobs/JLANDDA_PREP_BMAT +++ b/jobs/JLANDDA_PREP_BMAT @@ -13,6 +13,7 @@ set -xue export USHlandda="${HOMElandda}/ush" export EXEClandda="${HOMElandda}/exec" export PARMlandda="${HOMElandda}/parm" +export FIXlandda="${HOMElandda}/fix" export SCRIPTSlandda="${HOMElandda}/scripts" # #----------------------------------------------------------------------- diff --git a/jobs/JLANDDA_PREP_EXP b/jobs/JLANDDA_PREP_EXP index af82c244..5748908f 100755 --- a/jobs/JLANDDA_PREP_EXP +++ b/jobs/JLANDDA_PREP_EXP @@ -13,6 +13,7 @@ set -xue export USHlandda="${HOMElandda}/ush" export EXEClandda="${HOMElandda}/exec" export PARMlandda="${HOMElandda}/parm" +export FIXlandda="${HOMElandda}/fix" export SCRIPTSlandda="${HOMElandda}/scripts" # #----------------------------------------------------------------------- diff --git a/jobs/JLANDDA_PREP_OBS b/jobs/JLANDDA_PREP_OBS index 5907dfcb..e75c71a4 100755 --- a/jobs/JLANDDA_PREP_OBS +++ b/jobs/JLANDDA_PREP_OBS @@ -13,6 +13,7 @@ set -xue export USHlandda="${HOMElandda}/ush" export EXEClandda="${HOMElandda}/exec" export PARMlandda="${HOMElandda}/parm" +export FIXlandda="${HOMElandda}/fix" export SCRIPTSlandda="${HOMElandda}/scripts" # #----------------------------------------------------------------------- diff --git a/land_mods b/land_mods deleted file mode 100644 index 423602af..00000000 --- a/land_mods +++ /dev/null @@ -1,5 +0,0 @@ -module purge -module load intel/2022.2.0 -module load impi/2022.2.0 -module load netcdf/4.7.0 -module load netcdf-hdf5parallel/4.7.4 diff --git a/module_check.sh b/module_check.sh deleted file mode 100755 index 0db71cca..00000000 --- a/module_check.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Script to ensure required modules are loaded for the land DA system. - -# find which machine we are on -if [[ ${HOSTNAME} == *"Orion"* ]]; then - MACHINE=orion -elif [[ ${HOSTNAME} == *"hfe"* ]]; then - MACHINE=hera -fi - -# check which modules are required and notify the user if they are not currently loaded in the environment. -if [[ ( ${MACHINE} == "orion" || ${MACHINE} == "hera" ) && ${USE_SINGULARITY} != "yes" ]]; then - env_mods=($(grep -o 'load("[^"]*")' ${HOMElandda}/modulefiles/build_${MACHINE}_intel.lua | sed 's/load("//;s/")//')) - - missing_mods=() - - for mod in ${env_mods[@]} - do - if ! module is-loaded "${mod}"; then - missing_mods+=("${mod}") - fi - done - - if [[ ${#missing_mods[@]} -gt 0 ]]; then - echo "Error: the following modules are not loaded in the current environment: ${missing_mods[@]}. Please load them via 'module use ${HOMElandda}/modulefiles; module load landda_${MACHINE}.intel' and then re-launch do_submit_cycle.sh." - exit 1 - else - echo "All modules properly loaded in environment. Continuing!" - fi - -elif [[ ${USE_SINGULARITY} =~ "yes" ]]; then - - if ! command -v mpiexec &> /dev/null; then - echo "Error: mpiexec is not in the current path. Please load an Intel MPI of version 2021 or newer." - exit 1 - elif ! mpiexec --version | grep -Eq "Intel.*2021|202[2-9][0-9]*"; then - echo "Warning: loaded Intel MPI is a version older than 2021. Intel MPIs prior to 2021 have not been tested." - echo "Current loaded version is $(mpiexec --version)" - else - echo "Intel MPI properly set for Singularity. Continuing." - fi - -fi diff --git a/datm_cdeps_lnd_gswp3_rst b/parm/datm_cdeps_lnd_gswp3_rst similarity index 100% rename from datm_cdeps_lnd_gswp3_rst rename to parm/datm_cdeps_lnd_gswp3_rst diff --git a/parm/land_analysis_era5_hera.yaml b/parm/land_analysis_era5_hera.yaml index 214861d2..d4eff5ac 100644 --- a/parm/land_analysis_era5_hera.yaml +++ b/parm/land_analysis_era5_hera.yaml @@ -13,7 +13,6 @@ workflow: EXP_NAME: "LETKF" EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test" JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi_skylabv7.0" - LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" FORCING: "era5" RES: "96" FCSTHR: "24" @@ -47,7 +46,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" @@ -75,7 +73,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" WORKDIR: "&WORKDIR;" model_ver: "&model_ver;" @@ -104,7 +101,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" WORKDIR: "&WORKDIR;" model_ver: "&model_ver;" @@ -136,7 +132,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" @@ -173,7 +168,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" diff --git a/parm/land_analysis_era5_orion.yaml b/parm/land_analysis_era5_orion.yaml index 4f1ed4af..08859eba 100644 --- a/parm/land_analysis_era5_orion.yaml +++ b/parm/land_analysis_era5_orion.yaml @@ -13,7 +13,6 @@ workflow: EXP_NAME: "LETKF" EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test" JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi_skylabv7.0" - LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs" FORCING: "era5" RES: "96" FCSTHR: "24" @@ -47,7 +46,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" @@ -75,7 +73,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" WORKDIR: "&WORKDIR;" model_ver: "&model_ver;" @@ -104,7 +101,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" WORKDIR: "&WORKDIR;" model_ver: "&model_ver;" @@ -136,7 +132,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" @@ -173,7 +168,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" diff --git a/parm/land_analysis_gswp3_hera.yaml b/parm/land_analysis_gswp3_hera.yaml index 2119b85d..8f58c142 100644 --- a/parm/land_analysis_gswp3_hera.yaml +++ b/parm/land_analysis_gswp3_hera.yaml @@ -13,7 +13,6 @@ workflow: EXP_NAME: "LETKF" EXP_BASEDIR: "/scratch2/NAGAPE/epic/{USER}/landda_test" JEDI_INSTALL: "/scratch2/NAGAPE/epic/UFS_Land-DA/jedi_skylabv7.0" - LANDDA_INPUTS: "/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" FORCING: "gswp3" RES: "96" FCSTHR: "24" @@ -47,7 +46,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" @@ -75,7 +73,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" WORKDIR: "&WORKDIR;" model_ver: "&model_ver;" @@ -104,7 +101,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" WORKDIR: "&WORKDIR;" model_ver: "&model_ver;" @@ -136,7 +132,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" @@ -173,7 +168,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" diff --git a/parm/land_analysis_gswp3_orion.yaml b/parm/land_analysis_gswp3_orion.yaml index b2d0dd7b..d6251707 100644 --- a/parm/land_analysis_gswp3_orion.yaml +++ b/parm/land_analysis_gswp3_orion.yaml @@ -13,7 +13,6 @@ workflow: EXP_NAME: "LETKF" EXP_BASEDIR: "/work/noaa/epic/{USER}/landda_test" JEDI_INSTALL: "/work/noaa/epic/UFS_Land-DA/jedi_skylabv7.0" - LANDDA_INPUTS: "/work/noaa/epic/UFS_Land-DA/inputs" FORCING: "gswp3" RES: "96" FCSTHR: "24" @@ -47,7 +46,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" @@ -75,7 +73,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" WORKDIR: "&WORKDIR;" model_ver: "&model_ver;" @@ -104,7 +101,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" WORKDIR: "&WORKDIR;" model_ver: "&model_ver;" @@ -136,7 +132,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" @@ -173,7 +168,6 @@ workflow: SCHED: "&SCHED;" ACCOUNT: "&ACCOUNT;" EXP_NAME: "&EXP_NAME;" - LANDDA_INPUTS: "&LANDDA_INPUTS;" ATMOS_FORC: "&FORCING;" RES: "&RES;" TSTUB: "&TSTUB;" @@ -197,7 +191,7 @@ workflow: command: '&HOMElandda;/parm/task_load_modules_run_jjob.sh "forecast" "&HOMElandda;" "&MACHINE;"' jobname: forecast nodes: "1:ppn=&NPROCS_FORECAST;" - walltime: 00:40:00 + walltime: 01:00:00 queue: batch join: "&LOGDIR;/forecast.log" dependency: diff --git a/run_container_executable.sh b/parm/run_container_executable.sh similarity index 82% rename from run_container_executable.sh rename to parm/run_container_executable.sh index 152b3de6..232a2e89 100755 --- a/run_container_executable.sh +++ b/parm/run_container_executable.sh @@ -8,8 +8,8 @@ img=${img:-${CONTAINERLOC}/ubuntu20.04-intel-ue-landda.img} CONTAINERBASE="/"`echo $img | xargs realpath | awk -F"/" '{print $2}'` cmd=$(basename "$0") arg="$@" -if [ ! -z "$LANDDA_INPUTS" ]; then - INPUTBASE="/"`echo $LANDDA_INPUTS | xargs realpath | awk -F"/" '{print $2}'` +if [ ! -z "$FIXlandda" ]; then + INPUTBASE="/"`echo $FIXlandda | xargs realpath | awk -F"/" '{print $2}'` INPUTBIND="-B $INPUTBASE:$INPUTBASE" else INPUTBIND="" diff --git a/parm/run_without_rocoto.sh b/parm/run_without_rocoto.sh index 69528430..87087e81 100755 --- a/parm/run_without_rocoto.sh +++ b/parm/run_without_rocoto.sh @@ -18,11 +18,9 @@ export FORCING="era5" if [ "${MACHINE}" = "hera" ]; then export EXP_BASEDIR="/scratch2/NAGAPE/epic/{USER}/landda_test" export JEDI_INSTALL="/scratch2/NAGAPE/epic/UFS_Land-DA/jedi_skylabv7.0" - export LANDDA_INPUTS="/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" elif [ "${MACHINE}" = "orion" ]; then export EXP_BASEDIR="/work/noaa/epic/chjeon/landda_test" export JEDI_INSTALL="/work/noaa/epic/UFS_Land-DA/jedi_skylabv7.0" - export LANDDA_INPUTS="/work/noaa/epic/UFS_Land-DA/inputs" fi export RES="96" diff --git a/parm/templates/template.jedi2ufs b/parm/templates/template.jedi2ufs index 08cd7ffd..476dc41b 100644 --- a/parm/templates/template.jedi2ufs +++ b/parm/templates/template.jedi2ufs @@ -18,7 +18,7 @@ restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" ! Path for static file - static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + static_filename = "FIXlandda/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" ! Location of vector restart file (vector2tile direction) diff --git a/parm/templates/template.tile2vector b/parm/templates/template.tile2vector index 4655a86d..fc164726 100644 --- a/parm/templates/template.tile2vector +++ b/parm/templates/template.tile2vector @@ -18,7 +18,7 @@ restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" ! Path for static file - static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + static_filename = "FIXlandda/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" ! Location of vector restart file (vector2tile direction) diff --git a/parm/templates/template.ufs-noahMP.namelist.era5 b/parm/templates/template.ufs-noahMP.namelist.era5 index 21fd7e5d..baa82c3f 100644 --- a/parm/templates/template.ufs-noahMP.namelist.era5 +++ b/parm/templates/template.ufs-noahMP.namelist.era5 @@ -1,8 +1,8 @@ &run_setup - static_file = "/LANDDA_INPUTS/forcing/era5/static/ufs-land_C96_static_fields.nc" - init_file = "/LANDDA_INPUTS/forcing/era5/init/ufs-land_C96_init_2010-12-31_23-00-00.nc" - forcing_dir = "/LANDDA_INPUTS/forcing/era5/datm/C96/" + static_file = "/FIXlandda/forcing/era5/static/ufs-land_C96_static_fields.nc" + init_file = "/FIXlandda/forcing/era5/init/ufs-land_C96_init_2010-12-31_23-00-00.nc" + forcing_dir = "/FIXlandda/forcing/era5/datm/C96/" separate_output = .false. output_dir = "./noahmp_output/" diff --git a/parm/templates/template.ufs2jedi b/parm/templates/template.ufs2jedi index b5028419..4ae8cdb9 100644 --- a/parm/templates/template.ufs2jedi +++ b/parm/templates/template.ufs2jedi @@ -18,7 +18,7 @@ restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" ! Path for static file - static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + static_filename = "FIXlandda/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" ! Location of vector restart file (vector2tile direction) diff --git a/parm/templates/template.vector2tile b/parm/templates/template.vector2tile index e175d2e0..183bf1f6 100644 --- a/parm/templates/template.vector2tile +++ b/parm/templates/template.vector2tile @@ -18,7 +18,7 @@ restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" ! Path for static file - static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + static_filename = "FIXlandda/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" ! Location of vector restart file (vector2tile direction) diff --git a/release.environment b/release.environment deleted file mode 100644 index 9c5f01ab..00000000 --- a/release.environment +++ /dev/null @@ -1,43 +0,0 @@ -#Common environment variable settings for LandDA release v1.0.0 - -#Set defaults -export LANDDAROOT=${LANDDAROOT:-`dirname $PWD`} -export LANDDA_INPUTS=${LANDDA_INPUTS:-${LANDDAROOT}/inputs} -export HOMElandda=$(pwd) -export LANDDA_EXPTS=${LANDDA_EXPTS:-${LANDDAROOT}/landda_expts} -if [[ ! $BASELINE =~ 'hera.internal' ]]; then - export PYTHON=`/usr/bin/which python` -fi -export BUILDDIR=${BUILDDIR:-${HOMElandda}/sorc/build} - -#Change some variables if working with a container -if [[ ${USE_SINGULARITY} =~ yes ]]; then - EPICHOME=/opt - #use the python that is built into the container. It has all the pythonpaths set and can run the ioda converters - export PYTHON=$PWD/sorc/build/bin/python - #JEDI is installed under /opt in the container - export JEDI_INSTALL=/opt - #Scripts that launch containerized versions of the executables are in $PWD/singularity/bin They should be called - #from the host system to be run (e.g. mpiexec -n 6 $BUILDDIR/bin/fv3jedi_letkf.x ) - export BUILDDIR=$PWD/sorc/build - export JEDI_EXECDIR=${HOMElandda}/sorc/build/bin - module try-load singularity - export SINGULARITYBIN=`/usr/bin/which singularity` - sed -i 's/singularity exec/${SINGULARITYBIN} exec/g' run_container_executable.sh -fi - -#Location of observations for DA -export OBSDIR=${LANDDA_INPUTS}/DA - -# temporary directory where experiment is run from -export WORKDIR=${WORKDIR:-${LANDDAROOT}/workdir} - -# directory where output is saved -export OUTDIR=${LANDDA_EXPTS}/${exp_name} - -# directory where ctest base dataset is located -export TEST_BASEDIR=${TEST_BASEDIR:-"${EPICHOME}/test_base/mem000/restarts/vector"} - -# JEDI directories -export JEDI_EXECDIR=${JEDI_EXECDIR:-"${JEDI_INSTALL}/build/bin"} -export JEDI_STATICDIR=${JEDI_INSTALL}/jedi-bundle/fv3-jedi/test/Data diff --git a/scripts/exlandda_analysis.sh b/scripts/exlandda_analysis.sh index 9abf5c19..1407e242 100755 --- a/scripts/exlandda_analysis.sh +++ b/scripts/exlandda_analysis.sh @@ -6,13 +6,13 @@ set -ex # copy restarts to workdir, convert to UFS tile for DA (all members) if [[ ${EXP_NAME} == "openloop" ]]; then - do_jedi="NO" + do_jedi="NO" else - do_jedi="YES" - SAVE_TILE="YES" + do_jedi="YES" + SAVE_TILE="YES" fi -TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ +TPATH=${FIXlandda}/forcing/${ATMOS_FORC}/orog_files/ YYYY=${PDY:0:4} MM=${PDY:4:2} DD=${PDY:6:2} @@ -25,8 +25,6 @@ HP=${PTIME:8:2} mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} -MEM_MODL_OUTDIR=${COMOUT}/${mem_ens} -RSTRDIR=${MEM_WORKDIR} JEDIWORKDIR=${WORKDIR}/mem000/jedi FILEDATE=${YYYY}${MM}${DD}.${HH}0000 JEDI_STATICDIR=${JEDI_INSTALL}/jedi-bundle/fv3-jedi/test/Data @@ -48,8 +46,11 @@ MPIEXEC=`which mpiexec` YAML_DA=construct GFSv17="NO" B=30 # back ground error std for LETKFOI + cd $JEDIWORKDIR +mkdir -p output/DA/hofx + ################################################ # 3. DETERMINE REQUESTED JEDI TYPE, CONSTRUCT YAMLS ################################################ @@ -58,73 +59,67 @@ do_DA="YES" do_HOFX="NO" if [[ $do_DA == "NO" && $do_HOFX == "NO" ]]; then - echo "do_landDA:No obs found, not calling JEDI" - exit 0 + echo "do_landDA:No obs found, not calling JEDI" + exit 0 fi # if yaml is specified by user, use that. Otherwise, build the yaml if [[ $do_DA == "YES" ]]; then - if [[ $YAML_DA == "construct" ]];then # construct the yaml - cp ${PARMlandda}/jedi/${DAtype}.yaml ${JEDIWORKDIR}/letkf_land.yaml - for obs in "${OBS_TYPES[@]}"; - do - cat ${PARMlandda}/jedi/${obs}.yaml >> letkf_land.yaml - done - else # use specified yaml - echo "Using user specified YAML: ${YAML_DA}" - cp ${PARMlandda}/jedi/${YAML_DA} ${JEDIWORKDIR}/letkf_land.yaml - fi - - sed -i -e "s/XXYYYY/${YYYY}/g" letkf_land.yaml - sed -i -e "s/XXMM/${MM}/g" letkf_land.yaml - sed -i -e "s/XXDD/${DD}/g" letkf_land.yaml - sed -i -e "s/XXHH/${HH}/g" letkf_land.yaml - - sed -i -e "s/XXYYYP/${YYYP}/g" letkf_land.yaml - sed -i -e "s/XXMP/${MP}/g" letkf_land.yaml - sed -i -e "s/XXDP/${DP}/g" letkf_land.yaml - sed -i -e "s/XXHP/${HP}/g" letkf_land.yaml - - sed -i -e "s/XXTSTUB/${TSTUB}/g" letkf_land.yaml - sed -i -e "s#XXTPATH#${TPATH}#g" letkf_land.yaml - sed -i -e "s/XXRES/${RES}/g" letkf_land.yaml - RESP1=$((RES+1)) - sed -i -e "s/XXREP/${RESP1}/g" letkf_land.yaml - - sed -i -e "s/XXHOFX/false/g" letkf_land.yaml # do DA + if [[ $YAML_DA == "construct" ]];then # construct the yaml + cp ${PARMlandda}/jedi/${DAtype}.yaml ${JEDIWORKDIR}/letkf_land.yaml + for obs in "${OBS_TYPES[@]}"; + do + cat ${PARMlandda}/jedi/${obs}.yaml >> letkf_land.yaml + done + else # use specified yaml + echo "Using user specified YAML: ${YAML_DA}" + cp ${PARMlandda}/jedi/${YAML_DA} ${JEDIWORKDIR}/letkf_land.yaml + fi + + sed -i -e "s/XXYYYY/${YYYY}/g" letkf_land.yaml + sed -i -e "s/XXMM/${MM}/g" letkf_land.yaml + sed -i -e "s/XXDD/${DD}/g" letkf_land.yaml + sed -i -e "s/XXHH/${HH}/g" letkf_land.yaml + sed -i -e "s/XXYYYP/${YYYP}/g" letkf_land.yaml + sed -i -e "s/XXMP/${MP}/g" letkf_land.yaml + sed -i -e "s/XXDP/${DP}/g" letkf_land.yaml + sed -i -e "s/XXHP/${HP}/g" letkf_land.yaml + sed -i -e "s/XXTSTUB/${TSTUB}/g" letkf_land.yaml + sed -i -e "s#XXTPATH#${TPATH}#g" letkf_land.yaml + sed -i -e "s/XXRES/${RES}/g" letkf_land.yaml + RESP1=$((RES+1)) + sed -i -e "s/XXREP/${RESP1}/g" letkf_land.yaml + sed -i -e "s/XXHOFX/false/g" letkf_land.yaml # do DA fi if [[ $do_HOFX == "YES" ]]; then - if [[ $YAML_HOFX == "construct" ]];then # construct the yaml - cp ${PARMlandda}/jedi/${DAtype}.yaml ${JEDIWORKDIR}/hofx_land.yaml - for obs in "${OBS_TYPES[@]}"; - do - cat ${PARMlandda}/jedi/${obs}.yaml >> hofx_land.yaml - done - else # use specified yaml - echo "Using user specified YAML: ${YAML_HOFX}" - cp ${PARMlandda}/jedi/${YAML_HOFX} ${JEDIWORKDIR}/hofx_land.yaml - fi - - sed -i -e "s/XXYYYY/${YYYY}/g" hofx_land.yaml - sed -i -e "s/XXMM/${MM}/g" hofx_land.yaml - sed -i -e "s/XXDD/${DD}/g" hofx_land.yaml - sed -i -e "s/XXHH/${HH}/g" hofx_land.yaml - - sed -i -e "s/XXYYYP/${YYYP}/g" hofx_land.yaml - sed -i -e "s/XXMP/${MP}/g" hofx_land.yaml - sed -i -e "s/XXDP/${DP}/g" hofx_land.yaml - sed -i -e "s/XXHP/${HP}/g" hofx_land.yaml - - sed -i -e "s#XXTPATH#${TPATH}#g" hofx_land.yaml - sed -i -e "s/XXTSTUB/${TSTUB}/g" hofx_land.yaml - sed -i -e "s/XXRES/${RES}/g" hofx_land.yaml - RESP1=$((RES+1)) - sed -i -e "s/XXREP/${RESP1}/g" hofx_land.yaml - - sed -i -e "s/XXHOFX/true/g" hofx_land.yaml # do HOFX + if [[ $YAML_HOFX == "construct" ]];then # construct the yaml + cp ${PARMlandda}/jedi/${DAtype}.yaml ${JEDIWORKDIR}/hofx_land.yaml + for obs in "${OBS_TYPES[@]}"; + do + cat ${PARMlandda}/jedi/${obs}.yaml >> hofx_land.yaml + done + else # use specified yaml + echo "Using user specified YAML: ${YAML_HOFX}" + cp ${PARMlandda}/jedi/${YAML_HOFX} ${JEDIWORKDIR}/hofx_land.yaml + fi + + sed -i -e "s/XXYYYY/${YYYY}/g" hofx_land.yaml + sed -i -e "s/XXMM/${MM}/g" hofx_land.yaml + sed -i -e "s/XXDD/${DD}/g" hofx_land.yaml + sed -i -e "s/XXHH/${HH}/g" hofx_land.yaml + sed -i -e "s/XXYYYP/${YYYP}/g" hofx_land.yaml + sed -i -e "s/XXMP/${MP}/g" hofx_land.yaml + sed -i -e "s/XXDP/${DP}/g" hofx_land.yaml + sed -i -e "s/XXHP/${HP}/g" hofx_land.yaml + sed -i -e "s#XXTPATH#${TPATH}#g" hofx_land.yaml + sed -i -e "s/XXTSTUB/${TSTUB}/g" hofx_land.yaml + sed -i -e "s/XXRES/${RES}/g" hofx_land.yaml + RESP1=$((RES+1)) + sed -i -e "s/XXREP/${RESP1}/g" hofx_land.yaml + sed -i -e "s/XXHOFX/true/g" hofx_land.yaml # do HOFX fi @@ -142,7 +137,7 @@ fi ################################################ if [[ ! -e Data ]]; then - ln -s $JEDI_STATICDIR Data + ln -s $JEDI_STATICDIR Data fi echo 'do_landDA: calling fv3-jedi' @@ -201,7 +196,6 @@ EOF echo "apply snow increment failed" exit 10 fi - fi fi @@ -210,12 +204,17 @@ fi # 7. CLEAN UP ################################################ +if [[ -d output/DA/hofx ]]; then + cp -r output/DA/hofx ${COMOUT}/${mem_ens} +fi + # keep increments if [ $SAVE_INCR == "YES" ] && [ $do_DA == "YES" ]; then - cp ${JEDIWORKDIR}/${FILEDATE}.xainc.sfc_data.tile*.nc ${COMOUT}/DA/jedi_incr/ + mkdir -p ${COMOUT}/${mem_ens}/jedi_incr + cp ${JEDIWORKDIR}/${FILEDATE}.xainc.sfc_data.tile*.nc ${COMOUT}/${mem_ens}/jedi_incr fi # clean up if [[ $KEEPJEDIDIR == "NO" ]]; then - rm -rf ${JEDIWORKDIR} + rm -rf ${JEDIWORKDIR} fi diff --git a/scripts/exlandda_forecast.sh b/scripts/exlandda_forecast.sh index a016ec08..a635362e 100755 --- a/scripts/exlandda_forecast.sh +++ b/scripts/exlandda_forecast.sh @@ -6,14 +6,14 @@ set -ex # copy restarts to workdir, convert to UFS tile for DA (all members) if [[ ${EXP_NAME} == "openloop" ]]; then - do_jedi="NO" + do_jedi="NO" else - do_jedi="YES" - SAVE_TILE="YES" + do_jedi="YES" + SAVE_TILE="YES" fi MACHINE_ID=${MACHINE} -TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ +TPATH=${FIXlandda}/forcing/${ATMOS_FORC}/orog_files/ YYYY=${PDY:0:4} MM=${PDY:4:2} DD=${PDY:6:2} @@ -29,10 +29,6 @@ nHH=${NTIME:8:2} mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} -MEM_MODL_OUTDIR=${COMOUT}/${mem_ens} -RSTRDIR=${MEM_WORKDIR} -FILEDATE=${YYYY}${MM}${DD}.${HH}0000 -SAVE_INCR="YES" FREQ=$((${FCSTHR}*3600)) RDD=$((${FCSTHR}/24)) RHH=$((${FCSTHR}%24)) @@ -53,13 +49,15 @@ MPIRUN=${MPIRUN:-`which mpiexec`} cd $MEM_WORKDIR # convert back to vector, run model (all members) convert back to vector, run model (all members) -if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then +if [[ ${do_jedi} == "YES" ]]; then + + if [[ ${ATMOS_FORC} == "era5" ]]; then echo '************************************************' echo 'calling tile2vector' cp ${PARMlandda}/templates/template.tile2vector tile2vector.namelist - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" tile2vector.namelist + sed -i "s|FIXlandda|${FIXlandda}|g" tile2vector.namelist sed -i -e "s/XXYYYY/${YYYY}/g" tile2vector.namelist sed -i -e "s/XXMM/${MM}/g" tile2vector.namelist sed -i -e "s/XXDD/${DD}/g" tile2vector.namelist @@ -80,59 +78,16 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then fi # save analysis restart - mkdir -p ${MEM_MODL_OUTDIR}/restarts/vector - cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.nc -fi + mkdir -p ${COMOUT}/${mem_ens}/restarts/vector + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc ${COMOUT}/${mem_ens}/restarts/vector/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.nc -# convert back to UFS tile, run model (all members) -if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then - echo '************************************************' - echo 'calling tile2tile' - - cp ${PARMlandda}/templates/template.jedi2ufs jedi2ufs.namelist - - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" jedi2ufs.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" jedi2ufs.namelist - sed -i -e "s/XXMM/${MM}/g" jedi2ufs.namelist - sed -i -e "s/XXDD/${DD}/g" jedi2ufs.namelist - sed -i -e "s/XXHH/${HH}/g" jedi2ufs.namelist - sed -i -e "s/MODEL_FORCING/${ATMOS_FORC}/g" jedi2ufs.namelist - sed -i -e "s/XXRES/${RES}/g" jedi2ufs.namelist - sed -i -e "s/XXTSTUB/${TSTUB}/g" jedi2ufs.namelist - sed -i -e "s#XXTPATH#${TPATH}#g" jedi2ufs.namelist - - export pgm="tile2tile_converter.exe" - . prep_step - ${EXEClandda}/$pgm jedi2ufs.namelist >>$pgmout 2>errfile - export err=$?; err_chk - cp errfile errfile_tile2tile - if [[ $err != 0 ]]; then - echo "tile2tile failed" - exit 10 - fi - - # save analysis restart - mkdir -p ${MEM_MODL_OUTDIR}/restarts/tile - for tile in 1 2 3 4 5 6 - do - cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc - cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc - done -fi - - -#jkimmmm -############################ -# run the forecast model - -if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then echo '************************************************' echo 'running the forecast model' # update model namelist cp ${PARMlandda}/templates/template.ufs-noahMP.namelist.${ATMOS_FORC} ufs-land.namelist - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs-land.namelist + sed -i "s|FIXlandda|${FIXlandda}|g" ufs-land.namelist sed -i -e "s/XXYYYY/${YYYY}/g" ufs-land.namelist sed -i -e "s/XXMM/${MM}/g" ufs-land.namelist sed -i -e "s/XXDD/${DD}/g" ufs-land.namelist @@ -155,11 +110,42 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "era5" ]]; then echo "ufsLand failed" exit 10 fi -fi -# no error codes on exit from model, check for restart below instead -if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then - + # convert back to UFS tile, run model (all members) + elif [[ ${ATMOS_FORC} == "gswp3" ]]; then + echo '************************************************' + echo 'calling tile2tile' + + cp ${PARMlandda}/templates/template.jedi2ufs jedi2ufs.namelist + + sed -i "s|FIXlandda|${FIXlandda}|g" jedi2ufs.namelist + sed -i -e "s/XXYYYY/${YYYY}/g" jedi2ufs.namelist + sed -i -e "s/XXMM/${MM}/g" jedi2ufs.namelist + sed -i -e "s/XXDD/${DD}/g" jedi2ufs.namelist + sed -i -e "s/XXHH/${HH}/g" jedi2ufs.namelist + sed -i -e "s/MODEL_FORCING/${ATMOS_FORC}/g" jedi2ufs.namelist + sed -i -e "s/XXRES/${RES}/g" jedi2ufs.namelist + sed -i -e "s/XXTSTUB/${TSTUB}/g" jedi2ufs.namelist + sed -i -e "s#XXTPATH#${TPATH}#g" jedi2ufs.namelist + + export pgm="tile2tile_converter.exe" + . prep_step + ${EXEClandda}/$pgm jedi2ufs.namelist >>$pgmout 2>errfile + export err=$?; err_chk + cp errfile errfile_tile2tile + if [[ $err != 0 ]]; then + echo "tile2tile failed" + exit 10 + fi + + # save analysis restart + mkdir -p ${COMOUT}/${mem_ens}/restarts/tile + for tile in 1 2 3 4 5 6 + do + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${COMOUT}/${mem_ens}/restarts/tile/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${COMOUT}/${mem_ens}/restarts/tile/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc + done + echo '************************************************' echo 'running the forecast model' @@ -169,26 +155,26 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then RT_COMPILER=${RT_COMPILER:-intel} ATOL="1e-7" - cp $HOMElandda/$TEST_NAME_RST ${PATHRT}/tests/$TEST_NAME_RST + cp $PARMlandda/$TEST_NAME_RST ${PATHRT}/tests/$TEST_NAME_RST source ${PATHRT}/rt_utils.sh source ${PATHRT}/default_vars.sh source ${PATHRT}/tests/$TEST_NAME_RST source ${PATHRT}/atparse.bash BL_DATE=20230816 - RTPWD=${RTPWD:-${LANDDA_INPUTS}/NEMSfv3gfs/develop-${BL_DATE}/INTEL/${TEST_NAME}} - INPUTDATA_ROOT=${INPUTDATA_ROOT:-${LANDDA_INPUTS}/NEMSfv3gfs/input-data-20221101} + RTPWD=${RTPWD:-${FIXlandda}/NEMSfv3gfs/develop-${BL_DATE}/INTEL/${TEST_NAME}} + INPUTDATA_ROOT=${INPUTDATA_ROOT:-${FIXlandda}/NEMSfv3gfs/input-data-20221101} echo "RTPWD= $RTPWD" echo "INPUTDATA_ROOT= $INPUTDATA_ROOT" if [[ ! -d ${INPUTDATA_ROOT} ]] || [[ ! -d ${RTPWD} ]]; then - echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!" - exit 1 + echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!" + exit 1 fi # create run folder - RUNDIR=${MEM_MODL_OUTDIR}/noahmp/${TEST_NAME_RST} + RUNDIR=${DATA}/noahmp/${TEST_NAME_RST} [[ -d ${RUNDIR} ]] && echo "Warning: remove old run folder!" && rm -rf ${RUNDIR} mkdir -p ${RUNDIR} cd ${RUNDIR} @@ -202,14 +188,11 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then export layout_x=1 export layout_y=1 - # FV3 executable: -# cp ${EXEClandda}/ufs_model ./ufs_model - cp ${HOMElandda}/fv3_run ./fv3_run - + # FV3 executable: if [[ $DATM_CDEPS = 'true' ]] || [[ $FV3 = 'true' ]] || [[ $S2S = 'true' ]]; then - if [[ $HAFS = 'false' ]] || [[ $FV3 = 'true' && $HAFS = 'true' ]]; then - atparse < ${PATHRT}/parm/${INPUT_NML:-input.nml.IN} > input.nml - fi + if [[ $HAFS = 'false' ]] || [[ $FV3 = 'true' && $HAFS = 'true' ]]; then + atparse < ${PATHRT}/parm/${INPUT_NML:-input.nml.IN} > input.nml + fi fi atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure @@ -220,23 +203,67 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then # diag table if [[ "Q${DIAG_TABLE:-}" != Q ]] ; then - atparse < ${PATHRT}/parm/diag_table/${DIAG_TABLE} > diag_table + atparse < ${PATHRT}/parm/diag_table/${DIAG_TABLE} > diag_table fi # Field table if [[ "Q${FIELD_TABLE:-}" != Q ]] ; then - cp ${PATHRT}/parm/field_table/${FIELD_TABLE} field_table + cp ${PATHRT}/parm/field_table/${FIELD_TABLE} field_table fi # Field Dictionary cp ${PATHRT}/parm/fd_ufs.yaml fd_ufs.yaml # Set up the run directory - source ./fv3_run + mkdir -p RESTART INPUT + cd INPUT + ln -nsf ${FIXlandda}/UFS_WM/DATM_GSWP3_input_data/* . + cd - + + SUFFIX=${RT_SUFFIX} + # restart + if [ $WARM_START = .true. ]; then + # NoahMP restart files + cp ${COMOUT}/${mem_ens}/restarts/tile/ufs.cpld.lnd.out.${RESTART_FILE_SUFFIX_SECS}.tile*.nc RESTART/. + + # CMEPS restart and pointer files + RFILE1=ufs.cpld.cpl.r.${RESTART_FILE_SUFFIX_SECS}.nc + cp ${FIXlandda}/restarts/gswp3/${RFILE1} RESTART/. + ls -1 "RESTART/${RFILE1}">rpointer.cpl + + # CDEPS restart and pointer files + RFILE2=ufs.cpld.datm.r.${RESTART_FILE_SUFFIX_SECS}.nc + cp ${FIXlandda}/restarts/gswp3/${RFILE2} RESTART/. + ls -1 "RESTART/${RFILE2}">rpointer.atm + fi + + cd INPUT + ln -nsf ${FIXlandda}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile1.nc C96.initial.tile1.nc + ln -nsf ${FIXlandda}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile2.nc C96.initial.tile2.nc + ln -nsf ${FIXlandda}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile3.nc C96.initial.tile3.nc + ln -nsf ${FIXlandda}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile4.nc C96.initial.tile4.nc + ln -nsf ${FIXlandda}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile5.nc C96.initial.tile5.nc + ln -nsf ${FIXlandda}/UFS_WM/NOAHMP_IC/ufs-land_C96_init_fields.tile6.nc C96.initial.tile6.nc + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/C96.maximum_snow_albedo.tile*.nc . + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/C96.slope_type.tile*.nc . + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/C96.soil_type.tile*.nc . + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/C96.soil_color.tile*.nc . + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/C96.substrate_temperature.tile*.nc . + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/C96.vegetation_greenness.tile*.nc . + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/C96.vegetation_type.tile*.nc . + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile1.nc oro_data.tile1.nc + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile2.nc oro_data.tile2.nc + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile3.nc oro_data.tile3.nc + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile4.nc oro_data.tile4.nc + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile5.nc oro_data.tile5.nc + ln -nsf ${FIXlandda}/UFS_WM/FV3_fix_tiled/C96/oro_C96.mx100.tile6.nc oro_data.tile6.nc + ln -nsf ${FIXlandda}/UFS_WM/FV3_input_data/INPUT/C96_grid.tile*.nc . + ln -nsf ${FIXlandda}/UFS_WM/FV3_input_data/INPUT/grid_spec.nc C96_mosaic.nc + cd - if [[ $DATM_CDEPS = 'true' ]]; then - atparse < ${PATHRT}/parm/${DATM_IN_CONFIGURE:-datm_in.IN} > datm_in - atparse < ${PATHRT}/parm/${DATM_STREAM_CONFIGURE:-datm.streams.IN} > datm.streams + atparse < ${PATHRT}/parm/${DATM_IN_CONFIGURE:-datm_in.IN} > datm_in + atparse < ${PATHRT}/parm/${DATM_STREAM_CONFIGURE:-datm.streams.IN} > datm.streams fi # NoahMP table file @@ -253,22 +280,18 @@ if [[ $do_jedi == "YES" && ${ATMOS_FORC} == "gswp3" ]]; then echo "ufs_model failed" exit 10 fi + fi fi -# no error codes on exit from model, check for restart below instead - ############################ # check model ouput (all members) - if [[ ${ATMOS_FORC} == "era5" ]]; then - if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then - cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc - fi -fi - -if [[ ${ATMOS_FORC} == "gswp3" ]]; then + if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then + cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${COMOUT}/${mem_ens}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc + fi +elif [[ ${ATMOS_FORC} == "gswp3" ]]; then for tile in 1 2 3 4 5 6 do - cp ${COMOUT}/${mem_ens}/noahmp/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc + cp ${RUNDIR}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${COMOUT}/${mem_ens}/restarts/tile/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc done fi diff --git a/scripts/exlandda_prep_bmat.sh b/scripts/exlandda_prep_bmat.sh index 4573cfc4..83781500 100755 --- a/scripts/exlandda_prep_bmat.sh +++ b/scripts/exlandda_prep_bmat.sh @@ -12,7 +12,7 @@ else SAVE_TILE="YES" fi -TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ +TPATH=${FIXlandda}/forcing/${ATMOS_FORC}/orog_files/ YYYY=${PDY:0:4} MM=${PDY:4:2} DD=${PDY:6:2} @@ -21,8 +21,6 @@ HH=${cyc} mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} -MEM_MODL_OUTDIR=${COMOUT}/${mem_ens} -RSTRDIR=${MEM_WORKDIR} JEDIWORKDIR=${WORKDIR}/mem000/jedi FILEDATE=${YYYY}${MM}${DD}.${HH}0000 @@ -47,34 +45,34 @@ cd $JEDIWORKDIR if [[ ${DAtype} == "letkfoi_snow" ]]; then - if [ $GFSv17 == "YES" ]; then - SNOWDEPTHVAR="snodl" - else - SNOWDEPTHVAR="snwdph" - # replace field overwrite file - cp ${PARMlandda}/jedi/gfs-land.yaml ${JEDIWORKDIR}/gfs-land.yaml + if [ $GFSv17 == "YES" ]; then + SNOWDEPTHVAR="snodl" + else + SNOWDEPTHVAR="snwdph" + # replace field overwrite file + cp ${PARMlandda}/jedi/gfs-land.yaml ${JEDIWORKDIR}/gfs-land.yaml + fi + # FOR LETKFOI, CREATE THE PSEUDO-ENSEMBLE + for ens in pos neg + do + if [ -e $JEDIWORKDIR/mem_${ens} ]; then + rm -r $JEDIWORKDIR/mem_${ens} fi - # FOR LETKFOI, CREATE THE PSEUDO-ENSEMBLE - for ens in pos neg + mkdir -p $JEDIWORKDIR/mem_${ens} + for tile in 1 2 3 4 5 6 do - if [ -e $JEDIWORKDIR/mem_${ens} ]; then - rm -r $JEDIWORKDIR/mem_${ens} - fi - mkdir -p $JEDIWORKDIR/mem_${ens} - for tile in 1 2 3 4 5 6 - do - cp ${JEDIWORKDIR}/${FILEDATE}.sfc_data.tile${tile}.nc ${JEDIWORKDIR}/mem_${ens}/${FILEDATE}.sfc_data.tile${tile}.nc - done - cp ${JEDIWORKDIR}/${FILEDATE}.coupler.res ${JEDIWORKDIR}/mem_${ens}/${FILEDATE}.coupler.res + cp ${JEDIWORKDIR}/${FILEDATE}.sfc_data.tile${tile}.nc ${JEDIWORKDIR}/mem_${ens}/${FILEDATE}.sfc_data.tile${tile}.nc done + cp ${JEDIWORKDIR}/${FILEDATE}.coupler.res ${JEDIWORKDIR}/mem_${ens}/${FILEDATE}.coupler.res + done - echo 'do_landDA: calling create ensemble' + echo 'do_landDA: calling create ensemble' - # using ioda mods to get a python version with netCDF4 - ${USHlandda}/letkf_create_ens.py $FILEDATE $SNOWDEPTHVAR $B - if [[ $? != 0 ]]; then - echo "letkf create failed" - exit 10 - fi + # using ioda mods to get a python version with netCDF4 + ${USHlandda}/letkf_create_ens.py $FILEDATE $SNOWDEPTHVAR $B + if [[ $? != 0 ]]; then + echo "letkf create failed" + exit 10 + fi fi diff --git a/scripts/exlandda_prep_exp.sh b/scripts/exlandda_prep_exp.sh index 7e532ce2..25787e74 100755 --- a/scripts/exlandda_prep_exp.sh +++ b/scripts/exlandda_prep_exp.sh @@ -12,8 +12,8 @@ else SAVE_TILE="YES" fi -echo ${LANDDA_INPUTS}, ${ATMOS_FORC} -TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ +echo ${FIXlandda}, ${ATMOS_FORC} +TPATH=${FIXlandda}/forcing/${ATMOS_FORC}/orog_files/ YYYY=${PDY:0:4} MM=${PDY:4:2} DD=${PDY:6:2} @@ -25,19 +25,12 @@ HP=${PTIME:8:2} mem_ens="mem000" MEM_WORKDIR=${WORKDIR}/${mem_ens} -MEM_MODL_OUTDIR=${COMOUT}/${mem_ens} -RSTRDIR=${MEM_WORKDIR} JEDIWORKDIR=${WORKDIR}/mem000/jedi FILEDATE=${YYYY}${MM}${DD}.${HH}0000 -if [[ ! -e ${MEM_WORKDIR} ]]; then - mkdir -p ${MEM_WORKDIR} -fi -if [[ ! -e ${MEM_MODL_OUTDIR} ]]; then - mkdir -p ${MEM_MODL_OUTDIR} -fi - -mkdir -p $MEM_WORKDIR/modulefiles; cp ${HOMElandda}/modulefiles/build_${MACHINE}_intel.lua $MEM_WORKDIR/modulefiles/modules.landda.lua +mkdir -p ${MEM_WORKDIR} +mkdir -p $MEM_WORKDIR/modulefiles +cp ${HOMElandda}/modulefiles/build_${MACHINE}_intel.lua $MEM_WORKDIR/modulefiles/modules.landda.lua cd $MEM_WORKDIR # load modulefiles @@ -48,12 +41,14 @@ fi module use modulefiles; module load modules.landda -if [[ $do_jedi == "YES" && $ATMOS_FORC == "era5" ]]; then - +if [[ $do_jedi == "YES" ]]; then + + if [[ $ATMOS_FORC == "era5" ]]; then + # vector2tile for DA # copy restarts into work directory - rst_in=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + rst_in=${COMOUT}/${mem_ens}/restarts/vector/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc if [[ ! -e ${rst_in} ]]; then - rst_in=${LANDDA_INPUTS}/restarts/${ATMOS_FORC}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + rst_in=${FIXlandda}/restarts/${ATMOS_FORC}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc fi rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc cp ${rst_in} ${rst_out} @@ -66,7 +61,7 @@ if [[ $do_jedi == "YES" && $ATMOS_FORC == "era5" ]]; then # update vec2tile and tile2vec namelists cp ${PARMlandda}/templates/template.vector2tile vector2tile.namelist - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" vector2tile.namelist + sed -i "s|FIXlandda|${FIXlandda}|g" vector2tile.namelist sed -i -e "s/XXYYYY/${YYYY}/g" vector2tile.namelist sed -i -e "s/XXMM/${MM}/g" vector2tile.namelist sed -i -e "s/XXDD/${DD}/g" vector2tile.namelist @@ -89,31 +84,29 @@ if [[ $do_jedi == "YES" && $ATMOS_FORC == "era5" ]]; then if [[ $err != 0 ]]; then echo "vec2tile failed" exit - fi -fi # vector2tile for DA - -if [[ $do_jedi == "YES" && $ATMOS_FORC == "gswp3" ]]; then - - echo '************************************************' - echo 'calling tile2tile' + fi + elif [[ $ATMOS_FORC == "gswp3" ]]; then + # tile2tile for DA + echo '************************************************' + echo 'calling tile2tile' - export MEM_WORKDIR + export MEM_WORKDIR - # copy restarts into work directory - for tile in 1 2 3 4 5 6 - do - rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - if [[ ! -e ${rst_in} ]]; then - rst_in=${LANDDA_INPUTS}/restarts/${ATMOS_FORC}/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc - fi - rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc - cp ${rst_in} ${rst_out} - done + # copy restarts into work directory + for tile in 1 2 3 4 5 6 + do + rst_in=${COMOUT}/${mem_ens}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc + if [[ ! -e ${rst_in} ]]; then + rst_in=${FIXlandda}/restarts/${ATMOS_FORC}/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc + fi + rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc + cp ${rst_in} ${rst_out} + done # update tile2tile namelist cp ${PARMlandda}/templates/template.ufs2jedi ufs2jedi.namelist - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs2jedi.namelist + sed -i "s|FIXlandda|${FIXlandda}|g" ufs2jedi.namelist sed -i -e "s/XXYYYY/${YYYY}/g" ufs2jedi.namelist sed -i -e "s/XXMM/${MM}/g" ufs2jedi.namelist sed -i -e "s/XXDD/${DD}/g" ufs2jedi.namelist @@ -134,53 +127,41 @@ if [[ $do_jedi == "YES" && $ATMOS_FORC == "gswp3" ]]; then echo "tile2tile failed" exit fi -fi # tile2tile for DA + fi -if [[ $do_jedi == "YES" ]]; then - if [[ ! -e ${COMOUT}/DA ]]; then - mkdir -p ${COMOUT}/DA/jedi_incr - mkdir -p ${COMOUT}/DA/logs - mkdir -p ${COMOUT}/DA/hofx - fi - if [[ ! -e $JEDIWORKDIR ]]; then - mkdir -p $JEDIWORKDIR - fi - cd $JEDIWORKDIR - - if [[ ! -e ${JEDIWORKDIR}/output ]]; then - ln -s ${COMOUT} ${JEDIWORKDIR}/output - fi - - if [[ $SAVE_TILE == "YES" ]]; then - for tile in 1 2 3 4 5 6 - do - cp ${RSTRDIR}/${FILEDATE}.sfc_data.tile${tile}.nc ${RSTRDIR}/${FILEDATE}.sfc_data_back.tile${tile}.nc - done - fi + if [[ ! -e $JEDIWORKDIR ]]; then + mkdir -p $JEDIWORKDIR + fi + cd $JEDIWORKDIR - #stage restarts for applying JEDI update (files will get directly updated) + if [[ $SAVE_TILE == "YES" ]]; then for tile in 1 2 3 4 5 6 do - ln -fs ${RSTRDIR}/${FILEDATE}.sfc_data.tile${tile}.nc ${JEDIWORKDIR}/${FILEDATE}.sfc_data.tile${tile}.nc + cp ${MEM_WORKDIR}/${FILEDATE}.sfc_data.tile${tile}.nc ${MEM_WORKDIR}/${FILEDATE}.sfc_data_back.tile${tile}.nc done + fi - cres_file=${JEDIWORKDIR}/${FILEDATE}.coupler.res - - if [[ -e ${RSTRDIR}/${FILEDATE}.coupler.res ]]; then - ln -sf ${RSTRDIR}/${FILEDATE}.coupler.res $cres_file - else # if not present, need to create coupler.res for JEDI - cp ${PARMlandda}/templates/template.coupler.res $cres_file - - sed -i -e "s/XXYYYY/${YYYY}/g" $cres_file - sed -i -e "s/XXMM/${MM}/g" $cres_file - sed -i -e "s/XXDD/${DD}/g" $cres_file - sed -i -e "s/XXHH/${HH}/g" $cres_file + #stage restarts for applying JEDI update (files will get directly updated) + for tile in 1 2 3 4 5 6 + do + ln -fs ${MEM_WORKDIR}/${FILEDATE}.sfc_data.tile${tile}.nc ${JEDIWORKDIR}/${FILEDATE}.sfc_data.tile${tile}.nc + done - sed -i -e "s/XXYYYP/${YYYP}/g" $cres_file - sed -i -e "s/XXMP/${MP}/g" $cres_file - sed -i -e "s/XXDP/${DP}/g" $cres_file - sed -i -e "s/XXHP/${HP}/g" $cres_file - - fi + cres_file=${JEDIWORKDIR}/${FILEDATE}.coupler.res + + if [[ -e ${MEM_WORKDIR}/${FILEDATE}.coupler.res ]]; then + ln -sf ${MEM_WORKDIR}/${FILEDATE}.coupler.res $cres_file + else # if not present, need to create coupler.res for JEDI + cp ${PARMlandda}/templates/template.coupler.res $cres_file + + sed -i -e "s/XXYYYY/${YYYY}/g" $cres_file + sed -i -e "s/XXMM/${MM}/g" $cres_file + sed -i -e "s/XXDD/${DD}/g" $cres_file + sed -i -e "s/XXHH/${HH}/g" $cres_file + sed -i -e "s/XXYYYP/${YYYP}/g" $cres_file + sed -i -e "s/XXMP/${MP}/g" $cres_file + sed -i -e "s/XXDP/${DP}/g" $cres_file + sed -i -e "s/XXHP/${HP}/g" $cres_file + fi fi # do_jedi setup diff --git a/scripts/exlandda_prep_obs.sh b/scripts/exlandda_prep_obs.sh index bb5b201e..2de0c8e0 100755 --- a/scripts/exlandda_prep_obs.sh +++ b/scripts/exlandda_prep_obs.sh @@ -6,13 +6,12 @@ set -ex # copy restarts to workdir, convert to UFS tile for DA (all members) if [[ ${EXP_NAME} == "openloop" ]]; then - do_jedi="NO" + do_jedi="NO" else - do_jedi="YES" - SAVE_TILE="YES" + do_jedi="YES" + SAVE_TILE="YES" fi -TPATH=${LANDDA_INPUTS}/forcing/${ATMOS_FORC}/orog_files/ YYYY=${PDY:0:4} MM=${PDY:4:2} DD=${PDY:6:2} @@ -24,15 +23,11 @@ HP=${PTIME:8:2} mem_ens="mem000" -MEM_WORKDIR=${WORKDIR}/${mem_ens} -MEM_MODL_OUTDIR=${COMOUT}/${mem_ens} -RSTRDIR=${MEM_WORKDIR} JEDIWORKDIR=${WORKDIR}/mem000/jedi -FILEDATE=${YYYY}${MM}${DD}.${HH}0000 cd $JEDIWORKDIR -OBSDIR=${LANDDA_INPUTS}/DA +OBSDIR=${FIXlandda}/DA ################################################ # 2. PREPARE OBS FILES ################################################ diff --git a/settings_DA_cycle_era5 b/settings_DA_cycle_era5 deleted file mode 100644 index 34d04991..00000000 --- a/settings_DA_cycle_era5 +++ /dev/null @@ -1,49 +0,0 @@ -# Settings file for submit_cycle, for running the DA_GHCN_test - -# experiment name -export exp_name=DA_ERA5_test -#export BASELINE=hera.internal - -STARTDATE=2019122100 -ENDDATE=2019122200 - -# Get commmon variables -source ./release.environment -############################ - -#forcing options: gswp3, era5 -export atmos_forc=era5 - -# for LETKF, this is size of ensemble. -# for LETKF-OI pseudo ensemble, or non-ensemble runs use 1 -export ensemble_size=1 - -# length of each forecast -export FCSTHR=24 - -#FV3 resolution -export RES=96 -if [[ $BASELINE =~ 'hera.internal' ]]; then - export TPATH=/scratch2/NCEPDEV/land/data/fix/C96.mx100_frac/ -else - export TPATH="$LANDDA_INPUTS/forcing/${atmos_forc}/orog_files/" -fi -export TSTUB="oro_C96.mx100" # file stub for orography files in $TPATH - # oro_C${RES} for atm only, oro_C${RES}.mx100 for atm/ocean. - -# number of cycles to submit in a single job -export cycles_per_job=1 - -# directory with initial conditions -# can find some here:/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/offline_ICS/single -export ICSDIR=$LANDDAROOT/inputs/forcing/${atmos_forc}/orog_files/ - -# namelist for do_landDA.sh -# set to "openloop" to not call do_landDA.sh -export DA_config="settings_DA_test" - -# if want different DA at different times, list here. -export DA_config00=${DA_config} -export DA_config06=${DA_config} -export DA_config12=${DA_config} -export DA_config18=${DA_config} diff --git a/settings_DA_cycle_gswp3 b/settings_DA_cycle_gswp3 deleted file mode 100644 index d77cb54d..00000000 --- a/settings_DA_cycle_gswp3 +++ /dev/null @@ -1,53 +0,0 @@ -# Settings file for submit_cycle, for running the DA_GHCN_test - -# experiment name -export exp_name=DA_GSWP3_test -#export BASELINE=hera.internal - -#Machine options: orion, hera -export MACHINE_ID=hera - -# experiment period -STARTDATE=2000010300 -ENDDATE=2000010400 - -# Get commmon variables -source ./release.environment -############################ - -#forcing options: gswp3, era5 -export atmos_forc=gswp3 - -# for LETKF, this is size of ensemble. -# for LETKF-OI pseudo ensemble, or non-ensemble runs use 1 -export ensemble_size=1 - -# length of each forecast -export FCSTHR=24 - -#FV3 resolution -export RES=96 -if [[ $BASELINE =~ 'hera.internal' ]]; then - export TPATH=/scratch2/NCEPDEV/land/data/fix/C96.mx100_frac/ -else - export TPATH="$LANDDA_INPUTS/forcing/${atmos_forc}/orog_files/" -fi -export TSTUB="oro_C96.mx100" # file stub for orography files in $TPATH - # oro_C${RES} for atm only, oro_C${RES}.mx100 for atm/ocean. - -# number of cycles to submit in a single job -export cycles_per_job=1 - -# directory with initial conditions -# can find some here:/scratch2/BMC/gsienkf/Clara.Draper/DA_test_cases/land-offline_workflow/offline_ICS/single -export ICSDIR=$LANDDAROOT/inputs/forcing/${atmos_forc}/orog_files/ - -# namelist for do_landDA.sh -# set to "openloop" to not call do_landDA.sh -export DA_config="settings_DA_test" - -# if want different DA at different times, list here. -export DA_config00=${DA_config} -export DA_config06=${DA_config} -export DA_config12=${DA_config} -export DA_config18=${DA_config} diff --git a/settings_DA_test b/settings_DA_test deleted file mode 100644 index 103f8655..00000000 --- a/settings_DA_test +++ /dev/null @@ -1,45 +0,0 @@ -# settings for do_landDA.sh for IMS test case - -############################ -# if calling from submit_cycle,vars in this section will already be set. Otherwise, need to be set here. - -# exp_name= # experiment name -# THISDATE= # date YYYYMMDDHH -# OUTDIR=/scratch2/BMC/gsienkf/${USER}/cycle_land/${exp_name}/ # temporary directory where experiment is run from -# RES= #FV3 resolution -#export TPATH= -#export TSTUB= # file stub for orography files in $TPATH - # oro_C${RES} for atm only, oro_C${RES}.mx100 for atm/ocean. - -# if calling from submit_cycle.sh, do not change these three: -JEDIWORKDIR=${WORKDIR}/jedi/ -LANDDADIR=${HOMElandda}/sorc/DA_update/ # directory where do_landDA.sh script is -RSTRDIR=${MEM_WORKDIR} - -############################ -# DA options - -# DA algorithm and state being updated -# options: "letkfoi_snow" , "letkf_snow" -DAtype=letkfoi_snow - -# JEDI input obs. options : IMS, GHCN, GTS, SYNTH -OBS_TYPES=("GHCN" ) # format: ("OBS1" "OBS2") -# JEDI call type for each obs_type above. options: DA, HOFX -JEDI_TYPES=("DA" ) # format ("DA" "HOFX") - -# DA window lenth. Will generally be the same as the FCSTLEN -WINLEN=24 - -# YAMLS. Options, either: -# 1. "construct" to construct the YAML name, based on requested obs types and their availability -# 2. enter the desired YAML name (will not test for availability of obs) -YAML_DA=construct -YAML_HOFX=construct - -# OPTIONAL: CHANGE JEDI DIRECTORIES -# if using different JEDI VERSION, will likely need to edit your yamls. -#JEDI_EXECDIR= # JEDI FV3 build directory -#IODA_BUILD_DIR= # JEDI IODA-converter source directory -#IMShr= # is assimilating IMS at hour other than 18, specify here. -#OBSDIR= # to override default OBSDIR diff --git a/sorc/app_build.sh b/sorc/app_build.sh index 993f4243..f5f10dad 100755 --- a/sorc/app_build.sh +++ b/sorc/app_build.sh @@ -392,4 +392,12 @@ else fi fi +# Link land-DA input files to FIXlandda directory +if [ "${PLATFORM}" = "hera" ]; then + landda_fix_orig="/scratch2/NAGAPE/epic/UFS_Land-DA/inputs" +elif [ "${PLATFORM}" = "orion" ]; then + landda_fix_orig="/work/noaa/epic/UFS_Land-DA/inputs" +fi +ln -nsf ${landda_fix_orig}/* ${HOME_DIR}/fix + exit 0 diff --git a/sorc/test/CMakeLists.txt b/sorc/test/CMakeLists.txt index e2f305d3..305002b0 100644 --- a/sorc/test/CMakeLists.txt +++ b/sorc/test/CMakeLists.txt @@ -4,7 +4,7 @@ add_test(NAME test_vector2tile WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test) set_tests_properties(test_vector2tile PROPERTIES - ENVIRONMENT "LANDDA_INPUTS=$ENV{LANDDA_INPUTS}") + ENVIRONMENT "FIXlandda=$ENV{FIXlandda}") # test for creating pseudo ensemble for use in letkfoi add_test(NAME test_create_ens @@ -13,7 +13,7 @@ add_test(NAME test_create_ens set_tests_properties(test_create_ens PROPERTIES DEPENDS "test_vector2title" - ENVIRONMENT "LANDDA_INPUTS=$ENV{LANDDA_INPUTS}") + ENVIRONMENT "FIXlandda=$ENV{FIXlandda}") # test for running letkfoi to assimilate snow DA add_test(NAME test_letkfoi_snowda @@ -22,7 +22,7 @@ add_test(NAME test_letkfoi_snowda set_tests_properties(test_letkfoi_snowda PROPERTIES DEPENDS "test_create_ens" - ENVIRONMENT "LANDDA_INPUTS=$ENV{LANDDA_INPUTS}") + ENVIRONMENT "FIXlandda=$ENV{FIXlandda}") # test for adding jedi increment add_test(NAME test_apply_jediincr @@ -31,7 +31,7 @@ add_test(NAME test_apply_jediincr set_tests_properties(test_apply_jediincr PROPERTIES DEPENDS "test_letkfoi_snowda" - ENVIRONMENT "LANDDA_INPUTS=$ENV{LANDDA_INPUTS}") + ENVIRONMENT "FIXlandda=$ENV{FIXlandda}") # test for tile2vector for use in ufs-land-driver add_test(NAME test_tile2vector @@ -40,7 +40,7 @@ add_test(NAME test_tile2vector set_tests_properties(test_tile2vector PROPERTIES DEPENDS "test_apply_jediincr" - ENVIRONMENT "LANDDA_INPUTS=$ENV{LANDDA_INPUTS}; + ENVIRONMENT "FIXlandda=$ENV{FIXlandda}; TOL=$ENV{TOL}") # test for ufs-land-driver @@ -50,7 +50,7 @@ add_test(NAME test_land_driver set_tests_properties(test_land_driver PROPERTIES DEPENDS "test_tile2vector" - ENVIRONMENT "LANDDA_INPUTS=$ENV{LANDDA_INPUTS}; + ENVIRONMENT "FIXlandda=$ENV{FIXlandda}; TOL=$ENV{TOL}") # test for ufs-datm-lnd model diff --git a/sorc/test/ci/Dockerfile b/sorc/test/ci/Dockerfile index ce233b50..112d25d6 100644 --- a/sorc/test/ci/Dockerfile +++ b/sorc/test/ci/Dockerfile @@ -10,7 +10,7 @@ RUN rm -rf $HOME/land-offline_workflow COPY . $HOME/land-offline_workflow # set env vars -ENV LANDDA_INPUTS=$HOME/inputs +ENV FIXlandda=$HOME/land-DA_workflow/fix ENV EPICHOME=/opt ENV JEDI_INSTALL=${EPICHOME}/jedi_skylabv7.0 ENV TEST_BASEDIR=${EPICHOME}/test_base/mem000/restarts/vector"} diff --git a/sorc/test/runtime_vars.sh b/sorc/test/runtime_vars.sh index 12183e32..8fdb99b8 100755 --- a/sorc/test/runtime_vars.sh +++ b/sorc/test/runtime_vars.sh @@ -33,7 +33,7 @@ export nHH=`echo $NYMDH | cut -c9-10` # set paths export WORKDIR=$project_binary_dir/test export EXECDIR=${EXECDIR:-$project_binary_dir/bin} -export LANDDA_INPUTS=${LANDDA_INPUTS:-"`dirname $project_source_dir`/../inputs"} +export FIXlandda=${FIXlandda:-"`dirname $project_source_dir`/fix"} # set IODA path export IODA_BUILD_DIR=${IODA_BUILD_DIR:-"${JEDI_INSTALL}/build"} @@ -51,7 +51,7 @@ export PYTHON_EXEC=${PYTHON_EXEC:-`which python`} # configurations export RES=96 export atmos_forc=era5 -export TPATH="$LANDDA_INPUTS/forcing/${atmos_forc}/orog_files/" +export TPATH="$FIXlandda/forcing/${atmos_forc}/orog_files/" export TSTUB="oro_C${RES}.mx100" export GFSv17=NO export OBS_TYPES=("GHCN") diff --git a/sorc/test/test_letkfoi_snowda.sh b/sorc/test/test_letkfoi_snowda.sh index be197cc9..8a973448 100755 --- a/sorc/test/test_letkfoi_snowda.sh +++ b/sorc/test/test_letkfoi_snowda.sh @@ -10,7 +10,7 @@ source ${project_source_dir}/test/runtime_vars.sh ${project_binary_dir} ${projec # set extra paths OROG_PATH=$TPATH -OBSDIR=${LANDDA_INPUTS}/DA/ +OBSDIR=${FIXlandda}/DA/ # set executables JEDI_EXEC=${JEDI_EXEC:-$JEDI_EXECDIR/fv3jedi_letkf.x} diff --git a/sorc/test/test_ufs_land_driver.sh b/sorc/test/test_ufs_land_driver.sh index bba955b7..62554eb1 100755 --- a/sorc/test/test_ufs_land_driver.sh +++ b/sorc/test/test_ufs_land_driver.sh @@ -36,7 +36,7 @@ done # update model namelist cp $project_source_dir/../parm/templates/template.ufs-noahMP.namelist.${atmos_forc} ufs-land.namelist -sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs-land.namelist +sed -i "s|FIXlandda|${FIXlandda}|g" ufs-land.namelist sed -i -e "s/XXYYYY/${YY}/g" ufs-land.namelist sed -i -e "s/XXMM/${MM}/g" ufs-land.namelist sed -i -e "s/XXDD/${DD}/g" ufs-land.namelist diff --git a/sorc/test/test_vector2tile.sh b/sorc/test/test_vector2tile.sh index cfde003c..c5150c97 100755 --- a/sorc/test/test_vector2tile.sh +++ b/sorc/test/test_vector2tile.sh @@ -28,7 +28,7 @@ cd $WORKDIR # copy vector restarts mkdir -p $WORKDIR/${prefix}/restarts/vector -source_restart=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${YY}-${MM}-${DD}_${HH}-00-00.nc +source_restart=${FIXlandda}/restarts/${atmos_forc}/ufs_land_restart.${YY}-${MM}-${DD}_${HH}-00-00.nc target_restart=$WORKDIR/${prefix}/restarts/vector/ufs_land_restart.${YY}-${MM}-${DD}_${HH}-00-00.nc cp $source_restart $target_restart @@ -72,7 +72,7 @@ esac # update namelist cp ${project_source_dir}/test/testinput/template.vector2tile vector2tile.namelist sed -i "s|DIRECTION|${DIRECTION}|g" vector2tile.namelist -sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" vector2tile.namelist +sed -i "s|FIXlandda|${FIXlandda}|g" vector2tile.namelist sed -i "s|VECTOR_PATH|${VECTOR_PATH}|g" vector2tile.namelist sed -i "s|TILE_PATH|${TILE_PATH}|g" vector2tile.namelist sed -i "s|OUTPUT_PATH|${OUTPUT_PATH}|g" vector2tile.namelist diff --git a/sorc/test/testinput/template.vector2tile b/sorc/test/testinput/template.vector2tile index 1287c8c9..26aed30a 100644 --- a/sorc/test/testinput/template.vector2tile +++ b/sorc/test/testinput/template.vector2tile @@ -18,7 +18,7 @@ restart_date = "XXYYYY-XXMM-XXDD XXHH:00:00" ! Path for static file - static_filename = "LANDDA_INPUTS/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" + static_filename = "FIXlandda/forcing/MODEL_FORCING/static/ufs-land_CXXRES_static_fields.nc" ! Location of vector restart file (vector2tile direction) diff --git a/submit_cycle.sh b/submit_cycle.sh deleted file mode 100755 index a9dcd1e2..00000000 --- a/submit_cycle.sh +++ /dev/null @@ -1,418 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ufs_land_da -#SBATCH --account=da-cpu -#SBATCH --qos=debug -#SBATCH --nodes=1 -#SBATCH --tasks-per-node=6 -#SBATCH --cpus-per-task=1 -#SBATCH -t 00:06:00 -#SBATCH -o log_noahmp.%j.log -#SBATCH -e err_noahmp.%j.err - -############################ -# loop over time steps - -source $analdate - -THISDATE=$STARTDATE -date_count=0 - -while [ $date_count -lt $cycles_per_job ]; do - - if [ $THISDATE -ge $ENDDATE ]; then - echo "All done, at date ${THISDATE}" >> $logfile - cd $HOMElandda - if [ $KEEPWORKDIR == "NO" ]; then - rm -rf $WORKDIR - fi - exit - fi - - echo "starting $THISDATE" - - # get DA settings - - this_config=DA_config$HH - DA_config=${!this_config} - - if [ $DA_config == "openloop" ]; then do_jedi="NO" ; else do_jedi="YES" ; fi - - # substringing to get yr, mon, day, hr info - YYYY=`echo $THISDATE | cut -c1-4` - MM=`echo $THISDATE | cut -c5-6` - DD=`echo $THISDATE | cut -c7-8` - HH=`echo $THISDATE | cut -c9-10` - - # substringing to get yr, mon, day, hr info for previous cycle - PREVDATE=`${incdate} $THISDATE -6` - YYYP=`echo $PREVDATE | cut -c1-4` - MP=`echo $PREVDATE | cut -c5-6` - DP=`echo $PREVDATE | cut -c7-8` - HP=`echo $PREVDATE | cut -c9-10` - - # substring for next cycle - NEXTDATE=`${incdate} $THISDATE $FCSTHR` - nYYYY=`echo $NEXTDATE | cut -c1-4` - nMM=`echo $NEXTDATE | cut -c5-6` - nDD=`echo $NEXTDATE | cut -c7-8` - nHH=`echo $NEXTDATE | cut -c9-10` - - ############################ - # copy restarts to workdir, convert to UFS tile for DA (all members) - - mem_ens="mem000" - - MEM_WORKDIR=${WORKDIR}/${mem_ens} - MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} - - cd $MEM_WORKDIR - - if [[ $do_jedi == "YES" && $atmos_forc == "era5" ]]; then - - # copy restarts into work directory - rst_in=${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - if [[ ! -e ${rst_in} ]]; then - rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - fi - rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - cp ${rst_in} ${rst_out} - - echo '************************************************' - echo 'calling vector2tile' - - export MEM_WORKDIR - - # update vec2tile and tile2vec namelists - cp ${HOMElandda}/parm/templates/template.vector2tile vector2tile.namelist - - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" vector2tile.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" vector2tile.namelist - sed -i -e "s/XXMM/${MM}/g" vector2tile.namelist - sed -i -e "s/XXDD/${DD}/g" vector2tile.namelist - sed -i -e "s/XXHH/${HH}/g" vector2tile.namelist - sed -i -e "s/XXHH/${HH}/g" vector2tile.namelist - sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" vector2tile.namelist - sed -i -e "s/XXRES/${RES}/g" vector2tile.namelist - sed -i -e "s/XXTSTUB/${TSTUB}/g" vector2tile.namelist - sed -i -e "s#XXTPATH#${TPATH}#g" vector2tile.namelist - - # submit vec2tile - echo '************************************************' - echo 'calling vector2tile' - - if [[ $BASELINE =~ 'hera.internal' ]]; then - source ${HOMElandda}/land_mods - fi - $vec2tileexec vector2tile.namelist - if [[ $? != 0 ]]; then - echo "vec2tile failed" - exit - fi - fi # vector2tile for DA - - if [[ $do_jedi == "YES" && $atmos_forc == "gswp3" ]]; then - - echo '************************************************' - echo 'calling tile2tile' - - export MEM_WORKDIR - - # copy restarts into work directory - - for tile in 1 2 3 4 5 6 - do - rst_in=${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - if [[ ! -e ${rst_in} ]]; then - rst_in=${LANDDA_INPUTS}/restarts/${atmos_forc}/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc - fi - rst_out=${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc - cp ${rst_in} ${rst_out} - done - - # update tile2tile namelist - cp ${HOMElandda}/parm/templates/template.ufs2jedi ufs2jedi.namelist - - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs2jedi.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" ufs2jedi.namelist - sed -i -e "s/XXMM/${MM}/g" ufs2jedi.namelist - sed -i -e "s/XXDD/${DD}/g" ufs2jedi.namelist - sed -i -e "s/XXHH/${HH}/g" ufs2jedi.namelist - sed -i -e "s/XXHH/${HH}/g" ufs2jedi.namelist - sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" ufs2jedi.namelist - sed -i -e "s/XXRES/${RES}/g" ufs2jedi.namelist - sed -i -e "s/XXTSTUB/${TSTUB}/g" ufs2jedi.namelist - sed -i -e "s#XXTPATH#${TPATH}#g" ufs2jedi.namelist - - # submit tile2tile - - if [[ $BASELINE =~ 'hera.internal' ]]; then - source ${HOMElandda}/land_mods - fi - $tile2tileexec ufs2jedi.namelist - if [[ $? != 0 ]]; then - echo "tile2tile failed" - exit - fi - fi # tile2tile for DA - - ############################ - # do DA update - - if [[ $do_jedi == "YES" ]]; then - - # submit snow DA - echo '************************************************' - echo 'calling snow DA' - - cd $WORKDIR - - export THISDATE - export EXECdir="${HOMElandda}/exec" - $DAscript ${HOMElandda}/$DA_config - if [[ $? != 0 ]]; then - echo "land DA script failed" - exit - fi - fi - - ############################ - # convert back to vector/tile - - mem_ens="mem000" - - MEM_WORKDIR=${WORKDIR}/${mem_ens} - MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} - - cd $MEM_WORKDIR - - # convert back to vector, run model (all members) convert back to vector, run model (all members) - if [[ $do_jedi == "YES" && $atmos_forc == "era5" ]]; then - echo '************************************************' - echo 'calling tile2vector' - - if [[ $BASELINE =~ 'hera.internal' ]]; then - source ${HOMElandda}/land_mods - fi - - cp ${HOMElandda}/parm/templates/template.tile2vector tile2vector.namelist - - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" tile2vector.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" tile2vector.namelist - sed -i -e "s/XXMM/${MM}/g" tile2vector.namelist - sed -i -e "s/XXDD/${DD}/g" tile2vector.namelist - sed -i -e "s/XXHH/${HH}/g" tile2vector.namelist - sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" vector2tile.namelist - sed -i -e "s/XXRES/${RES}/g" tile2vector.namelist - sed -i -e "s/XXTSTUB/${TSTUB}/g" tile2vector.namelist - sed -i -e "s#XXTPATH#${TPATH}#g" tile2vector.namelist - - $vec2tileexec tile2vector.namelist - if [[ $? != 0 ]]; then - echo "tile2vector failed" - exit - fi - - # save analysis restart - cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.nc - fi - - # convert back to UFS tile, run model (all members) - if [[ $do_jedi == "YES" && $atmos_forc == "gswp3" ]]; then - echo '************************************************' - echo 'calling tile2tile' - - if [[ $BASELINE =~ 'hera.internal' ]]; then - source ${HOMElandda}/land_mods - fi - - cp ${HOMElandda}/parm/templates/template.jedi2ufs jedi2ufs.namelist - - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" jedi2ufs.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" jedi2ufs.namelist - sed -i -e "s/XXMM/${MM}/g" jedi2ufs.namelist - sed -i -e "s/XXDD/${DD}/g" jedi2ufs.namelist - sed -i -e "s/XXHH/${HH}/g" jedi2ufs.namelist - sed -i -e "s/MODEL_FORCING/${atmos_forc}/g" jedi2ufs.namelist - sed -i -e "s/XXRES/${RES}/g" jedi2ufs.namelist - sed -i -e "s/XXTSTUB/${TSTUB}/g" jedi2ufs.namelist - sed -i -e "s#XXTPATH#${TPATH}#g" jedi2ufs.namelist - - $tile2tileexec jedi2ufs.namelist - if [[ $? != 0 ]]; then - echo "tile2tile failed" - exit - fi - - # save analysis restart - for tile in 1 2 3 4 5 6 - do - cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_anal.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc - cp ${MEM_WORKDIR}/ufs_land_restart.${YYYY}-${MM}-${DD}_${HH}-00-00.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs.cpld.lnd.out.${YYYY}-${MM}-${DD}-00000.tile${tile}.nc - done - fi - - ############################ - # run the forecast model - - if [[ $do_jedi == "YES" && $atmos_forc == "era5" ]]; then - echo '************************************************' - echo 'running the forecast model' - - set -x - - # update model namelist - cp ${HOMElandda}/parm/templates/template.ufs-noahMP.namelist.${atmos_forc} ufs-land.namelist - - sed -i "s|LANDDA_INPUTS|${LANDDA_INPUTS}|g" ufs-land.namelist - sed -i -e "s/XXYYYY/${YYYY}/g" ufs-land.namelist - sed -i -e "s/XXMM/${MM}/g" ufs-land.namelist - sed -i -e "s/XXDD/${DD}/g" ufs-land.namelist - sed -i -e "s/XXHH/${HH}/g" ufs-land.namelist - sed -i -e "s/XXFREQ/${FREQ}/g" ufs-land.namelist - sed -i -e "s/XXRDD/${RDD}/g" ufs-land.namelist - sed -i -e "s/XXRHH/${RHH}/g" ufs-land.namelist - - # submit model - echo $MEM_WORKDIR - - nt=$SLURM_NTASKS - if [[ $BASELINE =~ 'hera.internal' ]]; then - source ${HOMElandda}/land_mods - fi - - if [[ $BASELINE =~ 'hera.internal' ]]; then - srun '--export=ALL' --label -K -n $nt $LSMexec - else - ${MPIEXEC} -n 1 $LSMexec - fi - fi - # no error codes on exit from model, check for restart below instead - - if [[ $do_jedi == "YES" && $atmos_forc == "gswp3" ]]; then - set -e - - echo '************************************************' - echo 'running the forecast model' - - TEST_NAME=datm_cdeps_lnd_gswp3 - TEST_NAME_RST=datm_cdeps_lnd_gswp3_rst - PATHRT=${HOMElandda}/sorc/ufs_model.fd/tests - RT_COMPILER=${RT_COMPILER:-intel} - ATOL="1e-7" - - cp $HOMElandda/$TEST_NAME_RST ${PATHRT}/tests/$TEST_NAME_RST - source ${PATHRT}/detect_machine.sh - source ${PATHRT}/rt_utils.sh - source ${PATHRT}/default_vars.sh - source ${PATHRT}/tests/$TEST_NAME_RST - source ${PATHRT}/atparse.bash - - BL_DATE=20230816 - RTPWD=${RTPWD:-${LANDDA_INPUTS}/NEMSfv3gfs/develop-${BL_DATE}/INTEL/${TEST_NAME}} - INPUTDATA_ROOT=${INPUTDATA_ROOT:-${LANDDA_INPUTS}/NEMSfv3gfs/input-data-20221101} - - echo "RTPWD= $RTPWD" - echo "INPUTDATA_ROOT= $INPUTDATA_ROOT" - - if [[ ! -d ${INPUTDATA_ROOT} ]] || [[ ! -d ${RTPWD} ]]; then - echo "Error: cannot find either folder for INPUTDATA_ROOT or RTPWD, please check!" - exit 1 - fi - - # create run folder - RUNDIR=${MEM_MODL_OUTDIR}/noahmp/${TEST_NAME_RST} - [[ -d ${RUNDIR} ]] && echo "Warning: remove old run folder!" && rm -rf ${RUNDIR} - mkdir -p ${RUNDIR} - cd ${RUNDIR} - - echo "NoahMP run dir= $RUNDIR" - - # modify some env variables - reduce core usage - export ATM_compute_tasks=0 - export ATM_io_tasks=1 - export LND_tasks=6 - export layout_x=1 - export layout_y=1 - - # FV3 executable: - cp ${HOMElandda}/exec/ufs_model . - cp ${HOMElandda}/fv3_run ./fv3_run - - if [[ $DATM_CDEPS = 'true' ]] || [[ $FV3 = 'true' ]] || [[ $S2S = 'true' ]]; then - if [[ $HAFS = 'false' ]] || [[ $FV3 = 'true' && $HAFS = 'true' ]]; then - atparse < ${PATHRT}/parm/${INPUT_NML:-input.nml.IN} > input.nml - fi - fi - - atparse < ${PATHRT}/parm/${MODEL_CONFIGURE:-model_configure.IN} > model_configure - - compute_petbounds_and_tasks - - atparse < ${PATHRT}/parm/${UFS_CONFIGURE:-ufs.configure} > ufs.configure - - # diag table - if [[ "Q${DIAG_TABLE:-}" != Q ]] ; then - atparse < ${PATHRT}/parm/diag_table/${DIAG_TABLE} > diag_table - fi - - # Field table - if [[ "Q${FIELD_TABLE:-}" != Q ]] ; then - cp ${PATHRT}/parm/field_table/${FIELD_TABLE} field_table - fi - - # Field Dictionary - cp ${PATHRT}/parm/fd_ufs.yaml fd_ufs.yaml - - # Set up the run directory - source ./fv3_run - - if [[ $DATM_CDEPS = 'true' ]]; then - atparse < ${PATHRT}/parm/${DATM_IN_CONFIGURE:-datm_in.IN} > datm_in - atparse < ${PATHRT}/parm/${DATM_STREAM_CONFIGURE:-datm.streams.IN} > datm.streams - fi - - # NoahMP table file - cp ${PATHRT}/parm/noahmptable.tbl noahmptable.tbl - - # start runs - echo "Start ufs-cdeps-land model run with TASKS: ${TASKS}" - export MPIRUN=${MPIRUN:-`which mpiexec`} - ${MPIRUN} -n ${TASKS} ./ufs_model - fi - - # no error codes on exit from model, check for restart below instead - - ############################ - # check model ouput (all members) - - mem_ens="mem000" - - MEM_WORKDIR=${WORKDIR}/${mem_ens} - MEM_MODL_OUTDIR=${OUTDIR}/${mem_ens} - - if [[ $atmos_forc == "era5" ]]; then - # if [[ -e ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ]]; then - cp ${MEM_WORKDIR}/ufs_land_restart.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc ${MEM_MODL_OUTDIR}/restarts/vector/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.nc - else - for tile in 1 2 3 4 5 6 - do - cp ${MEM_WORKDIR}/noahmp_output/${TEST_NAME_RST}/ufs.cpld.lnd.out.${nYYYY}-${nMM}-${nDD}-00000.tile${tile}.nc ${MEM_MODL_OUTDIR}/restarts/tile/ufs_land_restart_back.${nYYYY}-${nMM}-${nDD}_${nHH}-00-00.tile${tile}.nc - done - fi - - THISDATE=$NEXTDATE - date_count=$((date_count+1)) - -done # date_count -lt cycles_per_job - - -############################ -# resubmit script - -if [ $THISDATE -lt $ENDDATE ]; then - echo "STARTDATE=${THISDATE}" > ${analdate} - echo "ENDDATE=${ENDDATE}" >> ${analdate} - cd ${HOMElandda} - sbatch ${HOMElandda}/submit_cycle.sh -fi -