From 3d94ae985949a326d3183e7717e96bffcde3ab9c Mon Sep 17 00:00:00 2001
From: BinbinZhou-NOAA <67291029+BinbinZhou-NOAA@users.noreply.github.com>
Date: Fri, 17 May 2024 10:10:10 -0400
Subject: [PATCH] Feature/sref split prepbufr (#477)

* Add split prepbufr files before run PN2NC in ush/mesoscale/evs_prepare_sref.sh

* Add fix the mising data issue for  href PB2NC over Hawaii

* move export bufr_ver=12.0.0 in alphabetical order in run.ver

* Remove bufr_ver=12.0.0 from build.ver
---
 dev/modulefiles/mesoscale/mesoscale_stats.sh  |  1 +
 .../Pb2nc_obsGFS_Prepbufr_Profile.conf        |  2 +-
 .../grid2obs/Pb2nc_obsGFS_Prepbufr.conf       |  4 ++--
 ush/mesoscale/evs_prepare_sref.sh             | 20 +++++++++++++++----
 versions/run.ver                              |  1 +
 5 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/dev/modulefiles/mesoscale/mesoscale_stats.sh b/dev/modulefiles/mesoscale/mesoscale_stats.sh
index cebedaef4e..a98ab585e9 100644
--- a/dev/modulefiles/mesoscale/mesoscale_stats.sh
+++ b/dev/modulefiles/mesoscale/mesoscale_stats.sh
@@ -26,5 +26,6 @@ module load grib_util/${grib_util_ver}
 module load wgrib2/${wgrib2_ver}
 module load met/${met_ver}
 module load metplus/${metplus_ver}
+module load bufr/${bufr_ver}
 
 module list
diff --git a/parm/metplus_config/stats/cam/grid2obs/Pb2nc_obsGFS_Prepbufr_Profile.conf b/parm/metplus_config/stats/cam/grid2obs/Pb2nc_obsGFS_Prepbufr_Profile.conf
index f22435b536..2773c6cdea 100644
--- a/parm/metplus_config/stats/cam/grid2obs/Pb2nc_obsGFS_Prepbufr_Profile.conf
+++ b/parm/metplus_config/stats/cam/grid2obs/Pb2nc_obsGFS_Prepbufr_Profile.conf
@@ -66,7 +66,7 @@ PB2NC_WINDOW_END = 1800
 PB2NC_GRID = {ENV[verif_grid]}
 PB2NC_POLY =
 PB2NC_STATION_ID =
-PB2NC_MESSAGE_TYPE = ADPUPA
+PB2NC_MESSAGE_TYPE = ADPUPA, ADPSFC
 
 PB2NC_LEVEL_RANGE_BEG =  -1000
 PB2NC_LEVEL_RANGE_END = 100000
diff --git a/parm/metplus_config/stats/mesoscale/grid2obs/Pb2nc_obsGFS_Prepbufr.conf b/parm/metplus_config/stats/mesoscale/grid2obs/Pb2nc_obsGFS_Prepbufr.conf
index 56d55c90b0..7cdebfbe08 100644
--- a/parm/metplus_config/stats/mesoscale/grid2obs/Pb2nc_obsGFS_Prepbufr.conf
+++ b/parm/metplus_config/stats/mesoscale/grid2obs/Pb2nc_obsGFS_Prepbufr.conf
@@ -114,7 +114,7 @@ PB2NC_OBS_BUFR_MAP =  { key = "ZOB"; val = "HGT"; },{ key = "TOB"; val = "TMP";
 CONFIG_DIR = {PARM_BASE}/met_config
 
 # directory containing input to PB2NC
-PB2NC_INPUT_DIR = {ENV[COMINobsproc]}
+PB2NC_INPUT_DIR = {ENV[bufrpath]}
 
 # directory to write output from PB2NC
 PB2NC_OUTPUT_DIR = {OUTPUT_BASE}/prepbufr_nc
@@ -123,7 +123,7 @@ METPLUS_CONF = {OUTPUT_BASE}/final_pb2nc_{ENV[vbeg]}.conf
 # End of [dir] section and start of [filename_templates] section
 [filename_templates]
 # Template to look for forecast input to PB2NC relative to PB2NC_INPUT_DIR
-PB2NC_INPUT_TEMPLATE = gfs.{da_init?fmt=%Y%m%d}/{da_init?fmt=%H}/atmos/gfs.t{da_init?fmt=%H}z.prepbufr
+PB2NC_INPUT_TEMPLATE = prepbufr.{da_init?fmt=%Y%m%d}/gdas.t{da_init?fmt=%H}z.prepbufr
 
 # Template to use to write output from PB2NC
 PB2NC_OUTPUT_TEMPLATE = prepbufr.t{da_init?fmt=%H}z.grid212.nc
diff --git a/ush/mesoscale/evs_prepare_sref.sh b/ush/mesoscale/evs_prepare_sref.sh
index eb25d47ba4..6438f5827d 100755
--- a/ush/mesoscale/evs_prepare_sref.sh
+++ b/ush/mesoscale/evs_prepare_sref.sh
@@ -2,7 +2,12 @@
 #**************************************************************************
 #  Purpose: Get required input forecast and validation data files
 #           for sref stat jobs
-#  Last update: 10/30/2023, by Binbin Zhou Lynker@EMC/NCEP
+#  Last update: 
+#               05/04/2024, (1) change gfs to gdas for prepbufr files
+#                           (2) split the prepbufr files before running METplus PB2NC
+#                               to save walltime
+#                             by Binbin Zhou Lynker@EMC/NCEP
+#               10/30/2023, by Binbin Zhou Lynker@EMC/NCEP
 #************************************************************************
 set -x
 
@@ -201,13 +206,20 @@ if [ $modnam = prepbufr ] && [ ! -e $DATA/prepbufr.missing ] ; then
 
 export output_base=${WORK}/pb2nc
 
- if [ -s ${COMINobsproc}/gfs.${vday}/??/atmos/gfs.t??z.prepbufr ] ; then
+ if [ -s ${COMINobsproc}/gdas.${vday}/??/atmos/gdas.t??z.prepbufr ] ; then
 
    for vhr in 00  06  12  18  ; do
 
      export vbeg=${vhr}
      export vend=${vhr}
 
+     #Split the prepbufr data files into specifiically required data types to reduce
+     #the walltime
+     >$WORK/prepbufr.$vday/gdas.t${vhr}z.prepbufr
+     split_by_subset ${COMINobsproc}/gdas.${vday}/$vhr/atmos/gdas.t${vhr}z.prepbufr
+     cat $WORK/ADPSFC $WORK/SFCSHP $WORK/ADPUPA >> $WORK/prepbufr.$vday/gdas.t${vhr}z.prepbufr
+
+     export bufrpath=$WORK
      ${METPLUS_PATH}/ush/run_metplus.py -c ${PARMevs}/metplus_config/machine.conf -c ${GRID2OBS_CONF}/Pb2nc_obsGFS_Prepbufr.conf
      export err=$?; err_chk
      if [ -s ${WORK}/pb2nc/prepbufr_nc/*.nc ] ; then
@@ -216,11 +228,11 @@ export output_base=${WORK}/pb2nc
    done
 
  else
-  echo "WARNING: Missing file is ${COMINobsproc}/gfs.${vday}/??/atmos/gfs.t??z.prepbufr"
+  echo "WARNING: Missing file is ${COMINobsproc}/gdas.${vday}/??/atmos/gdas.t??z.prepbufr"
   if [ $SENDMAIL = YES ] ; then
    export subject="Prepbufr Data Missing for EVS ${COMPONENT}"
    echo "WARNING:  No Prepbufr data available for ${VDATE}" > mailmsg
-   echo "Missing file is ${COMINobsproc}/gfs.${vday}/??/atmos/gfs.t??z.prepbufr"  >> mailmsg
+   echo "Missing file is ${COMINobsproc}/gdas.${vday}/??/atmos/gdas.t??z.prepbufr"  >> mailmsg
    echo "Job ID: $jobid" >> mailmsg
    cat mailmsg | mail -s "$subject" $MAILTO 
   fi
diff --git a/versions/run.ver b/versions/run.ver
index fea0404a49..34238a4db3 100644
--- a/versions/run.ver
+++ b/versions/run.ver
@@ -1,6 +1,7 @@
 export evs_ver=v1.0.8
 
 export bacio_ver=2.4.1
+export bufr_ver=12.0.0
 export cdo_ver=1.9.8
 export cfp_ver=2.0.4
 export craympich_ver=8.1.19