Merge branch 'NOAA-EMC:develop' into develop

.gitignore

@@ -35,8 +35,10 @@ fix/gsi
 fix/lut
 fix/mom6
 fix/orog
+fix/orog_nest
 fix/sfc_climo
 fix/ugwd
+fix/ugwd_nest
 fix/verif
 fix/wave
 
@@ -164,6 +166,14 @@ scripts/exemcsfc_global_sfc_prep.sh
 scripts/exgdas_global_marine_analysis_ecen.py
 scripts/exglobal_prep_ocean_obs.py
 # ush symlinks
+ush/bufr2ioda_insitu_profile_argo.py
+ush/bufr2ioda_insitu_profile_bathy.py
+ush/bufr2ioda_insitu_profile_glider.py
+ush/bufr2ioda_insitu_profile_marinemammal.py
+ush/bufr2ioda_insitu_profile_tesac.py
+ush/bufr2ioda_insitu_profile_xbtctd.py
+ush/bufr2ioda_insitu_surface_altkob.py
+ush/bufr2ioda_insitu_surface_trkob.py
 ush/chgres_cube.sh
 ush/emcsfc_ice_blend.sh
 ush/emcsfc_snow.sh
@@ -174,6 +184,8 @@ ush/fv3gfs_make_grid.sh
 ush/fv3gfs_make_orog.sh
 ush/gen_bufr2ioda_json.py
 ush/gen_bufr2ioda_yaml.py
+ush/bufr2ioda_insitu_profile*.py
+ush/bufr2ioda_insitu_surface*.py
 ush/global_chgres.sh
 ush/global_chgres_driver.sh
 ush/global_cycle.sh

.gitmodules

@@ -26,11 +26,3 @@
 [submodule "sorc/gsi_monitor.fd"]
   path = sorc/gsi_monitor.fd
   url = https://github.com/NOAA-EMC/GSI-Monitor.git
-[submodule "sorc/upp.fd"]
-  path = sorc/upp.fd
-  url = https://github.com/NOAA-EMC/UPP.git
-  ignore = dirty
-[submodule "sorc/jcb"]
-  path = sorc/jcb
-  url = https://github.com/noaa-emc/jcb
-  fetchRecurseSubmodules = false

ci/Jenkinsfile

@@ -76,7 +76,7 @@ pipeline {
                     Machine = machine[0].toUpperCase() + machine.substring(1)
                     echo "Getting Common Workspace for ${Machine}"
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
-                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         HOME = "${WORKSPACE}"
                         sh(script: "mkdir -p ${HOME}/RUNTESTS;rm -Rf ${HOME}/RUNTESTS/*")
                         sh(script: """${GH} pr edit ${env.CHANGE_ID} --repo ${repo_url} --add-label "CI-${Machine}-Building" --remove-label "CI-${Machine}-Ready" """)

ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml

@@ -0,0 +1,18 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 384
+  resdetocean: 0.25
+  nens: 0
+  gfs_cyc: 4
+  start: cold
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  idate: 2021063000
+  edate: 2021070306
+  icsdir: /scratch1/NCEPDEV/climate/Jessica.Meixner/cycling/IC_2021063000_V2
+  yaml: {{ HOMEgfs }}/ci/cases/gfsv17/ocnanal.yaml

ci/cases/gfsv17/ocnanal.yaml

@@ -0,0 +1,30 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+
+base:
+  DOIAU: "YES"
+  DO_JEDIATMVAR: "NO"
+  DO_JEDIATMENS: "NO"
+  DO_JEDIOCNVAR: "YES"
+  DO_JEDISNOWDA: "NO"
+  DO_MERGENSST: "NO"
+  DOHYBVAR: "NO"
+  DO_FIT2OBS: "YES"
+  DO_VERFOZN: "YES"
+  DO_VERFRAD: "YES"
+  DO_VRFY_OCEANDA: "NO"
+  FHMAX_GFS: 240
+
+ocnanal:
+  SOCA_INPUT_FIX_DIR: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/static/1440x1080x75/soca
+  CASE_ANL: 'C24'
+  SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
+  SOCA_NINNER: 100
+  SABER_BLOCKS_YAML: ''
+  NICAS_RESOL: 1
+  NICAS_GRID_SIZE: 15000
+
+prepoceanobs:
+  SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml 
+  OBSPREP_YAML: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml
+  DMPDIR: /scratch1/NCEPDEV/da/common/

ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -21,5 +21,4 @@ skip_ci_on_hosts:
   - hera
   - orion
   - hercules
-  - wcoss2
 

ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml

@@ -1,6 +1,5 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
-  DOIAU: "NO"
   DO_JEDISNOWDA: "YES"
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}

ci/cases/yamls/gefs_ci_defaults.yaml

@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml
 base:
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  HPC_ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}

ci/cases/yamls/gfs_defaults_ci.yaml

@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}

ci/cases/yamls/gfs_extended_ci.yaml

@@ -2,11 +2,12 @@ defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 
 base:
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
   DO_GOES: "YES"
   DO_BUFRSND: "YES"
   DO_GEMPAK: "YES"
   DO_AWIPS: "NO"
   DO_NPOESS: "YES"
   DO_GENESIS_FSU: "NO"
   FHMAX_GFS: 384
+  FHMAX_HF_GFS: 120

ci/cases/yamls/soca_gfs_defaults_ci.yaml

@@ -1,5 +1,5 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
   DO_JEDIOCNVAR: "YES"

ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

@@ -4,7 +4,7 @@ base:
   DOIAU: "NO"
   DO_JEDIATMVAR: "YES"
   DO_JEDIATMENS: "YES"
-  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
 atmanl:
   LAYOUT_X_ATMANL: 4
   LAYOUT_Y_ATMANL: 4

ci/platforms/config.hera

@@ -2,8 +2,9 @@
 
 export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
 export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
-export STMP="/scratch1/NCEPDEV/stmp2/${USER}"
-export PTMP="/scratch1/NCEPDEV/stmp2/${USER}"
-export SLURM_ACCOUNT=nems
+export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4
+
+export JENKINS_AGENT_LANUCH_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness/Jenkins
+export JENKINS_WORK_DIR=/scratch1/NCEPDEV/global/Terry.McGuinness

ci/platforms/config.hercules

@@ -2,8 +2,9 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/HERCULES
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export STMP="/work2/noaa/stmp/${USER}/HERCULES"
-export PTMP="/work2/noaa/stmp/${USER}/HERCULES"
-export SLURM_ACCOUNT=nems
+export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4
+
+export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry
+export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS

ci/platforms/config.orion

@@ -2,8 +2,9 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export STMP="/work2/noaa/stmp/${USER}/ORION"
-export PTMP="/work2/noaa/stmp/${USER}/ORION"
-export SLURM_ACCOUNT=nems
+export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4
+
+export JENKINS_AGENT_LANUCH_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS/AGENT_mterry
+export JENKINS_WORK_DIR=/home/role-nems/GFS_CI_ROOT_JENKINS

ci/platforms/config.wcoss2

@@ -2,8 +2,6 @@
 
 export GFS_CI_ROOT=/lfs/h2/emc/global/noscrub/globalworkflow.ci/GFS_CI_ROOT
 export ICSDIR_ROOT=/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR
-export STMP="/lfs/h2/emc/stmp/${USER}"
-export PTMP="/lfs/h2/emc/ptmp/${USER}"
-export SLURM_ACCOUNT=GFS-DEV
+export HPC_ACCOUNT=GFS-DEV
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/scripts/clone-build_ci.sh

@@ -74,7 +74,7 @@ set +e
 source "${HOMEgfs}/ush/module-setup.sh"
 export BUILD_JOBS=8
 rm -rf log.build
-./build_all.sh -gk  >> log.build 2>&1
+./build_all.sh -guk  >> log.build 2>&1
 build_status=$?
 
 DATE=$(date +'%D %r')

ci/scripts/driver.sh

@@ -228,7 +228,7 @@ for pr in ${pr_list}; do
         } >> "${output_ci}"
         continue
       fi
-      "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml"  > "${LOGFILE_PATH}" 2>&1
+      "${HOMEgfs}/workflow/create_experiment.py" --yaml "${HOMEgfs}/ci/cases/pr/${case}.yaml" --overwrite  > "${LOGFILE_PATH}" 2>&1
       ci_status=$?
       set -e
       if [[ ${ci_status} -eq 0 ]]; then

ci/scripts/run-check_ci.sh

@@ -64,7 +64,7 @@ while true; do
   rocotorun -v "${ROCOTO_VERBOSE:-0}" -w "${xml}" -d "${db}"
 
   # Wait before running rocotostat
-  sleep 10
+  sleep 60
 
   # Get job statistics
   echo "Gather Rocoto statistics"

ci/scripts/utils/ci_utils.sh

@@ -122,7 +122,8 @@ function create_experiment () {
   source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
   source "${HOMEgfs}/workflow/gw_setup.sh"
 
-  # system=$(grep "system:" "${yaml_config}" | cut -d":" -f2 | tr -d " ") || true
+  # Remove RUNDIRS dir incase this is a retry
+  rm -Rf "${STMP}/RUNDIRS/${pslot}"
 
   "${HOMEgfs}/${system}/workflow/create_experiment.py" --overwrite --yaml "${yaml_config}"
 

ci/scripts/utils/launch_java_agent.sh

@@ -0,0 +1,77 @@
+#!/bin/env bash
+set -e
+
+controller_url="https://jenkins.epic.oarcloud.noaa.gov"
+controller_user="terry.mcguinness"
+HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
+host=$(hostname)
+
+#########################################################################
+#  Set up runtime environment varibles for accounts on supproted machines
+#########################################################################
+
+source "${HOMEgfs}/ush/detect_machine.sh"
+case ${MACHINE_ID} in
+  hera | orion | hercules | wcoss2)
+    echo "Launch Jenkins Java Controler on ${MACHINE_ID}"
+    ;;
+  *)
+    echo "Unsupported platform. Exiting with error."
+    exit 1
+    ;;
+esac
+
+LOG=lanuched_agent-$(date +%Y%m%d%M).log
+rm -f "${LOG}"
+
+source "${HOMEgfs}/ush/module-setup.sh"
+module use "${HOMEgfs}/modulefiles"
+module load "module_gwsetup.${MACHINE_ID}"
+source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
+
+JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10"
+if [[ ! -d "${JAVA_HOME}" ]]; then
+  JAVA_HOME=/usr/lib/jvm/jre-17
+  if [[ ! -d "${JAVA_HOME}" ]]; then
+    echo "ERROR: JAVA_HOME not found. Exiting with error."
+    exit 1
+  fi
+fi
+
+JAVA="${JAVA_HOME}/bin/java"
+echo "JAVA VERSION: "
+${JAVA} -version
+
+export GH="${HOME}/bin/gh"
+command -v "${GH}"
+${GH} --version
+
+if [[ -d "${JENKINS_AGENT_LANUCH_DIR}" ]]; then
+  echo "Jenkins Agent Lanuch Directory: ${JENKINS_AGENT_LANUCH_DIR}"
+else
+  echo "ERROR: Jenkins Agent Lanuch Directory not found. Exiting with error."
+  exit 1
+fi
+cd "${JENKINS_AGENT_LANUCH_DIR}"
+
+if ! [[ -f agent.jar ]]; then
+  curl -sO "${controller_url}/jnlpJars/agent.jar"
+fi
+
+JENKINS_TOKEN=$(cat jenkins_token)
+
+# 
+offline=$(curl --silent -u "${controller_user}:${JENKINS_TOKEN}" "${controller_url}/computer/${MACHINE_ID^}-EMC/api/json?pretty=true" | grep '\"offline\"' | awk '{gsub(/,/,"");print $3}') || true
+echo "Jenkins Agent offline setting: ${offline}"
+
+if [[ "${offline}" == "true" ]]; then
+  echo "Jenkins Agent is offline. Lanuching Jenkins Agent on ${host}"
+  command="nohup ${JAVA} -jar agent.jar -jnlpUrl ${controller_url}/computer/${MACHINE_ID^}-EMC/jenkins-agent.jnlp  -secret @jenkins-secret-file -workDir ${JENKINS_WORK_DIR}"
+  echo -e "Lanuching Jenkins Agent on ${host} with the command:\n${command}" >& "${LOG}"
+  ${command} >> "${LOG}" 2>&1 &
+  nohup_PID=$!
+  echo "Java agent running on PID: ${nohup_PID}" >> "${LOG}" 2>&1
+  echo "Java agent running on PID: ${nohup_PID}"
+else
+  echo "Jenkins Agent is online (nothing done)"
+fi