diff --git a/env/AWSPW.env b/env/AWSPW.env index e366128a1d..f365695f85 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -33,7 +33,29 @@ else exit 2 fi -if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then +if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then + + export POE="NO" + export BACK="NO" + export sys_tp="AWSPW" + export launcher_PREP="srun" + +elif [[ "${step}" = "prepsnowobs" ]]; then + + export APRUN_CALCFIMS="${APRUN_default}" + +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN="${APRUN_default}" + +elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then + + export CFP_MP="YES" + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi + export wavempexec=${launcher} + export wave_mpmd=${mpmd_opt} + +elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then export launcher="srun --mpi=pmi2 -l" @@ -52,52 +74,16 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} elif [[ "${step}" = "post" ]]; then - export NTHREADS_NP=${NTHREADS1} - export APRUN_NP="${APRUN_default}" - - export NTHREADS_DWN=${threads_per_task_dwn:-1} - [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} - export APRUN_DWN="${launcher} -n ${ntasks_dwn}" - -elif [[ "${step}" = "atmos_products" ]]; then - - export USE_CFP="YES" # Use MPMD for downstream product generation on Hera + export NTHREADS_UPP=${NTHREADS1} + export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}" elif [[ "${step}" = "oceanice_products" ]]; then export NTHREADS_OCNICEPOST=${NTHREADS1} export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" -elif [[ "${step}" = "ecen" ]]; then - - export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN_default}" - - export NTHREADS_CHGRES=${threads_per_task_chgres:-12} - [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} - export APRUN_CHGRES="time" - - export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} - [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN_default}" - -elif [[ "${step}" = "esfc" ]]; then - - export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN_default}" - - export NTHREADS_CYCLE=${threads_per_task_cycle:-14} - [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN_default}" - -elif [[ "${step}" = "epos" ]]; then - - export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN_default}" - -elif [[ "${step}" = "fit2obs" ]]; then +elif [[ "${step}" = "atmos_products" ]]; then - export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN_default}" + export USE_CFP="YES" # Use MPMD for downstream product generation on AWS fi diff --git a/env/AZUREPW.env b/env/AZUREPW.env index c2faeb2bf6..b2b4063ff3 100755 --- a/env/AZUREPW.env +++ b/env/AZUREPW.env @@ -15,6 +15,7 @@ export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment export OMP_STACKSIZE=2048000 export NTHSTACK=1024000000 +export UCX_TLS=ud,sm,self ulimit -s unlimited ulimit -a @@ -50,6 +51,10 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} export wavempexec=${launcher} export wave_mpmd=${mpmd_opt} +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN="${APRUN_default}" + elif [[ "${step}" = "post" ]]; then export NTHREADS_NP=${NTHREADS1} @@ -71,7 +76,7 @@ elif [[ "${step}" = "oceanice_products" ]]; then elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN}" + export APRUN_ECEN="${APRUN_default}" export NTHREADS_CHGRES=${threads_per_task_chgres:-12} [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} @@ -79,25 +84,25 @@ elif [[ "${step}" = "ecen" ]]; then export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN}" + export APRUN_CALCINC="${APRUN_default}" elif [[ "${step}" = "esfc" ]]; then export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN}" + export APRUN_ESFC="${APRUN_default}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN}" + export APRUN_CYCLE="${APRUN_default}" elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN}" + export APRUN_EPOS="${APRUN_default}" elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN}" + export MPIRUN="${APRUN_default}" fi diff --git a/env/GOOGLEPW.env b/env/GOOGLEPW.env index c3b5ec806a..d84008d648 100755 --- a/env/GOOGLEPW.env +++ b/env/GOOGLEPW.env @@ -45,7 +45,7 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then elif [[ "${step}" = "prep_emissions" ]]; then - export APRUN + export APRUN="${APRUN_default}" elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then @@ -102,6 +102,6 @@ elif [[ "${step}" = "epos" ]]; then elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN}" + export MPIRUN="${APRUN_default}" fi diff --git a/modulefiles/module_base.noaacloud.lua b/modulefiles/module_base.noaacloud.lua index 7997b618e4..3a7cc75d7a 100644 --- a/modulefiles/module_base.noaacloud.lua +++ b/modulefiles/module_base.noaacloud.lua @@ -5,8 +5,11 @@ Load environment to run GFS on noaacloud local spack_mod_path=(os.getenv("spack_mod_path") or "None") prepend_path("MODULEPATH", spack_mod_path) +load("gnu") load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) +unload("gnu") + load(pathJoin("python", (os.getenv("python_ver") or "None"))) load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) diff --git a/modulefiles/module_gwci.noaacloud.lua b/modulefiles/module_gwci.noaacloud.lua index c3142cd60d..2ac284ef85 100644 --- a/modulefiles/module_gwci.noaacloud.lua +++ b/modulefiles/module_gwci.noaacloud.lua @@ -2,10 +2,10 @@ help([[ Load environment to run GFS workflow setup scripts on noaacloud ]]) -prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-env/install/modulefiles/Core") -load(pathJoin("stack-intel", os.getenv("2021.3.0"))) -load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.3.0"))) +load(pathJoin("stack-intel", os.getenv("2021.10.0"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.10.0"))) load(pathJoin("netcdf-c", os.getenv("4.9.2"))) load(pathJoin("netcdf-fortran", os.getenv("4.6.1"))) diff --git a/modulefiles/module_gwsetup.noaacloud.lua b/modulefiles/module_gwsetup.noaacloud.lua index f3845e8d72..e2aa4050a3 100644 --- a/modulefiles/module_gwsetup.noaacloud.lua +++ b/modulefiles/module_gwsetup.noaacloud.lua @@ -4,17 +4,18 @@ Load environment to run GFS workflow setup scripts on noaacloud load(pathJoin("rocoto")) -prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core") -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0" -local python_ver=os.getenv("python_ver") or "3.10.3" +load("gnu") +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0" +local stack_mpi_ver=os.getenv("stack_mpi_ver") or "2021.10.0" load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("python", python_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_mpi_ver)) +unload("gnu") + load("py-jinja2") load("py-pyyaml") load("py-numpy") -local git_ver=os.getenv("git_ver") or "1.8.3.1" -load(pathJoin("git", git_ver)) whatis("Description: GFS run setup environment") diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index e1b9a036de..68f81c1039 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -41,15 +41,15 @@ case ${machine} in ;; "AWSPW") export PARTITION_BATCH="compute" - max_tasks_per_node=36 + max_tasks_per_node=48 ;; "AZUREPW") export PARTITION_BATCH="compute" - max_tasks_per_node=24 + max_tasks_per_node=36 ;; "GOOGLEPW") export PARTITION_BATCH="compute" - max_tasks_per_node=32 + max_tasks_per_node=30 ;; *) echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}" diff --git a/parm/config/gefs/config.resources.AWSPW b/parm/config/gefs/config.resources.AWSPW index a735c7622d..f91460b6aa 100644 --- a/parm/config/gefs/config.resources.AWSPW +++ b/parm/config/gefs/config.resources.AWSPW @@ -9,3 +9,61 @@ unset memory for mem_var in $(env | grep '^memory_' | cut -d= -f1); do unset "${mem_var}" done + +step=$1 + +case ${step} in + "fcst" | "efcs") + export PARTITION_BATCH="compute" + max_tasks_per_node=48 + ;; + + "arch") + export PARTITION_BATCH="process" + max_tasks_per_node=24 + ;; + + "prep_emissions") + export PARTITION_BATCH="process" + max_tasks_per_node=24 + export ntasks=1 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + + "waveinit") + export PARTITION_BATCH="process" + max_tasks_per_node=24 + export ntasks=12 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export NTASKS=${ntasks} + ;; + + "wavepostpnt") + export PARTITION_BATCH="compute" + max_tasks_per_node=48 + export ntasks=240 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export NTASKS=${ntasks} + ;; + + "wavepostsbs" | "wavepostbndpnt" | "wavepostbndpntbll") + export PARTITION_BATCH="process" + max_tasks_per_node=24 + export ntasks=24 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export NTASKS=${ntasks} + ;; + + *) + export PARTITION_BATCH="process" + max_tasks_per_node=24 + ;; + +esac + +export max_tasks_per_node + diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 230872b8f3..eeb33716c0 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -107,16 +107,16 @@ case ${machine} in ;; "AWSPW") export PARTITION_BATCH="compute" - npe_node_max=36 - max_tasks_per_node=36 + npe_node_max=48 + max_tasks_per_node=48 # TODO Supply a max mem/node value for AWS # shellcheck disable=SC2034 mem_node_max="" ;; "AZUREPW") export PARTITION_BATCH="compute" - npe_node_max=24 - max_tasks_per_node=24 + npe_node_max=36 + max_tasks_per_node=36 # TODO Supply a max mem/node value for AZURE # shellcheck disable=SC2034 mem_node_max="" diff --git a/parm/config/gfs/config.resources.AWSPW b/parm/config/gfs/config.resources.AWSPW index a735c7622d..22fe110670 100644 --- a/parm/config/gfs/config.resources.AWSPW +++ b/parm/config/gfs/config.resources.AWSPW @@ -9,3 +9,27 @@ unset memory for mem_var in $(env | grep '^memory_' | cut -d= -f1); do unset "${mem_var}" done + +step=$1 + +case ${step} in + "fcst" | "efcs") + export PARTITION_BATCH="compute" + max_tasks_per_node=48 + ;; + + "arch") + export PARTITION_BATCH="process" + max_tasks_per_node=24 + ;; + + + *) + export PARTITION_BATCH="process" + max_tasks_per_node=24 + ;; + +esac + +export max_tasks_per_node + diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh index 773c104be3..3b0b3ed638 100755 --- a/sorc/build_ufs.sh +++ b/sorc/build_ufs.sh @@ -12,7 +12,7 @@ EXEC_NAME="gfs_model.x" while getopts ":da:fj:e:vwy" option; do case "${option}" in - d) BUILD_TYPE="Debug";; + d) BUILD_TYPE="DEBUG";; a) APP="${OPTARG}";; f) FASTER="ON";; j) BUILD_JOBS="${OPTARG}";; diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 856a42076a..4848ecbb5e 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 856a42076a65256aaae9b29f4891532cb4a3fbca +Subproject commit 4848ecbb5e713b16127433e11f7d3edc6ac784c4 diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 06eec5b6f6..2323761084 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 06eec5b6f636123835e2dfd9fc5229980c006735 +Subproject commit 23237610845c3a4438b21b25e9b3dc25c4c15b73 diff --git a/versions/build.noaacloud.ver b/versions/build.noaacloud.ver index fc288b76b5..b5fd272b4b 100644 --- a/versions/build.noaacloud.ver +++ b/versions/build.noaacloud.ver @@ -1,5 +1,5 @@ -export stack_intel_ver=2021.3.0 -export stack_impi_ver=2021.3.0 +export stack_intel_ver=2021.10.0 +export stack_impi_ver=2021.10.0 export spack_env=gsi-addon-env source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" +export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" diff --git a/versions/run.noaacloud.ver b/versions/run.noaacloud.ver index 1fc3779b2e..98ec2b36f9 100644 --- a/versions/run.noaacloud.ver +++ b/versions/run.noaacloud.ver @@ -1,8 +1,8 @@ -export stack_intel_ver=2021.3.0 -export stack_impi_ver=2021.3.0 +export stack_intel_ver=2021.10.0 +export stack_impi_ver=2021.10.0 export spack_env=gsi-addon-env source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" +export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" export cdo_ver=2.2.0 diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index b98c838faa..c80800725a 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -27,5 +27,5 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -SUPPORT_WAVES: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C192', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. +AERO_INPUTS_DIR: /contrib/global-workflow-shared-data/data/gocart_emissions diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index 4725e28962..d7c064dc60 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -24,5 +24,7 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AZURE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORT_WAVES: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. +DO_TRACKER: 'NO' +DO_GENESIS: 'NO' +DO_METP: 'NO' +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index 1b979b6bc9..8ba8e18e74 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -24,5 +24,7 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from GOOGLE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORT_WAVES: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. +DO_TRACKER: 'NO' +DO_GENESIS: 'NO' +DO_METP: 'NO' +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 574dc0d91a..09bc1c90ac 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -372,7 +372,6 @@ def query_and_clean(dirname, force_clean=False): def validate_user_request(host, inputs): supp_res = host.info['SUPPORTED_RESOLUTIONS'] - supp_waves = host.info.get('SUPPORT_WAVES', 'YES') machine = host.machine for attr in ['resdetatmos', 'resensatmos']: try: @@ -382,9 +381,6 @@ def validate_user_request(host, inputs): if expt_res not in supp_res: raise NotImplementedError(f"Supported resolutions on {machine} are:\n{', '.join(supp_res)}") - if "W" in inputs.app and supp_waves == "NO": - raise NotImplementedError(f"Waves are not supported on {machine}") - def get_ocean_resolution(resdetatmos): """