diff --git a/data_override/include/data_override.inc b/data_override/include/data_override.inc index 17360d0b8..9b2db57f7 100644 --- a/data_override/include/data_override.inc +++ b/data_override/include/data_override.inc @@ -599,6 +599,10 @@ subroutine read_table_yaml(data_table) if (index(trim(filename), "ens_") .ne. 0) then if (file_exists(filename) .and. file_exists("data_table.yaml")) & call mpp_error(FATAL, "Both data_table.yaml and "//trim(filename)//" exists, pick one!") + + !< If the end_* file does not exist, revert back to "data_table.yaml" + !! where every ensemble is using the same yaml + if (.not. file_exists(filename)) filename = "data_table.yaml" endif file_id = open_and_parse_file(trim(filename)) diff --git a/diag_manager/fms_diag_yaml.F90 b/diag_manager/fms_diag_yaml.F90 index 5288e7f45..c36bcc93c 100644 --- a/diag_manager/fms_diag_yaml.F90 +++ b/diag_manager/fms_diag_yaml.F90 @@ -36,7 +36,7 @@ module fms_diag_yaml_mod time_diurnal, time_power, time_none, r8, i8, r4, i4, DIAG_NOT_REGISTERED, & middle_time, begin_time, end_time, MAX_STR_LEN use yaml_parser_mod, only: open_and_parse_file, get_value_from_key, get_num_blocks, get_nkeys, & - get_block_ids, get_key_value, get_key_ids, get_key_name + get_block_ids, get_key_value, get_key_ids, get_key_name, missing_file_error_code use fms_yaml_output_mod, only: fmsYamlOutKeys_type, fmsYamlOutValues_type, write_yaml_from_struct_3, & yaml_out_add_level2key, initialize_key_struct, initialize_val_struct use mpp_mod, only: mpp_error, FATAL, NOTE, mpp_pe, mpp_root_pe, stdout @@ -394,8 +394,13 @@ subroutine diag_yaml_object_init(diag_subset_output) if (index(trim(yamlfilename), "ens_") .ne. 0) then if (file_exists(yamlfilename) .and. file_exists("diag_table.yaml")) & call mpp_error(FATAL, "Both diag_table.yaml and "//trim(yamlfilename)//" exists, pick one!") + !< If the end_* file does not exist, revert back to "diag_table.yaml" + !! where every ensemble is using the same yaml + if (.not. file_exists(yamlfilename)) yamlfilename = "diag_table.yaml" endif diag_yaml_id = open_and_parse_file(trim(yamlfilename)) + if (diag_yaml_id .eq. missing_file_error_code) & + call mpp_error(FATAL, "The "//trim(yamlfilename)//" is not present and it is required!") call diag_get_value_from_key(diag_yaml_id, 0, "title", diag_yaml%diag_title) call get_value_from_key(diag_yaml_id, 0, "base_date", diag_yaml%diag_basedate) diff --git a/field_manager/field_manager.F90 b/field_manager/field_manager.F90 index ba2641719..407581bb0 100644 --- a/field_manager/field_manager.F90 +++ b/field_manager/field_manager.F90 @@ -618,6 +618,10 @@ subroutine read_field_table_yaml(nfields, table_name) if (index(trim(filename), "ens_") .ne. 0) then if (file_exists(filename) .and. file_exists(tbl_name)) & call mpp_error(FATAL, "Both "//trim(tbl_name)//" and "//trim(filename)//" exists, pick one!") + + !< If the end_* file does not exist, revert back to tbl_name + !! where every ensemble is using the same yaml + if (.not. file_exists(filename)) filename = tbl_name endif if (.not. file_exists(trim(filename))) then diff --git a/parser/yaml_parser.F90 b/parser/yaml_parser.F90 index a34c5d4e2..b99d615f7 100644 --- a/parser/yaml_parser.F90 +++ b/parser/yaml_parser.F90 @@ -39,6 +39,7 @@ module yaml_parser_mod private public :: open_and_parse_file +public :: missing_file_error_code public :: get_num_unique_blocks public :: get_unique_block_ids public :: get_block_name @@ -52,6 +53,8 @@ module yaml_parser_mod !public :: clean_up !> @} +integer, parameter :: missing_file_error_code = 999 + !> @brief Dermine the value of a key from a keyname !> @ingroup yaml_parser_mod interface get_value_from_key @@ -253,7 +256,7 @@ function open_and_parse_file(filename) & inquire(file=trim(filename), EXIST=yaml_exists) if (.not. yaml_exists) then - file_id = 999 + file_id = missing_file_error_code call mpp_error(NOTE, "The yaml file:"//trim(filename)//" does not exist, hopefully this is your intent!") return end if diff --git a/test_fms/data_override/test_data_override_ensembles.sh b/test_fms/data_override/test_data_override_ensembles.sh index afcdcd458..0fed41c5b 100755 --- a/test_fms/data_override/test_data_override_ensembles.sh +++ b/test_fms/data_override/test_data_override_ensembles.sh @@ -86,7 +86,7 @@ data_table: fieldname_in_model: runoff override_file: - fieldname_in_file: runoff - file_name: INPUT/runoff.daitren.clim.1440x1080.v20180328_ens_02.nc + file_name: INPUT/runoff.daitren.clim.1440x1080.v20180328_ens_01.nc interp_method: none factor: 1.0 _EOF @@ -94,6 +94,28 @@ _EOF test_expect_failure "test_data_override with both data_table.yaml and data_table.ens_xx.yaml files" ' mpirun -n 12 ../test_data_override_ongrid_${KIND} ' + +cat <<_EOF > input.nml +&data_override_nml + use_data_table_yaml = .True. +/ + +&test_data_override_ongrid_nml + test_case = 6 + write_only = .False. +/ + +&ensemble_nml + ensemble_size = 2 +/ +_EOF + +rm -rf INPUT/. +rm -rf data_table.ens_01.yaml data_table.ens_02.yaml +test_expect_success "test_data_override with two ensembles, same yaml file (${KIND})" ' + mpirun -n 12 ../test_data_override_ongrid_${KIND} + ' + rm -rf INPUT fi test_done diff --git a/test_fms/data_override/test_data_override_ongrid.F90 b/test_fms/data_override/test_data_override_ongrid.F90 index d8e3864ba..86feda79b 100644 --- a/test_fms/data_override/test_data_override_ongrid.F90 +++ b/test_fms/data_override/test_data_override_ongrid.F90 @@ -55,6 +55,7 @@ program test_data_override_ongrid integer, parameter :: scalar = 3 integer, parameter :: weight_file = 4 integer, parameter :: ensemble_case = 5 +integer, parameter :: ensemble_same_yaml = 6 integer :: test_case = ongrid integer :: npes integer, allocatable :: pelist(:) @@ -82,7 +83,7 @@ program test_data_override_ongrid call mpp_get_current_pelist(pelist) select case (test_case) -case (ensemble_case) +case (ensemble_case, ensemble_same_yaml) call set_up_ensemble_case() end select @@ -93,7 +94,7 @@ program test_data_override_ongrid call mpp_get_data_domain(Domain, is, ie, js, je) select case (test_case) -case (ensemble_case) +case (ensemble_case, ensemble_same_yaml) ! Go back to the full pelist call mpp_set_current_pelist(pelist) end select @@ -108,7 +109,7 @@ program test_data_override_ongrid call generate_scalar_input_file () case (weight_file) call generate_weight_input_file () - case (ensemble_case) + case (ensemble_case, ensemble_same_yaml) call generate_ensemble_input_file() end select @@ -117,7 +118,7 @@ program test_data_override_ongrid else select case (test_case) - case (ensemble_case) + case (ensemble_case, ensemble_same_yaml) !< Go back to the ensemble pelist call mpp_set_current_pelist(pelist_ens) end select @@ -134,7 +135,7 @@ program test_data_override_ongrid call scalar_test() case (weight_file) call weight_file_test() - case (ensemble_case) + case (ensemble_case, ensemble_same_yaml) call ensemble_test() call mpp_set_current_pelist(pelist) end select @@ -709,26 +710,34 @@ subroutine ensemble_test() real(lkind) :: expected_result !< Expected result from data_override type(time_type) :: Time !< Time real(lkind), allocatable, dimension(:,:) :: runoff !< Data to be written + integer :: scale_fac !< Scale factor to use when determining + !! the expected answer + logical :: sucessful !< .True. if the data_override was sucessful allocate(runoff(is:ie,js:je)) + scale_fac = ensemble_id + if (test_case .eq. ensemble_same_yaml) scale_fac = 1 + runoff = 999._lkind !< Run it when time=3 Time = set_date(1,1,4,0,0,0) - call data_override('OCN','runoff',runoff, Time) + call data_override('OCN','runoff',runoff, Time, override=sucessful) + if (.not. sucessful) call mpp_error(FATAL, "The data was not overriden correctly") !< Because you are getting the data when time=3, and this is an "ongrid" case, the expected result is just - !! equal to the data at time=3, which is 3+ensemble_id. - expected_result = 3._lkind + real(ensemble_id,kind=lkind) + !! equal to the data at time=3, which is 3+scale_fac. + expected_result = 3._lkind + real(scale_fac,kind=lkind) call compare_data(Domain, runoff, expected_result) !< Run it when time=4 runoff = 999._lkind Time = set_date(1,1,5,0,0,0) - call data_override('OCN','runoff',runoff, Time) - !< You are getting the data when time=4, the data at time=3 is 3+ensemble_id. and at time=5 is 4+ensemble_id., + call data_override('OCN','runoff',runoff, Time, override=sucessful) + if (.not. sucessful) call mpp_error(FATAL, "The data was not overriden correctly") + !< You are getting the data when time=4, the data at time=3 is 3+scale_fac. and at time=5 is 4+scale_fac., !! so the expected result is the average of the 2 (because this is is an "ongrid" case and there !! is no horizontal interpolation). - expected_result = (3._lkind + real(ensemble_id,kind=lkind) + 4._lkind + real(ensemble_id,kind=lkind)) / 2._lkind + expected_result = (3._lkind + real(scale_fac,kind=lkind) + 4._lkind + real(scale_fac,kind=lkind)) / 2._lkind call compare_data(Domain, runoff, expected_result) deallocate(runoff) diff --git a/test_fms/diag_manager/test_ens_runs.F90 b/test_fms/diag_manager/test_ens_runs.F90 index 621016430..f1c57c7a1 100644 --- a/test_fms/diag_manager/test_ens_runs.F90 +++ b/test_fms/diag_manager/test_ens_runs.F90 @@ -25,7 +25,8 @@ program test_ens_runs register_diag_field, diag_manager_init, diag_manager_end, register_static_field, & diag_axis_init use time_manager_mod, only: time_type, operator(+), JULIAN, set_time, set_calendar_type, set_date - use mpp_mod, only: FATAL, mpp_error, mpp_npes, mpp_pe, mpp_get_current_pelist, mpp_set_current_pelist + use mpp_mod, only: FATAL, mpp_error, mpp_npes, mpp_pe, mpp_get_current_pelist, mpp_set_current_pelist, & + input_nml_file use fms2_io_mod, only: FmsNetcdfFile_t, open_file, close_file, read_data, get_dimension_size, & set_filename_appendix, get_instance_filename use ensemble_manager_mod, only: get_ensemble_size, ensemble_manager_init @@ -47,7 +48,18 @@ program test_ens_runs character(len=10) :: text !< The filename appendix integer :: expected_ntimes + integer :: io_status !< Status after reading the namelist + + !< Configuration parameters + logical :: using_ens_yaml = .true. !< Indicates whether or not ensembles yamls are used in the test + + namelist / test_ens_runs_nml / using_ens_yaml + call fms_init + + read (input_nml_file, test_ens_runs_nml, iostat=io_status) + if (io_status > 0) call mpp_error(FATAL,'=>test_ens_runs: Error reading input.nml') + call ensemble_manager_init npes = mpp_npes() if (npes .ne. 2) & @@ -69,6 +81,7 @@ program test_ens_runs ensemble_id = 2 call mpp_set_current_pelist((/1/)) expected_ntimes = 24 + if (.not. using_ens_yaml) expected_ntimes = 48 endif write( text,'(a,i2.2)' ) 'ens_', ensemble_id @@ -105,6 +118,7 @@ subroutine check_output() real, allocatable :: var_data(:) !< Buffer to read variable data to integer :: j !< For looping character(len=255) :: filename !< Name of the diag file + integer :: scale_fac !< Scale factor to use when determining the expected answer call get_instance_filename("test_ens.nc", filename) if (.not. open_file(fileobj, filename, "read")) & @@ -116,9 +130,12 @@ subroutine check_output() allocate(var_data(var_size)) var_data = -999.99 + scale_fac = 1 + if (using_ens_yaml) scale_fac = ensemble_id + call read_data(fileobj, "var0", var_data) do j = 1, var_size - if (var_data(j) .ne. real(j * ensemble_id))& + if (var_data(j) .ne. real(j * scale_fac))& call mpp_error(FATAL, "The variable data for var1 at time level:"//& string(j)//" is not the correct value!") enddo diff --git a/test_fms/diag_manager/test_ens_runs.sh b/test_fms/diag_manager/test_ens_runs.sh index b2e262b69..f73825d5b 100755 --- a/test_fms/diag_manager/test_ens_runs.sh +++ b/test_fms/diag_manager/test_ens_runs.sh @@ -77,10 +77,10 @@ cat <<_EOF > diag_table.yaml title: test_diag_manager_01 base_date: 2 1 1 0 0 0 diag_files: -- file_name: test_0days +- file_name: test_ens time_units: days unlimdim: time - freq: 0 days + freq: 1 hours varlist: - module: ocn_mod var_name: var0 @@ -93,5 +93,24 @@ test_expect_failure "Running diag_manager with both diag_table.yaml and diag_tab mpirun -n 2 ../test_ens_runs ' +# Both ensembles have the same diag table yaml +cat <<_EOF > input.nml +&diag_manager_nml + use_modern_diag = .True. +/ + +&ensemble_nml + ensemble_size = 2 +/ +&test_ens_runs_nml + using_ens_yaml = .false. +/ +_EOF +rm -rf diag_table.ens_01.yaml diag_table.ens_02.yaml +my_test_count=`expr $my_test_count + 1` +test_expect_success "Running diag_manager with 2 ensembles, both ensembles have the same yaml (test $my_test_count)" ' + mpirun -n 2 ../test_ens_runs +' + fi test_done diff --git a/test_fms/field_manager/test_field_manager2.sh b/test_fms/field_manager/test_field_manager2.sh index 313c830a7..29b77ff19 100755 --- a/test_fms/field_manager/test_field_manager2.sh +++ b/test_fms/field_manager/test_field_manager2.sh @@ -148,6 +148,33 @@ _EOF rm -rf field_table.yaml test_expect_success "field manager test with 2 ensembles" 'mpirun -n 2 ./test_field_table_read' + +cat <<_EOF > input.nml +&field_manager_nml + use_field_table_yaml = .true. +/ +&test_field_table_read_nml + test_case = 2 +/ +&ensemble_nml + ensemble_size = 2 +/ +_EOF + + cat <<_EOF > field_table.yaml +field_table: +- field_type: tracer + modlist: + - model_type: atmos_mod + varlist: + - variable: radon + - variable: radon2 + - variable: radon3 + longname: bad radon! +_EOF + +rm -rf field_table.ens_01.yaml field_table.ens_02.yaml +test_expect_success "field manager test with 2 ensembles same yaml" 'mpirun -n 2 ./test_field_table_read' fi test_done diff --git a/test_fms/field_manager/test_field_table_read.F90 b/test_fms/field_manager/test_field_table_read.F90 index bb46256cb..4f2b993f3 100644 --- a/test_fms/field_manager/test_field_table_read.F90 +++ b/test_fms/field_manager/test_field_table_read.F90 @@ -53,6 +53,7 @@ program test_field_table_read character(len=10) :: text integer, parameter :: default_test = 0 integer, parameter :: ensemble_test = 1 +integer, parameter :: ensemble_same_yaml_test = 2 ! namelist parameters integer :: test_case = default_test @@ -69,7 +70,7 @@ program test_field_table_read nfields_expected = 4 select case (test_case) -case (ensemble_test) +case (ensemble_test, ensemble_same_yaml_test) if (npes .ne. 2) & call mpp_error(FATAL, "test_field_table_read:: this test requires 2 PEs!") @@ -78,16 +79,16 @@ program test_field_table_read if (ens_siz(1) .ne. 2) & call mpp_error(FATAL, "This test requires 2 ensembles") + nfields_expected = 3 if (mpp_pe() .eq. 0) then !PEs 0 is the first ensemble ensemble_id = 1 call mpp_set_current_pelist((/0/)) - nfields_expected = 3 else !PEs 1 is the second ensemble ensemble_id = 2 call mpp_set_current_pelist((/1/)) - nfields_expected = 4 + if (test_case .eq. ensemble_test) nfields_expected = 4 endif write( text,'(a,i2.2)' ) 'ens_', ensemble_id