From 36d9166d91fd3e7b5110f1c48e96a2aa30cf2f9a Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Wed, 15 Jun 2022 13:23:13 +0000 Subject: [PATCH 1/3] point to Rustys dycore branch --- .gitmodules | 6 ++++-- atmos_cubed_sphere | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6bb663df1..93113bf7c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,9 @@ [submodule "atmos_cubed_sphere"] path = atmos_cubed_sphere - url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere - branch = dev/emc + #url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere + #branch = dev/emc + url = https://github.com/bensonr/GFDL_atmos_cubed_sphere + branch = emc_io_fixes [submodule "ccpp/framework"] path = ccpp/framework url = https://github.com/NCAR/ccpp-framework diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index fad4c9f1f..619e5d8e4 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit fad4c9f1fc29c0cbb47df9a07a573249155a1c42 +Subproject commit 619e5d8e472af1928e84a63541245c97d3e3b00e From 703010510d3773ee1bbb24516d20c53d757493a5 Mon Sep 17 00:00:00 2001 From: Jun Wang Date: Wed, 15 Jun 2022 13:36:48 +0000 Subject: [PATCH 2/3] From Rusty: update atmos_model and FV3GFS_io read performance when io_layout=1,1 and allow one to override data integrity checks in FMS restart logic --- atmos_model.F90 | 13 +++++++++---- io/FV3GFS_io.F90 | 35 +++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/atmos_model.F90 b/atmos_model.F90 index e0ea26292..4677ea719 100644 --- a/atmos_model.F90 +++ b/atmos_model.F90 @@ -146,6 +146,7 @@ module atmos_model_mod real(kind=8), pointer, dimension(:,:) :: area real(kind=8), pointer, dimension(:,:,:) :: layer_hgt, level_hgt type(domain2d) :: domain ! domain decomposition + type(domain2d) :: domain_for_read ! domain decomposition type(time_type) :: Time ! current time type(time_type) :: Time_step ! atmospheric time step. type(time_type) :: Time_init ! reference time. @@ -173,7 +174,9 @@ module atmos_model_mod !logical :: debug = .true. logical :: sync = .false. real :: avg_max_length=3600. -namelist /atmos_model_nml/ blocksize, chksum_debug, dycore_only, debug, sync, ccpp_suite, avg_max_length +logical :: ignore_rst_cksum = .false. +namelist /atmos_model_nml/ blocksize, chksum_debug, dycore_only, debug, sync, ccpp_suite, avg_max_length, & + ignore_rst_cksum type (time_type) :: diag_time, diag_time_fhzero @@ -553,7 +556,8 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step) !----------------------------------------------------------------------- call atmosphere_resolution (nlon, nlat, global=.false.) call atmosphere_resolution (mlon, mlat, global=.true.) - call atmosphere_domain (Atmos%domain, Atmos%layout, Atmos%regional, Atmos%nested, & + call atmosphere_domain (Atmos%domain, Atmos%domain_for_read, Atmos%layout, & + Atmos%regional, Atmos%nested, & Atmos%moving_nest_parent, Atmos%is_moving_nest, & Atmos%ngrids, Atmos%mygrid, Atmos%pelist) call atmosphere_diag_axes (Atmos%axes) @@ -718,7 +722,8 @@ subroutine atmos_model_init (Atmos, Time_init, Time, Time_step) call GFS_restart_populate (GFS_restart_var, GFS_control, GFS_data%Statein, GFS_data%Stateout, GFS_data%Sfcprop, & GFS_data%Coupling, GFS_data%Grid, GFS_data%Tbd, GFS_data%Cldprop, GFS_data%Radtend, & GFS_data%IntDiag, Init_parm, GFS_Diag) - call FV3GFS_restart_read (GFS_data, GFS_restart_var, Atm_block, GFS_control, Atmos%domain, Atm(mygrid)%flagstruct%warm_start) + call FV3GFS_restart_read (GFS_data, GFS_restart_var, Atm_block, GFS_control, Atmos%domain_for_read, & + Atm(mygrid)%flagstruct%warm_start, ignore_rst_cksum) if(GFS_control%do_ca .and. Atm(mygrid)%flagstruct%warm_start)then call read_ca_restart (Atmos%domain,GFS_control%ncells,GFS_control%nca,GFS_control%ncells_g,GFS_control%nca_g) endif @@ -3190,7 +3195,7 @@ subroutine atmos_model_get_nth_domain_info(n, layout, nx, ny, pelist) integer, pointer, intent(out) :: pelist(:) call get_nth_domain_info(n, layout, nx, ny, pelist) - + end subroutine atmos_model_get_nth_domain_info end module atmos_model_mod diff --git a/io/FV3GFS_io.F90 b/io/FV3GFS_io.F90 index ef7cbf008..97d942fb3 100644 --- a/io/FV3GFS_io.F90 +++ b/io/FV3GFS_io.F90 @@ -122,19 +122,20 @@ module FV3GFS_io_mod !-------------------- ! FV3GFS_restart_read !-------------------- - subroutine FV3GFS_restart_read (GFS_Data, GFS_Restart, Atm_block, Model, fv_domain, warm_start) + subroutine FV3GFS_restart_read (GFS_Data, GFS_Restart, Atm_block, Model, fv_domain, warm_start, ignore_rst_cksum) type(GFS_data_type), intent(inout) :: GFS_Data(:) type(GFS_restart_type), intent(inout) :: GFS_Restart type(block_control_type), intent(in) :: Atm_block type(GFS_control_type), intent(inout) :: Model type(domain2d), intent(in) :: fv_domain logical, intent(in) :: warm_start + logical, intent(in) :: ignore_rst_cksum !--- read in surface data from chgres - call sfc_prop_restart_read (GFS_Data%Sfcprop, Atm_block, Model, fv_domain, warm_start) + call sfc_prop_restart_read (GFS_Data%Sfcprop, Atm_block, Model, fv_domain, warm_start, ignore_rst_cksum) !--- read in physics restart data - call phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain) + call phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_rst_cksum) end subroutine FV3GFS_restart_read @@ -508,13 +509,14 @@ end subroutine FV3GFS_GFS_checksum ! opens: oro_data.tile?.nc, sfc_data.tile?.nc ! !---------------------------------------------------------------------- - subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_start) + subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_start, ignore_rst_cksum) !--- interface variable definitions type(GFS_sfcprop_type), intent(inout) :: Sfcprop(:) type (block_control_type), intent(in) :: Atm_block type(GFS_control_type), intent(inout) :: Model type (domain2d), intent(in) :: fv_domain logical, intent(in) :: warm_start + logical, intent(in) :: ignore_rst_cksum !--- local variables integer :: i, j, k, ix, lsoil, num, nb, i_start, j_start, i_end, j_end integer :: isc, iec, jsc, jec, npz, nx, ny @@ -537,8 +539,8 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta character(37) :: infile !--- fms2_io file open logic logical :: amiopen - logical :: is_lsoil - + logical :: is_lsoil + nvar_o2 = 19 nvar_oro_ls_ss = 10 nvar_s2o = 18 @@ -635,7 +637,7 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta !--- read the orography restart/data call mpp_error(NOTE,'reading topographic/orographic information from INPUT/oro_data.tile*.nc') - call read_restart(Oro_restart) + call read_restart(Oro_restart, ignore_checksum=ignore_rst_cksum) call close_file(Oro_restart) @@ -887,11 +889,11 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta !--- read new GSL created orography restart/data call mpp_error(NOTE,'reading topographic/orographic information from & &INPUT/oro_data_ls.tile*.nc') - call read_restart(Oro_ls_restart) + call read_restart(Oro_ls_restart, ignore_checksum=ignore_rst_cksum) call close_file(Oro_ls_restart) call mpp_error(NOTE,'reading topographic/orographic information from & &INPUT/oro_data_ss.tile*.nc') - call read_restart(Oro_ss_restart) + call read_restart(Oro_ss_restart, ignore_checksum=ignore_rst_cksum) call close_file(Oro_ss_restart) @@ -1121,7 +1123,7 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta call register_axis(Sfc_restart, 'xaxis_1', 'X') call register_axis(Sfc_restart, 'yaxis_1', 'Y') call register_axis(Sfc_restart, 'zaxis_1', dimension_length=Model%kice) - + if (Model%lsm == Model%lsm_noah .or. Model%lsm == Model%lsm_noahmp) then call register_axis(Sfc_restart, 'zaxis_2', dimension_length=Model%lsoil) else if(Model%lsm == Model%lsm_ruc) then @@ -1247,7 +1249,7 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta end if end if enddo - + if (Model%lsm == Model%lsm_noahmp) then mand = .false. do num = nvar_s3+1,nvar_s3+3 @@ -1280,7 +1282,7 @@ subroutine sfc_prop_restart_read (Sfcprop, Atm_block, Model, fv_domain, warm_sta !--- read the surface restart/data call mpp_error(NOTE,'reading surface properties data from INPUT/sfc_data.tile*.nc') - call read_restart(Sfc_restart) + call read_restart(Sfc_restart, ignore_checksum=ignore_rst_cksum) call close_file(Sfc_restart) ! write(0,*)' stype read in min,max=',minval(sfc_var2(:,:,35)),maxval(sfc_var2(:,:,35)),' sfc_name2=',sfc_name2(35) @@ -2225,11 +2227,11 @@ subroutine sfc_prop_restart_write (Sfcprop, Atm_block, Model, fv_domain, timesta var3_p2 => sfc_var3eq(:,:,:,7) call register_restart_field(Sfc_restart, sfc_name3(7), var3_p2, dimensions=(/'xaxis_1', 'yaxis_1', 'zaxis_2', 'Time '/),& &is_optional=.not.mand) - + var3_p3 => sfc_var3zn(:,:,:,8) call register_restart_field(Sfc_restart, sfc_name3(8), var3_p3, dimensions=(/'xaxis_1', 'yaxis_1', 'zaxis_4', 'Time '/),& &is_optional=.not.mand) - + nullify(var3_p1) nullify(var3_p2) nullify(var3_p3) @@ -2435,12 +2437,13 @@ end subroutine sfc_prop_restart_write ! opens: phys_data.tile?.nc ! !---------------------------------------------------------------------- - subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain) + subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain, ignore_rst_cksum) !--- interface variable definitions type(GFS_restart_type), intent(in) :: GFS_Restart type(block_control_type), intent(in) :: Atm_block type(GFS_control_type), intent(in) :: Model type(domain2d), intent(in) :: fv_domain + logical, intent(in) :: ignore_rst_cksum !--- local variables integer :: i, j, k, nb, ix, num integer :: isc, iec, jsc, jec, npz, nx, ny @@ -2501,7 +2504,7 @@ subroutine phys_restart_read (GFS_Restart, Atm_block, Model, fv_domain) !--- read the surface restart/data call mpp_error(NOTE,'reading physics restart data from INPUT/phy_data.tile*.nc') - call read_restart(Phy_restart) + call read_restart(Phy_restart, ignore_checksum=ignore_rst_cksum) call close_file(Phy_restart) !--- place the data into the block GFS containers From 8aaeb5226f6258e700f106f7b8430caca5a8cb29 Mon Sep 17 00:00:00 2001 From: "Jun.Wang" Date: Tue, 28 Jun 2022 03:09:24 +0000 Subject: [PATCH 3/3] update dycore --- .gitmodules | 6 ++---- atmos_cubed_sphere | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 93113bf7c..6bb663df1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,7 @@ [submodule "atmos_cubed_sphere"] path = atmos_cubed_sphere - #url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere - #branch = dev/emc - url = https://github.com/bensonr/GFDL_atmos_cubed_sphere - branch = emc_io_fixes + url = https://github.com/NOAA-GFDL/GFDL_atmos_cubed_sphere + branch = dev/emc [submodule "ccpp/framework"] path = ccpp/framework url = https://github.com/NCAR/ccpp-framework diff --git a/atmos_cubed_sphere b/atmos_cubed_sphere index 619e5d8e4..0963bdec9 160000 --- a/atmos_cubed_sphere +++ b/atmos_cubed_sphere @@ -1 +1 @@ -Subproject commit 619e5d8e472af1928e84a63541245c97d3e3b00e +Subproject commit 0963bdec95f1419af3bec38c5802554389527f36