diff --git a/CMakeLists.txt b/CMakeLists.txt index d8643d6..e2e685c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,8 +42,8 @@ find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8) # Core JEDI repositories # ---------------------- -#ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH feature/ufs-stateset) -ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH feature/letkf-stateset) +#ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" BRANCH develop UPDATE ) ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE ) @@ -112,13 +112,14 @@ else() endif() +#set(UFS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model-cont) set(UFS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model) set(UFS_REPO https://github.com/ufs-community/ufs-weather-model) set(UFS_TAG "develop" ) if(UFS_APP MATCHES "^(NG-GODAS)$") set(patch_command ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/ng-godas.patch && cd && git apply ng-godas.patch) else() - set(patch_command "") + set(patch_command ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/ufs-wm.patch && cd && git apply ufs-wm.patch) endif() ExternalProject_Add(ufs-weather-model @@ -185,8 +186,8 @@ target_link_libraries( fv3atm INTERFACE fms ) ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.3.0 ) if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(ATM)$") option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/letkf-stateset UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/letkf-stateset UPDATE ) elseif(UFS_APP MATCHES "^(NG-GODAS)$") ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom_update UPDATE ) add_dependencies(soca ufs-weather-model) diff --git a/patches/ufs-wm.patch b/patches/ufs-wm.patch new file mode 100644 index 0000000..e8c256e --- /dev/null +++ b/patches/ufs-wm.patch @@ -0,0 +1,81 @@ +diff --git a/FV3/atmos_cubed_sphere/model/fv_control.F90 b/FV3/atmos_cubed_sphere/model/fv_control.F90 +index bf111af..574e233 100644 +--- a/FV3/atmos_cubed_sphere/model/fv_control.F90 ++++ b/FV3/atmos_cubed_sphere/model/fv_control.F90 +@@ -457,6 +457,7 @@ module fv_control_mod + allocate(grids_on_this_pe(ngrids)) + grids_on_this_pe(:) = .false. + ++ grid_pes(1) = npes + do n=1,ngrids + + if (ngrids == 1 .or. grid_pes(n) == 0) then +diff --git a/FV3/module_fcst_grid_comp.F90 b/FV3/module_fcst_grid_comp.F90 +index 27cdf95..49825d7 100644 +--- a/FV3/module_fcst_grid_comp.F90 ++++ b/FV3/module_fcst_grid_comp.F90 +@@ -575,7 +575,7 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE + type(ESMF_DistGrid) :: distgrid + integer :: jsc, jec, isc, iec, nlev + type(domain2D) :: domain +- integer :: n, fcstNpes, tmpvar, k ++ integer :: n, fcstNpes, tmpvar, k, ii + logical :: freq_restart, fexist + integer, allocatable, dimension(:) :: isl, iel, jsl, jel + integer, allocatable, dimension(:,:,:) :: deBlockList +@@ -600,7 +600,7 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE + Time_restart, Time_step_restart + type(time_type) :: iautime + integer :: io_unit, calendar_type_res, date_res(6), date_init_res(6) +- ++ integer,allocatable :: pelistZero(:) + integer,allocatable :: grid_number_on_all_pets(:) + logical,allocatable :: is_moving_on_all_pets(:), is_moving(:) + character(len=7) :: nest_suffix +@@ -865,8 +865,9 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE + + pelist => null() + call atmos_model_get_nth_domain_info(n, layout, nx, ny, pelist) +- call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=pelist(1), rc=rc); ESMF_ERR_ABORT(rc) +- ++! call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=pelist(1), rc=rc); ESMF_ERR_ABORT(rc) ++ call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=0, rc=rc); ESMF_ERR_ABORT(rc) ++ write(6,*) 'HEY, mype, mpp_pe and petlist are',mype, mpp_pe(), pelist + if (n==1) then + ! on grid==1 (top level parent) determine if the domain is global or regional + top_parent_is_global = .true. +@@ -875,11 +876,16 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE + endif + call mpi_bcast(top_parent_is_global, 1, MPI_LOGICAL, 0, fcst_mpi_comm, rc) + endif +- ++ + if (n==1 .and. top_parent_is_global) then +- +- fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelist, rc=rc); ESMF_ERR_ABORT(rc) +- ++ allocate(pelistZero(fcst_ntasks)) ++ do ii=1,fcst_ntasks ++ pelistZero(ii) = ii - 1 ++ enddo ++ write(6,*) 'HEY, mpp_pe and petlistZero are',mpp_pe(), pelistZero,fcst_ntasks ++ fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelistZero, rc=rc); ESMF_ERR_ABORT(rc) ++! fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelist, rc=rc); ESMF_ERR_ABORT(rc) ++ deallocate(pelistZero) + call ESMF_InfoGetFromHost(fcstGridComp(n), info=info, rc=rc); ESMF_ERR_ABORT(rc) + call ESMF_InfoSet(info, key="layout", values=layout, rc=rc); ESMF_ERR_ABORT(rc) + call ESMF_InfoSet(info, key="tilesize", value=Atmos%mlon, rc=rc); ESMF_ERR_ABORT(rc) +diff --git a/stochastic_physics/mpi_wrapper.F90 b/stochastic_physics/mpi_wrapper.F90 +index 7ccd5a7..8961474 100644 +--- a/stochastic_physics/mpi_wrapper.F90 ++++ b/stochastic_physics/mpi_wrapper.F90 +@@ -81,7 +81,8 @@ contains + integer, intent(in) :: mpiroot + type(MPI_Comm), intent(in) :: mpicomm + if (initialized) return +- root = mpiroot ++ root = 0 ++! root = mpiroot + comm = mpicomm + call MPI_COMM_RANK(comm, mype, ierror) + call MPI_COMM_SIZE(comm, npes, ierror)