Add patch for ufs-weather-model to allow ensemble control

CMakeLists.txt

@@ -42,8 +42,8 @@ find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8)
 
 # Core JEDI repositories
 # ----------------------
-#ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH feature/ufs-stateset)
-ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE )
+ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH feature/letkf-stateset)
+#ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE )
 ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" BRANCH develop UPDATE )
 ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE )
 
@@ -112,13 +112,14 @@ else()
 endif()
 
 
+#set(UFS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model-cont)
 set(UFS_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ufs-weather-model)
 set(UFS_REPO https://github.com/ufs-community/ufs-weather-model)
 set(UFS_TAG "develop" )
 if(UFS_APP MATCHES "^(NG-GODAS)$")
  set(patch_command ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/ng-godas.patch <SOURCE_DIR> && cd <SOURCE_DIR> && git apply ng-godas.patch)
 else()
- set(patch_command "")
+  set(patch_command ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/ufs-wm.patch <SOURCE_DIR> && cd <SOURCE_DIR> && git apply ufs-wm.patch)
 endif()
 
 ExternalProject_Add(ufs-weather-model
@@ -185,8 +186,8 @@ target_link_libraries( fv3atm INTERFACE fms )
 ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.3.0 )
 if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(ATM)$")
  option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON)
- ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE )
- ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH develop UPDATE )
+ ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/letkf-stateset UPDATE )
+ ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH feature/letkf-stateset UPDATE )
 elseif(UFS_APP MATCHES "^(NG-GODAS)$")
  ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom_update UPDATE )
  add_dependencies(soca ufs-weather-model)

patches/ufs-wm.patch

@@ -0,0 +1,81 @@
+diff --git a/FV3/atmos_cubed_sphere/model/fv_control.F90 b/FV3/atmos_cubed_sphere/model/fv_control.F90
+index bf111af..574e233 100644
+--- a/FV3/atmos_cubed_sphere/model/fv_control.F90
++++ b/FV3/atmos_cubed_sphere/model/fv_control.F90
+@@ -457,6 +457,7 @@ module fv_control_mod
+ allocate(grids_on_this_pe(ngrids))
+ grids_on_this_pe(:) = .false.
+
++ grid_pes(1) = npes
+ do n=1,ngrids
+
+ if (ngrids == 1 .or. grid_pes(n) == 0) then
+diff --git a/FV3/module_fcst_grid_comp.F90 b/FV3/module_fcst_grid_comp.F90
+index 27cdf95..49825d7 100644
+--- a/FV3/module_fcst_grid_comp.F90
++++ b/FV3/module_fcst_grid_comp.F90
+@@ -575,7 +575,7 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE
+ type(ESMF_DistGrid) :: distgrid
+ integer :: jsc, jec, isc, iec, nlev
+ type(domain2D) :: domain
+- integer :: n, fcstNpes, tmpvar, k
++ integer :: n, fcstNpes, tmpvar, k, ii
+ logical :: freq_restart, fexist
+ integer, allocatable, dimension(:) :: isl, iel, jsl, jel
+ integer, allocatable, dimension(:,:,:) :: deBlockList
+@@ -600,7 +600,7 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE
+ Time_restart, Time_step_restart
+ type(time_type) :: iautime
+ integer :: io_unit, calendar_type_res, date_res(6), date_init_res(6)
+-
++ integer,allocatable :: pelistZero(:)
+ integer,allocatable :: grid_number_on_all_pets(:)
+ logical,allocatable :: is_moving_on_all_pets(:), is_moving(:)
+ character(len=7) :: nest_suffix
+@@ -865,8 +865,9 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE
+
+ pelist => null()
+ call atmos_model_get_nth_domain_info(n, layout, nx, ny, pelist)
+- call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=pelist(1), rc=rc); ESMF_ERR_ABORT(rc)
+-
++! call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=pelist(1), rc=rc); ESMF_ERR_ABORT(rc)
++ call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=0, rc=rc); ESMF_ERR_ABORT(rc)
++ write(6,*) 'HEY, mype, mpp_pe and petlist are',mype, mpp_pe(), pelist
+ if (n==1) then
+ ! on grid==1 (top level parent) determine if the domain is global or regional
+ top_parent_is_global = .true.
+@@ -875,11 +876,16 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE
+ endif
+ call mpi_bcast(top_parent_is_global, 1, MPI_LOGICAL, 0, fcst_mpi_comm, rc)
+ endif
+-
++ 
+ if (n==1 .and. top_parent_is_global) then
+-
+- fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelist, rc=rc); ESMF_ERR_ABORT(rc)
+-
++ allocate(pelistZero(fcst_ntasks))
++ do ii=1,fcst_ntasks
++ pelistZero(ii) = ii - 1
++ enddo
++ write(6,*) 'HEY, mpp_pe and petlistZero are',mpp_pe(), pelistZero,fcst_ntasks
++ fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelistZero, rc=rc); ESMF_ERR_ABORT(rc)
++! fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelist, rc=rc); ESMF_ERR_ABORT(rc)
++ deallocate(pelistZero)
+ call ESMF_InfoGetFromHost(fcstGridComp(n), info=info, rc=rc); ESMF_ERR_ABORT(rc)
+ call ESMF_InfoSet(info, key="layout", values=layout, rc=rc); ESMF_ERR_ABORT(rc)
+ call ESMF_InfoSet(info, key="tilesize", value=Atmos%mlon, rc=rc); ESMF_ERR_ABORT(rc)
+diff --git a/stochastic_physics/mpi_wrapper.F90 b/stochastic_physics/mpi_wrapper.F90
+index 7ccd5a7..8961474 100644
+--- a/stochastic_physics/mpi_wrapper.F90
++++ b/stochastic_physics/mpi_wrapper.F90
+@@ -81,7 +81,8 @@ contains
+ integer, intent(in) :: mpiroot
+ type(MPI_Comm), intent(in) :: mpicomm
+ if (initialized) return
+- root = mpiroot
++ root = 0
++! root = mpiroot
+ comm = mpicomm
+ call MPI_COMM_RANK(comm, mype, ierror)
+ call MPI_COMM_SIZE(comm, npes, ierror)