diff --git a/.gitignore b/.gitignore index f306e88a3c..2db1c24b31 100644 --- a/.gitignore +++ b/.gitignore @@ -203,6 +203,7 @@ test_quad_reg_interp test_table_read test_ran_unif test_kde_dist +test_window # Directories to NOT IGNORE ... same as executable names # as far as I know, these must be listed after the executables diff --git a/CHANGELOG.rst b/CHANGELOG.rst index b466545457..b15c62115b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -22,17 +22,24 @@ individual files. The changes are now listed with the most recent at the top. -**October 22 2024 :: Bug-fixes: WRF and GOES. Tag 11.8.2** +**November 7 2024 :: MPI window memory reduction. Tag v11.8.3** + +- Removes unnecessary copy of state into mpi window. +- Removes cray pointer version of the mpi window. +- | Fortran-testanything included in developer tests. + | *From dennisdjensen: see developer_tests/contrib/fortran-testanything/LICENSE.txt* + +**October 22 2024 :: Bug-fixes: WRF and GOES. Tag v11.8.2** - Force THM to be the WRF-DART temperature variable - Remove offset on GOES observation converter -**September 27 2024 :: MOM6 mask bug-fix. Tag 11.8.1** +**September 27 2024 :: MOM6 mask bug-fix. Tag v11.8.1** - Fix for MOM6 CESM3 workhorse 2/3 degree grid TL319_t232 to mask missing geolon|lat|u|v|t values -**September 10 2024 :: MARBL_column. Tag 11.8.0** +**September 10 2024 :: MARBL_column. Tag v11.8.0** - Interface for MARBL_column for DART: @@ -48,7 +55,7 @@ Bugfix: - fix for IO for NetCDF files when only some variables have the unlimited dimension -**August 29 2024 :: Bug fixes for shortest_time_between_assimilations and get_close_init. Tag 11.7.1** +**August 29 2024 :: Bug fixes for shortest_time_between_assimilations and get_close_init. Tag v11.7.1** Bug fixes: @@ -64,7 +71,7 @@ Doc fixes: - GitHub template for reporting documentation issues -**August 26 2024 :: KQCEF. Tag 11.7.0** +**August 26 2024 :: KQCEF. Tag v11.7.0** - Adds a Quantile-Conserving Ensemble Filter Based on Kernel-Density Estimation to DART. - New distribution module kde_distribution_mod. diff --git a/assimilation_code/modules/utilities/cray_win_mod.f90 b/assimilation_code/modules/utilities/cray_win_mod.f90 deleted file mode 100644 index fa04c1284b..0000000000 --- a/assimilation_code/modules/utilities/cray_win_mod.f90 +++ /dev/null @@ -1,219 +0,0 @@ -! DART software - Copyright UCAR. This open source software is provided -! by UCAR, "as is", without charge, subject to all terms of use at -! http://www.image.ucar.edu/DAReS/DART/DART_download - -!> Contains the window information for the state. Two windows: -!> One for all copies, one for the mean. -!> Not sure whether we should just have one window to avoid multiple synchronizations. - -module window_mod - -!> \defgroup window window_mod -!> @{ -use mpi_utilities_mod, only : datasize, my_task_id -use types_mod, only : r8 -use ensemble_manager_mod, only : ensemble_type, map_pe_to_task, get_var_owner_index, & - copies_in_window, init_ensemble_manager, & - get_allow_transpose, end_ensemble_manager, & - set_num_extra_copies, all_copies_to_all_vars, & - all_vars_to_all_copies - -use mpi - -implicit none - -private -public :: create_mean_window, create_state_window, free_mean_window, & - free_state_window, data_count, mean_win, state_win, current_win, & - mean_ens_handle, NO_WINDOW, MEAN_WINDOW, STATE_WINDOW - -! mpi window handles -integer :: state_win !! window for the forward operator -integer :: mean_win !! window for the mean -integer :: current_win !! keep track of current window, start out assuming an invalid window -!>@todo the number of copies in the window is sloppy. You need to make this better. - -! parameters for keeping track of which window is open -integer, parameter :: NO_WINDOW = -1 -integer, parameter :: MEAN_WINDOW = 0 -integer, parameter :: STATE_WINDOW = 2 - -integer :: data_count !! number of copies in the window -integer(KIND=MPI_ADDRESS_KIND) window_size -logical :: use_distributed_mean = .false. ! initialize to false - -real(r8) :: duplicate_state(*) ! duplicate array for cray pointer fwd -pointer(a, duplicate_state) - -real(r8) :: duplicate_mean(*) ! duplicate array for cray pointer vert convert -pointer(b, duplicate_mean) -type(ensemble_type) :: mean_ens_handle - -contains - -!------------------------------------------------------------- -!> Create the window for the ensemble complete state vector -!> I think you have to pass it the state ensemble handle -subroutine create_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handle) - -type(ensemble_type), intent(inout) :: state_ens_handle !< state ensemble handle -type(ensemble_type), intent(inout), optional :: fwd_op_ens_handle -type(ensemble_type), intent(inout), optional :: qc_ens_handle - -integer :: ii, jj, count, ierr -integer :: bytesize !< size in bytes of each element in the window -integer :: my_num_vars - -! Find out how many copies to put in the window -! copies_in_window is not necessarily equal to ens_handle%num_copies -data_count = copies_in_window(state_ens_handle) - -if (get_allow_transpose(state_ens_handle)) then - call all_copies_to_all_vars(state_ens_handle) - if (present(fwd_op_ens_handle)) then - call all_copies_to_all_vars(fwd_op_ens_handle) - endif - if (present(qc_ens_handle)) then - call all_copies_to_all_vars(qc_ens_handle) - endif -else - ! find how many variables I have - my_num_vars = state_ens_handle%my_num_vars - - ! allocate some RDMA accessible memory - ! using MPI_ALLOC_MEM because the MPI standard allows vendors to require MPI_ALLOC_MEM for remote memory access - call mpi_type_size(datasize, bytesize, ierr) - window_size = my_num_vars*data_count*bytesize - a = malloc(my_num_vars*data_count) - call MPI_ALLOC_MEM(window_size, MPI_INFO_NULL, a, ierr) - - count = 1 - ! create a duplicate copies array for remote memory access - ! Doing this because you cannot use a cray pointer with an allocatable array - ! Can't do array assignment with a cray pointer, so you need to loop - do ii = 1, my_num_vars - do jj = 1, data_count - duplicate_state(count) = state_ens_handle%copies(jj,ii) - count = count + 1 - enddo - enddo - - ! Expose local memory to RMA operation by other processes in a communicator. - call mpi_win_create(duplicate_state, window_size, bytesize, MPI_INFO_NULL, mpi_comm_world, state_win, ierr) - -endif - -! Set the current window to the state window -current_win = STATE_WINDOW - -data_count = copies_in_window(state_ens_handle) - -end subroutine create_state_window - -!------------------------------------------------------------- -!> Create the window for the ensemble complete state vector -!> I think you have to pass it the state ensemble handle -subroutine create_mean_window(state_ens_handle, mean_copy, distribute_mean) - -type(ensemble_type), intent(in) :: state_ens_handle -integer, intent(in) :: mean_copy -logical, intent(in) :: distribute_mean - -integer :: ii, ierr -integer :: bytesize -integer :: my_num_vars - -! find out how many variables I have -my_num_vars = state_ens_handle%my_num_vars - -! create an ensemble handle of just the mean copy. -use_distributed_mean = distribute_mean - -if (use_distributed_mean) then - - call init_ensemble_manager(mean_ens_handle, 1, state_ens_handle%num_vars) ! distributed ensemble - call set_num_extra_copies(mean_ens_handle, 0) - mean_ens_handle%copies(1,:) = state_ens_handle%copies(mean_copy, :) - - ! find out how many variables I have - my_num_vars = mean_ens_handle%my_num_vars - call mpi_type_size(datasize, bytesize, ierr) - window_size = my_num_vars*bytesize - ! allocate some RDMA accessible memory - ! using MPI_ALLOC_MEM because the MPI standard allows vendors to require MPI_ALLOC_MEM for remote memory access - ! Have a look at MPI-3, I think this removes cray pointers. - b = malloc(my_num_vars) - call MPI_ALLOC_MEM(window_size, MPI_INFO_NULL, b, ierr) - - do ii = 1, my_num_vars - duplicate_mean(ii) = mean_ens_handle%copies(1, ii) - enddo - - ! Expose local memory to RMA operation by other processes in a communicator. - call mpi_win_create(duplicate_mean, window_size, bytesize, MPI_INFO_NULL, mpi_comm_world, mean_win, ierr) - -else - - call init_ensemble_manager(mean_ens_handle, 1, state_ens_handle%num_vars, transpose_type_in = 3) - call set_num_extra_copies(mean_ens_handle, 0) - mean_ens_handle%copies(1,:) = state_ens_handle%copies(mean_copy, :) - call all_copies_to_all_vars(mean_ens_handle) ! this is a transpose-duplicate - -endif - -! Set the current window to the state window -current_win = MEAN_WINDOW - -data_count = copies_in_window(mean_ens_handle) ! One - -end subroutine create_mean_window - -!------------------------------------------------------------- -!> End epoch of state access. -!> Need to transpose qc and fwd operator back to copy complete -subroutine free_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handle) - -type(ensemble_type), intent(inout) :: state_ens_handle -type(ensemble_type), intent(inout), optional :: fwd_op_ens_handle -type(ensemble_type), intent(inout), optional :: qc_ens_handle - -integer :: ierr - -if(get_allow_transpose(state_ens_handle)) then ! the forward operators were done var complete - !transpose back if allowing transposes - if (present(fwd_op_ens_handle)) & - call all_vars_to_all_copies(fwd_op_ens_handle) - if (present(qc_ens_handle)) & - call all_vars_to_all_copies(qc_ens_handle) -else - ! close mpi window - call mpi_win_free(state_win, ierr) - call MPI_FREE_MEM(duplicate_state, ierr) -endif - -current_win = NO_WINDOW - -end subroutine free_state_window - -!--------------------------------------------------------- -!> Free the mpi window -subroutine free_mean_window() - -integer :: ierr - -if(get_allow_transpose(mean_ens_handle)) then - call end_ensemble_manager(mean_ens_handle) -else - call mpi_win_free(mean_win, ierr) - call MPI_FREE_MEM(duplicate_mean, ierr) - call end_ensemble_manager(mean_ens_handle) -endif - -current_win = NO_WINDOW - -end subroutine free_mean_window - -!--------------------------------------------------------- -!> @} -end module window_mod - diff --git a/assimilation_code/modules/utilities/distributed_state_mod.f90 b/assimilation_code/modules/utilities/distributed_state_mod.f90 index 340afa1510..9038bc3f7d 100644 --- a/assimilation_code/modules/utilities/distributed_state_mod.f90 +++ b/assimilation_code/modules/utilities/distributed_state_mod.f90 @@ -110,7 +110,7 @@ subroutine get_fwd(x, my_index, state_ens_handle) !x = get_local_state(element_index) x = state_ens_handle%copies(1:data_count, element_index) else - call get_from_fwd(owner_of_state, state_win, element_index, data_count, x) + call get_from_fwd(owner_of_state, state_win, element_index, state_ens_handle%num_copies, data_count, x) endif endif diff --git a/assimilation_code/modules/utilities/mpi_utilities_mod.f90 b/assimilation_code/modules/utilities/mpi_utilities_mod.f90 index 90797018e8..619d8d1d55 100644 --- a/assimilation_code/modules/utilities/mpi_utilities_mod.f90 +++ b/assimilation_code/modules/utilities/mpi_utilities_mod.f90 @@ -1968,13 +1968,14 @@ end subroutine get_from_mean !----------------------------------------------------------------------------- -subroutine get_from_fwd(owner, window, mindex, num_rows, x) +subroutine get_from_fwd(owner, window, mindex, rows_in_window, num_rows, x) integer, intent(in) :: owner ! task in the window that owns the memory integer, intent(in) :: window ! window object integer, intent(in) :: mindex ! index in the tasks memory -integer, intent(in) :: num_rows ! number of rows in the window -real(r8), intent(out) :: x(:) ! result +integer, intent(in) :: rows_in_window ! number of rows in the window +integer, intent(in) :: num_rows ! number of rows to get from the window +real(r8), intent(out) :: x(num_rows) ! result integer(KIND=MPI_ADDRESS_KIND) :: target_disp integer :: errcode @@ -1983,7 +1984,7 @@ subroutine get_from_fwd(owner, window, mindex, num_rows, x) ! to have occured until the call to mpi_win_unlock. ! => Don't do anything with x in between mpi_get and mpi_win_lock -target_disp = (mindex - 1)*num_rows +target_disp = (mindex - 1)*rows_in_window call mpi_win_lock(MPI_LOCK_SHARED, owner, 0, window, errcode) call mpi_get(x, num_rows, datasize, owner, target_disp, num_rows, datasize, window, errcode) call mpi_win_unlock(owner, window, errcode) diff --git a/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 b/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 index dee8c618d1..45d628fa46 100644 --- a/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 +++ b/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 @@ -1969,12 +1969,13 @@ end subroutine get_from_mean !----------------------------------------------------------------------------- -subroutine get_from_fwd(owner, window, mindex, num_rows, x) +subroutine get_from_fwd(owner, window, mindex, rows_in_window, num_rows, x) integer, intent(in) :: owner ! task in the window that owns the memory type(MPI_Win), intent(in) :: window ! window object integer, intent(in) :: mindex ! index in the tasks memory -integer, intent(in) :: num_rows ! number of rows in the window +integer, intent(in) :: rows_in_window ! number of rows in the window +integer, intent(in) :: num_rows ! number of rows to get from the window real(r8), intent(out) :: x(num_rows) ! result integer(KIND=MPI_ADDRESS_KIND) :: target_disp @@ -1984,7 +1985,7 @@ subroutine get_from_fwd(owner, window, mindex, num_rows, x) ! to have occured until the call to mpi_win_unlock. ! => Don't do anything with x in between mpi_get and mpi_win_lock -target_disp = (mindex - 1)*num_rows +target_disp = (mindex - 1)*rows_in_window call mpi_win_lock(MPI_LOCK_SHARED, owner, 0, window, errcode) call mpi_get(x, num_rows, datasize, owner, target_disp, num_rows, datasize, window, errcode) call mpi_win_unlock(owner, window, errcode) diff --git a/assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 b/assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 index cc729c23aa..94e2c20881 100644 --- a/assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 +++ b/assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 @@ -634,13 +634,14 @@ end subroutine get_from_mean !----------------------------------------------------------------------------- -subroutine get_from_fwd(owner, window, mindex, num_rows, x) +subroutine get_from_fwd(owner, window, mindex, rows_in_window, num_rows, x) integer, intent(in) :: owner ! task in the window that owns the memory integer, intent(in) :: window ! window object integer, intent(in) :: mindex ! index in the tasks memory -integer, intent(in) :: num_rows ! number of rows in the window -real(r8), intent(out) :: x(num_rows) ! result +integer, intent(in) :: rows_in_window ! number of rows in the window +integer, intent(in) :: num_rows ! number of rows to get from the window +real(r8), intent(out) :: x(num_rows) ! result call error_handler(E_ERR,'get_from_fwd', 'cannot be used in serial mode', source) diff --git a/assimilation_code/modules/utilities/no_cray_win_mod.f90 b/assimilation_code/modules/utilities/win_mod.f90 similarity index 87% rename from assimilation_code/modules/utilities/no_cray_win_mod.f90 rename to assimilation_code/modules/utilities/win_mod.f90 index 6090621f83..e965261006 100644 --- a/assimilation_code/modules/utilities/no_cray_win_mod.f90 +++ b/assimilation_code/modules/utilities/win_mod.f90 @@ -2,7 +2,6 @@ ! by UCAR, "as is", without charge, subject to all terms of use at ! http://www.image.ucar.edu/DAReS/DART/DART_download -!> Window without cray pointer. Should you point the window at contigous memory? module window_mod !> \defgroup window window_mod @@ -30,21 +29,16 @@ module window_mod integer :: current_win !< keep track of current window, start out assuming an invalid window ! parameters for keeping track of which window is open -!>@todo should this be in the window_mod? you will have to change in both cray -!> and non cray versions integer, parameter :: NO_WINDOW = -1 integer, parameter :: MEAN_WINDOW = 0 integer, parameter :: STATE_WINDOW = 2 -integer :: data_count !! number of copies in the window +integer :: data_count !! number of copies required integer(KIND=MPI_ADDRESS_KIND) :: window_size logical :: use_distributed_mean = .false. ! initialize to false ! Global memory to stick the mpi window to. ! Need a simply contiguous piece of memory to pass to mpi_win_create -! Openmpi 1.10.0 will not compile with ifort 16 if -! you create a window with a 2d array. -real(r8), allocatable :: contiguous_fwd(:) real(r8), allocatable :: mean_1d(:) type(ensemble_type) :: mean_ens_handle @@ -65,8 +59,7 @@ subroutine create_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handl integer :: bytesize !< size in bytes of each element in the window integer :: my_num_vars !< my number of vars -! Find out how many copies to put in the window -! copies_in_window is not necessarily equal to ens_handle%num_copies +! Find out how many copies to get, maybe different to state_ens_handle%num_copies data_count = copies_in_window(state_ens_handle) if (get_allow_transpose(state_ens_handle)) then @@ -82,20 +75,15 @@ subroutine create_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handl my_num_vars = state_ens_handle%my_num_vars call mpi_type_size(datasize, bytesize, ierr) - window_size = my_num_vars*data_count*bytesize - - allocate(contiguous_fwd(data_count*my_num_vars)) - contiguous_fwd = reshape(state_ens_handle%copies(1:data_count, :), (/my_num_vars*data_count/)) + window_size = my_num_vars*state_ens_handle%num_copies*bytesize ! Expose local memory to RMA operation by other processes in a communicator. - call mpi_win_create(contiguous_fwd, window_size, bytesize, MPI_INFO_NULL, get_dart_mpi_comm(), state_win, ierr) + call mpi_win_create(state_ens_handle%copies, window_size, bytesize, MPI_INFO_NULL, get_dart_mpi_comm(), state_win, ierr) endif ! Set the current window to the state window current_win = STATE_WINDOW -data_count = copies_in_window(state_ens_handle) - end subroutine create_state_window !------------------------------------------------------------- @@ -163,7 +151,6 @@ subroutine free_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handle) else ! close mpi window call mpi_win_free(state_win, ierr) - deallocate(contiguous_fwd) endif current_win = NO_WINDOW diff --git a/assimilation_code/modules/utilities/no_cray_winf08_mod.f90 b/assimilation_code/modules/utilities/winf08_mod.f90 similarity index 87% rename from assimilation_code/modules/utilities/no_cray_winf08_mod.f90 rename to assimilation_code/modules/utilities/winf08_mod.f90 index 7ecfd90830..54f14de3db 100644 --- a/assimilation_code/modules/utilities/no_cray_winf08_mod.f90 +++ b/assimilation_code/modules/utilities/winf08_mod.f90 @@ -2,7 +2,6 @@ ! by UCAR, "as is", without charge, subject to all terms of use at ! http://www.image.ucar.edu/DAReS/DART/DART_download -!> Window without cray pointer. Should you point the window at contigous memory? module window_mod !> \defgroup window window_mod @@ -30,21 +29,16 @@ module window_mod integer :: current_win !< keep track of current window, start out assuming an invalid window ! parameters for keeping track of which window is open -!>@todo should this be in the window_mod? you will have to change in both cray -!> and non cray versions integer, parameter :: NO_WINDOW = -1 integer, parameter :: MEAN_WINDOW = 0 integer, parameter :: STATE_WINDOW = 2 -integer :: data_count !! number of copies in the window +integer :: data_count !! number of copies required integer(KIND=MPI_ADDRESS_KIND) :: window_size logical :: use_distributed_mean = .false. ! initialize to false ! Global memory to stick the mpi window to. ! Need a simply contiguous piece of memory to pass to mpi_win_create -! Openmpi 1.10.0 will not compile with ifort 16 if -! you create a window with a 2d array. -real(r8), allocatable :: contiguous_fwd(:) real(r8), allocatable :: mean_1d(:) type(ensemble_type) :: mean_ens_handle @@ -65,8 +59,7 @@ subroutine create_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handl integer :: bytesize !< size in bytes of each element in the window integer :: my_num_vars !< my number of vars -! Find out how many copies to put in the window -! copies_in_window is not necessarily equal to ens_handle%num_copies +! Find out how many copies to get, maybe different to state_ens_handle%num_copies data_count = copies_in_window(state_ens_handle) if (get_allow_transpose(state_ens_handle)) then @@ -82,20 +75,15 @@ subroutine create_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handl my_num_vars = state_ens_handle%my_num_vars call mpi_type_size(datasize, bytesize, ierr) - window_size = my_num_vars*data_count*bytesize - - allocate(contiguous_fwd(data_count*my_num_vars)) - contiguous_fwd = reshape(state_ens_handle%copies(1:data_count, :), (/my_num_vars*data_count/)) + window_size = my_num_vars*state_ens_handle%num_copies*bytesize ! Expose local memory to RMA operation by other processes in a communicator. - call mpi_win_create(contiguous_fwd, window_size, bytesize, MPI_INFO_NULL, get_dart_mpi_comm(), state_win, ierr) + call mpi_win_create(state_ens_handle%copies, window_size, bytesize, MPI_INFO_NULL, get_dart_mpi_comm(), state_win, ierr) endif ! Set the current window to the state window current_win = STATE_WINDOW -data_count = copies_in_window(state_ens_handle) - end subroutine create_state_window !------------------------------------------------------------- @@ -163,7 +151,6 @@ subroutine free_state_window(state_ens_handle, fwd_op_ens_handle, qc_ens_handle) else ! close mpi window call mpi_win_free(state_win, ierr) - deallocate(contiguous_fwd) endif current_win = NO_WINDOW diff --git a/build_templates/buildconvfunctions.sh b/build_templates/buildconvfunctions.sh index 8b194c7c90..91de56d7e5 100644 --- a/build_templates/buildconvfunctions.sh +++ b/build_templates/buildconvfunctions.sh @@ -25,7 +25,6 @@ # # The GSI obs converter needs mpi # mpisrc="null_mpi" -# windowsrc="" # m="" #------------------------- set -e @@ -34,7 +33,6 @@ source "$DART"/build_templates/buildpreprocess.sh # Defaults mpisrc="null_mpi" -windowsrc="" m="" LIBRARIES="" EXTRA="" @@ -123,28 +121,22 @@ local mpi="$DART"/assimilation_code/modules/utilities/mpi_utilities_mod.f90 local mpif08="$DART"/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 local nullmpi="$DART"/assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 local nullwin="$DART"/assimilation_code/modules/utilities/null_win_mod.f90 -local craywin="$DART"/assimilation_code/modules/utilities/cray_win_mod.f90 -local nocraywin="$DART"/assimilation_code/modules/utilities/no_cray_win_mod.f90 -local no_cray_winf08="$DART"/assimilation_code/modules/utilities/no_cray_winf08_mod.f90 +local win="$DART"/assimilation_code/modules/utilities/win_mod.f90 +local winf08="$DART"/assimilation_code/modules/utilities/winf08_mod.f90 if [ "$mpisrc" == "mpi" ]; then core=${core//$nullmpi/} core=${core//$nullwin/} core=${core//$mpif08/} - core=${core//$no_cray_winf08/} - if [ "$windowsrc" == "craywin" ]; then - core=${core//$nocraywin/} - else #nocraywin - core=${core//$craywin/} - fi + core=${core//$winf08/} + else #nompi core=${core//$mpi/} core=${core//$mpif08/} - core=${core//$nocraywin/} - core=${core//$no_cray_winf08/} - core=${core//$craywin/} + core=${core//$win/} + core=${core//$winf08/} fi convsrc="${core} ${conv} ${obserrsrc} ${modelsrc} ${misc} ${loc}" diff --git a/build_templates/buildfunctions.sh b/build_templates/buildfunctions.sh index f8188a8cad..1440922da4 100644 --- a/build_templates/buildfunctions.sh +++ b/build_templates/buildfunctions.sh @@ -105,7 +105,6 @@ fi # Default to build with mpi (non f08 version) mpisrc=mpi -windowsrc=no_cray_win m="-w" # mkmf wrapper arg # if the first argument is help, nompi, mpi, mpif08, clean @@ -116,20 +115,17 @@ case $1 in nompi) mpisrc="null_mpi" - windowsrc="" m="" shift 1 ;; mpi) mpisrc="mpi" - windowsrc="no_cray_win" shift 1 ;; mpif08) mpisrc="mpif08" - windowsrc="no_cray_winf08" shift 1 ;; @@ -168,37 +164,29 @@ local mpi="$DART"/assimilation_code/modules/utilities/mpi_utilities_mod.f90 local mpif08="$DART"/assimilation_code/modules/utilities/mpif08_utilities_mod.f90 local nullmpi="$DART"/assimilation_code/modules/utilities/null_mpi_utilities_mod.f90 local nullwin="$DART"/assimilation_code/modules/utilities/null_win_mod.f90 -local craywin="$DART"/assimilation_code/modules/utilities/cray_win_mod.f90 -local nocraywin="$DART"/assimilation_code/modules/utilities/no_cray_win_mod.f90 -local no_cray_winf08="$DART"/assimilation_code/modules/utilities/no_cray_winf08_mod.f90 +local win="$DART"/assimilation_code/modules/utilities/win_mod.f90 +local winf08="$DART"/assimilation_code/modules/utilities/winf08_mod.f90 if [ "$mpisrc" == "mpi" ]; then core=${core//$nullmpi/} core=${core//$nullwin/} core=${core//$mpif08/} - core=${core//$no_cray_winf08/} - if [ "$windowsrc" == "craywin" ]; then - core=${core//$nocraywin/} - else #nocraywin - core=${core//$craywin/} - fi + core=${core//$winf08/} elif [ "$mpisrc" == "mpif08" ]; then core=${core//$nullmpi/} core=${core//$nullwin/} core=${core//$mpi/} - core=${core//$craywin/} - core=${core//$nocraywin/} + core=${core//$win/} else #nompi core=${core//$mpi/} core=${core//$mpif08/} - core=${core//$nocraywin/} - core=${core//$no_cray_winf08/} - core=${core//$craywin/} + core=${core//$win/} + core=${core//$winf08/} fi dartsrc="${core} ${modelsrc} ${loc} ${misc}" @@ -234,8 +222,10 @@ done function dartbuild() { local program +local devlibs if [ $dev_test -eq 0 ]; then + devlibs="" #look in $program directory for {main}.f90 if [ $1 == "obs_diag" ]; then program=$DART/assimilation_code/programs/obs_diag/$LOCATION @@ -247,11 +237,13 @@ if [ $dev_test -eq 0 ]; then else # For developer tests {main}.f90 is in developer_tests program=$DART/developer_tests/$TEST/$1.f90 + devlibs=$DART/developer_tests/contrib/fortran-testanything fi $DART/build_templates/mkmf -x -a $DART $m -p $1 \ $dartsrc \ $EXTRA \ + $devlibs \ $program } @@ -291,7 +283,6 @@ if [ ! -z "$single_prog" ] ; then # build a single program elif [[ " ${serial_programs[*]} " =~ " ${single_prog} " ]]; then echo "building serial dart program " $single_prog mpisrc="null_mpi" - windowsrc="" m="" findsrc dartbuild $single_prog @@ -304,7 +295,6 @@ if [ ! -z "$single_prog" ] ; then # build a single program elif [[ " ${model_serial_programs[*]} " =~ " ${single_prog} " ]];then echo "building model program" $single_prog mpisrc="null_mpi" - windowsrc="" m="" findsrc modelbuild $single_prog @@ -340,7 +330,6 @@ done [ $mpisrc == "mpi" ] && \rm -f *.o *.mod mpisrc="null_mpi" -windowsrc="" m="" # Serial programs diff --git a/conf.py b/conf.py index 902f7875d9..23c29119d6 100644 --- a/conf.py +++ b/conf.py @@ -21,7 +21,7 @@ author = 'Data Assimilation Research Section' # The full version, including alpha/beta/rc tags -release = '11.8.2' +release = '11.8.3' root_doc = 'index' # -- General configuration --------------------------------------------------- diff --git a/developer_tests/contrib/fortran-testanything/LICENSE.txt b/developer_tests/contrib/fortran-testanything/LICENSE.txt new file mode 100644 index 0000000000..aaf4092f79 --- /dev/null +++ b/developer_tests/contrib/fortran-testanything/LICENSE.txt @@ -0,0 +1,14 @@ +Copyright 2015 Dennis Decker Jensen + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + diff --git a/developer_tests/contrib/fortran-testanything/is_i.inc b/developer_tests/contrib/fortran-testanything/is_i.inc new file mode 100644 index 0000000000..7f98a09089 --- /dev/null +++ b/developer_tests/contrib/fortran-testanything/is_i.inc @@ -0,0 +1,24 @@ +! Template parameter: wp (working precision) +! Template free identifiers: testline, tests +subroutine is(got, expected, msg) + integer(kind=wp), intent(in) :: got, expected + character(len=*), intent(in), optional :: msg + character(len=:), allocatable :: testmsg, idmsg + character(len=120) gotmsg, expectedmsg + logical good + + if (present(msg)) then + allocate(character(len=len_trim(msg)+20) :: testmsg, idmsg) + write (unit=idmsg, fmt='(A,A,A)') 'Failed test: "', trim(msg), '"' + testmsg = trim(msg) + else + allocate(character(len=30) :: testmsg, idmsg) + write (unit=idmsg, fmt='(A,I0)') 'Failed test no. ', tests + 1 + testmsg = "" + end if + write (unit=gotmsg, fmt='(A,I0)') ' got: ', got + write (unit=expectedmsg, fmt='(A,I0)') 'expected: ', expected + + good = got == expected + call testline(good, testmsg, idmsg, gotmsg, expectedmsg) +end diff --git a/developer_tests/contrib/fortran-testanything/is_r.inc b/developer_tests/contrib/fortran-testanything/is_r.inc new file mode 100644 index 0000000000..98599716b4 --- /dev/null +++ b/developer_tests/contrib/fortran-testanything/is_r.inc @@ -0,0 +1,83 @@ +! Template parameter: wp (working precision) +! Template free identifiers: testline, tests +subroutine isabs(got, expected, eps, msg) + real(kind=wp), intent(in) :: got, expected + character(len=*), intent(in), optional :: msg + real(kind=wp), intent(in), optional :: eps + character(len=:), allocatable :: testmsg, idmsg + character(len=120) gotmsg, expectedmsg + real(kind=wp) tolerance + logical good + + if (present(msg)) then + allocate(character(len=len_trim(msg)+20) :: testmsg, idmsg) + write (unit=idmsg, fmt='(A,A,A)') 'Failed test: "', trim(msg), '"' + testmsg = trim(msg) + else + allocate(character(len=30) :: testmsg, idmsg) + write (unit=idmsg, fmt='(A,I0)') 'Failed test no. ', tests + 1 + testmsg = "" + end if + write (unit=gotmsg, fmt='(A,G0)') ' got: ', got + write (unit=expectedmsg, fmt='(A,G0)') 'expected: ', expected + + if (present(eps)) then + tolerance = eps + else + tolerance = epsilon(got) + end if + ! eps = 0.5e-10_wp + ! Absolute accuracy within the 10 least significant digits + good = abs(got - expected) < tolerance + call testline(good, testmsg, idmsg, gotmsg, expectedmsg) +end + +subroutine isrel(got, expected, eps, msg) + real(kind=wp), intent(in) :: got, expected + character(len=*), intent(in), optional :: msg + real(kind=wp), intent(in), optional :: eps + real(kind=wp) tolerance + + ! eps = (abs(a) + abs(b)) * 0.5e-10_wp + ! Relative accuracy within the 10 most significant digits + tolerance = (abs(got) + abs(expected)) + if (present(eps)) then + tolerance = tolerance * eps + else + tolerance = tolerance * epsilon(got) + end if + call isabs(got, expected, tolerance, msg) +end + +subroutine isnear(got, expected, eps, msg) + real(kind=wp), intent(in) :: got, expected + character(len=*), intent(in), optional :: msg + real(kind=wp), intent(in), optional :: eps + character(len=:), allocatable :: testmsg, idmsg + character(len=120) gotmsg, expectedmsg + real(kind=wp) tolerance + logical good + + if (present(msg)) then + allocate(character(len=len_trim(msg)+20) :: testmsg, idmsg) + write (unit=idmsg, fmt='(A,A,A)') 'Failed test: "', trim(msg), '"' + testmsg = trim(msg) + else + allocate(character(len=30) :: testmsg, idmsg) + write (unit=idmsg, fmt='(A,I0)') 'Failed test no. ', tests + 1 + testmsg = "" + end if + write (unit=gotmsg, fmt='(A,G0)') ' got: ', got + write (unit=expectedmsg, fmt='(A,G0)') 'expected: ', expected + + if (present(eps)) then + tolerance = eps + else + tolerance = epsilon(got) ! minimun eps for which 1 + eps /= 1 + end if + ! Relative accuracy around 1.0_wp + ! Semantics of isnear means using <=, and not <, c.f. epsilon(got) + good = abs(got / expected - 1.0_wp) <= tolerance + call testline(good, testmsg, idmsg, gotmsg, expectedmsg) +end + diff --git a/developer_tests/contrib/fortran-testanything/test.f90 b/developer_tests/contrib/fortran-testanything/test.f90 new file mode 100644 index 0000000000..5b565779df --- /dev/null +++ b/developer_tests/contrib/fortran-testanything/test.f90 @@ -0,0 +1,373 @@ +! Copyright 2015 Dennis Decker Jensen +! See and +! Tectonics: gfortran -g -Wall -Wextra -std=f2008ts -c test.f08 + +module test_base + use, intrinsic :: iso_fortran_env, only: output_unit, error_unit + implicit none + + ! Kept as variables instead of aliases, + ! so that test output or diagnostic output can be redirected + integer :: test_unit = output_unit, diag_unit = error_unit + + integer :: tests = 0, todos = 0 + character(len=120) :: todomsg = "" + + interface todo + module procedure todo_i, todo_s, todo_s_i, todo + end interface + +contains + + subroutine diag(msg) + character(len=*), intent(in) :: msg + write (diag_unit, '("# ",A)') trim(msg) ! only trailing spaces + end + + subroutine note(msg) + character(len=*), intent(in) :: msg + write (test_unit, '("# ",A)') trim(msg) + end + + subroutine testline(ok, msg, idmsg, gotmsg, expectedmsg) + logical, intent(in) :: ok + character(len=*), intent(in) :: msg, idmsg, gotmsg, expectedmsg + + tests = tests + 1 + if (.not. ok) call out("not ") + write (test_unit, '("ok ",I0)', advance="NO") tests + + if (msg /= "" .or. todos > 0) call out(" - ") + + if (msg /= "") call out(trim(msg)) + + if (todos > 0) then + todos = todos - 1 + if (msg /= "") call out(" ") + call out("# TODO") + if (todomsg .ne. "") then + call out(": ") + call out(trim(todomsg)) + end if + end if + if (todos == 0) todomsg = "" + + write (test_unit, *) "" + + if (.not. ok) then + ! 3 spaces prepended = 4 spaces indentation after # on diag + if (idmsg /= "") call diag(" " // idmsg) + if (gotmsg /= "") call diag(" " // gotmsg) + if (expectedmsg /= "") call diag(" " // expectedmsg) + end if + contains + subroutine out(str) + character(len=*), intent(in) :: str + write (test_unit, '(A)', advance="NO") str + end + end subroutine testline + + subroutine ok(condition, msg) + logical, intent(in) :: condition + character(len=*), intent(in), optional :: msg + if (present(msg)) then + call testline(condition, msg, "", "", "") + else + call testline(condition, "", "", "", "") + end if + end + + subroutine pass(msg) + character(len=*), intent(in), optional :: msg + call ok(.true., msg) + end + + subroutine fail(msg) + character(len=*), intent(in), optional :: msg + call ok(.false., msg) + end + + subroutine todo_s_i(msg, howmany) + character(len=*), intent(in) :: msg + integer, intent(in) :: howmany + todomsg = msg + todos = howmany + end + + subroutine todo + call todo_s_i("", 1) + end + + subroutine todo_s(msg) + character(len=*), intent(in) :: msg + call todo_s_i(msg, 1) + end + + subroutine todo_i(howmany) + integer, intent(in) :: howmany + call todo_s_i("", howmany) + end + +end module test_base + +module test_planning + use test_base, only: test_unit, tests + implicit none + + integer, private :: planned = 0 + +contains + + subroutine bail_out(msg) + character(len=*), intent(in), optional :: msg + if (present(msg)) then + write (test_unit, '("Bail out! ",A)') msg + else + write (test_unit, '("Bail out!")') + end if + stop + end + + subroutine plan(tests) + integer, intent(in) :: tests + + select case (tests) + case (:-1) + call bail_out("A plan with a negative number of tests") + case (0) + write (test_unit, '("1..0")') + stop ! The same as skip_all without a given reason + case (1:) + if (planned > 0) & + & call bail_out("More than one plan in test output") + planned = tests + write (test_unit, '("1..",I0)') planned + end select + end + + subroutine done_testing(howmany) + integer, intent(in), optional :: howmany + + ! Put plan at the end of test output + if (present(howmany)) then + call plan(howmany) + else + if (planned == 0) call plan(tests) + ! else - We already have a plan + end if + end + + subroutine skip_all(msg) + character(len=*), intent(in), optional :: msg + if (present(msg)) then + write (test_unit, '("1..0 # Skipped: ",A)') msg + else + write (test_unit, '("1..0 # Skipped all")') + end if + stop + end + +end module test_planning + +! Template instances of integer kinds for "is" + +module is_i8_mod + use, intrinsic :: iso_fortran_env, only: wp => int8 + use, non_intrinsic :: test_base, only: testline, tests +contains + include "is_i.inc" +end + +module is_i16_mod + use, intrinsic :: iso_fortran_env, only: wp => int16 + use, non_intrinsic :: test_base, only: testline, tests +contains + include "is_i.inc" +end + +module is_i32_mod + use, intrinsic :: iso_fortran_env, only: wp => int32 + use, non_intrinsic :: test_base, only: testline, tests +contains + include "is_i.inc" +end + +module is_i64_mod + use, intrinsic :: iso_fortran_env, only: wp => int64 + use, non_intrinsic :: test_base, only: testline, tests +contains + include "is_i.inc" +end + +module is_i + use is_i8_mod, only: is_i8 => is + use is_i16_mod, only: is_i16 => is + use is_i32_mod, only: is_i32 => is + use is_i64_mod, only: is_i64 => is + interface is + module procedure is_i8, is_i16, is_i32, is_i64 + end interface +end + +! Template instances of real kinds for "is" + +module is_r32_mod + use, intrinsic :: iso_fortran_env, only: wp => real32 + use, non_intrinsic :: test_base, only: testline, tests +contains + include "is_r.inc" +end + +module is_r64_mod + use, intrinsic :: iso_fortran_env, only: wp => real64 + use, non_intrinsic :: test_base, only: testline, tests +contains + include "is_r.inc" +end + +module is_r128_mod + use, intrinsic :: iso_fortran_env, only: wp => real128 + use, non_intrinsic :: test_base, only: testline, tests +contains + include "is_r.inc" +end + +module is_r + use is_r32_mod, only: isrel_r32 => isrel, isabs_r32 => isabs, & + & isnear_r32 => isnear + use is_r64_mod, only: isrel_r64 => isrel, isabs_r64 => isabs, & + & isnear_r64 => isnear + use is_r128_mod, only: isrel_r128 => isrel, isabs_r128 => isabs, & + & isnear_r128 => isnear + interface isrel + module procedure isrel_r32, isrel_r64, isrel_r128 + end interface + + interface isabs + module procedure isabs_r32, isabs_r64, isabs_r128 + end interface + + interface isnear + module procedure isnear_r32, isnear_r64, isnear_r128 + end interface +end + +module test_more + use test_base, only: testline, tests, test_unit + use test_planning, only: bail_out ! for negative skips + use is_i, only: is, is_i8, is_i16, is_i32, is_i64 + use is_r, only: isabs, isrel, isnear, & + & isabs_r32, isrel_r32, isnear_r32, & + & isabs_r64, isrel_r64, isnear_r64, & + & isabs_r128, isrel_r128, isnear_r128 + + ! Complex numbers cannot be compared, hence no is_c module + + implicit none + + interface skip + module procedure skip_i, skip_s, skip_s_i, skip + end interface + + interface is + module procedure is_s, is_l + end interface + +contains + + subroutine skip_s_i(msg, howmany) + character(len=*), intent(in) :: msg + integer, intent(in) :: howmany + character(len=120) skipmsg + integer i + + if (howmany <= 0) then + call bail_out("Skipped non-positive number of tests") + end if + + if (msg == "") then + skipmsg = "# SKIP" + else + skipmsg = "# SKIP: " // trim(msg) + end if + + do i = 1, howmany + tests = tests + 1 + write (test_unit, '("ok ",I0," ",A)') tests, trim(skipmsg) + end do + end + + subroutine skip + call skip_s_i("", 1) + end + + subroutine skip_s(msg) + character(len=*), intent(in) :: msg + call skip_s_i(msg, 1) + end + + subroutine skip_i(howmany) + integer, intent(in) :: howmany + call skip_s_i("", howmany) + end + + ! Duplicates of is_i routines in file is_i.inc and ditto is_r + ! They are not factored any further, because it is easier + ! to see all the output together rather than in separate routines + + subroutine is_s(got, expected, msg) + character(len=*), intent(in) :: got + character(len=*), intent(in) :: expected + character(len=*), intent(in), optional :: msg + character(len=:), allocatable :: testmsg, idmsg + character(len=120) gotmsg, expectedmsg + logical good + + if (present(msg)) then + allocate(character(len=len_trim(msg)+20) :: testmsg, idmsg) + write (unit=idmsg, fmt='(A,A,A)') 'Failed test: "', trim(msg), '"' + testmsg = trim(msg) + else + allocate(character(len=30) :: testmsg, idmsg) + write (unit=idmsg, fmt='(A,I0)') 'Failed test no. ', tests + 1 + testmsg = "" + end if + write (unit=gotmsg, fmt='(A,A,A)') ' got: "', got, '"' + write (unit=expectedmsg, fmt='(A,A,A)') 'expected: "', expected, '"' + + good = got == expected + call testline(good, testmsg, idmsg, gotmsg, expectedmsg) + end + + subroutine is_l(got, expected, msg) + logical, intent(in) :: got, expected + character(len=*), intent(in), optional :: msg + character(len=:), allocatable :: testmsg, idmsg + character(len=120) gotmsg, expectedmsg + logical good + + if (present(msg)) then + allocate(character(len=len_trim(msg)+20) :: testmsg, idmsg) + write (unit=idmsg, fmt='(A,A,A)') 'Failed test: "', trim(msg), '"' + testmsg = trim(msg) + else + allocate(character(len=30) :: testmsg, idmsg) + write (unit=idmsg, fmt='(A,I0)') 'Failed test no. ', tests + 1 + testmsg = "" + end if + write (unit=gotmsg, fmt='(A,L1)') ' got: ', got + write (unit=expectedmsg, fmt='(A,L1)') 'expected: ', expected + + good = got .eqv. expected + call testline(good, testmsg, idmsg, gotmsg, expectedmsg) + end + +end module test_more + +module test + use test_base, only: test_unit, diag_unit, & + & ok, diag, note, pass, fail, todo + use test_planning, only: plan, done_testing, skip_all, bail_out + use test_more, only: is, isabs, isrel, isnear, skip +end module test + diff --git a/developer_tests/window/test_window.f90 b/developer_tests/window/test_window.f90 new file mode 100644 index 0000000000..b910d8b69d --- /dev/null +++ b/developer_tests/window/test_window.f90 @@ -0,0 +1,50 @@ +program test_window + +use mpi_utilities_mod, only : initialize_mpi_utilities, finalize_mpi_utilities, my_task_id, task_count, task_sync + +use ensemble_manager_mod, only : init_ensemble_manager, end_ensemble_manager, ensemble_type, set_num_extra_copies +use distributed_state_mod, only : create_state_window, free_state_window, get_state +use types_mod, only : i8, r8 + +use test ! fortran-testanything + +implicit none + +integer :: num_copies = 10 +integer :: real_ens_members = 3 +real(r8) :: res(3) +integer(i8) :: num_vars = 201 +type(ensemble_type) :: ens_handle + +call initialize_mpi_utilities('test_window') + +if (my_task_id() == 0 ) then + call plan(3*task_count()) +endif + +call init_ensemble_manager(ens_handle, num_copies, num_vars) +call set_num_extra_copies(ens_handle, num_copies - real_ens_members) + +ens_handle%copies(1:real_ens_members,:) = my_task_id() +ens_handle%copies(real_ens_members+1:num_copies,:) = -100 + +call create_state_window(ens_handle) + +! result should be index-1 mod task_count() for round robin distribution +res = get_state(1_i8, ens_handle) +call ok(res(1) == mod(1-1, task_count())) + +res = get_state(27_i8, ens_handle) +call ok(res(1) == mod(27-1, task_count())) + +res = get_state(198_i8, ens_handle) +call ok(res(1) == mod(198-1, task_count())) + +call free_state_window(ens_handle) + +call end_ensemble_manager(ens_handle) + +call finalize_mpi_utilities() + + +end program test_window \ No newline at end of file diff --git a/developer_tests/window/work/input.nml b/developer_tests/window/work/input.nml new file mode 100644 index 0000000000..71e9d827f1 --- /dev/null +++ b/developer_tests/window/work/input.nml @@ -0,0 +1,15 @@ +&ensemble_manager_nml +/ + +&utilities_nml + module_details = .false. + / + +&preprocess_nml + input_obs_qty_mod_file = '../../../assimilation_code/modules/observations/DEFAULT_obs_kind_mod.F90' + output_obs_qty_mod_file = '../../../assimilation_code/modules/observations/obs_kind_mod.f90' + input_obs_def_mod_file = '../../../observations/forward_operators/DEFAULT_obs_def_mod.F90' + output_obs_def_mod_file = '../../../observations/forward_operators/obs_def_mod.f90' + obs_type_files = '../../../observations/forward_operators/obs_def_gps_mod.f90' + quantity_files = '../../../assimilation_code/modules/observations/default_quantities_mod.f90' + / diff --git a/developer_tests/window/work/quickbuild.sh b/developer_tests/window/work/quickbuild.sh new file mode 100755 index 0000000000..5482d0f19a --- /dev/null +++ b/developer_tests/window/work/quickbuild.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# DART software - Copyright UCAR. This open source software is provided +# by UCAR, "as is", without charge, subject to all terms of use at +# http://www.image.ucar.edu/DAReS/DART/DART_download + +main() { + + +export DART=$(git rev-parse --show-toplevel) +source "$DART"/build_templates/buildfunctions.sh + +MODEL="none" +EXTRA=$DART/models/template/threed_model_mod.f90 +dev_test=1 +LOCATION="threed_sphere" +TEST="window" + +programs=( +test_window +) + +# quickbuild arguments +arguments "$@" + +# clean the directory +\rm -f -- *.o *.mod Makefile .cppdefs + +# build and run preprocess before making any other DART executables +buildpreprocess + +# build DART +buildit + +# clean up +\rm -f -- *.o *.mod + +} + +main "$@"