diff --git a/model/bin/build_utils.sh b/model/bin/build_utils.sh index fe65b6d61..9ba0c087b 100755 --- a/model/bin/build_utils.sh +++ b/model/bin/build_utils.sh @@ -802,7 +802,7 @@ create_file_list() prop= sourcet="w3parall w3triamd w3gridmd $stx $flx $nlx $btx $is $uostmd" IO='w3iogrmd' - aux="constants w3servmd w3arrymd w3dispmd w3gsrumd w3timemd w3nmlgridmd $pdlibyow $memcode" + aux="constants mpicomm w3servmd w3arrymd w3dispmd w3gsrumd w3timemd w3nmlgridmd $pdlibyow $memcode" if [ "$scrip" = 'SCRIP' ] then aux="$aux scrip_constants scrip_grids scrip_iounitsmod" @@ -819,42 +819,42 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $tr $trx $bt $setupcode $stx $flx $nlx $btx $is wmmdatmd w3parall $uostmd" IO='w3iogrmd w3iorsmd' - aux="constants w3triamd w3servmd w3arrymd w3dispmd w3gsrumd w3timemd" ;; + aux="constants mpicomm w3triamd w3servmd w3arrymd w3dispmd w3gsrumd w3timemd" ;; ww3_bound) core= data="w3adatmd $memcode w3gdatmd w3wdatmd w3idatmd w3odatmd" prop="$smcm" sourcet="$pdlibcode $pdlibyow $db $bt $setupcode $tr $trx $stx $nlx $btx $is wmmdatmd w3parall w3triamd $uostmd" IO='w3iobcmd w3iogrmd w3dispmd w3gsrumd' - aux="constants w3servmd w3timemd w3arrymd w3cspcmd w3nmlboundmd" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3cspcmd w3nmlboundmd" ;; ww3_bounc) core= data="w3adatmd $memcode w3gdatmd w3wdatmd w3idatmd w3odatmd" prop="$smcm" sourcet="$pdlibcode $pdlibyow $db $bt $setupcode $stx $nlx $btx $is wmmdatmd w3parall w3triamd $uostmd" IO='w3iobcmd w3iogrmd w3dispmd w3gsrumd' - aux="constants w3servmd w3arrymd w3timemd w3cspcmd w3nmlbouncmd" ;; + aux="constants mpicomm w3servmd w3arrymd w3timemd w3cspcmd w3nmlbouncmd" ;; ww3_prep) core='w3fldsmd' data="$memcode w3gdatmd w3adatmd w3idatmd w3odatmd w3wdatmd wmmdatmd" prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode w3triamd $stx $flx $nlx $btx $is $uostmd" IO="w3iogrmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants w3servmd w3timemd $tidecode w3arrymd w3dispmd w3gsrumd w3parall" ;; + aux="constants mpicomm w3servmd w3timemd $tidecode w3arrymd w3dispmd w3gsrumd w3parall" ;; ww3_prnc) core='w3fldsmd' data="$memcode w3gdatmd w3adatmd w3idatmd w3odatmd w3wdatmd wmmdatmd" prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode w3triamd $stx $flx $nlx $btx $is w3parall $uostmd" IO="w3iogrmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd w3tidemd w3nmlprncmd" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd w3tidemd w3nmlprncmd" ;; ww3_prtide) core='w3fldsmd' data="wmmdatmd $memcode w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd" prop="$pr $smcm" sourcet="$pdlibcode $pdlibyow $db $bt $setupcode w3triamd $stx $nlx $btx $is w3parall $uostmd" IO="w3iogrmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd $tidecode" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd $tidecode" ;; ww3_shel) core='w3fldsmd w3initmd w3wavemd w3wdasmd w3updtmd' data="wmmdatmd $memcode w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd" @@ -863,7 +863,7 @@ create_file_list() sourcet="$sourcet $is $db $tr $bs $refcode $igcode w3parall $uostmd" IO="w3iogrmd w3iogomd w3iopomd w3iotrmd w3iorsmd w3iobcmd $oasismd $agcmmd $ogcmmd $igcmmd" IO="$IO w3iosfmd w3partmd" - aux="constants w3servmd w3timemd $tidecode w3arrymd w3dispmd w3cspcmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd $tidecode w3arrymd w3dispmd w3cspcmd w3gsrumd" aux="$aux w3nmlshelmd $pdlibyow" ;; ww3_multi|ww3_multi_esmf) if [ "$prog" = "ww3_multi" ] @@ -879,7 +879,7 @@ create_file_list() sourcet="$pdlibcode $pdlibyow $setupcode w3parall w3triamd w3srcemd $dsx $flx $ln $st $nl $bt $ic $is $db $tr $bs $refcode $igcode $uostmd" IO='w3iogrmd w3iogomd w3iopomd wmiopomd' IO="$IO w3iotrmd w3iorsmd w3iobcmd w3iosfmd w3partmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants $tidecode w3servmd w3timemd w3arrymd w3dispmd w3cspcmd w3gsrumd $mprfaux" + aux="constants mpicomm $tidecode w3servmd w3timemd w3arrymd w3dispmd w3cspcmd w3gsrumd $mprfaux" aux="$aux wmunitmd w3nmlmultimd" if [ "$scrip" = 'SCRIP' ] then @@ -899,7 +899,7 @@ create_file_list() sourcet="$pdlibcode $pdlibyow w3triamd w3srcemd $dsx $flx $ln $st $nl $bt $db $tr $bs $refcode $igcode $is $ic $uostmd" IO='w3iogrmd w3iogomd w3iopomd wmiopomd' IO="$IO w3iotrmd w3iorsmd w3iobcmd w3iosfmd w3partmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3cspcmd w3gsrumd $mprfaux $tidecode" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3cspcmd w3gsrumd $mprfaux $tidecode" aux="$aux wmunitmd w3nmlmultimd" if [ "$scrip" = 'SCRIP' ] then @@ -917,7 +917,7 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode $tr $trx $stx $flx $nlx $btx $is $uostmd" IO='w3iogrmd w3iogomd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux" ;; ww3_ounf) core='w3initmd' @@ -925,7 +925,7 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode w3parall w3triamd $stx $flx $nlx $btx $is $uostmd" IO='w3iogrmd w3iogomd w3iorsmd w3iopomd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux w3nmlounfmd $smco w3ounfmetamd w3metamd" ;; ww3_outp) core= @@ -933,14 +933,14 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $setupcode $flx $ln $st $nl $bt $ic $is $db $tr $bs $igcode $uostmd" IO='w3bullmd w3iogrmd w3iopomd w3partmd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" ;; ww3_ounp) core= data="wmmdatmd w3parall w3triamd $memcode w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd" prop= sourcet="$pdlibcode $pdlibyow $setupcode $flx $ln $st $nl $bt $ic $is $db $tr $bs $igcode $uostmd" IO='w3bullmd w3iogrmd w3iopomd w3partmd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux w3nmlounpmd" ;; ww3_trck) core= @@ -948,21 +948,21 @@ create_file_list() prop= sourcet= IO= - aux="constants w3servmd w3timemd w3gsrumd" ;; + aux="constants mpicomm w3servmd w3timemd w3gsrumd" ;; ww3_trnc) core= data="$memcode w3gdatmd w3odatmd" prop= sourcet= IO= - aux="constants w3servmd w3timemd w3gsrumd w3nmltrncmd" ;; + aux="constants mpicomm w3servmd w3timemd w3gsrumd w3nmltrncmd" ;; ww3_grib) core= data="w3parall wmmdatmd w3triamd $memcode w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd" prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode $stx $flx $nlx $btx $is $uostmd" IO='w3iogrmd w3iogomd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux" ;; ww3_gspl) core='w3fldsmd' @@ -970,14 +970,14 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode wmmdatmd w3parall w3triamd $stx $flx $nlx $btx $is $uostmd" IO="w3iogrmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd $tidecode" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd $tidecode" ;; ww3_gint) core= data="w3parall wmmdatmd $memcode w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd" IO='w3iogrmd w3iogomd' prop= sourcet="$pdlibcode $pdlibyow $db $bt $st $nl $is $uostmd" - aux="constants w3triamd w3servmd w3arrymd w3dispmd w3timemd w3gsrumd" + aux="constants mpicomm w3triamd w3servmd w3arrymd w3dispmd w3timemd w3gsrumd" aux="$aux" ;; gx_outf) core= @@ -985,7 +985,7 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode wmmdatmd w3parall w3triamd $stx $flx $nlx $btx $tr $bs $is $uostmd" IO='w3iogrmd w3iogomd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux" ;; gx_outp) core= @@ -993,28 +993,28 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode wmmdatmd w3parall w3triamd $ln $flx $st $nlx $btx $tr $bs $is $ic $uostmd" IO='w3iogrmd w3iopomd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" ;; ww3_systrk) core='w3strkmd' data="$memcode w3gdatmd w3adatmd w3idatmd w3odatmd w3wdatmd" prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode wmmdatmd w3dispmd w3triamd $ln $stx $flx $nlx $btx $tr $bs $is $uostmd" IO= - aux="constants w3servmd w3timemd w3arrymd w3gsrumd w3parall" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3gsrumd w3parall" ;; libww3|libww3.so) core='w3fldsmd w3initmd w3wavemd w3wdasmd w3updtmd' data='wmmdatmd w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd' prop="$pr $smcm" sourcet="w3triamd w3srcemd $dsx $flx $ln $st $nl $bt $ic $is $db $tr $bs $refcode $igcode $uostmd" IO='w3iogrmd w3iogomd w3iopomd w3iotrmd w3iorsmd w3iobcmd w3iosfmd w3partmd' - aux="constants w3servmd w3timemd $tidecode w3arrymd w3dispmd w3cspcmd w3gsrumd" ;; + aux="constants mpicomm w3servmd w3timemd $tidecode w3arrymd w3dispmd w3cspcmd w3gsrumd" ;; ww3_uprstr) core= data='wmmdatmd w3triamd w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd' prop= sourcet="$memcode $pdlibcode $pdlibyow $flx $ln $st $nl $bt $ic $is $db $tr $bs $uostmd" IO='w3iogrmd w3iogomd w3iorsmd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux w3parall w3nmluprstrmd" ;; esac diff --git a/model/src/cmake/src_list.cmake b/model/src/cmake/src_list.cmake index d745be388..6bcb914c4 100644 --- a/model/src/cmake/src_list.cmake +++ b/model/src/cmake/src_list.cmake @@ -3,6 +3,7 @@ set(c_src w3getmem.c) # Core files always built set(ftn_src constants.F90 + mpicomm.F90 w3adatmd.F90 w3arrymd.F90 w3bullmd.F90 diff --git a/model/src/cmake/switches.json b/model/src/cmake/switches.json index a7b9bc94f..37cd52a30 100644 --- a/model/src/cmake/switches.json +++ b/model/src/cmake/switches.json @@ -282,7 +282,7 @@ }, { "name": "NL2", - "build_files": ["w3snl2md.F90", "mod_xnl4v5.f90", "serv_xnl4v5.f90", "mod_fileio.f90", "mod_constants.f90"], + "build_files": ["w3snl2md.F90", "mod_xnl4v5.f90", "serv_xnl4v5.f90", "mod_fileio.f90", "mod_constants.f90", "mod_mpicomm.f90"], "conflicts": ["OMPG", "OMPH"] }, { diff --git a/model/src/mpicomm.F90 b/model/src/mpicomm.F90 new file mode 100644 index 000000000..3132dece4 --- /dev/null +++ b/model/src/mpicomm.F90 @@ -0,0 +1,61 @@ +!> @file +!> @brief Defines MPI communicator id as constants for global use. +!> +!> @author H. L. Tolman @date 05-Jun-2018 +!> +#include "w3macros.h" + +!> +!> @brief Define some mpi constants for global use +!> +!> @author J. M. Sexton @date 19-Jul-2024 +!> +! +! +!/ ------------------------------------------------------------------- / +MODULE MPICOMM + !/ + !/ +-----------------------------------+ + !/ | WAVEWATCH III NOAA/NCEP | + !/ | H. L. Tolman | + !/ | FORTRAN 90 | + !/ | Last update : 19-Jul-2024 | + !/ +-----------------------------------+ + !/ + !/ 19-Jul-2024 : Add MPI SubCommunicator variable ( version 7.xx ) + !/ + !/ Copyright 2009-2024 National Weather Service (NWS), + !/ National Oceanic and Atmospheric Administration. All rights + !/ reserved. WAVEWATCH III is a trademark of the NWS. + !/ No unauthorized use without permission. + !/ + ! 1. Purpose : + ! + ! Define some mpi constants for global use + ! + ! 2. Variables and types : + ! + ! Name Type Scope Description + ! ---------------------------------------------------------------- + ! MPI_COMM_WW3 Int Global Value for mpi (sub)communicator for WW3 + ! IS_EXTERNAL_COMPONENT Bool Global General logical similar to IS_ESMF_COMPONENT + ! ---------------------------------------------------------------- + !/ ------------------------------------------------------------------- / + !/ + ! + + INTEGER :: MPI_COMM_WW3=0 !< MPI_COMM_WW3 + ! + ! Parameters in support of running as ESMF component + ! + ! --- Flag indicating whether or not the model has been invoked as an + ! external Component. This flag is set to true in the external + ! module during initialization. + LOGICAL :: IS_EXTERNAL_COMPONENT = .FALSE. !< IS_EXTERNAL_COMPONENT Flag for model invoked via external executable. + ! + CONTAINS + + !/ + !/ End of module MPICOMM ------------------------------------------- / + !/ +END MODULE MPICOMM diff --git a/model/src/w3gsrumd.F90 b/model/src/w3gsrumd.F90 index 9cd4beaf5..87a099d11 100644 --- a/model/src/w3gsrumd.F90 +++ b/model/src/w3gsrumd.F90 @@ -9631,7 +9631,7 @@ SUBROUTINE EXTCDE(IEXIT) LOGICAL :: RUN CALL MPI_INITIALIZED ( RUN, IERR_MPI ) IF ( RUN ) THEN - CALL MPI_ABORT ( MPI_COMM_WORLD, IEXIT, IERR_MPI ) + CALL MPI_ABORT ( MPI_COMM_WW3, IEXIT, IERR_MPI ) END IF #endif CALL EXIT(IEXIT) diff --git a/model/src/w3initmd.F90 b/model/src/w3initmd.F90 index 044a18760..f6e7676b5 100644 --- a/model/src/w3initmd.F90 +++ b/model/src/w3initmd.F90 @@ -410,6 +410,7 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD, UA, UD, U10, U10D, AS #ifdef W3_MPI USE W3ADATMD, ONLY: MPI_COMM_WAVE, MPI_COMM_WCMP + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif USE W3IDATMD, ONLY: FLLEV, FLCUR, FLWIND, FLICE, FLTAUA, FLRHOA,& FLMDN, FLMTH, FLMVS, FLIC1, FLIC2, FLIC3, & diff --git a/model/src/w3servmd.F90 b/model/src/w3servmd.F90 index 600e6e572..3ae73bcef 100644 --- a/model/src/w3servmd.F90 +++ b/model/src/w3servmd.F90 @@ -79,6 +79,10 @@ MODULE W3SERVMD !/ ------------------------------------------------------------------- / ! module default +#ifdef W3_MPI + USE MPICOMM, ONLY: MPI_COMM_WW3 +#endif + ! implicit none PUBLIC @@ -864,7 +868,7 @@ SUBROUTINE EXTCDE ( IEXIT, UNIT, MSG, FILE, LINE, COMM ) IF (PRESENT(COMM)) THEN WRITE(*,'(/A,I6/)') 'EXTCDE COMM=', COMM END IF - CALL MPI_ABORT ( MPI_COMM_WORLD, IEXIT, IERR_MPI ) + CALL MPI_ABORT ( MPI_COMM_WW3, IEXIT, IERR_MPI ) END IF END IF #endif diff --git a/model/src/w3strkmd.F90 b/model/src/w3strkmd.F90 index a776a36bb..d395419fc 100644 --- a/model/src/w3strkmd.F90 +++ b/model/src/w3strkmd.F90 @@ -28,6 +28,10 @@ MODULE W3STRKMD !/ No unauthorized use without permission. !/ use constants, only: file_endian +#ifdef W3_MPI + USE MPICOMM, ONLY: MPI_COMM_WW3 +#endif + ! IMPLICIT NONE ! ! 1. Purpose : @@ -470,8 +474,8 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & !/ ------------------------------------------------------------------- / #ifdef W3_MPI - CALL MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) - CALL MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierr) + CALL MPI_COMM_RANK(MPI_COMM_WW3, rank, ierr) + CALL MPI_COMM_SIZE(MPI_COMM_WW3, nproc, ierr) #endif NULLIFY( sysA ) NULLIFY( maxSys ) @@ -942,9 +946,9 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & #endif #ifdef W3_MPI - CALL MPI_BCAST(maxI,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxJ,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxTs,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(maxI,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxJ,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxTs,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) #endif ! Allocate the wsdat structure @@ -1177,13 +1181,13 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & wsdat(tsA)%lat(i,j), & wsdat(tsA)%lon(i,j)/) CALL MPI_SEND(COMMARR1,44,MPI_REAL,irank, & - (tag1+1),MPI_COMM_WORLD,IERR) + (tag1+1),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(6,*) '<< Receiving: rank,irank,tag1=', & ! rank,irank,(tag1+1) CALL MPI_RECV(COMMARR1,44,MPI_REAL,0,(tag1+1), & - MPI_COMM_WORLD,MPI_STATUS,IERR) + MPI_COMM_WW3,MPI_STATUS,IERR) wsdat(tsA)%par(i,j)%hs = COMMARR1(1:10) wsdat(tsA)%par(i,j)%tp = COMMARR1(11:20) wsdat(tsA)%par(i,j)%dir = COMMARR1(21:30) @@ -1197,12 +1201,12 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & IF (rank.EQ.0) THEN CALL MPI_SEND(wsdat(tsA)%date,1, & MPI_DOUBLE_PRECISION,irank, & - (tag1+2),MPI_COMM_WORLD,IERR) + (tag1+2),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.irank) THEN CALL MPI_RECV(wsdat(tsA)%date,1, & MPI_DOUBLE_PRECISION,0,(tag1+2), & - MPI_COMM_WORLD,MPI_STATUS,IERR) + MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.0) THEN @@ -1211,14 +1215,14 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & COMMARR2 = (/wsdat(tsA)%par(i,j)%ipart(:), & wsdat(tsA)%par(i,j)%checked/) CALL MPI_SEND(COMMARR2,11, & - MPI_INTEGER,irank,(tag1+3),MPI_COMM_WORLD,IERR) + MPI_INTEGER,irank,(tag1+3),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(6,*) '<< Receiving: rank,irank,tag1=', & ! rank,irank,(tag1+3) CALL MPI_RECV(COMMARR2,11, & MPI_INTEGER,0,(tag1+3), & - MPI_COMM_WORLD,MPI_STATUS,IERR) + MPI_COMM_WW3,MPI_STATUS,IERR) wsdat(tsA)%par(i,j)%ipart(:) = COMMARR2(1:10) wsdat(tsA)%par(i,j)%checked = COMMARR2(11) END IF @@ -1228,7 +1232,7 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & END IF END DO - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) #endif @@ -1416,7 +1420,7 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & #endif #ifdef W3_MPI - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) !! Define communicator for array of integers in structure "system" ! DOMSIZE = maxI*maxJ @@ -1447,13 +1451,13 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! Send results from current rank to rank 0 (blocking) ! WRITE(20,*) '>> Sending: rank,tsA,tag1=',rank,tsA,tag1 CALL MPI_SEND(maxSys(tsA),1,MPI_INTEGER,0,tag1, & - MPI_COMM_WORLD,IERR) + MPI_COMM_WW3,IERR) ! WRITE(20,*) 'Rank, IERR=',rank,IERR END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tsA,tag1=',rank,tsA,tag1 CALL MPI_RECV(maxSys(tsA),1,MPI_INTEGER, & - irank,tag1,MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,tag1,MPI_COMM_WW3,MPI_STATUS,IERR) ! Allocate structure at this time level ALLOCATE( sysA(tsA)%sys(maxSys(tsA)) ) DO ic = 1,maxSys(tsA) @@ -1501,14 +1505,14 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+1) CALL MPI_SEND(sysA(tsA)%sys(ic)%i(:),DOMSIZE, & - MPI_INTEGER,0,(tag2+1),MPI_COMM_WORLD,REQ(1),IERR) + MPI_INTEGER,0,(tag2+1),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+1) CALL MPI_RECV(sysA(tsA)%sys(ic)%i(:),DOMSIZE, & MPI_INTEGER,irank,(tag2+1), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) @@ -1516,92 +1520,92 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+2) CALL MPI_SEND(sysA(tsA)%sys(ic)%j(:),DOMSIZE, & - MPI_INTEGER,0,(tag2+2),MPI_COMM_WORLD,REQ(1),IERR) + MPI_INTEGER,0,(tag2+2),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+2) CALL MPI_RECV(sysA(tsA)%sys(ic)%j(:),DOMSIZE, & MPI_INTEGER,irank,(tag2+2), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+3) CALL MPI_SEND(sysA(tsA)%sys(ic)%lon(:),DOMSIZE, & - MPI_REAL,0,(tag2+3),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+3),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+3) CALL MPI_RECV(sysA(tsA)%sys(ic)%lon(:),DOMSIZE, & MPI_REAL,irank,(tag2+3), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+4) CALL MPI_SEND(sysA(tsA)%sys(ic)%lat(:),DOMSIZE, & - MPI_REAL,0,(tag2+4),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+4),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+4) CALL MPI_RECV(sysA(tsA)%sys(ic)%lat(:),DOMSIZE, & MPI_REAL,irank,(tag2+4), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+5) CALL MPI_SEND(sysA(tsA)%sys(ic)%hs(:),DOMSIZE, & - MPI_REAL,0,(tag2+5),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+5),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+5) CALL MPI_RECV(sysA(tsA)%sys(ic)%hs(:),DOMSIZE, & MPI_REAL,irank,(tag2+5), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+6) CALL MPI_SEND(sysA(tsA)%sys(ic)%tp(:),DOMSIZE, & - MPI_REAL,0,(tag2+6),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+6),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+6) CALL MPI_RECV(sysA(tsA)%sys(ic)%tp(:),DOMSIZE, & MPI_REAL,irank,(tag2+6), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+7) CALL MPI_SEND(sysA(tsA)%sys(ic)%dir(:),DOMSIZE, & - MPI_REAL,0,(tag2+7),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+7),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+7) CALL MPI_RECV(sysA(tsA)%sys(ic)%dir(:),DOMSIZE, & MPI_REAL,irank,(tag2+7), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+8) CALL MPI_SEND(sysA(tsA)%sys(ic)%dspr(:),DOMSIZE, & - MPI_REAL,0,(tag2+8),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+8),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+8) CALL MPI_RECV(sysA(tsA)%sys(ic)%dspr(:),DOMSIZE, & MPI_REAL,irank,(tag2+8), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) @@ -1609,85 +1613,85 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+9) CALL MPI_SEND(sysA(tsA)%sys(ic)%hsMean,1,MPI_REAL, & - 0,(tag2+9),MPI_COMM_WORLD,IERR) + 0,(tag2+9),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+9) CALL MPI_RECV(sysA(tsA)%sys(ic)%hsMean,1,MPI_REAL, & - irank,(tag2+9),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+9),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+10) CALL MPI_SEND(sysA(tsA)%sys(ic)%tpMean,1,MPI_REAL, & - 0,(tag2+10),MPI_COMM_WORLD,IERR) + 0,(tag2+10),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+10) CALL MPI_RECV(sysA(tsA)%sys(ic)%tpMean,1,MPI_REAL, & - irank,(tag2+10),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+10),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+11) CALL MPI_SEND(sysA(tsA)%sys(ic)%dirMean,1,MPI_REAL, & - 0,(tag2+11),MPI_COMM_WORLD,IERR) + 0,(tag2+11),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+11) CALL MPI_RECV(sysA(tsA)%sys(ic)%dirMean,1,MPI_REAL, & - irank,(tag2+11),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+11),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+12) CALL MPI_SEND(sysA(tsA)%sys(ic)%sysInd,1,MPI_INTEGER,& - 0,(tag2+12),MPI_COMM_WORLD,IERR) + 0,(tag2+12),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+12) CALL MPI_RECV(sysA(tsA)%sys(ic)%sysInd,1,MPI_INTEGER,& - irank,(tag2+12),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+12),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+13) CALL MPI_SEND(sysA(tsA)%sys(ic)%nPoints,1,MPI_INTEGER,& - 0,(tag2+13),MPI_COMM_WORLD,IERR) + 0,(tag2+13),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+13) CALL MPI_RECV(sysA(tsA)%sys(ic)%nPoints,1,MPI_INTEGER,& - irank,(tag2+13),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+13),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+14) CALL MPI_SEND(sysA(tsA)%sys(ic)%grp,1,MPI_INTEGER,& - 0,(tag2+14),MPI_COMM_WORLD,IERR) + 0,(tag2+14),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+14) CALL MPI_RECV(sysA(tsA)%sys(ic)%grp,1,MPI_INTEGER,& - irank,(tag2+14),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+14),MPI_COMM_WW3,MPI_STATUS,IERR) END IF END DO END IF END IF END DO - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) ! CALL MPI_TYPE_FREE(MPI_INT_DOMARR,IERR) ! CALL MPI_TYPE_FREE(MPI_REAL_DOMARR,IERR) diff --git a/model/src/ww3_multi.F90 b/model/src/ww3_multi.F90 index e3101b7c2..14e257aa6 100644 --- a/model/src/ww3_multi.F90 +++ b/model/src/ww3_multi.F90 @@ -95,6 +95,10 @@ PROGRAM W3MLTI USE OMP_LIB #endif !/ +#ifdef W3_MPI + USE MPICOMM, ONLY: MPI_COMM_WW3 +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -119,7 +123,7 @@ PROGRAM W3MLTI ! 0. Initialization necessary for driver ! 0.a General I/O: all can start with initialization in wmmdatmd ! - ! 0.b MPI environment: Here, we use MPI_COMM_WORLD + ! 0.b MPI environment: Here, we use MPI_COMM_WW3 ! #ifdef W3_OMPH FLHYBR = .TRUE. @@ -135,7 +139,9 @@ PROGRAM W3MLTI ENDIF #endif #ifdef W3_MPI - MPI_COMM = MPI_COMM_WORLD + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + + MPI_COMM = MPI_COMM_WW3 CALL MPI_COMM_SIZE ( MPI_COMM, NMPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM, IMPROC, IERR_MPI ) IMPROC = IMPROC + 1 diff --git a/model/src/ww3_ounp.F90 b/model/src/ww3_ounp.F90 index a1533c73b..bf1d55883 100644 --- a/model/src/ww3_ounp.F90 +++ b/model/src/ww3_ounp.F90 @@ -213,6 +213,10 @@ PROGRAM W3OUNP USE W3NMLOUNPMD USE NETCDF ! +#ifdef W3_MPI + USE MPICOMM, ONLY: MPI_COMM_WW3 +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -355,8 +359,9 @@ PROGRAM W3OUNP ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! diff --git a/model/src/ww3_prnc.F90 b/model/src/ww3_prnc.F90 index e107a1f43..78bb0175c 100644 --- a/model/src/ww3_prnc.F90 +++ b/model/src/ww3_prnc.F90 @@ -225,6 +225,10 @@ PROGRAM W3PRNC USE W3NMLPRNCMD USE NETCDF ! +#ifdef W3_MPI + USE MPICOMM, ONLY: MPI_COMM_WW3 +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -440,8 +444,9 @@ PROGRAM W3PRNC ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! @@ -1696,18 +1701,18 @@ PROGRAM W3PRNC IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(TIDE1DL, SLICE * TIDE_MF * NFIELDS * 2, MPI_REAL, & TIDE1D, NELEM * TIDE_MF * NFIELDS * 2, CUMUL * TIDE_MF * NFIELDS * 2, & - MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) #endif #ifdef W3_MPI IF (IAPROC.EQ.NAPOUT) THEN CALL MPI_GATHERV(MPI_IN_PLACE,NELEM(IAPROC), & MPI_LOGICAL, TIDALCOMP(:,IY), NELEM, CUMUL, MPI_LOGICAL, NAPOUT-1, & - MPI_COMM_WORLD, IERR_MPI) + MPI_COMM_WW3, IERR_MPI) ELSE CALL MPI_GATHERV(TIDALCOMP(CUMUL(IAPROC)+1:CUMUL(IAPROC)+NELEM(IAPROC),IY),NELEM(IAPROC), & MPI_LOGICAL, TIDALCOMP(:,IY), NELEM, CUMUL, MPI_LOGICAL, NAPOUT-1, & - MPI_COMM_WORLD, IERR_MPI) + MPI_COMM_WW3, IERR_MPI) END IF #endif diff --git a/model/src/ww3_prtide.F90 b/model/src/ww3_prtide.F90 index 1168cd1ab..dfea89471 100644 --- a/model/src/ww3_prtide.F90 +++ b/model/src/ww3_prtide.F90 @@ -139,6 +139,10 @@ PROGRAM W3PRTIDE USE W3TIDEMD USE W3IDATMD ! +#ifdef W3_MPI + USE MPICOMM, ONLY: MPI_COMM_WW3 +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -254,8 +258,9 @@ PROGRAM W3PRTIDE ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! @@ -673,11 +678,11 @@ PROGRAM W3PRTIDE #ifdef W3_MPI IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(FX1DL, SLICE, MPI_REAL, FX1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FY1DL, SLICE, MPI_REAL, FY1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FA1DL, SLICE, MPI_REAL, FA1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) ELSE FX1D = FX1DL FY1D = FY1DL @@ -768,11 +773,11 @@ PROGRAM W3PRTIDE #ifdef W3_MPI IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(FX1DL, SLICE, MPI_REAL, FX1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FY1DL, SLICE, MPI_REAL, FY1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FA1DL, SLICE, MPI_REAL, FA1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) ELSE FX1D = FX1DL FY1D = FY1DL diff --git a/model/src/ww3_sbs1.F90 b/model/src/ww3_sbs1.F90 index 94778b10d..1ca5571a9 100644 --- a/model/src/ww3_sbs1.F90 +++ b/model/src/ww3_sbs1.F90 @@ -177,6 +177,10 @@ PROGRAM W3SBS1 USE WMMDATMD, ONLY: MDSE, MDST, MDSS, NMPROC, IMPROC, NMPSCR, & NRGRD, STIME, ETIME !/ +#ifdef W3_MPI + USE MPICOMM, ONLY: MPI_COMM_WW3 +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -203,11 +207,11 @@ PROGRAM W3SBS1 ! 0. Initialization necessary for driver ! 0.a General I/O: all can start with initialization in wmmdatmd ! - ! 0.b MPI environment: Here, we use MPI_COMM_WORLD + ! 0.b MPI environment: Here, we use MPI_COMM_WW3 ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - MPI_COMM = MPI_COMM_WORLD + MPI_COMM = MPI_COMM_WW3 CALL MPI_COMM_SIZE ( MPI_COMM, NMPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM, IMPROC, IERR_MPI ) IMPROC = IMPROC + 1 diff --git a/model/src/ww3_shel.F90 b/model/src/ww3_shel.F90 index 4bb888b48..de14d2b82 100644 --- a/model/src/ww3_shel.F90 +++ b/model/src/ww3_shel.F90 @@ -307,6 +307,9 @@ PROGRAM W3SHEL #ifdef W3_OMPG USE OMP_LIB +#endif +#ifdef W3_MPI + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif IMPLICIT NONE ! @@ -487,7 +490,9 @@ PROGRAM W3SHEL #endif #ifdef W3_MPI - MPI_COMM = MPI_COMM_WORLD + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + + MPI_COMM = MPI_COMM_WW3 #endif #ifdef W3_OASIS END IF diff --git a/model/src/ww3_strt.F90 b/model/src/ww3_strt.F90 index b4bcad3ca..5e5b5ca68 100644 --- a/model/src/ww3_strt.F90 +++ b/model/src/ww3_strt.F90 @@ -241,6 +241,10 @@ PROGRAM W3STRT USE W3IDATMD, ONLY: W3NINP #endif !/ +#ifdef W3_MPI + USE MPICOMM, ONLY: MPI_COMM_WW3 +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -324,8 +328,9 @@ PROGRAM W3STRT ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 #endif ! diff --git a/model/src/ww3_systrk.F90 b/model/src/ww3_systrk.F90 index cb25a1e72..576977f3a 100644 --- a/model/src/ww3_systrk.F90 +++ b/model/src/ww3_systrk.F90 @@ -58,6 +58,10 @@ PROGRAM WW3_SYSTRK !/ USE W3STRKMD USE W3TIMEMD, ONLY: TDIFF +#ifdef W3_MPI + USE MPICOMM, ONLY: MPI_COMM_WW3 +#endif + ! IMPLICIT NONE #ifdef W3_MPI @@ -168,8 +172,10 @@ PROGRAM WW3_SYSTRK ! Start of parallel region CALL MPI_INIT(ierr) - CALL MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) - CALL MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierr) + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + + CALL MPI_COMM_RANK(MPI_COMM_WW3, rank, ierr) + CALL MPI_COMM_SIZE(MPI_COMM_WW3, nproc, ierr) #endif ! Open log file @@ -343,34 +349,34 @@ PROGRAM WW3_SYSTRK #ifdef W3_MPI ! MPI communication block - CALL MPI_BCAST(filename,80,MPI_CHARACTER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(filename,80,MPI_CHARACTER,0,MPI_COMM_WW3,IERR) CALL MPI_BCAST(tstart,1,MPI_DOUBLE_PRECISION,0, & - MPI_COMM_WORLD,IERR) + MPI_COMM_WW3,IERR) CALL MPI_BCAST(tend,1,MPI_DOUBLE_PRECISION,0, & - MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(dt,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(ntint,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(minlon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxlon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(minlat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxlat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(mxcwt,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(mycwt,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(dirKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(perKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(hsKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(wetPts,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(dirTimeKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(tpTimeKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(seedLon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(seedLat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(noutp,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(lonout,100,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(latout,100,MPI_REAL,0,MPI_COMM_WORLD,IERR) + MPI_COMM_WW3,IERR) + CALL MPI_BCAST(dt,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(ntint,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(minlon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxlon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(minlat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxlat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(mxcwt,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(mycwt,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(dirKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(perKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(hsKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(wetPts,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(dirTimeKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(tpTimeKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(seedLon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(seedLat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(noutp,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(lonout,100,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(latout,100,MPI_REAL,0,MPI_COMM_WW3,IERR) #endif #ifdef W3_MPI - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) #endif CALL waveTracking_NWS_V2 (intype ,tmax , &