From 1ef96e925470e5bbb041779abbe84dec291fd1aa Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Mon, 24 Jun 2024 10:00:10 -0700 Subject: [PATCH 01/16] Make a module mpicomm to store MPI_COMM_WW3 --- model/bin/build_utils.sh | 48 ++++++++--------- model/src/PDLIB/yowdatapool.F90 | 6 ++- model/src/cmake/src_list.cmake | 1 + model/src/cmake/switches.json | 2 +- model/src/mpicomm.F90 | 73 ++++++++++++++++++++++++++ model/src/w3gsrumd.F90 | 2 +- model/src/w3initmd.F90 | 3 ++ model/src/w3servmd.F90 | 6 ++- model/src/w3strkmd.F90 | 92 +++++++++++++++++---------------- model/src/ww3_multi.F90 | 8 ++- model/src/ww3_ounp.F90 | 8 ++- model/src/ww3_prnc.F90 | 14 +++-- model/src/ww3_prtide.F90 | 20 ++++--- model/src/ww3_sbs1.F90 | 8 ++- model/src/ww3_shel.F90 | 5 +- model/src/ww3_strt.F90 | 8 ++- model/src/ww3_systrk.F90 | 54 ++++++++++--------- 17 files changed, 239 insertions(+), 119 deletions(-) create mode 100644 model/src/mpicomm.F90 diff --git a/model/bin/build_utils.sh b/model/bin/build_utils.sh index fe65b6d61..9ba0c087b 100755 --- a/model/bin/build_utils.sh +++ b/model/bin/build_utils.sh @@ -802,7 +802,7 @@ create_file_list() prop= sourcet="w3parall w3triamd w3gridmd $stx $flx $nlx $btx $is $uostmd" IO='w3iogrmd' - aux="constants w3servmd w3arrymd w3dispmd w3gsrumd w3timemd w3nmlgridmd $pdlibyow $memcode" + aux="constants mpicomm w3servmd w3arrymd w3dispmd w3gsrumd w3timemd w3nmlgridmd $pdlibyow $memcode" if [ "$scrip" = 'SCRIP' ] then aux="$aux scrip_constants scrip_grids scrip_iounitsmod" @@ -819,42 +819,42 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $tr $trx $bt $setupcode $stx $flx $nlx $btx $is wmmdatmd w3parall $uostmd" IO='w3iogrmd w3iorsmd' - aux="constants w3triamd w3servmd w3arrymd w3dispmd w3gsrumd w3timemd" ;; + aux="constants mpicomm w3triamd w3servmd w3arrymd w3dispmd w3gsrumd w3timemd" ;; ww3_bound) core= data="w3adatmd $memcode w3gdatmd w3wdatmd w3idatmd w3odatmd" prop="$smcm" sourcet="$pdlibcode $pdlibyow $db $bt $setupcode $tr $trx $stx $nlx $btx $is wmmdatmd w3parall w3triamd $uostmd" IO='w3iobcmd w3iogrmd w3dispmd w3gsrumd' - aux="constants w3servmd w3timemd w3arrymd w3cspcmd w3nmlboundmd" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3cspcmd w3nmlboundmd" ;; ww3_bounc) core= data="w3adatmd $memcode w3gdatmd w3wdatmd w3idatmd w3odatmd" prop="$smcm" sourcet="$pdlibcode $pdlibyow $db $bt $setupcode $stx $nlx $btx $is wmmdatmd w3parall w3triamd $uostmd" IO='w3iobcmd w3iogrmd w3dispmd w3gsrumd' - aux="constants w3servmd w3arrymd w3timemd w3cspcmd w3nmlbouncmd" ;; + aux="constants mpicomm w3servmd w3arrymd w3timemd w3cspcmd w3nmlbouncmd" ;; ww3_prep) core='w3fldsmd' data="$memcode w3gdatmd w3adatmd w3idatmd w3odatmd w3wdatmd wmmdatmd" prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode w3triamd $stx $flx $nlx $btx $is $uostmd" IO="w3iogrmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants w3servmd w3timemd $tidecode w3arrymd w3dispmd w3gsrumd w3parall" ;; + aux="constants mpicomm w3servmd w3timemd $tidecode w3arrymd w3dispmd w3gsrumd w3parall" ;; ww3_prnc) core='w3fldsmd' data="$memcode w3gdatmd w3adatmd w3idatmd w3odatmd w3wdatmd wmmdatmd" prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode w3triamd $stx $flx $nlx $btx $is w3parall $uostmd" IO="w3iogrmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd w3tidemd w3nmlprncmd" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd w3tidemd w3nmlprncmd" ;; ww3_prtide) core='w3fldsmd' data="wmmdatmd $memcode w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd" prop="$pr $smcm" sourcet="$pdlibcode $pdlibyow $db $bt $setupcode w3triamd $stx $nlx $btx $is w3parall $uostmd" IO="w3iogrmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd $tidecode" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd $tidecode" ;; ww3_shel) core='w3fldsmd w3initmd w3wavemd w3wdasmd w3updtmd' data="wmmdatmd $memcode w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd" @@ -863,7 +863,7 @@ create_file_list() sourcet="$sourcet $is $db $tr $bs $refcode $igcode w3parall $uostmd" IO="w3iogrmd w3iogomd w3iopomd w3iotrmd w3iorsmd w3iobcmd $oasismd $agcmmd $ogcmmd $igcmmd" IO="$IO w3iosfmd w3partmd" - aux="constants w3servmd w3timemd $tidecode w3arrymd w3dispmd w3cspcmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd $tidecode w3arrymd w3dispmd w3cspcmd w3gsrumd" aux="$aux w3nmlshelmd $pdlibyow" ;; ww3_multi|ww3_multi_esmf) if [ "$prog" = "ww3_multi" ] @@ -879,7 +879,7 @@ create_file_list() sourcet="$pdlibcode $pdlibyow $setupcode w3parall w3triamd w3srcemd $dsx $flx $ln $st $nl $bt $ic $is $db $tr $bs $refcode $igcode $uostmd" IO='w3iogrmd w3iogomd w3iopomd wmiopomd' IO="$IO w3iotrmd w3iorsmd w3iobcmd w3iosfmd w3partmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants $tidecode w3servmd w3timemd w3arrymd w3dispmd w3cspcmd w3gsrumd $mprfaux" + aux="constants mpicomm $tidecode w3servmd w3timemd w3arrymd w3dispmd w3cspcmd w3gsrumd $mprfaux" aux="$aux wmunitmd w3nmlmultimd" if [ "$scrip" = 'SCRIP' ] then @@ -899,7 +899,7 @@ create_file_list() sourcet="$pdlibcode $pdlibyow w3triamd w3srcemd $dsx $flx $ln $st $nl $bt $db $tr $bs $refcode $igcode $is $ic $uostmd" IO='w3iogrmd w3iogomd w3iopomd wmiopomd' IO="$IO w3iotrmd w3iorsmd w3iobcmd w3iosfmd w3partmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3cspcmd w3gsrumd $mprfaux $tidecode" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3cspcmd w3gsrumd $mprfaux $tidecode" aux="$aux wmunitmd w3nmlmultimd" if [ "$scrip" = 'SCRIP' ] then @@ -917,7 +917,7 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode $tr $trx $stx $flx $nlx $btx $is $uostmd" IO='w3iogrmd w3iogomd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux" ;; ww3_ounf) core='w3initmd' @@ -925,7 +925,7 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode w3parall w3triamd $stx $flx $nlx $btx $is $uostmd" IO='w3iogrmd w3iogomd w3iorsmd w3iopomd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux w3nmlounfmd $smco w3ounfmetamd w3metamd" ;; ww3_outp) core= @@ -933,14 +933,14 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $setupcode $flx $ln $st $nl $bt $ic $is $db $tr $bs $igcode $uostmd" IO='w3bullmd w3iogrmd w3iopomd w3partmd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" ;; ww3_ounp) core= data="wmmdatmd w3parall w3triamd $memcode w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd" prop= sourcet="$pdlibcode $pdlibyow $setupcode $flx $ln $st $nl $bt $ic $is $db $tr $bs $igcode $uostmd" IO='w3bullmd w3iogrmd w3iopomd w3partmd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux w3nmlounpmd" ;; ww3_trck) core= @@ -948,21 +948,21 @@ create_file_list() prop= sourcet= IO= - aux="constants w3servmd w3timemd w3gsrumd" ;; + aux="constants mpicomm w3servmd w3timemd w3gsrumd" ;; ww3_trnc) core= data="$memcode w3gdatmd w3odatmd" prop= sourcet= IO= - aux="constants w3servmd w3timemd w3gsrumd w3nmltrncmd" ;; + aux="constants mpicomm w3servmd w3timemd w3gsrumd w3nmltrncmd" ;; ww3_grib) core= data="w3parall wmmdatmd w3triamd $memcode w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd" prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode $stx $flx $nlx $btx $is $uostmd" IO='w3iogrmd w3iogomd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux" ;; ww3_gspl) core='w3fldsmd' @@ -970,14 +970,14 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode wmmdatmd w3parall w3triamd $stx $flx $nlx $btx $is $uostmd" IO="w3iogrmd $oasismd $agcmmd $ogcmmd $igcmmd" - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd $tidecode" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd $tidecode" ;; ww3_gint) core= data="w3parall wmmdatmd $memcode w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd" IO='w3iogrmd w3iogomd' prop= sourcet="$pdlibcode $pdlibyow $db $bt $st $nl $is $uostmd" - aux="constants w3triamd w3servmd w3arrymd w3dispmd w3timemd w3gsrumd" + aux="constants mpicomm w3triamd w3servmd w3arrymd w3dispmd w3timemd w3gsrumd" aux="$aux" ;; gx_outf) core= @@ -985,7 +985,7 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode wmmdatmd w3parall w3triamd $stx $flx $nlx $btx $tr $bs $is $uostmd" IO='w3iogrmd w3iogomd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux" ;; gx_outp) core= @@ -993,28 +993,28 @@ create_file_list() prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode wmmdatmd w3parall w3triamd $ln $flx $st $nlx $btx $tr $bs $is $ic $uostmd" IO='w3iogrmd w3iopomd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" ;; ww3_systrk) core='w3strkmd' data="$memcode w3gdatmd w3adatmd w3idatmd w3odatmd w3wdatmd" prop= sourcet="$pdlibcode $pdlibyow $db $bt $setupcode wmmdatmd w3dispmd w3triamd $ln $stx $flx $nlx $btx $tr $bs $is $uostmd" IO= - aux="constants w3servmd w3timemd w3arrymd w3gsrumd w3parall" ;; + aux="constants mpicomm w3servmd w3timemd w3arrymd w3gsrumd w3parall" ;; libww3|libww3.so) core='w3fldsmd w3initmd w3wavemd w3wdasmd w3updtmd' data='wmmdatmd w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd' prop="$pr $smcm" sourcet="w3triamd w3srcemd $dsx $flx $ln $st $nl $bt $ic $is $db $tr $bs $refcode $igcode $uostmd" IO='w3iogrmd w3iogomd w3iopomd w3iotrmd w3iorsmd w3iobcmd w3iosfmd w3partmd' - aux="constants w3servmd w3timemd $tidecode w3arrymd w3dispmd w3cspcmd w3gsrumd" ;; + aux="constants mpicomm w3servmd w3timemd $tidecode w3arrymd w3dispmd w3cspcmd w3gsrumd" ;; ww3_uprstr) core= data='wmmdatmd w3triamd w3gdatmd w3wdatmd w3adatmd w3idatmd w3odatmd' prop= sourcet="$memcode $pdlibcode $pdlibyow $flx $ln $st $nl $bt $ic $is $db $tr $bs $uostmd" IO='w3iogrmd w3iogomd w3iorsmd' - aux="constants w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" + aux="constants mpicomm w3servmd w3timemd w3arrymd w3dispmd w3gsrumd" aux="$aux w3parall w3nmluprstrmd" ;; esac diff --git a/model/src/PDLIB/yowdatapool.F90 b/model/src/PDLIB/yowdatapool.F90 index 24ae18162..89dd0830d 100644 --- a/model/src/PDLIB/yowdatapool.F90 +++ b/model/src/PDLIB/yowdatapool.F90 @@ -37,7 +37,11 @@ ! !> Has fancy data module yowDatapool - use MPI, only: MPI_COMM_WORLD, MPI_INTEGER, MPI_REAL4, MPI_REAL8, MPI_STATUS_SIZE + use MPI, only: MPI_INTEGER, MPI_REAL4, MPI_REAL8, MPI_STATUS_SIZE +#ifdef W3_MPI + USE MPICOMM, only: MPI_COMM_WW3 => MPI_COMM_WORLD +#endif + ! implicit none !#ifdef USE_SINGLE ! !> single precision. Enable with compiler flag -DUSE_SINGLE diff --git a/model/src/cmake/src_list.cmake b/model/src/cmake/src_list.cmake index d745be388..6bcb914c4 100644 --- a/model/src/cmake/src_list.cmake +++ b/model/src/cmake/src_list.cmake @@ -3,6 +3,7 @@ set(c_src w3getmem.c) # Core files always built set(ftn_src constants.F90 + mpicomm.F90 w3adatmd.F90 w3arrymd.F90 w3bullmd.F90 diff --git a/model/src/cmake/switches.json b/model/src/cmake/switches.json index 30eca480c..6664a9ec3 100644 --- a/model/src/cmake/switches.json +++ b/model/src/cmake/switches.json @@ -282,7 +282,7 @@ }, { "name": "NL2", - "build_files": ["w3snl2md.F90", "mod_xnl4v5.f90", "serv_xnl4v5.f90", "mod_fileio.f90", "mod_constants.f90"], + "build_files": ["w3snl2md.F90", "mod_xnl4v5.f90", "serv_xnl4v5.f90", "mod_fileio.f90", "mod_constants.f90", "mod_mpicomm.f90"], "conflicts": ["OMPG", "OMPH"] }, { diff --git a/model/src/mpicomm.F90 b/model/src/mpicomm.F90 new file mode 100644 index 000000000..2c2ae7562 --- /dev/null +++ b/model/src/mpicomm.F90 @@ -0,0 +1,73 @@ +!> @file +!> @brief Defines MPI communicator id as constants for global use. +!> +!> @author H. L. Tolman @date 05-Jun-2018 +!> +#include "w3macros.h" + +!> +!> @brief Define some mpi constants for global use +!> +!> @author H. L. Tolman @date 05-Jun-2018 +!> +! +#ifndef ENDIANNESS +#define ENDIANNESS "native" +#endif +! +!/ ------------------------------------------------------------------- / +MODULE MPICOMM + !/ + !/ +-----------------------------------+ + !/ | WAVEWATCH III NOAA/NCEP | + !/ | H. L. Tolman | + !/ | FORTRAN 90 | + !/ | Last update : 05-Jun-2018 | + !/ +-----------------------------------+ + !/ + !/ 11-Nov-1999 : Fortran 90 version. ( version 2.00 ) + !/ 29-May-2009 : Preparing distribution version. ( version 3.14 ) + !/ 25-Jun-2011 : Adding Kelvin functions. ( version 4.05 ) + !/ 03-Sep-2012 : Adding TSTOUT flag. ( version 4.10 ) + !/ 28-Feb-2013 : Adding cap at 0.5 in FWTABLE ( version 4.08 ) + !/ 20-Jan-2017 : Add parameters for ESMF ( version 6.02 ) + !/ 01-Mar-2018 : Add UNDEF parameter ( version 6.02 ) + !/ 05-Jun-2018 : Add PDLIB parameters ( version 6.04 ) + !/ + !/ Copyright 2009-2012 National Weather Service (NWS), + !/ National Oceanic and Atmospheric Administration. All rights + !/ reserved. WAVEWATCH III is a trademark of the NWS. + !/ No unauthorized use without permission. + !/ + ! 1. Purpose : + ! + ! Define some mpi constants for global use + ! + ! 2. Variables and types : + ! + ! Name Type Scope Description + ! ---------------------------------------------------------------- + ! UNDEF Real Global Value for undefined variable in output + ! ---------------------------------------------------------------- + !/ ------------------------------------------------------------------- / + !/ + ! +!#ifdef W3_MPI +! INCLUDE "mpif.h" +!#endif + + INTEGER :: MPI_COMM_WW3=0 !< MPI_COMM_WW3 + ! + ! Parameters in support of running as ESMF component + ! + ! --- Flag indicating whether or not the model has been invoked as an + ! external Component. This flag is set to true in the external + ! module during initialization. + LOGICAL :: IS_EXTERNAL_COMPONENT = .FALSE. !< IS_EXTERNAL_COMPONENT Flag for model invoked via external executable. + ! + CONTAINS + + !/ + !/ End of module MPICOMM ------------------------------------------- / + !/ +END MODULE MPICOMM diff --git a/model/src/w3gsrumd.F90 b/model/src/w3gsrumd.F90 index 9cd4beaf5..87a099d11 100644 --- a/model/src/w3gsrumd.F90 +++ b/model/src/w3gsrumd.F90 @@ -9631,7 +9631,7 @@ SUBROUTINE EXTCDE(IEXIT) LOGICAL :: RUN CALL MPI_INITIALIZED ( RUN, IERR_MPI ) IF ( RUN ) THEN - CALL MPI_ABORT ( MPI_COMM_WORLD, IEXIT, IERR_MPI ) + CALL MPI_ABORT ( MPI_COMM_WW3, IEXIT, IERR_MPI ) END IF #endif CALL EXIT(IEXIT) diff --git a/model/src/w3initmd.F90 b/model/src/w3initmd.F90 index 044a18760..b906dd18f 100644 --- a/model/src/w3initmd.F90 +++ b/model/src/w3initmd.F90 @@ -410,6 +410,7 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD, UA, UD, U10, U10D, AS #ifdef W3_MPI USE W3ADATMD, ONLY: MPI_COMM_WAVE, MPI_COMM_WCMP + USE MPICOMM #endif USE W3IDATMD, ONLY: FLLEV, FLCUR, FLWIND, FLICE, FLTAUA, FLRHOA,& FLMDN, FLMTH, FLMVS, FLIC1, FLIC2, FLIC3, & @@ -562,6 +563,8 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD, #endif ! #ifdef W3_MPI + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + MPI_COMM_WAVE = MPI_COMM CALL MPI_COMM_SIZE ( MPI_COMM_WAVE, NTPROC, IERR_MPI ) NAPROC = NTPROC diff --git a/model/src/w3servmd.F90 b/model/src/w3servmd.F90 index 600e6e572..70dcc061f 100644 --- a/model/src/w3servmd.F90 +++ b/model/src/w3servmd.F90 @@ -79,6 +79,10 @@ MODULE W3SERVMD !/ ------------------------------------------------------------------- / ! module default +#ifdef W3_MPI + USE MPICOMM +#endif + ! implicit none PUBLIC @@ -864,7 +868,7 @@ SUBROUTINE EXTCDE ( IEXIT, UNIT, MSG, FILE, LINE, COMM ) IF (PRESENT(COMM)) THEN WRITE(*,'(/A,I6/)') 'EXTCDE COMM=', COMM END IF - CALL MPI_ABORT ( MPI_COMM_WORLD, IEXIT, IERR_MPI ) + CALL MPI_ABORT ( MPI_COMM_WW3, IEXIT, IERR_MPI ) END IF END IF #endif diff --git a/model/src/w3strkmd.F90 b/model/src/w3strkmd.F90 index a776a36bb..212e14ba5 100644 --- a/model/src/w3strkmd.F90 +++ b/model/src/w3strkmd.F90 @@ -28,6 +28,10 @@ MODULE W3STRKMD !/ No unauthorized use without permission. !/ use constants, only: file_endian +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! ! 1. Purpose : @@ -470,8 +474,8 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & !/ ------------------------------------------------------------------- / #ifdef W3_MPI - CALL MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) - CALL MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierr) + CALL MPI_COMM_RANK(MPI_COMM_WW3, rank, ierr) + CALL MPI_COMM_SIZE(MPI_COMM_WW3, nproc, ierr) #endif NULLIFY( sysA ) NULLIFY( maxSys ) @@ -942,9 +946,9 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & #endif #ifdef W3_MPI - CALL MPI_BCAST(maxI,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxJ,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxTs,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(maxI,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxJ,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxTs,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) #endif ! Allocate the wsdat structure @@ -1177,13 +1181,13 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & wsdat(tsA)%lat(i,j), & wsdat(tsA)%lon(i,j)/) CALL MPI_SEND(COMMARR1,44,MPI_REAL,irank, & - (tag1+1),MPI_COMM_WORLD,IERR) + (tag1+1),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(6,*) '<< Receiving: rank,irank,tag1=', & ! rank,irank,(tag1+1) CALL MPI_RECV(COMMARR1,44,MPI_REAL,0,(tag1+1), & - MPI_COMM_WORLD,MPI_STATUS,IERR) + MPI_COMM_WW3,MPI_STATUS,IERR) wsdat(tsA)%par(i,j)%hs = COMMARR1(1:10) wsdat(tsA)%par(i,j)%tp = COMMARR1(11:20) wsdat(tsA)%par(i,j)%dir = COMMARR1(21:30) @@ -1197,12 +1201,12 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & IF (rank.EQ.0) THEN CALL MPI_SEND(wsdat(tsA)%date,1, & MPI_DOUBLE_PRECISION,irank, & - (tag1+2),MPI_COMM_WORLD,IERR) + (tag1+2),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.irank) THEN CALL MPI_RECV(wsdat(tsA)%date,1, & MPI_DOUBLE_PRECISION,0,(tag1+2), & - MPI_COMM_WORLD,MPI_STATUS,IERR) + MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.0) THEN @@ -1211,14 +1215,14 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & COMMARR2 = (/wsdat(tsA)%par(i,j)%ipart(:), & wsdat(tsA)%par(i,j)%checked/) CALL MPI_SEND(COMMARR2,11, & - MPI_INTEGER,irank,(tag1+3),MPI_COMM_WORLD,IERR) + MPI_INTEGER,irank,(tag1+3),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(6,*) '<< Receiving: rank,irank,tag1=', & ! rank,irank,(tag1+3) CALL MPI_RECV(COMMARR2,11, & MPI_INTEGER,0,(tag1+3), & - MPI_COMM_WORLD,MPI_STATUS,IERR) + MPI_COMM_WW3,MPI_STATUS,IERR) wsdat(tsA)%par(i,j)%ipart(:) = COMMARR2(1:10) wsdat(tsA)%par(i,j)%checked = COMMARR2(11) END IF @@ -1228,7 +1232,7 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & END IF END DO - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) #endif @@ -1416,7 +1420,7 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & #endif #ifdef W3_MPI - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) !! Define communicator for array of integers in structure "system" ! DOMSIZE = maxI*maxJ @@ -1447,13 +1451,13 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! Send results from current rank to rank 0 (blocking) ! WRITE(20,*) '>> Sending: rank,tsA,tag1=',rank,tsA,tag1 CALL MPI_SEND(maxSys(tsA),1,MPI_INTEGER,0,tag1, & - MPI_COMM_WORLD,IERR) + MPI_COMM_WW3,IERR) ! WRITE(20,*) 'Rank, IERR=',rank,IERR END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tsA,tag1=',rank,tsA,tag1 CALL MPI_RECV(maxSys(tsA),1,MPI_INTEGER, & - irank,tag1,MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,tag1,MPI_COMM_WW3,MPI_STATUS,IERR) ! Allocate structure at this time level ALLOCATE( sysA(tsA)%sys(maxSys(tsA)) ) DO ic = 1,maxSys(tsA) @@ -1501,14 +1505,14 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+1) CALL MPI_SEND(sysA(tsA)%sys(ic)%i(:),DOMSIZE, & - MPI_INTEGER,0,(tag2+1),MPI_COMM_WORLD,REQ(1),IERR) + MPI_INTEGER,0,(tag2+1),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+1) CALL MPI_RECV(sysA(tsA)%sys(ic)%i(:),DOMSIZE, & MPI_INTEGER,irank,(tag2+1), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) @@ -1516,92 +1520,92 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+2) CALL MPI_SEND(sysA(tsA)%sys(ic)%j(:),DOMSIZE, & - MPI_INTEGER,0,(tag2+2),MPI_COMM_WORLD,REQ(1),IERR) + MPI_INTEGER,0,(tag2+2),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+2) CALL MPI_RECV(sysA(tsA)%sys(ic)%j(:),DOMSIZE, & MPI_INTEGER,irank,(tag2+2), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+3) CALL MPI_SEND(sysA(tsA)%sys(ic)%lon(:),DOMSIZE, & - MPI_REAL,0,(tag2+3),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+3),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+3) CALL MPI_RECV(sysA(tsA)%sys(ic)%lon(:),DOMSIZE, & MPI_REAL,irank,(tag2+3), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+4) CALL MPI_SEND(sysA(tsA)%sys(ic)%lat(:),DOMSIZE, & - MPI_REAL,0,(tag2+4),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+4),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+4) CALL MPI_RECV(sysA(tsA)%sys(ic)%lat(:),DOMSIZE, & MPI_REAL,irank,(tag2+4), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+5) CALL MPI_SEND(sysA(tsA)%sys(ic)%hs(:),DOMSIZE, & - MPI_REAL,0,(tag2+5),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+5),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+5) CALL MPI_RECV(sysA(tsA)%sys(ic)%hs(:),DOMSIZE, & MPI_REAL,irank,(tag2+5), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+6) CALL MPI_SEND(sysA(tsA)%sys(ic)%tp(:),DOMSIZE, & - MPI_REAL,0,(tag2+6),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+6),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+6) CALL MPI_RECV(sysA(tsA)%sys(ic)%tp(:),DOMSIZE, & MPI_REAL,irank,(tag2+6), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+7) CALL MPI_SEND(sysA(tsA)%sys(ic)%dir(:),DOMSIZE, & - MPI_REAL,0,(tag2+7),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+7),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+7) CALL MPI_RECV(sysA(tsA)%sys(ic)%dir(:),DOMSIZE, & MPI_REAL,irank,(tag2+7), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+8) CALL MPI_SEND(sysA(tsA)%sys(ic)%dspr(:),DOMSIZE, & - MPI_REAL,0,(tag2+8),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+8),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+8) CALL MPI_RECV(sysA(tsA)%sys(ic)%dspr(:),DOMSIZE, & MPI_REAL,irank,(tag2+8), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) @@ -1609,85 +1613,85 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+9) CALL MPI_SEND(sysA(tsA)%sys(ic)%hsMean,1,MPI_REAL, & - 0,(tag2+9),MPI_COMM_WORLD,IERR) + 0,(tag2+9),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+9) CALL MPI_RECV(sysA(tsA)%sys(ic)%hsMean,1,MPI_REAL, & - irank,(tag2+9),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+9),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+10) CALL MPI_SEND(sysA(tsA)%sys(ic)%tpMean,1,MPI_REAL, & - 0,(tag2+10),MPI_COMM_WORLD,IERR) + 0,(tag2+10),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+10) CALL MPI_RECV(sysA(tsA)%sys(ic)%tpMean,1,MPI_REAL, & - irank,(tag2+10),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+10),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+11) CALL MPI_SEND(sysA(tsA)%sys(ic)%dirMean,1,MPI_REAL, & - 0,(tag2+11),MPI_COMM_WORLD,IERR) + 0,(tag2+11),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+11) CALL MPI_RECV(sysA(tsA)%sys(ic)%dirMean,1,MPI_REAL, & - irank,(tag2+11),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+11),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+12) CALL MPI_SEND(sysA(tsA)%sys(ic)%sysInd,1,MPI_INTEGER,& - 0,(tag2+12),MPI_COMM_WORLD,IERR) + 0,(tag2+12),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+12) CALL MPI_RECV(sysA(tsA)%sys(ic)%sysInd,1,MPI_INTEGER,& - irank,(tag2+12),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+12),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+13) CALL MPI_SEND(sysA(tsA)%sys(ic)%nPoints,1,MPI_INTEGER,& - 0,(tag2+13),MPI_COMM_WORLD,IERR) + 0,(tag2+13),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+13) CALL MPI_RECV(sysA(tsA)%sys(ic)%nPoints,1,MPI_INTEGER,& - irank,(tag2+13),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+13),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+14) CALL MPI_SEND(sysA(tsA)%sys(ic)%grp,1,MPI_INTEGER,& - 0,(tag2+14),MPI_COMM_WORLD,IERR) + 0,(tag2+14),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+14) CALL MPI_RECV(sysA(tsA)%sys(ic)%grp,1,MPI_INTEGER,& - irank,(tag2+14),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+14),MPI_COMM_WW3,MPI_STATUS,IERR) END IF END DO END IF END IF END DO - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) ! CALL MPI_TYPE_FREE(MPI_INT_DOMARR,IERR) ! CALL MPI_TYPE_FREE(MPI_REAL_DOMARR,IERR) diff --git a/model/src/ww3_multi.F90 b/model/src/ww3_multi.F90 index e3101b7c2..f1c55169b 100644 --- a/model/src/ww3_multi.F90 +++ b/model/src/ww3_multi.F90 @@ -95,6 +95,10 @@ PROGRAM W3MLTI USE OMP_LIB #endif !/ +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -119,7 +123,7 @@ PROGRAM W3MLTI ! 0. Initialization necessary for driver ! 0.a General I/O: all can start with initialization in wmmdatmd ! - ! 0.b MPI environment: Here, we use MPI_COMM_WORLD + ! 0.b MPI environment: Here, we use MPI_COMM_WW3 ! #ifdef W3_OMPH FLHYBR = .TRUE. @@ -135,7 +139,7 @@ PROGRAM W3MLTI ENDIF #endif #ifdef W3_MPI - MPI_COMM = MPI_COMM_WORLD + MPI_COMM = MPI_COMM_WW3 CALL MPI_COMM_SIZE ( MPI_COMM, NMPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM, IMPROC, IERR_MPI ) IMPROC = IMPROC + 1 diff --git a/model/src/ww3_ounp.F90 b/model/src/ww3_ounp.F90 index c35ff6e98..9678768f2 100644 --- a/model/src/ww3_ounp.F90 +++ b/model/src/ww3_ounp.F90 @@ -213,6 +213,10 @@ PROGRAM W3OUNP USE W3NMLOUNPMD USE NETCDF ! +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -355,8 +359,8 @@ PROGRAM W3OUNP ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! diff --git a/model/src/ww3_prnc.F90 b/model/src/ww3_prnc.F90 index e107a1f43..ee347659e 100644 --- a/model/src/ww3_prnc.F90 +++ b/model/src/ww3_prnc.F90 @@ -225,6 +225,10 @@ PROGRAM W3PRNC USE W3NMLPRNCMD USE NETCDF ! +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -440,8 +444,8 @@ PROGRAM W3PRNC ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! @@ -1696,18 +1700,18 @@ PROGRAM W3PRNC IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(TIDE1DL, SLICE * TIDE_MF * NFIELDS * 2, MPI_REAL, & TIDE1D, NELEM * TIDE_MF * NFIELDS * 2, CUMUL * TIDE_MF * NFIELDS * 2, & - MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) #endif #ifdef W3_MPI IF (IAPROC.EQ.NAPOUT) THEN CALL MPI_GATHERV(MPI_IN_PLACE,NELEM(IAPROC), & MPI_LOGICAL, TIDALCOMP(:,IY), NELEM, CUMUL, MPI_LOGICAL, NAPOUT-1, & - MPI_COMM_WORLD, IERR_MPI) + MPI_COMM_WW3, IERR_MPI) ELSE CALL MPI_GATHERV(TIDALCOMP(CUMUL(IAPROC)+1:CUMUL(IAPROC)+NELEM(IAPROC),IY),NELEM(IAPROC), & MPI_LOGICAL, TIDALCOMP(:,IY), NELEM, CUMUL, MPI_LOGICAL, NAPOUT-1, & - MPI_COMM_WORLD, IERR_MPI) + MPI_COMM_WW3, IERR_MPI) END IF #endif diff --git a/model/src/ww3_prtide.F90 b/model/src/ww3_prtide.F90 index 1168cd1ab..0da03424a 100644 --- a/model/src/ww3_prtide.F90 +++ b/model/src/ww3_prtide.F90 @@ -139,6 +139,10 @@ PROGRAM W3PRTIDE USE W3TIDEMD USE W3IDATMD ! +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -254,8 +258,8 @@ PROGRAM W3PRTIDE ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! @@ -673,11 +677,11 @@ PROGRAM W3PRTIDE #ifdef W3_MPI IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(FX1DL, SLICE, MPI_REAL, FX1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FY1DL, SLICE, MPI_REAL, FY1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FA1DL, SLICE, MPI_REAL, FA1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) ELSE FX1D = FX1DL FY1D = FY1DL @@ -768,11 +772,11 @@ PROGRAM W3PRTIDE #ifdef W3_MPI IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(FX1DL, SLICE, MPI_REAL, FX1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FY1DL, SLICE, MPI_REAL, FY1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FA1DL, SLICE, MPI_REAL, FA1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) ELSE FX1D = FX1DL FY1D = FY1DL diff --git a/model/src/ww3_sbs1.F90 b/model/src/ww3_sbs1.F90 index 94778b10d..8da309c62 100644 --- a/model/src/ww3_sbs1.F90 +++ b/model/src/ww3_sbs1.F90 @@ -177,6 +177,10 @@ PROGRAM W3SBS1 USE WMMDATMD, ONLY: MDSE, MDST, MDSS, NMPROC, IMPROC, NMPSCR, & NRGRD, STIME, ETIME !/ +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -203,11 +207,11 @@ PROGRAM W3SBS1 ! 0. Initialization necessary for driver ! 0.a General I/O: all can start with initialization in wmmdatmd ! - ! 0.b MPI environment: Here, we use MPI_COMM_WORLD + ! 0.b MPI environment: Here, we use MPI_COMM_WW3 ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - MPI_COMM = MPI_COMM_WORLD + MPI_COMM = MPI_COMM_WW3 CALL MPI_COMM_SIZE ( MPI_COMM, NMPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM, IMPROC, IERR_MPI ) IMPROC = IMPROC + 1 diff --git a/model/src/ww3_shel.F90 b/model/src/ww3_shel.F90 index 4bb888b48..0f73ca337 100644 --- a/model/src/ww3_shel.F90 +++ b/model/src/ww3_shel.F90 @@ -308,6 +308,9 @@ PROGRAM W3SHEL #ifdef W3_OMPG USE OMP_LIB #endif + + USE MPICOMM + IMPLICIT NONE ! #ifdef W3_MPI @@ -487,7 +490,7 @@ PROGRAM W3SHEL #endif #ifdef W3_MPI - MPI_COMM = MPI_COMM_WORLD + MPI_COMM = MPI_COMM_WW3 #endif #ifdef W3_OASIS END IF diff --git a/model/src/ww3_strt.F90 b/model/src/ww3_strt.F90 index b4bcad3ca..9202792da 100644 --- a/model/src/ww3_strt.F90 +++ b/model/src/ww3_strt.F90 @@ -241,6 +241,10 @@ PROGRAM W3STRT USE W3IDATMD, ONLY: W3NINP #endif !/ +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -324,8 +328,8 @@ PROGRAM W3STRT ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 #endif ! diff --git a/model/src/ww3_systrk.F90 b/model/src/ww3_systrk.F90 index cb25a1e72..7f63a2e33 100644 --- a/model/src/ww3_systrk.F90 +++ b/model/src/ww3_systrk.F90 @@ -58,6 +58,10 @@ PROGRAM WW3_SYSTRK !/ USE W3STRKMD USE W3TIMEMD, ONLY: TDIFF +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE #ifdef W3_MPI @@ -168,8 +172,8 @@ PROGRAM WW3_SYSTRK ! Start of parallel region CALL MPI_INIT(ierr) - CALL MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) - CALL MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierr) + CALL MPI_COMM_RANK(MPI_COMM_WW3, rank, ierr) + CALL MPI_COMM_SIZE(MPI_COMM_WW3, nproc, ierr) #endif ! Open log file @@ -343,34 +347,34 @@ PROGRAM WW3_SYSTRK #ifdef W3_MPI ! MPI communication block - CALL MPI_BCAST(filename,80,MPI_CHARACTER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(filename,80,MPI_CHARACTER,0,MPI_COMM_WW3,IERR) CALL MPI_BCAST(tstart,1,MPI_DOUBLE_PRECISION,0, & - MPI_COMM_WORLD,IERR) + MPI_COMM_WW3,IERR) CALL MPI_BCAST(tend,1,MPI_DOUBLE_PRECISION,0, & - MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(dt,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(ntint,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(minlon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxlon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(minlat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxlat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(mxcwt,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(mycwt,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(dirKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(perKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(hsKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(wetPts,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(dirTimeKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(tpTimeKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(seedLon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(seedLat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(noutp,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(lonout,100,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(latout,100,MPI_REAL,0,MPI_COMM_WORLD,IERR) + MPI_COMM_WW3,IERR) + CALL MPI_BCAST(dt,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(ntint,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(minlon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxlon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(minlat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxlat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(mxcwt,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(mycwt,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(dirKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(perKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(hsKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(wetPts,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(dirTimeKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(tpTimeKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(seedLon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(seedLat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(noutp,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(lonout,100,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(latout,100,MPI_REAL,0,MPI_COMM_WW3,IERR) #endif #ifdef W3_MPI - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) #endif CALL waveTracking_NWS_V2 (intype ,tmax , & From 1e561f8b33179171d475d288b25d27c132498077 Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Thu, 11 Jul 2024 13:57:29 -0700 Subject: [PATCH 02/16] Test with debug --- .github/workflows/regtest_gnu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regtest_gnu.yml b/.github/workflows/regtest_gnu.yml index d5b71673c..71d60132b 100644 --- a/.github/workflows/regtest_gnu.yml +++ b/.github/workflows/regtest_gnu.yml @@ -110,7 +110,7 @@ jobs: ls -l ${GITHUB_WORKSPACE}/ww3/model/bin/ww3_from_ftp.sh -k cd regtests - ./bin/run_cmake_test -o all -S -T -s PR1_MPI -w work_PR1_MPI -f -p mpirun -n 24 ../model ww3_tp2.5 + ./bin/run_cmake_test -c gnu_debug -o all -S -T -s PR1_MPI -w work_PR1_MPI -f -p mpirun -n 24 ../model ww3_tp2.5 cd ww3_tp2.5 ls -l cd work_PR1_MPI From 6a4c276dd9dc4b7c089228d74c29b2ed8e24c0fd Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Thu, 11 Jul 2024 16:52:09 -0700 Subject: [PATCH 03/16] Test with change to matrix --- regtests/bin/matrix_cmake_datarmor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regtests/bin/matrix_cmake_datarmor b/regtests/bin/matrix_cmake_datarmor index 16c31e47c..21c97bbcd 100755 --- a/regtests/bin/matrix_cmake_datarmor +++ b/regtests/bin/matrix_cmake_datarmor @@ -46,7 +46,7 @@ fi #convert main_dir to absolute path main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`" - cmplr=datarmor_intel + cmplr=datarmor_intel_debug From 98b8425e45b279f1f6d7f79d07d0ad53fc9fcad1 Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Thu, 11 Jul 2024 16:55:35 -0700 Subject: [PATCH 04/16] Remove debug compiler, uncomment cmake line --- .github/workflows/regtest_gnu.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/regtest_gnu.yml b/.github/workflows/regtest_gnu.yml index 71d60132b..e909c3a9c 100644 --- a/.github/workflows/regtest_gnu.yml +++ b/.github/workflows/regtest_gnu.yml @@ -104,13 +104,13 @@ jobs: export OASISDIR=${GITHUB_WORKSPACE}/work_oasis3-mct # mkdir build && cd build export LD_LIBRARY_PATH="/home/runner/work/WW3/WW3/spack/var/spack/environments/ww3-gnu/.spack-env/view/:$LD_LIBRARY_PATH" - # cmake -DSWITCH=${GITHUB_WORKSPACE}/ww3/regtests/unittests/data/switch.io -DCMAKE_BUILD_TYPE=Debug .. - # make -j2 VERBOSE=1 + cmake -DSWITCH=${GITHUB_WORKSPACE}/ww3/regtests/unittests/data/switch.io -DCMAKE_BUILD_TYPE=Debug .. + make -j2 VERBOSE=1 cd ${GITHUB_WORKSPACE}/ww3 ls -l ${GITHUB_WORKSPACE}/ww3/model/bin/ww3_from_ftp.sh -k cd regtests - ./bin/run_cmake_test -c gnu_debug -o all -S -T -s PR1_MPI -w work_PR1_MPI -f -p mpirun -n 24 ../model ww3_tp2.5 + ./bin/run_cmake_test -o all -S -T -s PR1_MPI -w work_PR1_MPI -f -p mpirun -n 24 ../model ww3_tp2.5 cd ww3_tp2.5 ls -l cd work_PR1_MPI From f2810fb5cbc8a4c621313ba673d3f9c2d23dcb5f Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Thu, 11 Jul 2024 16:58:27 -0700 Subject: [PATCH 05/16] Fix path --- .github/workflows/regtest_gnu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/regtest_gnu.yml b/.github/workflows/regtest_gnu.yml index e909c3a9c..82a6c2b2e 100644 --- a/.github/workflows/regtest_gnu.yml +++ b/.github/workflows/regtest_gnu.yml @@ -102,7 +102,7 @@ jobs: export CC=mpicc export FC=mpif90 export OASISDIR=${GITHUB_WORKSPACE}/work_oasis3-mct - # mkdir build && cd build + mkdir build && cd build export LD_LIBRARY_PATH="/home/runner/work/WW3/WW3/spack/var/spack/environments/ww3-gnu/.spack-env/view/:$LD_LIBRARY_PATH" cmake -DSWITCH=${GITHUB_WORKSPACE}/ww3/regtests/unittests/data/switch.io -DCMAKE_BUILD_TYPE=Debug .. make -j2 VERBOSE=1 From 5805c9b202fcc95b6f3ecc9ae160c4e8271b2da6 Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Tue, 16 Jul 2024 10:56:18 -0700 Subject: [PATCH 06/16] Test revert using all of them --- model/src/w3gsrumd.F90 | 2 +- model/src/w3servmd.F90 | 6 +-- model/src/w3strkmd.F90 | 92 +++++++++++++++++++--------------------- model/src/ww3_multi.F90 | 8 +--- model/src/ww3_ounp.F90 | 8 +--- model/src/ww3_prnc.F90 | 14 +++--- model/src/ww3_prtide.F90 | 20 ++++----- model/src/ww3_sbs1.F90 | 8 +--- model/src/ww3_shel.F90 | 5 +-- model/src/ww3_strt.F90 | 8 +--- model/src/ww3_systrk.F90 | 54 +++++++++++------------ 11 files changed, 93 insertions(+), 132 deletions(-) diff --git a/model/src/w3gsrumd.F90 b/model/src/w3gsrumd.F90 index 87a099d11..9cd4beaf5 100644 --- a/model/src/w3gsrumd.F90 +++ b/model/src/w3gsrumd.F90 @@ -9631,7 +9631,7 @@ SUBROUTINE EXTCDE(IEXIT) LOGICAL :: RUN CALL MPI_INITIALIZED ( RUN, IERR_MPI ) IF ( RUN ) THEN - CALL MPI_ABORT ( MPI_COMM_WW3, IEXIT, IERR_MPI ) + CALL MPI_ABORT ( MPI_COMM_WORLD, IEXIT, IERR_MPI ) END IF #endif CALL EXIT(IEXIT) diff --git a/model/src/w3servmd.F90 b/model/src/w3servmd.F90 index 70dcc061f..600e6e572 100644 --- a/model/src/w3servmd.F90 +++ b/model/src/w3servmd.F90 @@ -79,10 +79,6 @@ MODULE W3SERVMD !/ ------------------------------------------------------------------- / ! module default -#ifdef W3_MPI - USE MPICOMM -#endif - ! implicit none PUBLIC @@ -868,7 +864,7 @@ SUBROUTINE EXTCDE ( IEXIT, UNIT, MSG, FILE, LINE, COMM ) IF (PRESENT(COMM)) THEN WRITE(*,'(/A,I6/)') 'EXTCDE COMM=', COMM END IF - CALL MPI_ABORT ( MPI_COMM_WW3, IEXIT, IERR_MPI ) + CALL MPI_ABORT ( MPI_COMM_WORLD, IEXIT, IERR_MPI ) END IF END IF #endif diff --git a/model/src/w3strkmd.F90 b/model/src/w3strkmd.F90 index 212e14ba5..a776a36bb 100644 --- a/model/src/w3strkmd.F90 +++ b/model/src/w3strkmd.F90 @@ -28,10 +28,6 @@ MODULE W3STRKMD !/ No unauthorized use without permission. !/ use constants, only: file_endian -#ifdef W3_MPI - USE MPICOMM -#endif - ! IMPLICIT NONE ! ! 1. Purpose : @@ -474,8 +470,8 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & !/ ------------------------------------------------------------------- / #ifdef W3_MPI - CALL MPI_COMM_RANK(MPI_COMM_WW3, rank, ierr) - CALL MPI_COMM_SIZE(MPI_COMM_WW3, nproc, ierr) + CALL MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) + CALL MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierr) #endif NULLIFY( sysA ) NULLIFY( maxSys ) @@ -946,9 +942,9 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & #endif #ifdef W3_MPI - CALL MPI_BCAST(maxI,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(maxJ,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(maxTs,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxI,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(maxJ,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(maxTs,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) #endif ! Allocate the wsdat structure @@ -1181,13 +1177,13 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & wsdat(tsA)%lat(i,j), & wsdat(tsA)%lon(i,j)/) CALL MPI_SEND(COMMARR1,44,MPI_REAL,irank, & - (tag1+1),MPI_COMM_WW3,IERR) + (tag1+1),MPI_COMM_WORLD,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(6,*) '<< Receiving: rank,irank,tag1=', & ! rank,irank,(tag1+1) CALL MPI_RECV(COMMARR1,44,MPI_REAL,0,(tag1+1), & - MPI_COMM_WW3,MPI_STATUS,IERR) + MPI_COMM_WORLD,MPI_STATUS,IERR) wsdat(tsA)%par(i,j)%hs = COMMARR1(1:10) wsdat(tsA)%par(i,j)%tp = COMMARR1(11:20) wsdat(tsA)%par(i,j)%dir = COMMARR1(21:30) @@ -1201,12 +1197,12 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & IF (rank.EQ.0) THEN CALL MPI_SEND(wsdat(tsA)%date,1, & MPI_DOUBLE_PRECISION,irank, & - (tag1+2),MPI_COMM_WW3,IERR) + (tag1+2),MPI_COMM_WORLD,IERR) END IF IF (rank.EQ.irank) THEN CALL MPI_RECV(wsdat(tsA)%date,1, & MPI_DOUBLE_PRECISION,0,(tag1+2), & - MPI_COMM_WW3,MPI_STATUS,IERR) + MPI_COMM_WORLD,MPI_STATUS,IERR) END IF IF (rank.EQ.0) THEN @@ -1215,14 +1211,14 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & COMMARR2 = (/wsdat(tsA)%par(i,j)%ipart(:), & wsdat(tsA)%par(i,j)%checked/) CALL MPI_SEND(COMMARR2,11, & - MPI_INTEGER,irank,(tag1+3),MPI_COMM_WW3,IERR) + MPI_INTEGER,irank,(tag1+3),MPI_COMM_WORLD,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(6,*) '<< Receiving: rank,irank,tag1=', & ! rank,irank,(tag1+3) CALL MPI_RECV(COMMARR2,11, & MPI_INTEGER,0,(tag1+3), & - MPI_COMM_WW3,MPI_STATUS,IERR) + MPI_COMM_WORLD,MPI_STATUS,IERR) wsdat(tsA)%par(i,j)%ipart(:) = COMMARR2(1:10) wsdat(tsA)%par(i,j)%checked = COMMARR2(11) END IF @@ -1232,7 +1228,7 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & END IF END DO - CALL MPI_Barrier(MPI_COMM_WW3,IERR) + CALL MPI_Barrier(MPI_COMM_WORLD,IERR) #endif @@ -1420,7 +1416,7 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & #endif #ifdef W3_MPI - CALL MPI_Barrier(MPI_COMM_WW3,IERR) + CALL MPI_Barrier(MPI_COMM_WORLD,IERR) !! Define communicator for array of integers in structure "system" ! DOMSIZE = maxI*maxJ @@ -1451,13 +1447,13 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! Send results from current rank to rank 0 (blocking) ! WRITE(20,*) '>> Sending: rank,tsA,tag1=',rank,tsA,tag1 CALL MPI_SEND(maxSys(tsA),1,MPI_INTEGER,0,tag1, & - MPI_COMM_WW3,IERR) + MPI_COMM_WORLD,IERR) ! WRITE(20,*) 'Rank, IERR=',rank,IERR END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tsA,tag1=',rank,tsA,tag1 CALL MPI_RECV(maxSys(tsA),1,MPI_INTEGER, & - irank,tag1,MPI_COMM_WW3,MPI_STATUS,IERR) + irank,tag1,MPI_COMM_WORLD,MPI_STATUS,IERR) ! Allocate structure at this time level ALLOCATE( sysA(tsA)%sys(maxSys(tsA)) ) DO ic = 1,maxSys(tsA) @@ -1505,14 +1501,14 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+1) CALL MPI_SEND(sysA(tsA)%sys(ic)%i(:),DOMSIZE, & - MPI_INTEGER,0,(tag2+1),MPI_COMM_WW3,REQ(1),IERR) + MPI_INTEGER,0,(tag2+1),MPI_COMM_WORLD,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+1) CALL MPI_RECV(sysA(tsA)%sys(ic)%i(:),DOMSIZE, & MPI_INTEGER,irank,(tag2+1), & - MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) @@ -1520,92 +1516,92 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+2) CALL MPI_SEND(sysA(tsA)%sys(ic)%j(:),DOMSIZE, & - MPI_INTEGER,0,(tag2+2),MPI_COMM_WW3,REQ(1),IERR) + MPI_INTEGER,0,(tag2+2),MPI_COMM_WORLD,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+2) CALL MPI_RECV(sysA(tsA)%sys(ic)%j(:),DOMSIZE, & MPI_INTEGER,irank,(tag2+2), & - MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+3) CALL MPI_SEND(sysA(tsA)%sys(ic)%lon(:),DOMSIZE, & - MPI_REAL,0,(tag2+3),MPI_COMM_WW3,REQ(1),IERR) + MPI_REAL,0,(tag2+3),MPI_COMM_WORLD,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+3) CALL MPI_RECV(sysA(tsA)%sys(ic)%lon(:),DOMSIZE, & MPI_REAL,irank,(tag2+3), & - MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+4) CALL MPI_SEND(sysA(tsA)%sys(ic)%lat(:),DOMSIZE, & - MPI_REAL,0,(tag2+4),MPI_COMM_WW3,REQ(1),IERR) + MPI_REAL,0,(tag2+4),MPI_COMM_WORLD,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+4) CALL MPI_RECV(sysA(tsA)%sys(ic)%lat(:),DOMSIZE, & MPI_REAL,irank,(tag2+4), & - MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+5) CALL MPI_SEND(sysA(tsA)%sys(ic)%hs(:),DOMSIZE, & - MPI_REAL,0,(tag2+5),MPI_COMM_WW3,REQ(1),IERR) + MPI_REAL,0,(tag2+5),MPI_COMM_WORLD,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+5) CALL MPI_RECV(sysA(tsA)%sys(ic)%hs(:),DOMSIZE, & MPI_REAL,irank,(tag2+5), & - MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+6) CALL MPI_SEND(sysA(tsA)%sys(ic)%tp(:),DOMSIZE, & - MPI_REAL,0,(tag2+6),MPI_COMM_WW3,REQ(1),IERR) + MPI_REAL,0,(tag2+6),MPI_COMM_WORLD,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+6) CALL MPI_RECV(sysA(tsA)%sys(ic)%tp(:),DOMSIZE, & MPI_REAL,irank,(tag2+6), & - MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+7) CALL MPI_SEND(sysA(tsA)%sys(ic)%dir(:),DOMSIZE, & - MPI_REAL,0,(tag2+7),MPI_COMM_WW3,REQ(1),IERR) + MPI_REAL,0,(tag2+7),MPI_COMM_WORLD,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+7) CALL MPI_RECV(sysA(tsA)%sys(ic)%dir(:),DOMSIZE, & MPI_REAL,irank,(tag2+7), & - MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+8) CALL MPI_SEND(sysA(tsA)%sys(ic)%dspr(:),DOMSIZE, & - MPI_REAL,0,(tag2+8),MPI_COMM_WW3,REQ(1),IERR) + MPI_REAL,0,(tag2+8),MPI_COMM_WORLD,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+8) CALL MPI_RECV(sysA(tsA)%sys(ic)%dspr(:),DOMSIZE, & MPI_REAL,irank,(tag2+8), & - MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) @@ -1613,85 +1609,85 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+9) CALL MPI_SEND(sysA(tsA)%sys(ic)%hsMean,1,MPI_REAL, & - 0,(tag2+9),MPI_COMM_WW3,IERR) + 0,(tag2+9),MPI_COMM_WORLD,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+9) CALL MPI_RECV(sysA(tsA)%sys(ic)%hsMean,1,MPI_REAL, & - irank,(tag2+9),MPI_COMM_WW3,MPI_STATUS,IERR) + irank,(tag2+9),MPI_COMM_WORLD,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+10) CALL MPI_SEND(sysA(tsA)%sys(ic)%tpMean,1,MPI_REAL, & - 0,(tag2+10),MPI_COMM_WW3,IERR) + 0,(tag2+10),MPI_COMM_WORLD,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+10) CALL MPI_RECV(sysA(tsA)%sys(ic)%tpMean,1,MPI_REAL, & - irank,(tag2+10),MPI_COMM_WW3,MPI_STATUS,IERR) + irank,(tag2+10),MPI_COMM_WORLD,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+11) CALL MPI_SEND(sysA(tsA)%sys(ic)%dirMean,1,MPI_REAL, & - 0,(tag2+11),MPI_COMM_WW3,IERR) + 0,(tag2+11),MPI_COMM_WORLD,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+11) CALL MPI_RECV(sysA(tsA)%sys(ic)%dirMean,1,MPI_REAL, & - irank,(tag2+11),MPI_COMM_WW3,MPI_STATUS,IERR) + irank,(tag2+11),MPI_COMM_WORLD,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+12) CALL MPI_SEND(sysA(tsA)%sys(ic)%sysInd,1,MPI_INTEGER,& - 0,(tag2+12),MPI_COMM_WW3,IERR) + 0,(tag2+12),MPI_COMM_WORLD,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+12) CALL MPI_RECV(sysA(tsA)%sys(ic)%sysInd,1,MPI_INTEGER,& - irank,(tag2+12),MPI_COMM_WW3,MPI_STATUS,IERR) + irank,(tag2+12),MPI_COMM_WORLD,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+13) CALL MPI_SEND(sysA(tsA)%sys(ic)%nPoints,1,MPI_INTEGER,& - 0,(tag2+13),MPI_COMM_WW3,IERR) + 0,(tag2+13),MPI_COMM_WORLD,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+13) CALL MPI_RECV(sysA(tsA)%sys(ic)%nPoints,1,MPI_INTEGER,& - irank,(tag2+13),MPI_COMM_WW3,MPI_STATUS,IERR) + irank,(tag2+13),MPI_COMM_WORLD,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+14) CALL MPI_SEND(sysA(tsA)%sys(ic)%grp,1,MPI_INTEGER,& - 0,(tag2+14),MPI_COMM_WW3,IERR) + 0,(tag2+14),MPI_COMM_WORLD,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+14) CALL MPI_RECV(sysA(tsA)%sys(ic)%grp,1,MPI_INTEGER,& - irank,(tag2+14),MPI_COMM_WW3,MPI_STATUS,IERR) + irank,(tag2+14),MPI_COMM_WORLD,MPI_STATUS,IERR) END IF END DO END IF END IF END DO - CALL MPI_Barrier(MPI_COMM_WW3,IERR) + CALL MPI_Barrier(MPI_COMM_WORLD,IERR) ! CALL MPI_TYPE_FREE(MPI_INT_DOMARR,IERR) ! CALL MPI_TYPE_FREE(MPI_REAL_DOMARR,IERR) diff --git a/model/src/ww3_multi.F90 b/model/src/ww3_multi.F90 index f1c55169b..e3101b7c2 100644 --- a/model/src/ww3_multi.F90 +++ b/model/src/ww3_multi.F90 @@ -95,10 +95,6 @@ PROGRAM W3MLTI USE OMP_LIB #endif !/ -#ifdef W3_MPI - USE MPICOMM -#endif - ! IMPLICIT NONE ! #ifdef W3_MPI @@ -123,7 +119,7 @@ PROGRAM W3MLTI ! 0. Initialization necessary for driver ! 0.a General I/O: all can start with initialization in wmmdatmd ! - ! 0.b MPI environment: Here, we use MPI_COMM_WW3 + ! 0.b MPI environment: Here, we use MPI_COMM_WORLD ! #ifdef W3_OMPH FLHYBR = .TRUE. @@ -139,7 +135,7 @@ PROGRAM W3MLTI ENDIF #endif #ifdef W3_MPI - MPI_COMM = MPI_COMM_WW3 + MPI_COMM = MPI_COMM_WORLD CALL MPI_COMM_SIZE ( MPI_COMM, NMPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM, IMPROC, IERR_MPI ) IMPROC = IMPROC + 1 diff --git a/model/src/ww3_ounp.F90 b/model/src/ww3_ounp.F90 index aa70f42d0..a1533c73b 100644 --- a/model/src/ww3_ounp.F90 +++ b/model/src/ww3_ounp.F90 @@ -213,10 +213,6 @@ PROGRAM W3OUNP USE W3NMLOUNPMD USE NETCDF ! -#ifdef W3_MPI - USE MPICOMM -#endif - ! IMPLICIT NONE ! #ifdef W3_MPI @@ -359,8 +355,8 @@ PROGRAM W3OUNP ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! diff --git a/model/src/ww3_prnc.F90 b/model/src/ww3_prnc.F90 index ee347659e..e107a1f43 100644 --- a/model/src/ww3_prnc.F90 +++ b/model/src/ww3_prnc.F90 @@ -225,10 +225,6 @@ PROGRAM W3PRNC USE W3NMLPRNCMD USE NETCDF ! -#ifdef W3_MPI - USE MPICOMM -#endif - ! IMPLICIT NONE ! #ifdef W3_MPI @@ -444,8 +440,8 @@ PROGRAM W3PRNC ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! @@ -1700,18 +1696,18 @@ PROGRAM W3PRNC IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(TIDE1DL, SLICE * TIDE_MF * NFIELDS * 2, MPI_REAL, & TIDE1D, NELEM * TIDE_MF * NFIELDS * 2, CUMUL * TIDE_MF * NFIELDS * 2, & - MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) + MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) #endif #ifdef W3_MPI IF (IAPROC.EQ.NAPOUT) THEN CALL MPI_GATHERV(MPI_IN_PLACE,NELEM(IAPROC), & MPI_LOGICAL, TIDALCOMP(:,IY), NELEM, CUMUL, MPI_LOGICAL, NAPOUT-1, & - MPI_COMM_WW3, IERR_MPI) + MPI_COMM_WORLD, IERR_MPI) ELSE CALL MPI_GATHERV(TIDALCOMP(CUMUL(IAPROC)+1:CUMUL(IAPROC)+NELEM(IAPROC),IY),NELEM(IAPROC), & MPI_LOGICAL, TIDALCOMP(:,IY), NELEM, CUMUL, MPI_LOGICAL, NAPOUT-1, & - MPI_COMM_WW3, IERR_MPI) + MPI_COMM_WORLD, IERR_MPI) END IF #endif diff --git a/model/src/ww3_prtide.F90 b/model/src/ww3_prtide.F90 index 0da03424a..1168cd1ab 100644 --- a/model/src/ww3_prtide.F90 +++ b/model/src/ww3_prtide.F90 @@ -139,10 +139,6 @@ PROGRAM W3PRTIDE USE W3TIDEMD USE W3IDATMD ! -#ifdef W3_MPI - USE MPICOMM -#endif - ! IMPLICIT NONE ! #ifdef W3_MPI @@ -258,8 +254,8 @@ PROGRAM W3PRTIDE ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! @@ -677,11 +673,11 @@ PROGRAM W3PRTIDE #ifdef W3_MPI IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(FX1DL, SLICE, MPI_REAL, FX1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) CALL MPI_GATHERV(FY1DL, SLICE, MPI_REAL, FY1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) CALL MPI_GATHERV(FA1DL, SLICE, MPI_REAL, FA1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) ELSE FX1D = FX1DL FY1D = FY1DL @@ -772,11 +768,11 @@ PROGRAM W3PRTIDE #ifdef W3_MPI IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(FX1DL, SLICE, MPI_REAL, FX1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) CALL MPI_GATHERV(FY1DL, SLICE, MPI_REAL, FY1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) CALL MPI_GATHERV(FA1DL, SLICE, MPI_REAL, FA1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) ELSE FX1D = FX1DL FY1D = FY1DL diff --git a/model/src/ww3_sbs1.F90 b/model/src/ww3_sbs1.F90 index 8da309c62..94778b10d 100644 --- a/model/src/ww3_sbs1.F90 +++ b/model/src/ww3_sbs1.F90 @@ -177,10 +177,6 @@ PROGRAM W3SBS1 USE WMMDATMD, ONLY: MDSE, MDST, MDSS, NMPROC, IMPROC, NMPSCR, & NRGRD, STIME, ETIME !/ -#ifdef W3_MPI - USE MPICOMM -#endif - ! IMPLICIT NONE ! #ifdef W3_MPI @@ -207,11 +203,11 @@ PROGRAM W3SBS1 ! 0. Initialization necessary for driver ! 0.a General I/O: all can start with initialization in wmmdatmd ! - ! 0.b MPI environment: Here, we use MPI_COMM_WW3 + ! 0.b MPI environment: Here, we use MPI_COMM_WORLD ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - MPI_COMM = MPI_COMM_WW3 + MPI_COMM = MPI_COMM_WORLD CALL MPI_COMM_SIZE ( MPI_COMM, NMPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM, IMPROC, IERR_MPI ) IMPROC = IMPROC + 1 diff --git a/model/src/ww3_shel.F90 b/model/src/ww3_shel.F90 index 0f73ca337..4bb888b48 100644 --- a/model/src/ww3_shel.F90 +++ b/model/src/ww3_shel.F90 @@ -308,9 +308,6 @@ PROGRAM W3SHEL #ifdef W3_OMPG USE OMP_LIB #endif - - USE MPICOMM - IMPLICIT NONE ! #ifdef W3_MPI @@ -490,7 +487,7 @@ PROGRAM W3SHEL #endif #ifdef W3_MPI - MPI_COMM = MPI_COMM_WW3 + MPI_COMM = MPI_COMM_WORLD #endif #ifdef W3_OASIS END IF diff --git a/model/src/ww3_strt.F90 b/model/src/ww3_strt.F90 index 9202792da..b4bcad3ca 100644 --- a/model/src/ww3_strt.F90 +++ b/model/src/ww3_strt.F90 @@ -241,10 +241,6 @@ PROGRAM W3STRT USE W3IDATMD, ONLY: W3NINP #endif !/ -#ifdef W3_MPI - USE MPICOMM -#endif - ! IMPLICIT NONE ! #ifdef W3_MPI @@ -328,8 +324,8 @@ PROGRAM W3STRT ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 #endif ! diff --git a/model/src/ww3_systrk.F90 b/model/src/ww3_systrk.F90 index 7f63a2e33..cb25a1e72 100644 --- a/model/src/ww3_systrk.F90 +++ b/model/src/ww3_systrk.F90 @@ -58,10 +58,6 @@ PROGRAM WW3_SYSTRK !/ USE W3STRKMD USE W3TIMEMD, ONLY: TDIFF -#ifdef W3_MPI - USE MPICOMM -#endif - ! IMPLICIT NONE #ifdef W3_MPI @@ -172,8 +168,8 @@ PROGRAM WW3_SYSTRK ! Start of parallel region CALL MPI_INIT(ierr) - CALL MPI_COMM_RANK(MPI_COMM_WW3, rank, ierr) - CALL MPI_COMM_SIZE(MPI_COMM_WW3, nproc, ierr) + CALL MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) + CALL MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierr) #endif ! Open log file @@ -347,34 +343,34 @@ PROGRAM WW3_SYSTRK #ifdef W3_MPI ! MPI communication block - CALL MPI_BCAST(filename,80,MPI_CHARACTER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(filename,80,MPI_CHARACTER,0,MPI_COMM_WORLD,IERR) CALL MPI_BCAST(tstart,1,MPI_DOUBLE_PRECISION,0, & - MPI_COMM_WW3,IERR) + MPI_COMM_WORLD,IERR) CALL MPI_BCAST(tend,1,MPI_DOUBLE_PRECISION,0, & - MPI_COMM_WW3,IERR) - CALL MPI_BCAST(dt,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(ntint,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(minlon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(maxlon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(minlat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(maxlat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(mxcwt,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(mycwt,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(dirKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(perKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(hsKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(wetPts,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(dirTimeKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(tpTimeKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(seedLon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(seedLat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(noutp,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(lonout,100,MPI_REAL,0,MPI_COMM_WW3,IERR) - CALL MPI_BCAST(latout,100,MPI_REAL,0,MPI_COMM_WW3,IERR) + MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(dt,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(ntint,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(minlon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(maxlon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(minlat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(maxlat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(mxcwt,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(mycwt,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(dirKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(perKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(hsKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(wetPts,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(dirTimeKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(tpTimeKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(seedLon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(seedLat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(noutp,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(lonout,100,MPI_REAL,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(latout,100,MPI_REAL,0,MPI_COMM_WORLD,IERR) #endif #ifdef W3_MPI - CALL MPI_Barrier(MPI_COMM_WW3,IERR) + CALL MPI_Barrier(MPI_COMM_WORLD,IERR) #endif CALL waveTracking_NWS_V2 (intype ,tmax , & From ba8db4c1c6ef8013b719ec3afb35ac92092926f3 Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Tue, 16 Jul 2024 13:07:36 -0700 Subject: [PATCH 07/16] Test revert PDLIB changes as well --- model/src/PDLIB/yowdatapool.F90 | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/model/src/PDLIB/yowdatapool.F90 b/model/src/PDLIB/yowdatapool.F90 index 89dd0830d..24ae18162 100644 --- a/model/src/PDLIB/yowdatapool.F90 +++ b/model/src/PDLIB/yowdatapool.F90 @@ -37,11 +37,7 @@ ! !> Has fancy data module yowDatapool - use MPI, only: MPI_INTEGER, MPI_REAL4, MPI_REAL8, MPI_STATUS_SIZE -#ifdef W3_MPI - USE MPICOMM, only: MPI_COMM_WW3 => MPI_COMM_WORLD -#endif - ! + use MPI, only: MPI_COMM_WORLD, MPI_INTEGER, MPI_REAL4, MPI_REAL8, MPI_STATUS_SIZE implicit none !#ifdef USE_SINGLE ! !> single precision. Enable with compiler flag -DUSE_SINGLE From 7805f33264af4fbfc34542fae2c4504f307f754f Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Tue, 16 Jul 2024 14:15:32 -0700 Subject: [PATCH 08/16] Small w3 add back --- model/src/w3gsrumd.F90 | 2 +- model/src/w3servmd.F90 | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/model/src/w3gsrumd.F90 b/model/src/w3gsrumd.F90 index 9cd4beaf5..87a099d11 100644 --- a/model/src/w3gsrumd.F90 +++ b/model/src/w3gsrumd.F90 @@ -9631,7 +9631,7 @@ SUBROUTINE EXTCDE(IEXIT) LOGICAL :: RUN CALL MPI_INITIALIZED ( RUN, IERR_MPI ) IF ( RUN ) THEN - CALL MPI_ABORT ( MPI_COMM_WORLD, IEXIT, IERR_MPI ) + CALL MPI_ABORT ( MPI_COMM_WW3, IEXIT, IERR_MPI ) END IF #endif CALL EXIT(IEXIT) diff --git a/model/src/w3servmd.F90 b/model/src/w3servmd.F90 index 600e6e572..70dcc061f 100644 --- a/model/src/w3servmd.F90 +++ b/model/src/w3servmd.F90 @@ -79,6 +79,10 @@ MODULE W3SERVMD !/ ------------------------------------------------------------------- / ! module default +#ifdef W3_MPI + USE MPICOMM +#endif + ! implicit none PUBLIC @@ -864,7 +868,7 @@ SUBROUTINE EXTCDE ( IEXIT, UNIT, MSG, FILE, LINE, COMM ) IF (PRESENT(COMM)) THEN WRITE(*,'(/A,I6/)') 'EXTCDE COMM=', COMM END IF - CALL MPI_ABORT ( MPI_COMM_WORLD, IEXIT, IERR_MPI ) + CALL MPI_ABORT ( MPI_COMM_WW3, IEXIT, IERR_MPI ) END IF END IF #endif From 53bbe185965962ee0814a6d9b59fde5e4c43d827 Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Tue, 16 Jul 2024 14:20:47 -0700 Subject: [PATCH 09/16] Big w3 add back --- model/src/w3strkmd.F90 | 92 ++++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/model/src/w3strkmd.F90 b/model/src/w3strkmd.F90 index a776a36bb..212e14ba5 100644 --- a/model/src/w3strkmd.F90 +++ b/model/src/w3strkmd.F90 @@ -28,6 +28,10 @@ MODULE W3STRKMD !/ No unauthorized use without permission. !/ use constants, only: file_endian +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! ! 1. Purpose : @@ -470,8 +474,8 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & !/ ------------------------------------------------------------------- / #ifdef W3_MPI - CALL MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) - CALL MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierr) + CALL MPI_COMM_RANK(MPI_COMM_WW3, rank, ierr) + CALL MPI_COMM_SIZE(MPI_COMM_WW3, nproc, ierr) #endif NULLIFY( sysA ) NULLIFY( maxSys ) @@ -942,9 +946,9 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & #endif #ifdef W3_MPI - CALL MPI_BCAST(maxI,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxJ,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxTs,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(maxI,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxJ,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxTs,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) #endif ! Allocate the wsdat structure @@ -1177,13 +1181,13 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & wsdat(tsA)%lat(i,j), & wsdat(tsA)%lon(i,j)/) CALL MPI_SEND(COMMARR1,44,MPI_REAL,irank, & - (tag1+1),MPI_COMM_WORLD,IERR) + (tag1+1),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(6,*) '<< Receiving: rank,irank,tag1=', & ! rank,irank,(tag1+1) CALL MPI_RECV(COMMARR1,44,MPI_REAL,0,(tag1+1), & - MPI_COMM_WORLD,MPI_STATUS,IERR) + MPI_COMM_WW3,MPI_STATUS,IERR) wsdat(tsA)%par(i,j)%hs = COMMARR1(1:10) wsdat(tsA)%par(i,j)%tp = COMMARR1(11:20) wsdat(tsA)%par(i,j)%dir = COMMARR1(21:30) @@ -1197,12 +1201,12 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & IF (rank.EQ.0) THEN CALL MPI_SEND(wsdat(tsA)%date,1, & MPI_DOUBLE_PRECISION,irank, & - (tag1+2),MPI_COMM_WORLD,IERR) + (tag1+2),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.irank) THEN CALL MPI_RECV(wsdat(tsA)%date,1, & MPI_DOUBLE_PRECISION,0,(tag1+2), & - MPI_COMM_WORLD,MPI_STATUS,IERR) + MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.0) THEN @@ -1211,14 +1215,14 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & COMMARR2 = (/wsdat(tsA)%par(i,j)%ipart(:), & wsdat(tsA)%par(i,j)%checked/) CALL MPI_SEND(COMMARR2,11, & - MPI_INTEGER,irank,(tag1+3),MPI_COMM_WORLD,IERR) + MPI_INTEGER,irank,(tag1+3),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(6,*) '<< Receiving: rank,irank,tag1=', & ! rank,irank,(tag1+3) CALL MPI_RECV(COMMARR2,11, & MPI_INTEGER,0,(tag1+3), & - MPI_COMM_WORLD,MPI_STATUS,IERR) + MPI_COMM_WW3,MPI_STATUS,IERR) wsdat(tsA)%par(i,j)%ipart(:) = COMMARR2(1:10) wsdat(tsA)%par(i,j)%checked = COMMARR2(11) END IF @@ -1228,7 +1232,7 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & END IF END DO - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) #endif @@ -1416,7 +1420,7 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & #endif #ifdef W3_MPI - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) !! Define communicator for array of integers in structure "system" ! DOMSIZE = maxI*maxJ @@ -1447,13 +1451,13 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! Send results from current rank to rank 0 (blocking) ! WRITE(20,*) '>> Sending: rank,tsA,tag1=',rank,tsA,tag1 CALL MPI_SEND(maxSys(tsA),1,MPI_INTEGER,0,tag1, & - MPI_COMM_WORLD,IERR) + MPI_COMM_WW3,IERR) ! WRITE(20,*) 'Rank, IERR=',rank,IERR END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tsA,tag1=',rank,tsA,tag1 CALL MPI_RECV(maxSys(tsA),1,MPI_INTEGER, & - irank,tag1,MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,tag1,MPI_COMM_WW3,MPI_STATUS,IERR) ! Allocate structure at this time level ALLOCATE( sysA(tsA)%sys(maxSys(tsA)) ) DO ic = 1,maxSys(tsA) @@ -1501,14 +1505,14 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+1) CALL MPI_SEND(sysA(tsA)%sys(ic)%i(:),DOMSIZE, & - MPI_INTEGER,0,(tag2+1),MPI_COMM_WORLD,REQ(1),IERR) + MPI_INTEGER,0,(tag2+1),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+1) CALL MPI_RECV(sysA(tsA)%sys(ic)%i(:),DOMSIZE, & MPI_INTEGER,irank,(tag2+1), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) @@ -1516,92 +1520,92 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+2) CALL MPI_SEND(sysA(tsA)%sys(ic)%j(:),DOMSIZE, & - MPI_INTEGER,0,(tag2+2),MPI_COMM_WORLD,REQ(1),IERR) + MPI_INTEGER,0,(tag2+2),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+2) CALL MPI_RECV(sysA(tsA)%sys(ic)%j(:),DOMSIZE, & MPI_INTEGER,irank,(tag2+2), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+3) CALL MPI_SEND(sysA(tsA)%sys(ic)%lon(:),DOMSIZE, & - MPI_REAL,0,(tag2+3),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+3),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+3) CALL MPI_RECV(sysA(tsA)%sys(ic)%lon(:),DOMSIZE, & MPI_REAL,irank,(tag2+3), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+4) CALL MPI_SEND(sysA(tsA)%sys(ic)%lat(:),DOMSIZE, & - MPI_REAL,0,(tag2+4),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+4),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+4) CALL MPI_RECV(sysA(tsA)%sys(ic)%lat(:),DOMSIZE, & MPI_REAL,irank,(tag2+4), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+5) CALL MPI_SEND(sysA(tsA)%sys(ic)%hs(:),DOMSIZE, & - MPI_REAL,0,(tag2+5),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+5),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+5) CALL MPI_RECV(sysA(tsA)%sys(ic)%hs(:),DOMSIZE, & MPI_REAL,irank,(tag2+5), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+6) CALL MPI_SEND(sysA(tsA)%sys(ic)%tp(:),DOMSIZE, & - MPI_REAL,0,(tag2+6),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+6),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+6) CALL MPI_RECV(sysA(tsA)%sys(ic)%tp(:),DOMSIZE, & MPI_REAL,irank,(tag2+6), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+7) CALL MPI_SEND(sysA(tsA)%sys(ic)%dir(:),DOMSIZE, & - MPI_REAL,0,(tag2+7),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+7),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+7) CALL MPI_RECV(sysA(tsA)%sys(ic)%dir(:),DOMSIZE, & MPI_REAL,irank,(tag2+7), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,tag2=',rank,(tag2+8) CALL MPI_SEND(sysA(tsA)%sys(ic)%dspr(:),DOMSIZE, & - MPI_REAL,0,(tag2+8),MPI_COMM_WORLD,REQ(1),IERR) + MPI_REAL,0,(tag2+8),MPI_COMM_WW3,REQ(1),IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,tag2=',rank,(tag2+8) CALL MPI_RECV(sysA(tsA)%sys(ic)%dspr(:),DOMSIZE, & MPI_REAL,irank,(tag2+8), & - MPI_COMM_WORLD,MPI_STATUS,REQ(2),IERR) + MPI_COMM_WW3,MPI_STATUS,REQ(2),IERR) END IF ! CALL MPI_WAITALL(2,REQ,ISTAT,IERR) @@ -1609,85 +1613,85 @@ SUBROUTINE waveTracking_NWS_V2 (intype ,tmax , & ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+9) CALL MPI_SEND(sysA(tsA)%sys(ic)%hsMean,1,MPI_REAL, & - 0,(tag2+9),MPI_COMM_WORLD,IERR) + 0,(tag2+9),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+9) CALL MPI_RECV(sysA(tsA)%sys(ic)%hsMean,1,MPI_REAL, & - irank,(tag2+9),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+9),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+10) CALL MPI_SEND(sysA(tsA)%sys(ic)%tpMean,1,MPI_REAL, & - 0,(tag2+10),MPI_COMM_WORLD,IERR) + 0,(tag2+10),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+10) CALL MPI_RECV(sysA(tsA)%sys(ic)%tpMean,1,MPI_REAL, & - irank,(tag2+10),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+10),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+11) CALL MPI_SEND(sysA(tsA)%sys(ic)%dirMean,1,MPI_REAL, & - 0,(tag2+11),MPI_COMM_WORLD,IERR) + 0,(tag2+11),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+11) CALL MPI_RECV(sysA(tsA)%sys(ic)%dirMean,1,MPI_REAL, & - irank,(tag2+11),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+11),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+12) CALL MPI_SEND(sysA(tsA)%sys(ic)%sysInd,1,MPI_INTEGER,& - 0,(tag2+12),MPI_COMM_WORLD,IERR) + 0,(tag2+12),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+12) CALL MPI_RECV(sysA(tsA)%sys(ic)%sysInd,1,MPI_INTEGER,& - irank,(tag2+12),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+12),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+13) CALL MPI_SEND(sysA(tsA)%sys(ic)%nPoints,1,MPI_INTEGER,& - 0,(tag2+13),MPI_COMM_WORLD,IERR) + 0,(tag2+13),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+13) CALL MPI_RECV(sysA(tsA)%sys(ic)%nPoints,1,MPI_INTEGER,& - irank,(tag2+13),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+13),MPI_COMM_WW3,MPI_STATUS,IERR) END IF IF (rank.EQ.irank) THEN ! WRITE(20,*) '>> Sending: rank,irank,tag2=', & ! rank,irank,(tag2+14) CALL MPI_SEND(sysA(tsA)%sys(ic)%grp,1,MPI_INTEGER,& - 0,(tag2+14),MPI_COMM_WORLD,IERR) + 0,(tag2+14),MPI_COMM_WW3,IERR) END IF IF (rank.EQ.0) THEN ! WRITE(20,*) '<< Receiving: rank,irank,tag2=', & ! rank,irank,(tag2+14) CALL MPI_RECV(sysA(tsA)%sys(ic)%grp,1,MPI_INTEGER,& - irank,(tag2+14),MPI_COMM_WORLD,MPI_STATUS,IERR) + irank,(tag2+14),MPI_COMM_WW3,MPI_STATUS,IERR) END IF END DO END IF END IF END DO - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) ! CALL MPI_TYPE_FREE(MPI_INT_DOMARR,IERR) ! CALL MPI_TYPE_FREE(MPI_REAL_DOMARR,IERR) From a285a6af03c60b0d06ce603e6ce5a98f5607bdd0 Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Tue, 16 Jul 2024 14:33:10 -0700 Subject: [PATCH 10/16] Add to ww3_shel --- model/src/ww3_shel.F90 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/model/src/ww3_shel.F90 b/model/src/ww3_shel.F90 index 4bb888b48..0f73ca337 100644 --- a/model/src/ww3_shel.F90 +++ b/model/src/ww3_shel.F90 @@ -308,6 +308,9 @@ PROGRAM W3SHEL #ifdef W3_OMPG USE OMP_LIB #endif + + USE MPICOMM + IMPLICIT NONE ! #ifdef W3_MPI @@ -487,7 +490,7 @@ PROGRAM W3SHEL #endif #ifdef W3_MPI - MPI_COMM = MPI_COMM_WORLD + MPI_COMM = MPI_COMM_WW3 #endif #ifdef W3_OASIS END IF From 9f34f74d731a0b442d6234bf059dc64ced82b74f Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Tue, 16 Jul 2024 14:42:00 -0700 Subject: [PATCH 11/16] Move world setup --- model/src/w3initmd.F90 | 2 -- model/src/ww3_shel.F90 | 6 ++++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/model/src/w3initmd.F90 b/model/src/w3initmd.F90 index b906dd18f..6247fe176 100644 --- a/model/src/w3initmd.F90 +++ b/model/src/w3initmd.F90 @@ -563,8 +563,6 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD, #endif ! #ifdef W3_MPI - MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 - MPI_COMM_WAVE = MPI_COMM CALL MPI_COMM_SIZE ( MPI_COMM_WAVE, NTPROC, IERR_MPI ) NAPROC = NTPROC diff --git a/model/src/ww3_shel.F90 b/model/src/ww3_shel.F90 index 0f73ca337..f18aff007 100644 --- a/model/src/ww3_shel.F90 +++ b/model/src/ww3_shel.F90 @@ -308,9 +308,9 @@ PROGRAM W3SHEL #ifdef W3_OMPG USE OMP_LIB #endif - +#ifdef W3_MPI USE MPICOMM - +#endif IMPLICIT NONE ! #ifdef W3_MPI @@ -490,6 +490,8 @@ PROGRAM W3SHEL #endif #ifdef W3_MPI + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + MPI_COMM = MPI_COMM_WW3 #endif #ifdef W3_OASIS From 863f97e162c85fad317d6cb84b50bac6ce2a608d Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Tue, 16 Jul 2024 14:55:15 -0700 Subject: [PATCH 12/16] Add all back and tweak multi similar to shel --- model/src/ww3_multi.F90 | 10 ++++++-- model/src/ww3_ounp.F90 | 8 ++++-- model/src/ww3_prnc.F90 | 14 +++++++---- model/src/ww3_prtide.F90 | 20 +++++++++------ model/src/ww3_sbs1.F90 | 8 ++++-- model/src/ww3_strt.F90 | 8 ++++-- model/src/ww3_systrk.F90 | 54 +++++++++++++++++++++------------------- 7 files changed, 76 insertions(+), 46 deletions(-) diff --git a/model/src/ww3_multi.F90 b/model/src/ww3_multi.F90 index e3101b7c2..046e8ba82 100644 --- a/model/src/ww3_multi.F90 +++ b/model/src/ww3_multi.F90 @@ -95,6 +95,10 @@ PROGRAM W3MLTI USE OMP_LIB #endif !/ +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -119,7 +123,7 @@ PROGRAM W3MLTI ! 0. Initialization necessary for driver ! 0.a General I/O: all can start with initialization in wmmdatmd ! - ! 0.b MPI environment: Here, we use MPI_COMM_WORLD + ! 0.b MPI environment: Here, we use MPI_COMM_WW3 ! #ifdef W3_OMPH FLHYBR = .TRUE. @@ -135,7 +139,9 @@ PROGRAM W3MLTI ENDIF #endif #ifdef W3_MPI - MPI_COMM = MPI_COMM_WORLD + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + + MPI_COMM = MPI_COMM_WW3 CALL MPI_COMM_SIZE ( MPI_COMM, NMPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM, IMPROC, IERR_MPI ) IMPROC = IMPROC + 1 diff --git a/model/src/ww3_ounp.F90 b/model/src/ww3_ounp.F90 index a1533c73b..aa70f42d0 100644 --- a/model/src/ww3_ounp.F90 +++ b/model/src/ww3_ounp.F90 @@ -213,6 +213,10 @@ PROGRAM W3OUNP USE W3NMLOUNPMD USE NETCDF ! +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -355,8 +359,8 @@ PROGRAM W3OUNP ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! diff --git a/model/src/ww3_prnc.F90 b/model/src/ww3_prnc.F90 index e107a1f43..ee347659e 100644 --- a/model/src/ww3_prnc.F90 +++ b/model/src/ww3_prnc.F90 @@ -225,6 +225,10 @@ PROGRAM W3PRNC USE W3NMLPRNCMD USE NETCDF ! +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -440,8 +444,8 @@ PROGRAM W3PRNC ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! @@ -1696,18 +1700,18 @@ PROGRAM W3PRNC IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(TIDE1DL, SLICE * TIDE_MF * NFIELDS * 2, MPI_REAL, & TIDE1D, NELEM * TIDE_MF * NFIELDS * 2, CUMUL * TIDE_MF * NFIELDS * 2, & - MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) #endif #ifdef W3_MPI IF (IAPROC.EQ.NAPOUT) THEN CALL MPI_GATHERV(MPI_IN_PLACE,NELEM(IAPROC), & MPI_LOGICAL, TIDALCOMP(:,IY), NELEM, CUMUL, MPI_LOGICAL, NAPOUT-1, & - MPI_COMM_WORLD, IERR_MPI) + MPI_COMM_WW3, IERR_MPI) ELSE CALL MPI_GATHERV(TIDALCOMP(CUMUL(IAPROC)+1:CUMUL(IAPROC)+NELEM(IAPROC),IY),NELEM(IAPROC), & MPI_LOGICAL, TIDALCOMP(:,IY), NELEM, CUMUL, MPI_LOGICAL, NAPOUT-1, & - MPI_COMM_WORLD, IERR_MPI) + MPI_COMM_WW3, IERR_MPI) END IF #endif diff --git a/model/src/ww3_prtide.F90 b/model/src/ww3_prtide.F90 index 1168cd1ab..0da03424a 100644 --- a/model/src/ww3_prtide.F90 +++ b/model/src/ww3_prtide.F90 @@ -139,6 +139,10 @@ PROGRAM W3PRTIDE USE W3TIDEMD USE W3IDATMD ! +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -254,8 +258,8 @@ PROGRAM W3PRTIDE ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC #endif ! @@ -673,11 +677,11 @@ PROGRAM W3PRTIDE #ifdef W3_MPI IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(FX1DL, SLICE, MPI_REAL, FX1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FY1DL, SLICE, MPI_REAL, FY1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FA1DL, SLICE, MPI_REAL, FA1D, NELEM, & - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) ELSE FX1D = FX1DL FY1D = FY1DL @@ -768,11 +772,11 @@ PROGRAM W3PRTIDE #ifdef W3_MPI IF (NAPROC.GT.1) THEN CALL MPI_GATHERV(FX1DL, SLICE, MPI_REAL, FX1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FY1DL, SLICE, MPI_REAL, FY1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) CALL MPI_GATHERV(FA1DL, SLICE, MPI_REAL, FA1D, NELEM,& - CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WORLD, IERR_MPI) + CUMUL, MPI_REAL, NAPOUT-1, MPI_COMM_WW3, IERR_MPI) ELSE FX1D = FX1DL FY1D = FY1DL diff --git a/model/src/ww3_sbs1.F90 b/model/src/ww3_sbs1.F90 index 94778b10d..8da309c62 100644 --- a/model/src/ww3_sbs1.F90 +++ b/model/src/ww3_sbs1.F90 @@ -177,6 +177,10 @@ PROGRAM W3SBS1 USE WMMDATMD, ONLY: MDSE, MDST, MDSS, NMPROC, IMPROC, NMPSCR, & NRGRD, STIME, ETIME !/ +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -203,11 +207,11 @@ PROGRAM W3SBS1 ! 0. Initialization necessary for driver ! 0.a General I/O: all can start with initialization in wmmdatmd ! - ! 0.b MPI environment: Here, we use MPI_COMM_WORLD + ! 0.b MPI environment: Here, we use MPI_COMM_WW3 ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - MPI_COMM = MPI_COMM_WORLD + MPI_COMM = MPI_COMM_WW3 CALL MPI_COMM_SIZE ( MPI_COMM, NMPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM, IMPROC, IERR_MPI ) IMPROC = IMPROC + 1 diff --git a/model/src/ww3_strt.F90 b/model/src/ww3_strt.F90 index b4bcad3ca..9202792da 100644 --- a/model/src/ww3_strt.F90 +++ b/model/src/ww3_strt.F90 @@ -241,6 +241,10 @@ PROGRAM W3STRT USE W3IDATMD, ONLY: W3NINP #endif !/ +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE ! #ifdef W3_MPI @@ -324,8 +328,8 @@ PROGRAM W3STRT ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) - CALL MPI_COMM_SIZE ( MPI_COMM_WORLD, NAPROC, IERR_MPI ) - CALL MPI_COMM_RANK ( MPI_COMM_WORLD, IAPROC, IERR_MPI ) + CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) + CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 #endif ! diff --git a/model/src/ww3_systrk.F90 b/model/src/ww3_systrk.F90 index cb25a1e72..7f63a2e33 100644 --- a/model/src/ww3_systrk.F90 +++ b/model/src/ww3_systrk.F90 @@ -58,6 +58,10 @@ PROGRAM WW3_SYSTRK !/ USE W3STRKMD USE W3TIMEMD, ONLY: TDIFF +#ifdef W3_MPI + USE MPICOMM +#endif + ! IMPLICIT NONE #ifdef W3_MPI @@ -168,8 +172,8 @@ PROGRAM WW3_SYSTRK ! Start of parallel region CALL MPI_INIT(ierr) - CALL MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr) - CALL MPI_COMM_SIZE(MPI_COMM_WORLD, nproc, ierr) + CALL MPI_COMM_RANK(MPI_COMM_WW3, rank, ierr) + CALL MPI_COMM_SIZE(MPI_COMM_WW3, nproc, ierr) #endif ! Open log file @@ -343,34 +347,34 @@ PROGRAM WW3_SYSTRK #ifdef W3_MPI ! MPI communication block - CALL MPI_BCAST(filename,80,MPI_CHARACTER,0,MPI_COMM_WORLD,IERR) + CALL MPI_BCAST(filename,80,MPI_CHARACTER,0,MPI_COMM_WW3,IERR) CALL MPI_BCAST(tstart,1,MPI_DOUBLE_PRECISION,0, & - MPI_COMM_WORLD,IERR) + MPI_COMM_WW3,IERR) CALL MPI_BCAST(tend,1,MPI_DOUBLE_PRECISION,0, & - MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(dt,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(ntint,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(minlon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxlon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(minlat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(maxlat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(mxcwt,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(mycwt,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(dirKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(perKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(hsKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(wetPts,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(dirTimeKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(tpTimeKnob,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(seedLon,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(seedLat,1,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(noutp,1,MPI_INTEGER,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(lonout,100,MPI_REAL,0,MPI_COMM_WORLD,IERR) - CALL MPI_BCAST(latout,100,MPI_REAL,0,MPI_COMM_WORLD,IERR) + MPI_COMM_WW3,IERR) + CALL MPI_BCAST(dt,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(ntint,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(minlon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxlon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(minlat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(maxlat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(mxcwt,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(mycwt,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(dirKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(perKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(hsKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(wetPts,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(dirTimeKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(tpTimeKnob,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(seedLon,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(seedLat,1,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(noutp,1,MPI_INTEGER,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(lonout,100,MPI_REAL,0,MPI_COMM_WW3,IERR) + CALL MPI_BCAST(latout,100,MPI_REAL,0,MPI_COMM_WW3,IERR) #endif #ifdef W3_MPI - CALL MPI_Barrier(MPI_COMM_WORLD,IERR) + CALL MPI_Barrier(MPI_COMM_WW3,IERR) #endif CALL waveTracking_NWS_V2 (intype ,tmax , & From ee61569be43ccb81c62461941cf4b030bb768f25 Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Tue, 16 Jul 2024 15:07:51 -0700 Subject: [PATCH 13/16] Tweak other ww3 similar to shel --- model/src/ww3_ounp.F90 | 1 + model/src/ww3_prnc.F90 | 1 + model/src/ww3_prtide.F90 | 1 + model/src/ww3_strt.F90 | 1 + model/src/ww3_systrk.F90 | 2 ++ 5 files changed, 6 insertions(+) diff --git a/model/src/ww3_ounp.F90 b/model/src/ww3_ounp.F90 index aa70f42d0..d673775ab 100644 --- a/model/src/ww3_ounp.F90 +++ b/model/src/ww3_ounp.F90 @@ -359,6 +359,7 @@ PROGRAM W3OUNP ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC diff --git a/model/src/ww3_prnc.F90 b/model/src/ww3_prnc.F90 index ee347659e..2646e0ea4 100644 --- a/model/src/ww3_prnc.F90 +++ b/model/src/ww3_prnc.F90 @@ -444,6 +444,7 @@ PROGRAM W3PRNC ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC diff --git a/model/src/ww3_prtide.F90 b/model/src/ww3_prtide.F90 index 0da03424a..65f68569c 100644 --- a/model/src/ww3_prtide.F90 +++ b/model/src/ww3_prtide.F90 @@ -258,6 +258,7 @@ PROGRAM W3PRTIDE ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 ! this is to have IAPROC between 1 and NAPROC diff --git a/model/src/ww3_strt.F90 b/model/src/ww3_strt.F90 index 9202792da..9f22c49f3 100644 --- a/model/src/ww3_strt.F90 +++ b/model/src/ww3_strt.F90 @@ -328,6 +328,7 @@ PROGRAM W3STRT ! #ifdef W3_MPI CALL MPI_INIT ( IERR_MPI ) + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 CALL MPI_COMM_SIZE ( MPI_COMM_WW3, NAPROC, IERR_MPI ) CALL MPI_COMM_RANK ( MPI_COMM_WW3, IAPROC, IERR_MPI ) IAPROC = IAPROC + 1 diff --git a/model/src/ww3_systrk.F90 b/model/src/ww3_systrk.F90 index 7f63a2e33..5ba131eb0 100644 --- a/model/src/ww3_systrk.F90 +++ b/model/src/ww3_systrk.F90 @@ -172,6 +172,8 @@ PROGRAM WW3_SYSTRK ! Start of parallel region CALL MPI_INIT(ierr) + MPI_COMM_WW3=MPI_COMM_WORLD !< MPI_COMM_WW3 + CALL MPI_COMM_RANK(MPI_COMM_WW3, rank, ierr) CALL MPI_COMM_SIZE(MPI_COMM_WW3, nproc, ierr) From c9bd9064fb34daea77d2e8a3c7840af9c388f2d6 Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Fri, 19 Jul 2024 14:53:01 -0700 Subject: [PATCH 14/16] Minor header updates --- model/src/mpicomm.F90 | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/model/src/mpicomm.F90 b/model/src/mpicomm.F90 index 2c2ae7562..7c321fe0f 100644 --- a/model/src/mpicomm.F90 +++ b/model/src/mpicomm.F90 @@ -8,7 +8,7 @@ !> !> @brief Define some mpi constants for global use !> -!> @author H. L. Tolman @date 05-Jun-2018 +!> @author J. M. Sexton @date 19-Jul-2024 !> ! #ifndef ENDIANNESS @@ -22,19 +22,12 @@ MODULE MPICOMM !/ | WAVEWATCH III NOAA/NCEP | !/ | H. L. Tolman | !/ | FORTRAN 90 | - !/ | Last update : 05-Jun-2018 | + !/ | Last update : 19-Jul-2024 | !/ +-----------------------------------+ !/ - !/ 11-Nov-1999 : Fortran 90 version. ( version 2.00 ) - !/ 29-May-2009 : Preparing distribution version. ( version 3.14 ) - !/ 25-Jun-2011 : Adding Kelvin functions. ( version 4.05 ) - !/ 03-Sep-2012 : Adding TSTOUT flag. ( version 4.10 ) - !/ 28-Feb-2013 : Adding cap at 0.5 in FWTABLE ( version 4.08 ) - !/ 20-Jan-2017 : Add parameters for ESMF ( version 6.02 ) - !/ 01-Mar-2018 : Add UNDEF parameter ( version 6.02 ) - !/ 05-Jun-2018 : Add PDLIB parameters ( version 6.04 ) + !/ 19-Jul-2024 : Add MPI SubCommunicator variable ( version 7.xx ) !/ - !/ Copyright 2009-2012 National Weather Service (NWS), + !/ Copyright 2009-2024 National Weather Service (NWS), !/ National Oceanic and Atmospheric Administration. All rights !/ reserved. WAVEWATCH III is a trademark of the NWS. !/ No unauthorized use without permission. @@ -45,9 +38,10 @@ MODULE MPICOMM ! ! 2. Variables and types : ! - ! Name Type Scope Description + ! Name Type Scope Description ! ---------------------------------------------------------------- - ! UNDEF Real Global Value for undefined variable in output + ! MPI_COMM_WW3 Int Global Value for mpi (sub)communicator for WW3 + ! IS_EXTERNAL_COMPONENT Bool Global General logical similar to IS_ESMF_COMPONENT ! ---------------------------------------------------------------- !/ ------------------------------------------------------------------- / !/ From cf3d25c1c53fc859ffa357f3686885a9502dc730 Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Tue, 23 Jul 2024 10:23:51 -0700 Subject: [PATCH 15/16] Revert unneeded changes and remove unneeded comment --- .github/workflows/regtest_gnu.yml | 6 +++--- model/src/mpicomm.F90 | 6 ------ regtests/bin/matrix_cmake_datarmor | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/.github/workflows/regtest_gnu.yml b/.github/workflows/regtest_gnu.yml index 82a6c2b2e..d5b71673c 100644 --- a/.github/workflows/regtest_gnu.yml +++ b/.github/workflows/regtest_gnu.yml @@ -102,10 +102,10 @@ jobs: export CC=mpicc export FC=mpif90 export OASISDIR=${GITHUB_WORKSPACE}/work_oasis3-mct - mkdir build && cd build + # mkdir build && cd build export LD_LIBRARY_PATH="/home/runner/work/WW3/WW3/spack/var/spack/environments/ww3-gnu/.spack-env/view/:$LD_LIBRARY_PATH" - cmake -DSWITCH=${GITHUB_WORKSPACE}/ww3/regtests/unittests/data/switch.io -DCMAKE_BUILD_TYPE=Debug .. - make -j2 VERBOSE=1 + # cmake -DSWITCH=${GITHUB_WORKSPACE}/ww3/regtests/unittests/data/switch.io -DCMAKE_BUILD_TYPE=Debug .. + # make -j2 VERBOSE=1 cd ${GITHUB_WORKSPACE}/ww3 ls -l ${GITHUB_WORKSPACE}/ww3/model/bin/ww3_from_ftp.sh -k diff --git a/model/src/mpicomm.F90 b/model/src/mpicomm.F90 index 7c321fe0f..3132dece4 100644 --- a/model/src/mpicomm.F90 +++ b/model/src/mpicomm.F90 @@ -11,9 +11,6 @@ !> @author J. M. Sexton @date 19-Jul-2024 !> ! -#ifndef ENDIANNESS -#define ENDIANNESS "native" -#endif ! !/ ------------------------------------------------------------------- / MODULE MPICOMM @@ -46,9 +43,6 @@ MODULE MPICOMM !/ ------------------------------------------------------------------- / !/ ! -!#ifdef W3_MPI -! INCLUDE "mpif.h" -!#endif INTEGER :: MPI_COMM_WW3=0 !< MPI_COMM_WW3 ! diff --git a/regtests/bin/matrix_cmake_datarmor b/regtests/bin/matrix_cmake_datarmor index 21c97bbcd..16c31e47c 100755 --- a/regtests/bin/matrix_cmake_datarmor +++ b/regtests/bin/matrix_cmake_datarmor @@ -46,7 +46,7 @@ fi #convert main_dir to absolute path main_dir="`cd $main_dir 1>/dev/null 2>&1 && pwd`" - cmplr=datarmor_intel_debug + cmplr=datarmor_intel From 18a9e514aff6942d75e00dd0416c3b179a483e26 Mon Sep 17 00:00:00 2001 From: "Jean M. Sexton" Date: Tue, 23 Jul 2024 10:25:08 -0700 Subject: [PATCH 16/16] Change to using ONLY --- model/src/w3initmd.F90 | 2 +- model/src/w3servmd.F90 | 2 +- model/src/w3strkmd.F90 | 2 +- model/src/ww3_multi.F90 | 2 +- model/src/ww3_ounp.F90 | 2 +- model/src/ww3_prnc.F90 | 2 +- model/src/ww3_prtide.F90 | 2 +- model/src/ww3_sbs1.F90 | 2 +- model/src/ww3_shel.F90 | 2 +- model/src/ww3_strt.F90 | 2 +- model/src/ww3_systrk.F90 | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/model/src/w3initmd.F90 b/model/src/w3initmd.F90 index 6247fe176..f6e7676b5 100644 --- a/model/src/w3initmd.F90 +++ b/model/src/w3initmd.F90 @@ -410,7 +410,7 @@ SUBROUTINE W3INIT ( IMOD, IsMulti, FEXT, MDS, MTRACE, ODAT, FLGRD, FLGR2, FLGD, UA, UD, U10, U10D, AS #ifdef W3_MPI USE W3ADATMD, ONLY: MPI_COMM_WAVE, MPI_COMM_WCMP - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif USE W3IDATMD, ONLY: FLLEV, FLCUR, FLWIND, FLICE, FLTAUA, FLRHOA,& FLMDN, FLMTH, FLMVS, FLIC1, FLIC2, FLIC3, & diff --git a/model/src/w3servmd.F90 b/model/src/w3servmd.F90 index 70dcc061f..3ae73bcef 100644 --- a/model/src/w3servmd.F90 +++ b/model/src/w3servmd.F90 @@ -80,7 +80,7 @@ MODULE W3SERVMD ! module default #ifdef W3_MPI - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif ! implicit none diff --git a/model/src/w3strkmd.F90 b/model/src/w3strkmd.F90 index 212e14ba5..d395419fc 100644 --- a/model/src/w3strkmd.F90 +++ b/model/src/w3strkmd.F90 @@ -29,7 +29,7 @@ MODULE W3STRKMD !/ use constants, only: file_endian #ifdef W3_MPI - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif ! IMPLICIT NONE diff --git a/model/src/ww3_multi.F90 b/model/src/ww3_multi.F90 index 046e8ba82..14e257aa6 100644 --- a/model/src/ww3_multi.F90 +++ b/model/src/ww3_multi.F90 @@ -96,7 +96,7 @@ PROGRAM W3MLTI #endif !/ #ifdef W3_MPI - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif ! IMPLICIT NONE diff --git a/model/src/ww3_ounp.F90 b/model/src/ww3_ounp.F90 index d673775ab..bf1d55883 100644 --- a/model/src/ww3_ounp.F90 +++ b/model/src/ww3_ounp.F90 @@ -214,7 +214,7 @@ PROGRAM W3OUNP USE NETCDF ! #ifdef W3_MPI - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif ! IMPLICIT NONE diff --git a/model/src/ww3_prnc.F90 b/model/src/ww3_prnc.F90 index 2646e0ea4..78bb0175c 100644 --- a/model/src/ww3_prnc.F90 +++ b/model/src/ww3_prnc.F90 @@ -226,7 +226,7 @@ PROGRAM W3PRNC USE NETCDF ! #ifdef W3_MPI - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif ! IMPLICIT NONE diff --git a/model/src/ww3_prtide.F90 b/model/src/ww3_prtide.F90 index 65f68569c..dfea89471 100644 --- a/model/src/ww3_prtide.F90 +++ b/model/src/ww3_prtide.F90 @@ -140,7 +140,7 @@ PROGRAM W3PRTIDE USE W3IDATMD ! #ifdef W3_MPI - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif ! IMPLICIT NONE diff --git a/model/src/ww3_sbs1.F90 b/model/src/ww3_sbs1.F90 index 8da309c62..1ca5571a9 100644 --- a/model/src/ww3_sbs1.F90 +++ b/model/src/ww3_sbs1.F90 @@ -178,7 +178,7 @@ PROGRAM W3SBS1 NRGRD, STIME, ETIME !/ #ifdef W3_MPI - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif ! IMPLICIT NONE diff --git a/model/src/ww3_shel.F90 b/model/src/ww3_shel.F90 index f18aff007..de14d2b82 100644 --- a/model/src/ww3_shel.F90 +++ b/model/src/ww3_shel.F90 @@ -309,7 +309,7 @@ PROGRAM W3SHEL USE OMP_LIB #endif #ifdef W3_MPI - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif IMPLICIT NONE ! diff --git a/model/src/ww3_strt.F90 b/model/src/ww3_strt.F90 index 9f22c49f3..5e5b5ca68 100644 --- a/model/src/ww3_strt.F90 +++ b/model/src/ww3_strt.F90 @@ -242,7 +242,7 @@ PROGRAM W3STRT #endif !/ #ifdef W3_MPI - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif ! IMPLICIT NONE diff --git a/model/src/ww3_systrk.F90 b/model/src/ww3_systrk.F90 index 5ba131eb0..576977f3a 100644 --- a/model/src/ww3_systrk.F90 +++ b/model/src/ww3_systrk.F90 @@ -59,7 +59,7 @@ PROGRAM WW3_SYSTRK USE W3STRKMD USE W3TIMEMD, ONLY: TDIFF #ifdef W3_MPI - USE MPICOMM + USE MPICOMM, ONLY: MPI_COMM_WW3 #endif ! IMPLICIT NONE