Merge branch 'develop' into develop

doc/Sphinx/Use/installation.rst

@@ -143,6 +143,47 @@ executed before compilation. If you successfully write such a file for
 a common supercomputer, please share it with developpers so that it can
 be included in the next release of :program:`Smilei`.
 
+
+.. rubric:: Compilation for GPU accelerated nodes:
+
+As each supercomputer has a different environnment to compile for GPUs and since the nvhpc + CUDA/ cray + HIP modules evolve quickly, a machine file is required for the compilation.
+Several machine files are already available as an example in smilei/scripts/compile_tools/machine/ ; such as: jean_zay_gpu_V100, jean_zay_gpu_A100, adastra, ruche_gpu2.
+
+Typically we need it to specify ACCELERATOR_GPU_FLAGS += -ta=tesla:cc80 for nvhpc <23.4 and ACCELERATOR_GPU_FLAGS += -gpu=cc80 -acc for the more recent versions of nvhpc.
+
+.. code-block:: bash
+
+	make -j 12 machine="jean_zay_gpu_A100" config="gpu_nvidia noopenmp verbose" # for Nvidia GPU
+	make -j 12 machine="adastra" config="gpu_amd" 			            # for AMD GPU
+
+
+Furthermore, here are 2 examples of known working ennvironments, first for AMD GPUs, second for Nvidia GPUs:
+
+.. code-block:: bash
+
+	module purge
+	module load craype-accel-amd-gfx90a craype-x86-trento
+	module load PrgEnv-cray/8.3.3
+	module load cpe/23.02
+	module load cray-mpich/8.1.24 cray-hdf5-parallel/1.12.2.1 cray-python/3.9.13.1
+	module load amd-mixed/5.2.3
+
+.. code-block:: bash
+
+	module purge
+	module load anaconda-py3/2020.11  # python is fine as well if you can pip install the required modules
+	module load nvidia-compilers/23.1
+	module load cuda/11.2
+	module load openmpi/4.1.1-cuda
+	module load hdf5/1.12.0-mpi-cuda
+	# For HDF5, note that module show can give you the right path
+	export HDF5_ROOT_DIR=/DIRECTORY_NAME/hdf5/1.12.0/pgi-20.4-HASH/
+
+Note: 
+
+* we are aware of issues with CUDA >12.0, fixes are being tested but are not deployed yet. We recommend CUDA 11.x at the moment.
+* The hdf5 module should be compiled with the nvidia/cray compiler ; openmpi as well, but depending on the nvhpc module it might not be needed as it can be included in the nvhpc module 
+
 ----
 
 .. _vectorization_flags:

doc/Sphinx/Use/run.rst

@@ -141,10 +141,14 @@ With AMD GPUs using cray on Adastra:
 
 For the binding scripts themselves, as it depends completely on the node
 architecture, please contact your admin support team.
+
+Binding script for adastra can be found here: https://github.com/SmileiPIC/Smilei/issues/672#issuecomment-1820677606 with the example of a slurm script.
+it can be used as a template for other AMD GPUs based supercomputers/clusters.
 
 Be aware that GPU support is in development and not all features are currently available.
 Please refer to the list of current supported features.
 
+
 ----
 
 Debugging

happi/_Diagnostics/Diagnostic.py

@@ -1,5 +1,7 @@
 from .._Utils import *
 
+from matplotlib import ticker
+
 class Diagnostic(object):
 	"""Mother class for all Diagnostics.
 	To create a diagnostic, refer to the doc of the SmileiSimulation class.
@@ -832,23 +834,18 @@ def _setAxesOptions(self, ax):
 		for option, value in self.options.labels.items():
 			getattr(ax, "set_"+option)( value, self.options.labels_font[option] )
 		# Ticklabels + fonts
-		for option, value in self.options.ticklabels_font.items():
-			if option in self.options.ticklabels:
-				getattr(ax, "set_"+option)( value, self.options.ticklabels_font[option] )
-			else: # manage tick label fonts even when not setting tick labels first
-				ticklabels = getattr(ax, "get_"+option)()
-				self._plt.setp(ticklabels, **self.options.ticklabels_font[option])
+		for tl in ["xticklabels", "yticklabels"]:
+			font = self.options.ticklabels_font[tl] if tl in self.options.ticklabels_font else {}
+			if tl in self.options.ticklabels:
+				getattr(ax, "set_"+tl)( self.options.ticklabels[tl], fontdict=font )
+			elif font: # manage tick label fonts even when not setting tick labels first
+				ticklabels = getattr(ax, "get_"+tl)()
+				self._plt.setp(ticklabels, **font)
 		# Tick formatting
-		try:
-			if self.options.xtick: ax.ticklabel_format(axis="x",**self.options.xtick)
-		except Exception as e:
-			if self._verbose: print("Cannot format x ticks (typically happens with log-scale)")
-			self.options.xtick = []
-		try:
-			if self.options.ytick: ax.ticklabel_format(axis="y",**self.options.ytick)
-		except Exception as e:
-			if self._verbose: print("Cannot format y ticks (typically happens with log-scale)")
-			self.options.ytick = []
+		if type(ax.xaxis.get_major_formatter()) == ticker.ScalarFormatter:
+			ax.ticklabel_format(axis="x",**self.options.xtick)
+		if type(ax.yaxis.get_major_formatter()) == ticker.ScalarFormatter:
+			ax.ticklabel_format(axis="y",**self.options.ytick)
 	def _setColorbarOptions(self, ax):
 		# Colorbar tick font
 		if self.options.colorbar_font:

happi/_Utils.py

@@ -195,7 +195,7 @@ def set(self, **kwargs):
 				kw = kwa[:-5]
 				self.labels_font[kw] = val
 			elif kwa in ["xticklabels", "yticklabels"]:
-				self.ticklabels[kw] = val
+				self.ticklabels[kwa] = val
 			elif kwa in ["xticklabels_font", "yticklabels_font"]:
 				kw = kwa[:-5]
 				self.ticklabels_font[kw] = val

makefile

@@ -532,9 +532,9 @@ help:
 	@echo '    inspector                    : to compile for Intel Inspector analysis'
 	@echo '    gpu_nvidia                   : to compile for GPU (uses OpenACC)'
 	@echo '    gpu_amd                      : to compile for GPU (uses OpenMP)'
-	@echo '    omptasks                     : to compile with OpenMP tasks'
-	@echo '    part_event_tracing_tasks_on  : to compile particle event tracing and OpenMP tasks'
-	@echo '    part_event_tracing_tasks_off : to compile particle event tracing without OpenMP tasks'
+#	@echo '    omptasks                     : to compile with OpenMP tasks'
+#	@echo '    part_event_tracing_tasks_on  : to compile particle event tracing and OpenMP tasks'
+#	@echo '    part_event_tracing_tasks_off : to compile particle event tracing without OpenMP tasks'
 #	@echo '    omptasks                     : to compile with OpenMP tasks'
 #	@echo '    part_event_tracing_tasks_on  : to compile particle event tracing and OpenMP tasks'
 #	@echo '    part_event_tracing_tasks_off : to compile particle event tracing without OpenMP tasks'

src/ElectroMagnBC/ElectroMagnBCAM_BM.cpp

@@ -188,23 +188,13 @@ void ElectroMagnBCAM_BM::apply( ElectroMagn *EMfields, double, Patch *patch )
 
             unsigned int j= n_d[1]-2;
 
-
-            //   MESSAGE("JGLOB "<< patch->getCellStartingGlobalIndex(1)+j);
-            //std::cout<<"come heree "<<patch->getCellStartingGlobalIndex(1)<<"  "<<j<<" \n " ;
-            //std::cout<<"come here "<<n_p[1] <<" nr*d[1] "<<n_p[1]*d[1]<<" \n " ;
             // for Bl^(p,d)
             for( unsigned int i=0 ; i<n_p[0]-1; i++ ) {
                 ( *Bl )( i, j+1 ) = ( *Bl_old )( i, j )
                                     -      Alpha_Bl_Rmax * ( ( *Bl )( i, j ) - ( *Bl_old )( i, j+1 ) )
                                     +      Gamma_Bl_Rmax * ( ( *Br )( i+1, j ) + ( *Br_old )( i+1, j ) - ( *Br )( i, j ) - ( *Br_old )( i, j ) )
                                     -      Beta_Bl_Rmax * Icpx * ( double )imode * ( ( *Er )( i, j+1 ) + ( *Er )( i, j ) )
                                     - 2. * Beta_Bl_Rmax * ( *Et )( i, j );
-                //if (std::abs((*Bl)(i,j+1))>1.){
-                //MESSAGE("BlBM");
-                //MESSAGE(i);
-                //MESSAGE(j+1);
-                //MESSAGE((*Bl)(i,j+1));
-                //}
             }//i  ---end Bl
 
             // for Bt^(d,d)
@@ -215,12 +205,6 @@ void ElectroMagnBCAM_BM::apply( ElectroMagn *EMfields, double, Patch *patch )
                                         + Gamma_Bt_Rmax * ( *Bt_old )( i, j )
                                         - Icpx * ( double )imode * CB_BM * Epsilon_Bt_Rmax  * ( ( *Br )( i, j ) + ( *Br_old )( i, j ) )
                                         - CE_BM * Delta_Bt_Rmax * ( ( *Er )( i, j+1 )+( *Er )( i, j )-( *Er )( i-1, j+1 ) -( *Er )( i-1, j ) ) ;
-                //if (std::abs((*Bt)(i,j+1))>1.){
-                //    MESSAGE("BtMF");
-                //    MESSAGE(i);
-                //    MESSAGE(j+1);
-                //    MESSAGE((*Bt)(i,j+1));
-                //}
             }//i  ---end Bt
         }
     }

src/ElectroMagnBC/ElectroMagnBCAM_SM.cpp

@@ -203,8 +203,8 @@ void ElectroMagnBCAM_SM::apply( ElectroMagn *EMfields, double time_dual, Patch *
                 unsigned int i=0;
                 ( *Br )( i, j ) = Alpha_Xmin   * ( *Et )( i, j )
                                   +              Beta_Xmin    * ( *Br )( i+1, j )
-                                  +              Gamma_Xmin   * byW;
-                +              Delta_Xmin   *( ( *Bl )( i, j+1 )- ( *Bl )( i, j ) );
+                                  +              Gamma_Xmin   * byW
+                                  +              Delta_Xmin   *( ( *Bl )( i, j+1 )- ( *Bl )( i, j ) );
             }//j  ---end compute Br
 
 
@@ -258,7 +258,7 @@ void ElectroMagnBCAM_SM::apply( ElectroMagn *EMfields, double time_dual, Patch *
                 ( *Br )( i, j ) = - Alpha_Xmax   * ( *Et )( i-1, j )
                                   +                   Beta_Xmax    * ( *Br )( i-1, j )
                                   +                   Gamma_Xmax   * byE
-                                  +                   Delta_Xmax   * ( ( *Bl )( i-1, j+1 )- ( *Bl )( i-1, j ) ); // Check x-index
+                                  -                   Delta_Xmax   * ( ( *Bl )( i-1, j+1 )- ( *Bl )( i-1, j ) ); // Check x-index
 
             }//j  ---end compute Br
 

src/Patch/VectorPatch.cpp

@@ -339,10 +339,6 @@ void VectorPatch::dynamics( Params &params,
 
     timers.particles.restart();
     ostringstream t;
-#ifdef _PARTEVENTTRACING
-    bool diag_PartEventTracing {false};
-    double reference_time;
-#endif
 
 #ifdef _OMPTASKS
     #pragma omp single
@@ -362,7 +358,7 @@ void VectorPatch::dynamics( Params &params,
     bool diag_PartEventTracing {false};
     if( !params.Laser_Envelope_model ) {
         diag_PartEventTracing = smpi->diagPartEventTracing( time_dual, params.timestep);
-        if (diag_PartEventTracing) smpi->reference_time = MPI_Wtime();
+        if (diag_PartEventTracing) smpi->reference_time_ = MPI_Wtime();
     }
 #endif
 
@@ -4559,7 +4555,7 @@ void VectorPatch::ponderomotiveUpdateSusceptibilityAndMomentum( Params &params,
 
 #  ifdef _PARTEVENTTRACING
     bool diag_PartEventTracing = smpi->diagPartEventTracing( time_dual, params.timestep);
-    if (diag_PartEventTracing) smpi->reference_time = MPI_Wtime();
+    if (diag_PartEventTracing) smpi->reference_time_ = MPI_Wtime();
 #  endif
 
 
@@ -4601,7 +4597,7 @@ void VectorPatch::ponderomotiveUpdatePositionAndCurrents( Params &params,
 #ifdef _PARTEVENTTRACING
     bool diag_PartEventTracing {false};
     diag_PartEventTracing = smpi->diagPartEventTracing( time_dual, params.timestep);
-    // if (diag_PartEventTracing) smpi->reference_time = MPI_Wtime();
+    // if (diag_PartEventTracing) smpi->reference_time_ = MPI_Wtime();
 #endif
 
 #ifdef _OMPTASKS