Merge commit 'd5eadb44ad81b974c52b9d61a9473903c57f33a8' into develop

.gitlab-ci.yml

@@ -8,14 +8,14 @@ stages:
     - compile_debug
     - compile_no_mpi_threadmultiple
     - compile_no_openmp
-    - compile_omptasks
-    - run_omptasks
+#    - compile_omptasks
+#    - run_omptasks
 
 install:
   stage: install
   only:
     - develop
-
+ 
   script:
     # Force workdir cleaning in case of retried
     - echo "CI_PIPELINE_ID = " $CI_PIPELINE_ID
@@ -33,7 +33,7 @@ compile_default:
   stage: compile_default
   only:
     - develop
-
+ 
   script:
     # Move in test dir
     - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei/validation
@@ -44,7 +44,7 @@ runQuick:
   stage: run_quick
   only:
     - develop
-
+ 
   script:
     # Move in test dir
     - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei/validation
@@ -55,7 +55,7 @@ run1D:
   stage: run_default
   only:
     - develop
-
+ 
   script:
     # Move in test dir
     - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei/validation
@@ -67,7 +67,7 @@ run2D:
   stage: run_default
   only:
     - develop
-
+ 
   script:
     # Move in test dir
     - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei/validation
@@ -81,7 +81,7 @@ run3D:
   stage: run_default
   only:
     - develop
-
+ 
   script:
     # Move in test dir
     - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei/validation
@@ -96,7 +96,7 @@ runAM:
   stage: run_default
   only:
     - develop
-
+ 
   script:
     # Move in test dir
     - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei/validation
@@ -108,7 +108,7 @@ runCollisions:
   stage: run_default
   only:
     - develop
-
+ 
   script:
     # Move in test dir
     - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei/validation
@@ -164,21 +164,21 @@ compile_no_openmp:
     - make clean
     - python validation/validation.py -k noopenmp -c -v
 
-compile_omptasks:
-  stage: compile_omptasks
-  only:
-    - develop
-
-  script:
-    - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei
-    - make clean
-    - python validation/validation.py -k omptasks -c -v
-
-run_omptasks:
-  stage: run_omptasks
-  only:
-    - develop
-
-  script:
-    - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei/validation
-    - python validation.py -k omptasks -b "tst2d_tasks_01_radiation_pressure_acc.py" -m 4 -o 4 -n 1 -v
+#compile_omptasks:
+#  stage: compile_omptasks
+#  only:
+#    - develop
+#
+#  script:
+#    - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei
+#    - make clean
+#    - python validation/validation.py -k omptasks -c -v
+#
+#run_omptasks:
+#  stage: run_omptasks
+#  only:
+#    - develop
+#    
+#  script:
+#    - cd /sps3/gitlab-runner/$CI_PIPELINE_ID/smilei/validation
+#    - python validation.py -k omptasks -b "tst2d_tasks_01_radiation_pressure_acc.py" -m 4 -o 4 -n 1 -v

doc/Sphinx/Use/namelist.rst

@@ -1148,6 +1148,9 @@ Each species has to be defined in a ``Species`` block::
   :ref:`tracking <DiagTrackParticles>`. The available fields are ``"Ex"``, ``"Ey"``, ``"Ez"``, 
   ``"Bx"``, ``"By"`` and ``"Bz"``.
 
+  Note that magnetic field components, as they originate from the interpolator,
+  are shifted by half a timestep compared to those from the *Fields* diagnostics.
+
   Additionally, the work done by each component of the electric field is available as
   ``"Wx"``, ``"Wy"`` and ``"Wz"``. Contrary to the other interpolated fields, these quantities
   are accumulated over time.
@@ -2716,7 +2719,8 @@ or several points arranged in a 2-D or 3-D grid.
   * **In "AMcylindrical" geometry**, probes are defined with 3D Cartesian coordinates
     and cannot be separated per mode. Use Field diagnostics for cylindrical coordinates and
     information per mode.
-
+  * **Probes rely on the particle interpolator to compute fields** so that the
+    magnetic field is shifted by half a timestep compared to that of *Fields* diagnostics.
 
 To add one probe diagnostic, include the block ``DiagProbe``::
 
@@ -3343,19 +3347,20 @@ for instance::
     def my_filter(particles):
         return (particles.px>-1.)*(particles.px<1.) + (particles.pz>3.)
 
-.. Warning:: The ``px``, ``py`` and ``pz`` quantities are not exactly the momenta.
-  They are actually the velocities multiplied by the lorentz factor, i.e.,
-  :math:`\gamma v_x`, :math:`\gamma v_y` and :math:`\gamma v_z`. This is true only
-  inside the ``filter`` function (not for the output of the diagnostic).
-
-.. Note:: The ``id`` attribute contains the :doc:`particles identification number<ids>`.
-  This number is set to 0 at the beginning of the simulation. **Only after particles have
-  passed the filter**, they acquire a positive ``id``.
-
-.. Note:: For advanced filtration, Smilei provides the quantity ``Main.iteration``,
-  accessible within the ``filter`` function. Its value is always equal to the current
-  iteration number of the PIC loop. The current time of the simulation is thus
-  ``Main.iteration * Main.timestep``.
+.. Note::
+
+  * In the ``filter`` function only, the ``px``, ``py`` and ``pz`` quantities
+    are not exactly the momenta.
+    They are actually the velocities multiplied by the lorentz factor, i.e.,
+    :math:`\gamma v_x`, :math:`\gamma v_y` and :math:`\gamma v_z`.
+    This is *not* true for the output of the diagnostic.
+  * The ``id`` attribute contains the :doc:`particles identification number<ids>`.
+    This number is set to 0 at the beginning of the simulation. **Only after particles have
+    passed the filter**, they acquire a positive ``id``.
+  * For advanced filtration, Smilei provides the quantity ``Main.iteration``,
+    accessible within the ``filter`` function. Its value is always equal to the current
+    iteration number of the PIC loop. The current time of the simulation is thus
+    ``Main.iteration * Main.timestep``.
 
 .. py:data:: attributes
 
@@ -3368,6 +3373,11 @@ for instance::
   (``"chi"``, only for species with radiation losses) or the fields interpolated
   at their  positions (``"Ex"``, ``"Ey"``, ``"Ez"``, ``"Bx"``, ``"By"``, ``"Bz"``).
 
+.. Note:: Here, interpolated fields are normally computed after the Maxwell solver.
+  They may thus differ by half a timestep from those computed at the middle of the
+  timestep to push particles. When exact values are needed, use the option
+  :py:data:`keep_interpolated_fields`.
+
 ----
 
 .. rst-class:: experimental

doc/Sphinx/implementation.rst

@@ -547,7 +547,7 @@ file ``Smilei.cpp`` thought calls to different ``vecPatches`` methods.
 
 .. code-block:: c++
 
-    vecPatches.finalizeAndSortParticles( params, &smpi, simWindow,
+    vecPatches.finalizeExchParticlesAndSort( params, &smpi, simWindow,
                                                  time_dual, timers, itime );
 
 * **Particle merging**: merging process for particles (still experimental)

happi/_Diagnostics/TrackParticles.py

@@ -447,8 +447,9 @@ def _orderFiles( self, fileOrdered, chunksize, sort ):
 						for k, name in self._short_properties_from_raw.items():
 							if k not in group: continue
 							ordered = self._np.empty((nparticles_to_write, ), dtype=group[k].dtype)
-							if k == "id": ordered.fill(0)
-							else        : ordered.fill(self._np.nan)
+							if k == "id"      : ordered.fill(0)
+							elif k == "charge": ordered.fill(9999)
+							else              : ordered.fill(self._np.nan)
 							ordered[locs] = group[k][()][selectedIndices]
 							f0[name].write_direct(ordered, dest_sel=self._np.s_[it,:])
 
@@ -461,8 +462,9 @@ def _orderFiles( self, fileOrdered, chunksize, sort ):
 					for first_o, last_o, npart_o in ChunkedRange(nparticles_to_write, chunksize):
 						for k, name in self._short_properties_from_raw.items():
 							if k not in group: continue
-							if k == "id": data[k].fill(0)
-							else        : data[k].fill(self._np.nan)
+							if k == "id"      : data[k].fill(0)
+							elif k == "charge": data[k].fill(9999)
+							else              : data[k].fill(self._np.nan)
 						# Loop chunks of the input
 						for first_i, last_i, npart_i in ChunkedRange(nparticles, chunksize):
 							# Obtain IDs
@@ -538,7 +540,10 @@ def _generateRawData(self, times=None):
 								data[it,:] -= self._XmovedForTime[time]
 						else:
 							data = self._readUnstructuredH5(self._h5items[axis], self.selectedParticles, first_time, last_time)
-						data[deadParticles] = self._np.nan
+						if data.dtype == float:
+							data[deadParticles] = self._np.nan
+						else:
+							data[deadParticles] = 9999
 						self._rawData[axis] = data
 
 				if self._verbose: print("Process broken lines ...")

happi/_Utils.py

@@ -42,7 +42,10 @@ def updateMatplotLibColormaps():
 	if "smilei" in matplotlib.pyplot.colormaps(): return
 	def register(name, d):
 		cmap = matplotlib.colors.LinearSegmentedColormap(name, d, N=256, gamma=1.0)
-		matplotlib.pyplot.register_cmap(cmap=cmap)
+		try:
+			matplotlib.pyplot.register_cmap(cmap=cmap)
+		except Exception as e:
+			matplotlib.colormaps.register(cmap)
 	register(u"smilei", {
 			'red'  :((0., 0., 0.), (0.0625 , 0.091, 0.091), (0.09375, 0.118, 0.118), (0.125 , 0.127, 0.127), (0.1875 , 0.135, 0.135), (0.21875, 0.125, 0.125), (0.28125, 0.034, 0.034), (0.3125 , 0.010, 0.010), (0.34375, 0.009, 0.009), (0.4375 , 0.049, 0.049), (0.46875, 0.057, 0.057), (0.5 , 0.058, 0.058), (0.59375, 0.031, 0.031), (0.625 , 0.028, 0.028), (0.65625, 0.047, 0.047), (0.71875, 0.143, 0.143), (0.78125, 0.294, 0.294), (0.84375, 0.519, 0.519), (0.90625, 0.664, 0.664), (0.9375 , 0.760, 0.760), (0.96875, 0.880, 0.880), (1., 1., 1. )),
 			'green':((0., 0., 0.), (0.21875, 0.228, 0.228), (0.78125, 0.827, 0.827), (0.8125 , 0.852, 0.852), (0.84375, 0.869, 0.869), (0.9375 , 0.937, 0.937), (0.96875, 0.967, 0.967), (1. , 1. , 1. )),
@@ -398,7 +401,11 @@ def __init__(self, operation, QuantityTranslator, ureg):
 							raise Exception("Quantity "+q+" not understood")
 			# Calculate the total units and its inverse
 			locals().update(self.imports)
-			units = eval("".join(basic_op)).units
+			units = eval("".join(basic_op))
+			if isinstance(units, (int, float)):
+				units = ureg.Quantity(1) # dimensionless
+			else:
+				units = units.units
 			self.translated_units = units.format_babel(locale="en")
 			# Make the operation string
 			self.translated_operation = "".join(full_op)

makefile

@@ -52,7 +52,7 @@ DIRS := $(shell find src -type d)
 SRCS := $(shell find src/* -name \*.cpp)
 OBJS := $(addprefix $(BUILD_DIR)/, $(SRCS:.cpp=.o))
 DEPS := $(addprefix $(BUILD_DIR)/, $(SRCS:.cpp=.d))
-SITEDIR = $(shell $(PYTHONEXE) -c 'import site; site._script()' --user-site)
+SITEDIR = $(shell d=`$(PYTHONEXE) -m site --user-site` && echo $$d || $(PYTHONEXE) -c "import sysconfig; print(sysconfig.get_path('purelib'))")
 
 # Smilei tools
 TABLES_DIR := tools/tables
@@ -202,9 +202,9 @@ endif
 ifneq (,$(call parse_config,gpu_nvidia))
 	override config += noopenmp # Prevent openmp for nvidia
 
-	CXXFLAGS += -DSMILEI_ACCELERATOR_MODE -DSMILEI_OPENACC_MODE
+	CXXFLAGS += -DSMILEI_ACCELERATOR_GPU -DSMILEI_ACCELERATOR_GPU_OACC
 	GPU_COMPILER ?= nvcc
-	GPU_COMPILER_FLAGS += -x cu -DSMILEI_ACCELERATOR_MODE -DSMILEI_OPENACC_MODE $(DIRS:%=-I%)
+	GPU_COMPILER_FLAGS += -x cu -DSMILEI_ACCELERATOR_GPU -DSMILEI_ACCELERATOR_GPU_OACC $(DIRS:%=-I%)
 	GPU_COMPILER_FLAGS += -I$(BUILD_DIR)/src/Python $(PY_CXXFLAGS)
 	GPU_KERNEL_SRCS := $(shell find src/* -name \*.cu)
 	GPU_KERNEL_OBJS := $(addprefix $(BUILD_DIR)/, $(GPU_KERNEL_SRCS:.cu=.o))
@@ -214,9 +214,9 @@ endif
 
 # AMD GPUs
 ifneq (,$(call parse_config,gpu_amd))
-	CXXFLAGS += -DSMILEI_ACCELERATOR_MODE
+	CXXFLAGS += -DSMILEI_ACCELERATOR_GPU -DSMILEI_ACCELERATOR_GPU_OMP
 	GPU_COMPILER ?= $(CC)
-	GPU_COMPILER_FLAGS += -x hip -DSMILEI_ACCELERATOR_MODE -std=c++14 $(DIRS:%=-I%) #$(PY_FLAGS)
+	GPU_COMPILER_FLAGS += -x hip -DSMILEI_ACCELERATOR_GPU -DSMILEI_ACCELERATOR_GPU_OMP -std=c++14 $(DIRS:%=-I%)
 	GPU_COMPILER_FLAGS += -I$(BUILD_DIR)/src/Python $(PY_CXXFLAGS)
 	GPU_KERNEL_SRCS := $(shell find src/* -name \*.cu)
 	GPU_KERNEL_OBJS := $(addprefix $(BUILD_DIR)/, $(GPU_KERNEL_SRCS:.cu=.o))

scripts/compile_tools/machine/adastra

@@ -85,7 +85,6 @@ ADASTRA_DEBUG_FLAGS           := -g -ggdb $(ADASTRA_DEBUG_SANITIZER_FLAGS) -v #
 
 ifneq (,$(call parse_config,gpu_amd))
     # When using OMP
-    ADASTRA_ACCELERATOR_GPU_OMP_DEFINE_FLAGS := -DSMILEI_ACCELERATOR_GPU_OMP=1
 
     # ADASTRA_ACCELERATOR_GPU_TARGET := gfx908
 	# ADASTRA_ACCELERATOR_GPU_TARGET := gfx908:xnack-

scripts/compile_tools/machine/ruche_gpu2

@@ -26,7 +26,7 @@ GPU_COMPILER_FLAGS += -arch=sm_80 #sm_89 # first compile completely with sm_80 t
 CXXFLAGS        += -Minfo=accel # what is offloaded/copied
 # CXXFLAGS        += -Minfo=all   # very verbose output
 
-# To turn on the OpenMP support, uncomment these 3 lines and comment the line just above defining 'SMILEI_OPENACC_MODE'
+# To turn on the OpenMP support, uncomment these 3 lines and comment the line just above defining 'SMILEI_ACCELERATOR_GPU_OACC'
 # CXXFLAGS        += -mp=gpu -DSMILEI_ACCELERATOR_GPU_OMP
 # GPU_COMPILER_FLAGS +=         -DSMILEI_ACCELERATOR_GPU_OMP # Can't we pass the -mp=gpu to nvcc when compiling a .cu file ?
 # LDFLAGS                      += -mp=gpu

src/Checkpoint/Checkpoint.cpp

@@ -233,7 +233,7 @@ void Checkpoint::dumpAll( VectorPatch &vecPatches, Region &region, unsigned int
     MESSAGE( " Checkpoint #" << num_dump << " at iteration " << itime << " dumped" );
 #endif
 
-#if defined( SMILEI_ACCELERATOR_GPU_OMP ) || defined( SMILEI_OPENACC_MODE )
+#if defined( SMILEI_ACCELERATOR_GPU_OMP ) || defined( SMILEI_ACCELERATOR_GPU_OACC )
     MESSAGE( " Copying device data in main memory" );
     // TODO(Etienne M): This may very well be redundant if we did a diagnostic
     // during the last iteration. Indeed, we copy everything from the device to

src/Diagnostic/DiagnosticProbes.cpp

@@ -740,7 +740,7 @@ void DiagnosticProbes::run( SmileiMPI *smpi, VectorPatch &vecPatches, int itime,
         // Interpolate all usual fields on probe ("fake") particles of current patch
         unsigned int iPart_MPI = offset_in_MPI[ipatch];
         unsigned int maxPart_MPI = offset_in_MPI[ipatch] + npart;
-#if defined( SMILEI_ACCELERATOR_MODE )
+#if defined( SMILEI_ACCELERATOR_GPU )
         smpi->resizeDeviceBuffers( ithread,
                                    nDim_particle,
                                    npart );