Update test files for nvidia-smi parsing

Update to the new nvidia-smi pmon output fields
Add a pycuda GPU burner script for tests

README.md

@@ -254,4 +254,4 @@ to CMake using `Gperftools_ROOT_DIR`.
 
 # Copyright
 
-Copyright (c) 2018-2023 CERN.
+Copyright (c) 2018-2024 CERN.

package/scripts/gpu-burner.py

@@ -0,0 +1,38 @@
+#! /usr/bin/env python
+#
+# This is a slightly adapted "hello, world" script from
+# pycuda, that can be used for stressing a CUDA GPU for
+# tests
+#
+# pycuda is required!
+#
+
+import pycuda.autoinit
+import pycuda.driver as drv
+import numpy
+from time import time
+
+from pycuda.compiler import SourceModule
+mod = SourceModule("""
+__global__ void multiply_them(float *dest, float *a, float *b, float *c)
+{
+  const int i = threadIdx.x;
+  dest[i] = a[i] * b[i] + c[i];
+}
+""")
+
+multiply_them = mod.get_function("multiply_them")
+
+a = numpy.random.randn(1024).astype(numpy.float32)
+b = numpy.random.randn(1024).astype(numpy.float32)
+c = numpy.random.randn(1024).astype(numpy.float32)
+
+dest = numpy.zeros_like(a)
+
+start = time()
+while (time() - start < 20):
+    multiply_them(
+            drv.Out(dest), drv.In(a), drv.In(b), drv.In(c),
+            block=(1024,1,1), grid=(1,1))
+
+print(dest-a*b+c)

package/scripts/precook_test.py

@@ -96,18 +96,21 @@ def make_net(proc_net, fixed_value, rand=False):
 def make_nvidia(proc_nvidia, fixed_value, rand=False):
     # idx
     smi_fname = os.path.join(proc_nvidia, "smi")
-    memory_lim = 10000
+    memory_lim = 100
     with open(smi_fname, "w") as f:
         params = [
             0,  # idx
             pid,  # pid
             "G",  # type
             random.randint(0, memory_lim) if rand else fixed_value,  # sm
             random.randint(0, memory_lim) if rand else fixed_value,  # mem
-            # enc, dec are not monitored metrics
-            0,  # enc
-            0,  # dec
-            random.randint(0, memory_lim) if rand else fixed_value,  # fb
+            # The following are not monitored metrics
+            "-",  # enc
+            "-",  # dec
+            "-",  # jpg
+            "-",  # ofa
+            random.randint(0, memory_lim*100) if rand else fixed_value,  # fb
+            0, # ccpm
             "python3",  # command
         ]
         for param in params:

package/scripts/precooked_tests/drop/1/nvidia/smi

@@ -1 +1 @@
-0 1729 G 50 50 0 0 50 python3 
+0 1729 G 86 53 - - - - 9077 0 python3 

package/scripts/precooked_tests/drop/2/nvidia/smi

@@ -1 +1 @@
-0 1729 G 100 100 0 0 100 python3 
+0 1729 G 17 93 - - - - 4374 0 python3 

package/scripts/precooked_tests/drop/3/nvidia/smi

@@ -1 +1 @@
-0 1729 G 20 20 0 0 20 python3 
+0 1729 G 2 69 - - - - 4629 0 python3