Don't log gpu metrics to mlfoundry metrics

truefoundry · Dec 28, 2023 · d1f44f5 · d1f44f5
1 parent 79dad36
commit d1f44f5
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/mlfoundry_utils.py b/mlfoundry_utils.py
@@ -134,6 +134,8 @@ def on_log(self, args, state, control, logs, model=None, **kwargs):
 
         metrics = {}
         for k, v in logs.items():
+            if k.startswith("system/gpu"):
+                continue
             if isinstance(v, (int, float, np.integer, np.floating)) and math.isfinite(v):
                 metrics[k] = v
             else: