From d1f44f545cf7e0e0f96ed02f732d553f10409119 Mon Sep 17 00:00:00 2001
From: Chirag Jain <jain.chirag925@gmail.com>
Date: Thu, 28 Dec 2023 11:36:23 +0000
Subject: [PATCH] Don't log gpu metrics to mlfoundry metrics

---
 mlfoundry_utils.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mlfoundry_utils.py b/mlfoundry_utils.py
index 87db15d..1a50eb4 100644
--- a/mlfoundry_utils.py
+++ b/mlfoundry_utils.py
@@ -134,6 +134,8 @@ def on_log(self, args, state, control, logs, model=None, **kwargs):
 
         metrics = {}
         for k, v in logs.items():
+            if k.startswith("system/gpu"):
+                continue
             if isinstance(v, (int, float, np.integer, np.floating)) and math.isfinite(v):
                 metrics[k] = v
             else: