vast_dashboard(14GPU).json

{
  "__inputs": [
    {
      "name": "DS_PROMETHEUS",
      "label": "Prometheus",
      "description": "",
      "type": "datasource",
      "pluginId": "prometheus",
      "pluginName": "Prometheus"
    }
  ],
  "__elements": {},
  "__requires": [
    {
      "type": "panel",
      "id": "gauge",
      "name": "Gauge",
      "version": ""
    },
    {
      "type": "grafana",
      "id": "grafana",
      "name": "Grafana",
      "version": "10.3.3"
    },
    {
      "type": "datasource",
      "id": "prometheus",
      "name": "Prometheus",
      "version": "1.0.0"
    },
    {
      "type": "panel",
      "id": "stat",
      "name": "Stat",
      "version": ""
    },
    {
      "type": "panel",
      "id": "state-timeline",
      "name": "State timeline",
      "version": ""
    },
    {
      "type": "panel",
      "id": "table",
      "name": "Table",
      "version": ""
    },
    {
      "type": "panel",
      "id": "timeseries",
      "name": "Time series",
      "version": ""
    }
  ],
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "grafana",
          "uid": "-- Grafana --"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 0,
  "id": null,
  "links": [],
  "liveNow": false,
  "panels": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Reported by vast api. ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              }
            ]
          },
          "unit": "currencyUSD",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 2,
        "x": 0,
        "y": 0
      },
      "id": 1,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_current_total",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Pending Payout",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Based on incress of the pending balance over the laste hour",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 0.1
              }
            ]
          },
          "unit": "currencyUSD",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 2,
        "x": 2,
        "y": 0
      },
      "id": 12,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "deriv(vastai_current_total[1h]) * 3600",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Current Income per hour",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Reported by vast for all machines",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 0.1
              }
            ]
          },
          "unit": "currencyUSD",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 2,
        "x": 4,
        "y": 0
      },
      "id": 21,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(vastai_machine_earn_hour)/1.25",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Reported Income per day",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "orange",
                "value": 50
              },
              {
                "color": "yellow",
                "value": 80
              },
              {
                "color": "blue",
                "value": 90
              },
              {
                "color": "green",
                "value": 100
              }
            ]
          },
          "unit": "%",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 3,
        "x": 6,
        "y": 0
      },
      "id": 6,
      "options": {
        "minVizHeight": 75,
        "minVizWidth": 75,
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "sizing": "auto"
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(vast_machine_Listed)/ sum(vast_machine_hostname) *100",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Machines Listed",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "orange",
                "value": 50
              },
              {
                "color": "yellow",
                "value": 80
              },
              {
                "color": "blue",
                "value": 90
              },
              {
                "color": "green",
                "value": 100
              }
            ]
          },
          "unit": "%",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 3,
        "x": 9,
        "y": 0
      },
      "id": 8,
      "options": {
        "minVizHeight": 75,
        "minVizWidth": 75,
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "sizing": "auto"
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(vast_machine_Verification)/ sum(vast_machine_hostname) *100",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Machines verfied",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 2,
        "x": 12,
        "y": 0
      },
      "id": 9,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "count(vast_machine_hostname)",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Total Machines",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 2,
        "x": 14,
        "y": 0
      },
      "id": 3,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showPercentChange": false,
        "textMode": "auto",
        "wideLayout": true
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(vast_machine_gpu_name)",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Toal GPU's",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "This shows how many of the GPUs are detected by the systems vs what each machine has registered on vast. ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "green",
                "value": 100
              }
            ]
          },
          "unit": "%",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 2,
        "x": 16,
        "y": 0
      },
      "id": 5,
      "options": {
        "minVizHeight": 75,
        "minVizWidth": 75,
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "sizing": "auto"
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "count(DCGM_FI_DEV_GPU_UTIL{})/sum(vast_machine_gpu_name) * 100",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Actual vs Vast registred",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "orange",
                "value": 30
              },
              {
                "color": "yellow",
                "value": 50
              },
              {
                "color": "blue",
                "value": 70
              },
              {
                "color": "green",
                "value": 100
              }
            ]
          },
          "unit": "%",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 2,
        "x": 18,
        "y": 0
      },
      "id": 4,
      "options": {
        "minVizHeight": 75,
        "minVizWidth": 75,
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "sizing": "auto"
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(vastai_machine_gpu_rented_on_demand) / sum(vast_machine_gpu_name) * 100",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "GPU Used On-demand",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "orange",
                "value": 30
              },
              {
                "color": "yellow",
                "value": 50
              },
              {
                "color": "blue",
                "value": 70
              },
              {
                "color": "green",
                "value": 100
              }
            ]
          },
          "unit": "%",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 2,
        "x": 20,
        "y": 0
      },
      "id": 22,
      "options": {
        "minVizHeight": 75,
        "minVizWidth": 75,
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "sizing": "auto"
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(vastai_machine_gpu_rented_bid_demand) / sum(vast_machine_gpu_name) * 100",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "GPU Used Bid",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [],
          "max": 100,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "blue",
                "value": 20
              },
              {
                "color": "yellow",
                "value": 40
              },
              {
                "color": "orange",
                "value": 60
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "%",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 2,
        "x": 22,
        "y": 0
      },
      "id": 7,
      "options": {
        "minVizHeight": 75,
        "minVizWidth": 75,
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "showThresholdLabels": false,
        "showThresholdMarkers": true,
        "sizing": "auto"
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(vastai_machine_gpu_idle) / sum(vast_machine_gpu_name) * 100",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "GPU's Idle",
      "type": "gauge"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "As reported by vast api.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "type": "auto"
            },
            "inspect": false,
            "minWidth": 50,
            "width": 50
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Stored Jobs"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 90
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "GPU's"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 57
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "hostname"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 85
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Idle"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 50
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Run I"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 50
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Run D"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 55
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "gpu_name"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 102
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 80
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Listed"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 65
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Avail Space(GB)"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 125
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #H"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 60
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Value #I"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 65
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Reliability"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 60
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 30,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "id": 11,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "enablePagination": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": [
          {
            "desc": false,
            "displayName": "hostname"
          }
        ]
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vast_machine_gpu_name",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vast_machine_hostname",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_current_rentals_resident",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_idle",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_rented_bid_demand",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_rented_on_demand",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_avail_disk_space",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vast_machine_Listed",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vast_machine_Verification",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vast_machine_Reliability",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "J"
        }
      ],
      "timeFrom": "30s",
      "title": "Machine_Info",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "machine_id",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Time 1": true,
              "Time 10": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "Value #A": false,
              "Value #B": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "instance": true,
              "instance 1": true,
              "instance 10": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "instance 9": true,
              "job": true,
              "job 1": true,
              "job 10": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "job 9": true
            },
            "includeByName": {},
            "indexByName": {
              "Time": 45,
              "Time 10": 49,
              "Time 2": 5,
              "Time 3": 10,
              "Time 4": 15,
              "Time 5": 20,
              "Time 6": 25,
              "Time 7": 30,
              "Time 8": 34,
              "Time 9": 39,
              "Value #A": 48,
              "Value #B": 9,
              "Value #C": 14,
              "Value #D": 19,
              "Value #E": 24,
              "Value #F": 29,
              "Value #G": 44,
              "Value #H": 38,
              "Value #I": 43,
              "Value #J": 2,
              "__name__ 1": 4,
              "__name__ 10": 50,
              "__name__ 2": 6,
              "__name__ 3": 11,
              "__name__ 4": 16,
              "__name__ 5": 21,
              "__name__ 6": 26,
              "__name__ 7": 31,
              "__name__ 8": 35,
              "__name__ 9": 40,
              "gpu_name": 3,
              "hostname": 1,
              "instance": 46,
              "instance 10": 51,
              "instance 2": 7,
              "instance 3": 12,
              "instance 4": 17,
              "instance 5": 22,
              "instance 6": 27,
              "instance 7": 32,
              "instance 8": 36,
              "instance 9": 41,
              "job": 47,
              "job 10": 52,
              "job 2": 8,
              "job 3": 13,
              "job 4": 18,
              "job 5": 23,
              "job 6": 28,
              "job 7": 33,
              "job 8": 37,
              "job 9": 42,
              "machine_id": 0
            },
            "renameByName": {
              "Value #A": "GPU's",
              "Value #C": "Stored Jobs",
              "Value #D": "Idle",
              "Value #E": "Run I",
              "Value #F": "Run D",
              "Value #G": "Avail Space(GB)",
              "Value #H": "Listed",
              "Value #I": "Verifyd",
              "Value #J": "Reliability",
              "instance 6": "",
              "machine_id": "Machine"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "type": "auto"
            },
            "inspect": false,
            "minWidth": 50
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Hostname"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 85
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 30,
        "w": 3,
        "x": 12,
        "y": 8
      },
      "id": 29,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "count(DCGM_FI_DEV_FB_USED) by (Hostname)",
          "format": "table",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "timeFrom": "30s",
      "title": "Reported GPU Count",
      "transformations": [
        {
          "id": "groupBy",
          "options": {
            "fields": {
              "Hostname": {
                "aggregations": [],
                "operation": "groupby"
              },
              "Value": {
                "aggregations": [
                  "lastNotNull"
                ],
                "operation": "aggregate"
              }
            }
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {},
            "indexByName": {},
            "renameByName": {
              "Value (lastNotNull)": "GPU COUNT"
            }
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "As reported by vast api.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 13,
        "w": 9,
        "x": 15,
        "y": 8
      },
      "id": 10,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum (vast_machine_Reliability) by (machine_id)",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Machine Reliability",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Reported by vast api. ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 17,
        "w": 9,
        "x": 15,
        "y": 21
      },
      "id": 19,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum(vastai_machine_earn_hour)  by (machine_id)",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Current Machine earnings per day",
      "transformations": [
        {
          "filter": {
            "id": "byRefId",
            "options": ""
          },
          "id": "filterByValue",
          "options": {
            "filters": [],
            "match": "any",
            "type": "include"
          }
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "As reported by vast api.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "fillOpacity": 70,
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineWidth": 0,
            "spanNulls": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          },
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 12,
        "w": 12,
        "x": 0,
        "y": 38
      },
      "id": 13,
      "options": {
        "alignValue": "left",
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "mergeValues": false,
        "rowHeight": 0.9,
        "showValue": "auto",
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "9.5.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by(machine_id) (vast_machine_timeout)",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Machine Online Status",
      "type": "state-timeline"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "As reported by vast api.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "fillOpacity": 70,
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineWidth": 0,
            "spanNulls": false
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 10
              }
            ]
          },
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 12,
        "w": 12,
        "x": 12,
        "y": 38
      },
      "id": 36,
      "options": {
        "alignValue": "left",
        "legend": {
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": false
        },
        "mergeValues": false,
        "rowHeight": 0.9,
        "showValue": "auto",
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "9.5.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by(machine_id) (vastai_machine_ErrorDescription)",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Machine Error Status",
      "type": "state-timeline"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "watt",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 50
      },
      "id": 26,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "$CPU_Max_Power * count by (job)(node_cpu_temperature{package=~\"[0-9]+\"})/2 + sum by (job) (DCGM_FI_DEV_POWER_USAGE) + $BasePower +$CPU_Max_Power",
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Machines Power usage",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Avraged over 24h as reported by vast api. ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 27,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "always",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "max": 100,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "%",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 50
      },
      "id": 27,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "(sum(vastai_machine_gpu_rented_on_demand) by (machine_id)) / (sum(vast_machine_gpu_name) by (machine_id) )* 100",
          "hide": true,
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "avg_over_time(\r\n  (\r\n    sum(vastai_machine_gpu_rented_on_demand) by (machine_id) / \r\n    sum(vast_machine_gpu_name) by (machine_id)\r\n  )[2h:1m]\r\n) * 100\r\n",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        }
      ],
      "title": "Machine usage On-demand",
      "transformations": [
        {
          "filter": {
            "id": "byRefId",
            "options": ""
          },
          "id": "filterByValue",
          "options": {
            "filters": [],
            "match": "any",
            "type": "include"
          }
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "inspect": false,
            "minWidth": 55
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 51,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 70
              },
              "type": "range"
            },
            {
              "options": {
                "from": 71,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 80
              },
              "type": "range"
            },
            {
              "options": {
                "from": 81,
                "result": {
                  "color": "orange",
                  "index": 3
                },
                "to": 90
              },
              "type": "range"
            },
            {
              "options": {
                "from": 91,
                "result": {
                  "color": "red",
                  "index": 4
                },
                "to": 200
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": null
              }
            ]
          },
          "unit": "celsius",
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 75
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 35,
        "w": 12,
        "x": 0,
        "y": 58
      },
      "id": 32,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"8\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"9\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"10\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "K"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"11\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "L"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"12\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "M"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_HOT_SPOT_TEMP{gpu=\"13\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "N"
        }
      ],
      "timeFrom": "30s",
      "title": "Machine GPU Core Hot Spot",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "job",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "Hostname 1": true,
              "Hostname 2": true,
              "Hostname 3": true,
              "Hostname 4": true,
              "Hostname 5": true,
              "Hostname 6": true,
              "Hostname 7": true,
              "Hostname 8": true,
              "Time": true,
              "Time 1": true,
              "Time 10": true,
              "Time 11": true,
              "Time 12": true,
              "Time 13": true,
              "Time 14": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "UUID 1": true,
              "UUID 10": true,
              "UUID 11": true,
              "UUID 12": true,
              "UUID 13": true,
              "UUID 14": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "UUID 9": true,
              "Value #D": false,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 12": true,
              "__name__ 13": true,
              "__name__ 14": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu 1": true,
              "gpu 10": true,
              "gpu 11": true,
              "gpu 12": true,
              "gpu 13": true,
              "gpu 14": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "gpu 9": true,
              "instance": true,
              "instance 1": true,
              "instance 10": true,
              "instance 11": true,
              "instance 12": true,
              "instance 13": true,
              "instance 14": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "instance 9": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true
            },
            "includeByName": {},
            "indexByName": {
              "DCGM_FI_DRIVER_VERSION 1": 1,
              "DCGM_FI_DRIVER_VERSION 2": 13,
              "DCGM_FI_DRIVER_VERSION 3": 22,
              "DCGM_FI_DRIVER_VERSION 4": 32,
              "Hostname 1": 5,
              "Hostname 2": 14,
              "Hostname 3": 23,
              "Hostname 4": 33,
              "Time 1": 4,
              "Time 2": 12,
              "Time 3": 21,
              "Time 4": 31,
              "UUID 1": 6,
              "UUID 2": 15,
              "UUID 3": 24,
              "UUID 4": 34,
              "Value #A": 2,
              "Value #B": 3,
              "Value #C": 30,
              "Value #D": 40,
              "__name__ 1": 7,
              "__name__ 2": 16,
              "__name__ 3": 25,
              "__name__ 4": 35,
              "device 1": 8,
              "device 2": 17,
              "device 3": 26,
              "device 4": 36,
              "gpu 1": 9,
              "gpu 2": 18,
              "gpu 3": 27,
              "gpu 4": 37,
              "instance 1": 10,
              "instance 2": 19,
              "instance 3": 28,
              "instance 4": 38,
              "job": 0,
              "modelName 1": 11,
              "modelName 2": 20,
              "modelName 3": 29,
              "modelName 4": 39
            },
            "renameByName": {
              "Hostname 1": "",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7",
              "Value #I": "GPU8",
              "Value #J": "GPU9",
              "Value #K": "GPU10",
              "Value #L": "GPU11",
              "Value #M": "GPU12",
              "Value #N": "GPU13",
              "job": "Machine"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "inspect": false,
            "minWidth": 55
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "transparent",
                  "index": 0,
                  "text": "idle"
                },
                "to": 0
              },
              "type": "range"
            },
            {
              "options": {
                "from": 1,
                "result": {
                  "color": "yellow",
                  "index": 1,
                  "text": "I"
                },
                "to": 1
              },
              "type": "range"
            },
            {
              "options": {
                "from": 2,
                "result": {
                  "color": "green",
                  "index": 2,
                  "text": "D"
                },
                "to": 2
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": null
              }
            ]
          },
          "unit": "none",
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 75
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "job 1"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 55
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Hostname"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 78
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 35,
        "w": 12,
        "x": 12,
        "y": 58
      },
      "id": 37,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"8\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"9\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"10\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "K"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"11\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "L"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"12\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "M"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "vastai_machine_gpu_occupancy{gpu=\"13\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "N"
        }
      ],
      "timeFrom": "30s",
      "title": "Machine GPU Core Hot Spot",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "Hostname",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "Hostname 1": true,
              "Hostname 2": true,
              "Hostname 3": true,
              "Hostname 4": true,
              "Hostname 5": true,
              "Hostname 6": true,
              "Hostname 7": true,
              "Hostname 8": true,
              "Time": true,
              "Time 1": true,
              "Time 10": true,
              "Time 11": true,
              "Time 12": true,
              "Time 13": true,
              "Time 14": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "UUID 1": true,
              "UUID 10": true,
              "UUID 11": true,
              "UUID 12": true,
              "UUID 13": true,
              "UUID 14": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "UUID 9": true,
              "Value #D": false,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 12": true,
              "__name__ 13": true,
              "__name__ 14": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu 1": true,
              "gpu 10": true,
              "gpu 11": true,
              "gpu 12": true,
              "gpu 13": true,
              "gpu 14": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "gpu 9": true,
              "instance": true,
              "instance 1": true,
              "instance 10": true,
              "instance 11": true,
              "instance 12": true,
              "instance 13": true,
              "instance 14": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "instance 9": true,
              "job 1": true,
              "job 10": true,
              "job 11": true,
              "job 12": true,
              "job 13": true,
              "job 14": true,
              "job 2": true,
              "job 3": true,
              "job 4": true,
              "job 5": true,
              "job 6": true,
              "job 7": true,
              "job 8": true,
              "job 9": true,
              "machine_id 1": true,
              "machine_id 10": true,
              "machine_id 11": true,
              "machine_id 12": true,
              "machine_id 13": true,
              "machine_id 14": true,
              "machine_id 2": true,
              "machine_id 3": true,
              "machine_id 4": true,
              "machine_id 5": true,
              "machine_id 6": true,
              "machine_id 7": true,
              "machine_id 8": true,
              "machine_id 9": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true
            },
            "includeByName": {},
            "indexByName": {
              "Hostname": 0,
              "Time 1": 3,
              "Time 10": 64,
              "Time 11": 71,
              "Time 12": 78,
              "Time 13": 85,
              "Time 14": 92,
              "Time 2": 7,
              "Time 3": 11,
              "Time 4": 16,
              "Time 5": 29,
              "Time 6": 36,
              "Time 7": 43,
              "Time 8": 50,
              "Time 9": 57,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 15,
              "Value #D": 20,
              "Value #E": 35,
              "Value #F": 42,
              "Value #G": 49,
              "Value #H": 56,
              "Value #I": 63,
              "Value #J": 70,
              "Value #K": 77,
              "Value #L": 84,
              "Value #M": 91,
              "Value #N": 98,
              "__name__ 1": 4,
              "__name__ 10": 65,
              "__name__ 11": 72,
              "__name__ 12": 79,
              "__name__ 13": 86,
              "__name__ 14": 93,
              "__name__ 2": 8,
              "__name__ 3": 12,
              "__name__ 4": 17,
              "__name__ 5": 30,
              "__name__ 6": 37,
              "__name__ 7": 44,
              "__name__ 8": 51,
              "__name__ 9": 58,
              "gpu 1": 5,
              "gpu 10": 66,
              "gpu 11": 73,
              "gpu 12": 80,
              "gpu 13": 87,
              "gpu 14": 94,
              "gpu 2": 9,
              "gpu 3": 13,
              "gpu 4": 18,
              "gpu 5": 31,
              "gpu 6": 38,
              "gpu 7": 45,
              "gpu 8": 52,
              "gpu 9": 59,
              "instance 1": 6,
              "instance 10": 67,
              "instance 11": 74,
              "instance 12": 81,
              "instance 13": 88,
              "instance 14": 95,
              "instance 2": 10,
              "instance 3": 14,
              "instance 4": 19,
              "instance 5": 32,
              "instance 6": 39,
              "instance 7": 46,
              "instance 8": 53,
              "instance 9": 60,
              "job 1": 21,
              "job 10": 68,
              "job 11": 75,
              "job 12": 82,
              "job 13": 89,
              "job 14": 96,
              "job 2": 23,
              "job 3": 25,
              "job 4": 27,
              "job 5": 33,
              "job 6": 40,
              "job 7": 47,
              "job 8": 54,
              "job 9": 61,
              "machine_id 1": 22,
              "machine_id 10": 69,
              "machine_id 11": 76,
              "machine_id 12": 83,
              "machine_id 13": 90,
              "machine_id 14": 97,
              "machine_id 2": 24,
              "machine_id 3": 26,
              "machine_id 4": 28,
              "machine_id 5": 34,
              "machine_id 6": 41,
              "machine_id 7": 48,
              "machine_id 8": 55,
              "machine_id 9": 62
            },
            "renameByName": {
              "Hostname 1": "",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7",
              "Value #I": "GPU8",
              "Value #J": "GPU9",
              "Value #K": "GPU10",
              "Value #L": "GPU11",
              "Value #M": "GPU12",
              "Value #N": "GPU13",
              "job": "Machine"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "inspect": false,
            "minWidth": 55
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 51,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 70
              },
              "type": "range"
            },
            {
              "options": {
                "from": 71,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 80
              },
              "type": "range"
            },
            {
              "options": {
                "from": 81,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 200
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": null
              }
            ]
          },
          "unit": "celsius",
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 75
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 27,
        "w": 12,
        "x": 0,
        "y": 93
      },
      "id": 25,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"8\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"9\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"10\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "K"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"11\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "L"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"12\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "M"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_GPU_TEMP{gpu=\"13\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "N"
        }
      ],
      "timeFrom": "30s",
      "title": "Machine GPU Temps",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "job",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "Hostname": true,
              "Hostname 1": true,
              "Hostname 10": true,
              "Hostname 11": true,
              "Hostname 12": true,
              "Hostname 13": true,
              "Hostname 14": true,
              "Hostname 2": true,
              "Hostname 3": true,
              "Hostname 4": true,
              "Hostname 5": true,
              "Hostname 6": true,
              "Hostname 7": true,
              "Hostname 8": true,
              "Hostname 9": true,
              "Time 1": true,
              "Time 10": true,
              "Time 11": true,
              "Time 12": true,
              "Time 13": true,
              "Time 14": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "UUID 1": true,
              "UUID 10": true,
              "UUID 11": true,
              "UUID 12": true,
              "UUID 13": true,
              "UUID 14": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "UUID 9": true,
              "Value #D": false,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 11": true,
              "__name__ 12": true,
              "__name__ 13": true,
              "__name__ 14": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "device 1": true,
              "device 10": true,
              "device 11": true,
              "device 12": true,
              "device 13": true,
              "device 14": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "device 9": true,
              "gpu 1": true,
              "gpu 10": true,
              "gpu 11": true,
              "gpu 12": true,
              "gpu 13": true,
              "gpu 14": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "gpu 9": true,
              "instance": true,
              "instance 1": true,
              "instance 10": true,
              "instance 12": true,
              "instance 13": true,
              "instance 14": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "instance 9": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 10": true,
              "modelName 11": true,
              "modelName 12": true,
              "modelName 13": true,
              "modelName 14": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "modelName 9": true
            },
            "includeByName": {},
            "indexByName": {
              "DCGM_FI_DRIVER_VERSION 1": 1,
              "DCGM_FI_DRIVER_VERSION 2": 13,
              "DCGM_FI_DRIVER_VERSION 3": 22,
              "DCGM_FI_DRIVER_VERSION 4": 32,
              "Hostname 1": 5,
              "Hostname 2": 14,
              "Hostname 3": 23,
              "Hostname 4": 33,
              "Time 1": 4,
              "Time 2": 12,
              "Time 3": 21,
              "Time 4": 31,
              "UUID 1": 6,
              "UUID 2": 15,
              "UUID 3": 24,
              "UUID 4": 34,
              "Value #A": 2,
              "Value #B": 3,
              "Value #C": 30,
              "Value #D": 40,
              "__name__ 1": 7,
              "__name__ 2": 16,
              "__name__ 3": 25,
              "__name__ 4": 35,
              "device 1": 8,
              "device 2": 17,
              "device 3": 26,
              "device 4": 36,
              "gpu 1": 9,
              "gpu 2": 18,
              "gpu 3": 27,
              "gpu 4": 37,
              "instance 1": 10,
              "instance 2": 19,
              "instance 3": 28,
              "instance 4": 38,
              "job": 0,
              "modelName 1": 11,
              "modelName 2": 20,
              "modelName 3": 29,
              "modelName 4": 39
            },
            "renameByName": {
              "Hostname 1": "",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7",
              "Value #I": "GPU8",
              "Value #J": "GPU9",
              "Value #K": "GPU10",
              "Value #L": "GPU11",
              "Value #M": "GPU12",
              "Value #N": "GPU13",
              "job": "Machine"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "inspect": false,
            "minWidth": 55
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 51,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 70
              },
              "type": "range"
            },
            {
              "options": {
                "from": 71,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 80
              },
              "type": "range"
            },
            {
              "options": {
                "from": 81,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 100
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": null
              }
            ]
          },
          "unit": "%",
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 72
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 27,
        "w": 12,
        "x": 12,
        "y": 93
      },
      "id": 28,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"8\"}",
          "format": "table",
          "hide": false,
          "instant": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"9\"}",
          "format": "table",
          "hide": false,
          "instant": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"10\"}",
          "format": "table",
          "hide": false,
          "instant": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "K"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"11\"}",
          "hide": false,
          "instant": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "L"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"12\"}",
          "format": "table",
          "hide": false,
          "instant": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "M"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_FAN_SPEED{gpu=\"13\"}",
          "format": "table",
          "hide": false,
          "instant": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "N"
        }
      ],
      "timeFrom": "30s",
      "title": "Machine GPU FAN Speeds",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "job",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "Hostname": true,
              "Hostname 1": true,
              "Hostname 10": true,
              "Hostname 12": true,
              "Hostname 13": true,
              "Hostname 2": true,
              "Hostname 3": true,
              "Hostname 4": true,
              "Hostname 5": true,
              "Hostname 6": true,
              "Hostname 7": true,
              "Hostname 8": true,
              "Hostname 9": true,
              "Time 1": true,
              "Time 10": true,
              "Time 11": true,
              "Time 12": true,
              "Time 13": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "UUID 1": true,
              "UUID 10": true,
              "UUID 11": true,
              "UUID 12": true,
              "UUID 13": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "UUID 9": true,
              "Value #D": false,
              "Value #J": false,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 11": true,
              "__name__ 12": true,
              "__name__ 13": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "device 1": true,
              "device 10": true,
              "device 11": true,
              "device 12": true,
              "device 13": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "device 9": true,
              "gpu 1": true,
              "gpu 10": true,
              "gpu 11": true,
              "gpu 12": true,
              "gpu 13": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "gpu 9": true,
              "instance": true,
              "instance 1": true,
              "instance 10": true,
              "instance 11": true,
              "instance 12": true,
              "instance 13": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "instance 9": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 10": true,
              "modelName 11": true,
              "modelName 12": true,
              "modelName 13": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "modelName 9": true
            },
            "includeByName": {},
            "indexByName": {
              "Hostname": 33,
              "Hostname 2": 9,
              "Hostname 3": 17,
              "Hostname 4": 25,
              "Hostname 5": 38,
              "Hostname 6": 47,
              "Hostname 7": 56,
              "Hostname 8": 65,
              "Time 1": 4,
              "Time 2": 8,
              "Time 3": 16,
              "Time 4": 24,
              "Time 5": 37,
              "Time 6": 46,
              "Time 7": 55,
              "Time 8": 64,
              "UUID 1": 5,
              "UUID 2": 10,
              "UUID 3": 18,
              "UUID 4": 26,
              "UUID 5": 39,
              "UUID 6": 48,
              "UUID 7": 57,
              "UUID 8": 66,
              "Value #A": 1,
              "Value #B": 2,
              "Value #C": 3,
              "Value #D": 32,
              "Value #E": 45,
              "Value #F": 54,
              "Value #G": 63,
              "Value #H": 72,
              "__name__": 34,
              "__name__ 2": 11,
              "__name__ 3": 19,
              "__name__ 4": 27,
              "__name__ 5": 40,
              "__name__ 6": 49,
              "__name__ 7": 58,
              "__name__ 8": 67,
              "device 1": 6,
              "device 2": 12,
              "device 3": 20,
              "device 4": 28,
              "device 5": 41,
              "device 6": 50,
              "device 7": 59,
              "device 8": 68,
              "gpu 1": 7,
              "gpu 2": 13,
              "gpu 3": 21,
              "gpu 4": 29,
              "gpu 5": 42,
              "gpu 6": 51,
              "gpu 7": 60,
              "gpu 8": 69,
              "instance": 35,
              "instance 2": 14,
              "instance 3": 22,
              "instance 4": 30,
              "instance 5": 43,
              "instance 6": 52,
              "instance 7": 61,
              "instance 8": 70,
              "job": 0,
              "modelName": 36,
              "modelName 2": 15,
              "modelName 3": 23,
              "modelName 4": 31,
              "modelName 5": 44,
              "modelName 6": 53,
              "modelName 7": 62,
              "modelName 8": 71
            },
            "renameByName": {
              "Hostname 1": "",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7",
              "Value #I": "GPU9",
              "Value #J": "GPU10",
              "Value #K": "GPU11",
              "Value #M": "GPU12",
              "Value #N": "GPU13",
              "job": "Machine"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "inspect": false,
            "minWidth": 55
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50.999
              },
              "type": "range"
            },
            {
              "options": {
                "from": 51,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 200.999
              },
              "type": "range"
            },
            {
              "options": {
                "from": 201,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 300.999
              },
              "type": "range"
            },
            {
              "options": {
                "from": 301,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 500
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": null
              }
            ]
          },
          "unit": "watt",
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 75
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 28,
        "w": 12,
        "x": 0,
        "y": 120
      },
      "id": 30,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"8\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"9\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"10\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "K"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"11\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "L"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"12\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "M"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_POWER_USAGE{gpu=\"13\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "N"
        }
      ],
      "timeFrom": "30s",
      "title": "Machine GPU Power",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "job",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "Hostname": true,
              "Hostname 1": true,
              "Hostname 10": true,
              "Hostname 11": true,
              "Hostname 12": true,
              "Hostname 13": true,
              "Hostname 14": true,
              "Hostname 2": true,
              "Hostname 3": true,
              "Hostname 4": true,
              "Hostname 5": true,
              "Hostname 6": true,
              "Hostname 7": true,
              "Hostname 8": true,
              "Hostname 9": true,
              "Time 1": true,
              "Time 10": true,
              "Time 11": true,
              "Time 12": true,
              "Time 13": true,
              "Time 14": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "UUID 1": true,
              "UUID 10": true,
              "UUID 11": true,
              "UUID 12": true,
              "UUID 13": true,
              "UUID 14": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "UUID 9": true,
              "Value #D": false,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 11": true,
              "__name__ 12": true,
              "__name__ 13": true,
              "__name__ 14": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "device 1": true,
              "device 10": true,
              "device 11": true,
              "device 12": true,
              "device 13": true,
              "device 14": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "device 9": true,
              "gpu 1": true,
              "gpu 10": true,
              "gpu 11": true,
              "gpu 12": true,
              "gpu 13": true,
              "gpu 14": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "gpu 9": true,
              "instance": true,
              "instance 1": true,
              "instance 10": true,
              "instance 11": true,
              "instance 12": true,
              "instance 13": true,
              "instance 14": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "instance 9": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 10": true,
              "modelName 11": true,
              "modelName 12": true,
              "modelName 13": true,
              "modelName 14": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "modelName 9": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true
            },
            "includeByName": {},
            "indexByName": {
              "DCGM_FI_DRIVER_VERSION 1": 1,
              "DCGM_FI_DRIVER_VERSION 2": 13,
              "DCGM_FI_DRIVER_VERSION 3": 22,
              "DCGM_FI_DRIVER_VERSION 4": 32,
              "Hostname 1": 5,
              "Hostname 2": 14,
              "Hostname 3": 23,
              "Hostname 4": 33,
              "Time 1": 4,
              "Time 2": 12,
              "Time 3": 21,
              "Time 4": 31,
              "UUID 1": 6,
              "UUID 2": 15,
              "UUID 3": 24,
              "UUID 4": 34,
              "Value #A": 2,
              "Value #B": 3,
              "Value #C": 30,
              "Value #D": 40,
              "__name__ 1": 7,
              "__name__ 2": 16,
              "__name__ 3": 25,
              "__name__ 4": 35,
              "device 1": 8,
              "device 2": 17,
              "device 3": 26,
              "device 4": 36,
              "gpu 1": 9,
              "gpu 2": 18,
              "gpu 3": 27,
              "gpu 4": 37,
              "instance 1": 10,
              "instance 2": 19,
              "instance 3": 28,
              "instance 4": 38,
              "job": 0,
              "modelName 1": 11,
              "modelName 2": 20,
              "modelName 3": 29,
              "modelName 4": 39
            },
            "renameByName": {
              "Hostname 1": "",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7",
              "Value #I": "GPU8",
              "Value #J": "GPU9",
              "Value #K": "GPU10",
              "Value #L": "GPU11",
              "Value #M": "GPU12",
              "Value #N": "GPU13",
              "job": "Machine"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "inspect": false,
            "minWidth": 50
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 50
              },
              "type": "range"
            },
            {
              "options": {
                "from": 51,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 70
              },
              "type": "range"
            },
            {
              "options": {
                "from": 71,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 90
              },
              "type": "range"
            },
            {
              "options": {
                "from": 91,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 200
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": null
              }
            ]
          },
          "unit": "celsius",
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 75
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 27,
        "w": 12,
        "x": 12,
        "y": 120
      },
      "id": 31,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"8\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"9\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"10\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "K"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"11\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "L"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"12\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "M"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_VRAM_TEMP{gpu=\"13\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "N"
        }
      ],
      "timeFrom": "30s",
      "title": "Machine GPU VRAM Temps ",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "job",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "Hostname 1": true,
              "Hostname 2": true,
              "Hostname 3": true,
              "Hostname 4": true,
              "Hostname 5": true,
              "Hostname 6": true,
              "Hostname 7": true,
              "Hostname 8": true,
              "Time 1": true,
              "Time 10": true,
              "Time 11": true,
              "Time 12": true,
              "Time 13": true,
              "Time 14": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "UUID 1": true,
              "UUID 10": true,
              "UUID 11": true,
              "UUID 12": true,
              "UUID 13": true,
              "UUID 14": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "UUID 9": true,
              "Value #D": false,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 11": true,
              "__name__ 12": true,
              "__name__ 13": true,
              "__name__ 14": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu 1": true,
              "gpu 10": true,
              "gpu 11": true,
              "gpu 12": true,
              "gpu 13": true,
              "gpu 14": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "gpu 9": true,
              "instance": true,
              "instance 1": true,
              "instance 10": true,
              "instance 11": true,
              "instance 12": true,
              "instance 13": true,
              "instance 14": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "instance 9": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true
            },
            "includeByName": {},
            "indexByName": {
              "DCGM_FI_DRIVER_VERSION 1": 1,
              "DCGM_FI_DRIVER_VERSION 2": 13,
              "DCGM_FI_DRIVER_VERSION 3": 22,
              "DCGM_FI_DRIVER_VERSION 4": 32,
              "Hostname 1": 5,
              "Hostname 2": 14,
              "Hostname 3": 23,
              "Hostname 4": 33,
              "Time 1": 4,
              "Time 2": 12,
              "Time 3": 21,
              "Time 4": 31,
              "UUID 1": 6,
              "UUID 2": 15,
              "UUID 3": 24,
              "UUID 4": 34,
              "Value #A": 2,
              "Value #B": 3,
              "Value #C": 30,
              "Value #D": 40,
              "__name__ 1": 7,
              "__name__ 2": 16,
              "__name__ 3": 25,
              "__name__ 4": 35,
              "device 1": 8,
              "device 2": 17,
              "device 3": 26,
              "device 4": 36,
              "gpu 1": 9,
              "gpu 2": 18,
              "gpu 3": 27,
              "gpu 4": 37,
              "instance 1": 10,
              "instance 2": 19,
              "instance 3": 28,
              "instance 4": 38,
              "job": 0,
              "modelName 1": 11,
              "modelName 2": 20,
              "modelName 3": 29,
              "modelName 4": 39
            },
            "renameByName": {
              "Hostname 1": "",
              "Time 13": "",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7",
              "Value #I": "GPU8",
              "Value #J": "GPU9",
              "Value #K": "GPU10",
              "Value #L": "GPU11",
              "Value #M": "GPU12",
              "Value #N": "GPU13",
              "job": "Machine"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "inspect": false,
            "minWidth": 54
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 0
              },
              "type": "range"
            },
            {
              "options": {
                "from": 1,
                "result": {
                  "color": "red",
                  "index": 1
                },
                "to": 100
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": null
              }
            ]
          },
          "unit": "bool",
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 75
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 28,
        "w": 12,
        "x": 12,
        "y": 147
      },
      "id": 33,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"4\"} ",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"8\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"9\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"10\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "K"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"11\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "L"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"12\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "M"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "DCGM_FI_DEV_CLOCKS_THROTTLE_REASON{reason=\"SwThermalSlowdown\", gpu=\"13\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "N"
        }
      ],
      "timeFrom": "30s",
      "title": "Machine GPU Thermal Throttle",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "job",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "Hostname 1": true,
              "Hostname 2": true,
              "Hostname 3": true,
              "Hostname 4": true,
              "Hostname 5": true,
              "Hostname 6": true,
              "Hostname 7": true,
              "Hostname 8": true,
              "Time": true,
              "Time 1": true,
              "Time 10": true,
              "Time 11": true,
              "Time 12": true,
              "Time 13": true,
              "Time 14": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "UUID 1": true,
              "UUID 10": true,
              "UUID 11": true,
              "UUID 12": true,
              "UUID 13": true,
              "UUID 14": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "UUID 9": true,
              "Value #D": false,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 11": true,
              "__name__ 12": true,
              "__name__ 13": true,
              "__name__ 14": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu 1": true,
              "gpu 10": true,
              "gpu 11": true,
              "gpu 12": true,
              "gpu 13": true,
              "gpu 14": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "gpu 9": true,
              "instance": true,
              "instance 1": true,
              "instance 10": true,
              "instance 11": true,
              "instance 12": true,
              "instance 13": true,
              "instance 14": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "instance 9": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "reason": true,
              "reason 10": true,
              "reason 11": true,
              "reason 12": true,
              "reason 13": true,
              "reason 14": true,
              "reason 9": true
            },
            "includeByName": {},
            "indexByName": {
              "DCGM_FI_DRIVER_VERSION 1": 1,
              "DCGM_FI_DRIVER_VERSION 2": 13,
              "DCGM_FI_DRIVER_VERSION 3": 22,
              "DCGM_FI_DRIVER_VERSION 4": 32,
              "Hostname 1": 5,
              "Hostname 2": 14,
              "Hostname 3": 23,
              "Hostname 4": 33,
              "Time 1": 4,
              "Time 2": 12,
              "Time 3": 21,
              "Time 4": 31,
              "UUID 1": 6,
              "UUID 2": 15,
              "UUID 3": 24,
              "UUID 4": 34,
              "Value #A": 2,
              "Value #B": 3,
              "Value #C": 30,
              "Value #D": 40,
              "__name__ 1": 7,
              "__name__ 2": 16,
              "__name__ 3": 25,
              "__name__ 4": 35,
              "device 1": 8,
              "device 2": 17,
              "device 3": 26,
              "device 4": 36,
              "gpu 1": 9,
              "gpu 2": 18,
              "gpu 3": 27,
              "gpu 4": 37,
              "instance 1": 10,
              "instance 2": 19,
              "instance 3": 28,
              "instance 4": 38,
              "job": 0,
              "modelName 1": 11,
              "modelName 2": 20,
              "modelName 3": 29,
              "modelName 4": 39
            },
            "renameByName": {
              "Hostname 1": "",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7",
              "Value #I": "GPU8",
              "Value #J": "GPU9",
              "Value #K": "GPU10",
              "Value #L": "GPU11",
              "Value #M": "GPU12",
              "Value #N": "GPU13",
              "job": "Machine"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "AER  Advanced Error Reporting by PCI Express devices",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "inspect": false,
            "minWidth": 55
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "blue",
                  "index": 0
                },
                "to": 1
              },
              "type": "range"
            },
            {
              "options": {
                "from": 1,
                "result": {
                  "color": "green",
                  "index": 1
                },
                "to": 10
              },
              "type": "range"
            },
            {
              "options": {
                "from": 10,
                "result": {
                  "color": "yellow",
                  "index": 2
                },
                "to": 100
              },
              "type": "range"
            },
            {
              "options": {
                "from": 100,
                "result": {
                  "color": "red",
                  "index": 3
                },
                "to": 4294967295
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": null
              }
            ]
          },
          "unit": "none",
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 75
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 28,
        "w": 12,
        "x": 0,
        "y": 148
      },
      "id": 35,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"4\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"8\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"9\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"10\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "K"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"11\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "L"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"12\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "M"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_AER_TOTAL_ERRORS{gpu=\"13\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "N"
        }
      ],
      "timeFrom": "30s",
      "title": "Machine GPU AER_ RRORS",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "job",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "Hostname": true,
              "Hostname 1": true,
              "Hostname 10": true,
              "Hostname 11": true,
              "Hostname 12": true,
              "Hostname 13": true,
              "Hostname 14": true,
              "Hostname 2": true,
              "Hostname 3": true,
              "Hostname 4": true,
              "Hostname 5": true,
              "Hostname 6": true,
              "Hostname 7": true,
              "Hostname 8": true,
              "Hostname 9": true,
              "Time": true,
              "Time 1": true,
              "Time 10": true,
              "Time 11": true,
              "Time 12": true,
              "Time 13": true,
              "Time 14": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "UUID 1": true,
              "UUID 10": true,
              "UUID 11": true,
              "UUID 12": true,
              "UUID 13": true,
              "UUID 14": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "UUID 9": true,
              "Value #D": false,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 11": true,
              "__name__ 12": true,
              "__name__ 13": true,
              "__name__ 14": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "device 1": true,
              "device 10": true,
              "device 11": true,
              "device 12": true,
              "device 13": true,
              "device 14": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "device 9": true,
              "gpu 1": true,
              "gpu 10": true,
              "gpu 11": true,
              "gpu 12": true,
              "gpu 13": true,
              "gpu 14": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "gpu 9": true,
              "instance": true,
              "instance 1": true,
              "instance 10": true,
              "instance 11": true,
              "instance 12": true,
              "instance 13": true,
              "instance 14": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "instance 9": true,
              "modelName": true,
              "modelName 1": true,
              "modelName 10": true,
              "modelName 11": true,
              "modelName 12": true,
              "modelName 13": true,
              "modelName 14": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "modelName 9": true,
              "reason 1": true,
              "reason 2": true,
              "reason 3": true,
              "reason 4": true,
              "reason 5": true,
              "reason 6": true
            },
            "includeByName": {},
            "indexByName": {
              "DCGM_FI_DRIVER_VERSION 1": 1,
              "DCGM_FI_DRIVER_VERSION 2": 13,
              "DCGM_FI_DRIVER_VERSION 3": 22,
              "DCGM_FI_DRIVER_VERSION 4": 32,
              "Hostname 1": 5,
              "Hostname 2": 14,
              "Hostname 3": 23,
              "Hostname 4": 33,
              "Time 1": 4,
              "Time 2": 12,
              "Time 3": 21,
              "Time 4": 31,
              "UUID 1": 6,
              "UUID 2": 15,
              "UUID 3": 24,
              "UUID 4": 34,
              "Value #A": 2,
              "Value #B": 3,
              "Value #C": 30,
              "Value #D": 40,
              "__name__ 1": 7,
              "__name__ 2": 16,
              "__name__ 3": 25,
              "__name__ 4": 35,
              "device 1": 8,
              "device 2": 17,
              "device 3": 26,
              "device 4": 36,
              "gpu 1": 9,
              "gpu 2": 18,
              "gpu 3": 27,
              "gpu 4": 37,
              "instance 1": 10,
              "instance 2": 19,
              "instance 3": 28,
              "instance 4": 38,
              "job": 0,
              "modelName 1": 11,
              "modelName 2": 20,
              "modelName 3": 29,
              "modelName 4": 39
            },
            "renameByName": {
              "Hostname 1": "",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7",
              "Value #I": "GPU8",
              "Value #J": "GPU9",
              "Value #K": "GPU10",
              "Value #L": "GPU11",
              "Value #M": "GPU12",
              "Value #N": "GPU13",
              "job": "Machine"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "cellOptions": {
              "mode": "gradient",
              "type": "color-background"
            },
            "filterable": false,
            "inspect": false,
            "minWidth": 54
          },
          "decimals": 0,
          "mappings": [
            {
              "options": {
                "from": 0,
                "result": {
                  "color": "transparent",
                  "index": 0,
                  "text": "Uknown"
                },
                "to": 0
              },
              "type": "range"
            },
            {
              "options": {
                "from": 1,
                "result": {
                  "color": "blue",
                  "index": 1,
                  "text": "OK"
                },
                "to": 1
              },
              "type": "range"
            },
            {
              "options": {
                "from": 2,
                "result": {
                  "color": "red",
                  "index": 2,
                  "text": "Error"
                },
                "to": 100
              },
              "type": "range"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "transparent",
                "value": null
              }
            ]
          },
          "unit": "bool",
          "unitScale": true
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Machine"
            },
            "properties": [
              {
                "id": "custom.width",
                "value": 75
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 25,
        "w": 12,
        "x": 12,
        "y": 175
      },
      "id": 34,
      "options": {
        "cellHeight": "sm",
        "footer": {
          "countRows": false,
          "fields": "",
          "reducer": [
            "sum"
          ],
          "show": false
        },
        "frameIndex": 0,
        "showHeader": true,
        "sortBy": []
      },
      "pluginVersion": "10.3.3",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{ gpu=\"0\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"1\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"2\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "C"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"3\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "D"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"4\"} ",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "E"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{ gpu=\"5\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "F"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{ gpu=\"6\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "G"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"7\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "H"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"8\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "I"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"9\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "J"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"10\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "K"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{ gpu=\"11\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "L"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{ gpu=\"12\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "M"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "GPU_ERROR_STATE{gpu=\"13\"}",
          "format": "table",
          "hide": false,
          "legendFormat": "__auto",
          "range": true,
          "refId": "N"
        }
      ],
      "timeFrom": "30s",
      "title": "Machine GPU Error State",
      "transformations": [
        {
          "id": "joinByField",
          "options": {
            "byField": "job",
            "mode": "outer"
          }
        },
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "DCGM_FI_DRIVER_VERSION 1": true,
              "DCGM_FI_DRIVER_VERSION 2": true,
              "DCGM_FI_DRIVER_VERSION 3": true,
              "DCGM_FI_DRIVER_VERSION 4": true,
              "Hostname 1": true,
              "Hostname 2": true,
              "Hostname 3": true,
              "Hostname 4": true,
              "Hostname 5": true,
              "Hostname 6": true,
              "Hostname 7": true,
              "Hostname 8": true,
              "Time": true,
              "Time 1": true,
              "Time 10": true,
              "Time 11": true,
              "Time 12": true,
              "Time 13": true,
              "Time 14": true,
              "Time 2": true,
              "Time 3": true,
              "Time 4": true,
              "Time 5": true,
              "Time 6": true,
              "Time 7": true,
              "Time 8": true,
              "Time 9": true,
              "UUID 1": true,
              "UUID 10": true,
              "UUID 11": true,
              "UUID 12": true,
              "UUID 13": true,
              "UUID 14": true,
              "UUID 2": true,
              "UUID 3": true,
              "UUID 4": true,
              "UUID 5": true,
              "UUID 6": true,
              "UUID 7": true,
              "UUID 8": true,
              "UUID 9": true,
              "Value #D": false,
              "__name__": true,
              "__name__ 1": true,
              "__name__ 10": true,
              "__name__ 11": true,
              "__name__ 12": true,
              "__name__ 13": true,
              "__name__ 14": true,
              "__name__ 2": true,
              "__name__ 3": true,
              "__name__ 4": true,
              "__name__ 5": true,
              "__name__ 6": true,
              "__name__ 7": true,
              "__name__ 8": true,
              "__name__ 9": true,
              "device 1": true,
              "device 2": true,
              "device 3": true,
              "device 4": true,
              "device 5": true,
              "device 6": true,
              "device 7": true,
              "device 8": true,
              "gpu 1": true,
              "gpu 10": true,
              "gpu 11": true,
              "gpu 12": true,
              "gpu 13": true,
              "gpu 14": true,
              "gpu 2": true,
              "gpu 3": true,
              "gpu 4": true,
              "gpu 5": true,
              "gpu 6": true,
              "gpu 7": true,
              "gpu 8": true,
              "gpu 9": true,
              "instance": true,
              "instance 1": true,
              "instance 10": true,
              "instance 11": true,
              "instance 12": true,
              "instance 13": true,
              "instance 14": true,
              "instance 2": true,
              "instance 3": true,
              "instance 4": true,
              "instance 5": true,
              "instance 6": true,
              "instance 7": true,
              "instance 8": true,
              "instance 9": true,
              "modelName 1": true,
              "modelName 2": true,
              "modelName 3": true,
              "modelName 4": true,
              "modelName 5": true,
              "modelName 6": true,
              "modelName 7": true,
              "modelName 8": true,
              "reason": true,
              "reason 10": true,
              "reason 11": true,
              "reason 12": true,
              "reason 13": true,
              "reason 14": true,
              "reason 9": true
            },
            "includeByName": {},
            "indexByName": {
              "DCGM_FI_DRIVER_VERSION 1": 1,
              "DCGM_FI_DRIVER_VERSION 2": 13,
              "DCGM_FI_DRIVER_VERSION 3": 22,
              "DCGM_FI_DRIVER_VERSION 4": 32,
              "Hostname 1": 5,
              "Hostname 2": 14,
              "Hostname 3": 23,
              "Hostname 4": 33,
              "Time 1": 4,
              "Time 2": 12,
              "Time 3": 21,
              "Time 4": 31,
              "UUID 1": 6,
              "UUID 2": 15,
              "UUID 3": 24,
              "UUID 4": 34,
              "Value #A": 2,
              "Value #B": 3,
              "Value #C": 30,
              "Value #D": 40,
              "__name__ 1": 7,
              "__name__ 2": 16,
              "__name__ 3": 25,
              "__name__ 4": 35,
              "device 1": 8,
              "device 2": 17,
              "device 3": 26,
              "device 4": 36,
              "gpu 1": 9,
              "gpu 2": 18,
              "gpu 3": 27,
              "gpu 4": 37,
              "instance 1": 10,
              "instance 2": 19,
              "instance 3": 28,
              "instance 4": 38,
              "job": 0,
              "modelName 1": 11,
              "modelName 2": 20,
              "modelName 3": 29,
              "modelName 4": 39
            },
            "renameByName": {
              "Hostname 1": "",
              "Value #A": "GPU0",
              "Value #B": "GPU1",
              "Value #C": "GPU2",
              "Value #D": "GPU3",
              "Value #E": "GPU4",
              "Value #F": "GPU5",
              "Value #G": "GPU6",
              "Value #H": "GPU7",
              "Value #I": "GPU8",
              "Value #J": "GPU9",
              "Value #K": "GPU10",
              "Value #L": "GPU11",
              "Value #M": "GPU12",
              "Value #N": "GPU13",
              "job": "Machine"
            }
          }
        },
        {
          "id": "sortBy",
          "options": {
            "fields": {},
            "sort": [
              {
                "field": "Machine"
              }
            ]
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Reported by vast api. ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "MBs",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 176
      },
      "id": 23,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by(machine_id) (vast_machine_InetDown)",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Machine Internet Down speed",
      "transformations": [
        {
          "filter": {
            "id": "byRefId",
            "options": ""
          },
          "id": "filterByValue",
          "options": {
            "filters": [],
            "match": "any",
            "type": "include"
          }
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "As reported by vast api.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "decmbytes",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 184
      },
      "id": 18,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "(sum(vastai_machine_avail_disk_space)  by (machine_id) ) !=0",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Machine avail disk space reported by vast",
      "transformations": [
        {
          "filter": {
            "id": "byRefId",
            "options": ""
          },
          "id": "filterByValue",
          "options": {
            "filters": [],
            "match": "any",
            "type": "include"
          }
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "As reported by vast api for the day before. ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 192
      },
      "id": 17,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "(sum(vastai_per_machine_bwd_earn+vastai_per_machine_bwu_earn)  by (machine_id) ) !=0",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Historic Machine earnings Bandwith  per day",
      "transformations": [
        {
          "filter": {
            "id": "byRefId",
            "options": ""
          },
          "id": "filterByValue",
          "options": {
            "filters": [],
            "match": "any",
            "type": "include"
          }
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "As reported by vast api for the day before. ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 200
      },
      "id": 16,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "(sum(vastai_per_machine_sto_earn)  by (machine_id) ) !=0",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Historic Machine earnings storage  per day",
      "transformations": [
        {
          "filter": {
            "id": "byRefId",
            "options": ""
          },
          "id": "filterByValue",
          "options": {
            "filters": [],
            "match": "any",
            "type": "include"
          }
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "Reported by vast api. ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "MBs",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 200
      },
      "id": 24,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "sum by(machine_id) (vastai_machine_InetUp)",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Machine Internet Upload speed",
      "transformations": [
        {
          "filter": {
            "id": "byRefId",
            "options": ""
          },
          "id": "filterByValue",
          "options": {
            "filters": [],
            "match": "any",
            "type": "include"
          }
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "As reported by vast api for the day before. ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 208
      },
      "id": 15,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "(sum(vastai_per_machine_gpu_earn)  by (machine_id) ) !=0",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Historic Machine earnings for GPUs  per day",
      "transformations": [
        {
          "filter": {
            "id": "byRefId",
            "options": ""
          },
          "id": "filterByValue",
          "options": {
            "filters": [],
            "match": "any",
            "type": "include"
          }
        }
      ],
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "${DS_PROMETHEUS}"
      },
      "description": "As reported by vast api for the day before. ",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "currencyUSD",
          "unitScale": true
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 208
      },
      "id": 14,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${DS_PROMETHEUS}"
          },
          "editorMode": "code",
          "expr": "(sum(vastai_per_machine_gpu_earn+vastai_per_machine_bwd_earn+vastai_per_machine_bwu_earn+vastai_per_machine_sto_earn)  by (machine_id) ) !=0",
          "legendFormat": "{{machine_id}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Historic Machine earnings per day",
      "transformations": [
        {
          "filter": {
            "id": "byRefId",
            "options": ""
          },
          "id": "filterByValue",
          "options": {
            "filters": [],
            "match": "any",
            "type": "include"
          }
        }
      ],
      "type": "timeseries"
    }
  ],
  "refresh": "",
  "schemaVersion": 39,
  "tags": [
    "Prometheuse"
  ],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "300",
          "value": "300"
        },
        "hide": 0,
        "label": "CPU Max Power",
        "name": "CPU_Max_Power",
        "options": [
          {
            "selected": true,
            "text": "300",
            "value": "300"
          }
        ],
        "query": "300",
        "skipUrlSync": false,
        "type": "textbox"
      },
      {
        "current": {
          "selected": false,
          "text": "100",
          "value": "100"
        },
        "hide": 0,
        "label": "BasePower",
        "name": "BasePower",
        "options": [
          {
            "selected": true,
            "text": "100",
            "value": "100"
          }
        ],
        "query": "100",
        "skipUrlSync": false,
        "type": "textbox"
      }
    ]
  },
  "time": {
    "from": "now-24h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "Vast Dashboard",
  "uid": "d1e22d5b-dcf4-4f54-8629-972173cb24eb",
  "version": 41,
  "weekStart": ""
}