lcore: add cpu usage info LcoreMgr info command (#580)

output of ./build/app/LcoreMgr --info: MT: 2023-11-14 15:00:03, mtl_lcore_shm_print, MTL used lcores 3 MT: 2023-11-14 15:00:03, mtl_lcore_shm_print, lcore 28 active by xxx@xxx, pid: 236759(comm: RxTxApp) type: app_allocated MT: 2023-11-14 15:00:03, mtl_lcore_shm_print, lcore 29 active by xxx@xxx, pid: 236759(comm: RxTxApp) type: lib_sch MT: 2023-11-14 15:00:03, mtl_lcore_shm_print, lcore 30 active by xxx@xxx, pid: 236759(comm: RxTxApp) type: lib_sch MT: 2023-11-14 15:00:03, mtl_lcore_shm_print, collecting cpu usage MT: 2023-11-14 15:00:04, mtl_lcore_shm_print, lcore 28 cpu usage 4.08% MT: 2023-11-14 15:00:04, mtl_lcore_shm_print, lcore 29 cpu usage 100.00% MT: 2023-11-14 15:00:04, mtl_lcore_shm_print, lcore 30 cpu usage 100.00% Signed-off-by: Frank Du <[email protected]>
OpenVisualCloud · Nov 14, 2023 · 21109f1 · 21109f1
1 parent 4f46e49
commit 21109f1
Show file tree

Hide file tree

Showing 9 changed files with 187 additions and 26 deletions.
diff --git a/doc/lcore.md b/doc/lcore.md
@@ -21,4 +21,18 @@ IMTL also provides a tool which can be find from `./build/app/LcoreMgr` for manu
 
 When you use the `--clean_pid_auto_check` option, the tool will perform a loop check for all the active entries in the map. It checks whether the hostname and user match the current login environment and then verifies if the PID is still running. If the tool detects that the PID is no longer active, it will proceed to remove the lcore from the mapping.
 
-If you are deploying in a multi-container environment, the PID check becomes less useful as each container has its own process namespace. In such cases, you can use the --clean_lcore option to remove an entry based on the lcore ID. However, it's important to confirm that the lcore is not active before using this option.
+If you are deploying in a multi-container environment, the PID check becomes less useful as each container has its own process namespace. In such cases, you can use the `--clean_lcore` option to remove an entry based on the lcore ID. However, it's important to confirm that the lcore is not active before using this option.
+
+Before using the `--clean_lcore` option, please ensure that you double-check the current lcore status by running `./build/app/LcoreMgr --info`. An example output list is provided below. Pay close attention to the lcore usage details, including user, host, PID, process name, allocation information, and, most importantly, CPU usage.
+Typically, an active lcore allocated by the IMTL library should utilize 100% of the CPU resources.
+
+```bash
+MT: 2023-11-14 15:00:03, mtl_lcore_shm_print, MTL used lcores 3
+MT: 2023-11-14 15:00:03, mtl_lcore_shm_print, lcore 28 active by xxx@xxx, pid: 236759(comm: RxTxApp) type: app_allocated
+MT: 2023-11-14 15:00:03, mtl_lcore_shm_print, lcore 29 active by xxx@xxx, pid: 236759(comm: RxTxApp) type: lib_sch
+MT: 2023-11-14 15:00:03, mtl_lcore_shm_print, lcore 30 active by xxx@xxx, pid: 236759(comm: RxTxApp) type: lib_sch
+MT: 2023-11-14 15:00:03, mtl_lcore_shm_print, collecting cpu usage
+MT: 2023-11-14 15:00:04, mtl_lcore_shm_print, lcore 28 cpu usage 4.08%
+MT: 2023-11-14 15:00:04, mtl_lcore_shm_print, lcore 29 cpu usage 100.00%
+MT: 2023-11-14 15:00:04, mtl_lcore_shm_print, lcore 30 cpu usage 100.00%
+```
diff --git a/lib/src/mt_main.c b/lib/src/mt_main.c
@@ -674,7 +674,7 @@ int mtl_get_lcore(mtl_handle mt, unsigned int* lcore) {
     return -EIO;
   }
 
-  return mt_sch_get_lcore(impl, lcore);
+  return mt_sch_get_lcore(impl, lcore, MT_LCORE_TYPE_USER);
 }
 
 int mtl_put_lcore(mtl_handle mt, unsigned int lcore) {

diff --git a/lib/src/mt_main.h b/lib/src/mt_main.h
@@ -711,10 +711,26 @@ struct mt_interface {
   void* xdp;
 };
 
-struct mt_lcore_shm_entry {
+struct mt_user_info {
   char hostname[64];
   char user[32];
+  /* the current process name */
+  char comm[64];
   pid_t pid;
+};
+
+enum mt_lcore_type {
+  MT_LCORE_TYPE_SCH = 0, /* lib scheduler used */
+  MT_LCORE_TYPE_TAP,
+  MT_LCORE_TYPE_RXV_RING_LCORE,
+  MT_LCORE_TYPE_USER, /* allocated by application */
+  MT_LCORE_TYPE_MAX,
+};
+
+struct mt_lcore_shm_entry {
+  struct mt_user_info u_info;
+  pid_t pid;
+  enum mt_lcore_type type;
   bool active;
 };
 
@@ -1091,12 +1107,6 @@ struct mt_dp_impl {
   rte_spinlock_t txq_sys_entry_lock; /* protect txq_sys_entry */
 };
 
-struct mt_user_info {
-  char hostname[64];
-  char user[32];
-  pid_t pid;
-};
-
 struct mtl_main_impl {
   struct mt_interface inf[MTL_PORT_MAX];
 

diff --git a/lib/src/mt_sch.c b/lib/src/mt_sch.c
@@ -201,7 +201,7 @@ static int sch_start(struct mt_sch_impl* sch) {
   rte_atomic32_set(&sch->stopped, 0);
 
   if (!sch->run_in_thread) {
-    ret = mt_sch_get_lcore(sch->parent, &sch->lcore);
+    ret = mt_sch_get_lcore(sch->parent, &sch->lcore, MT_LCORE_TYPE_SCH);
     if (ret < 0) {
       err("%s(%d), get lcore fail %d\n", __func__, idx, ret);
       sch_unlock(sch);
@@ -563,7 +563,8 @@ static int sch_init_lcores(struct mt_sch_mgr* mgr) {
   return 0;
 }
 
-int mt_sch_get_lcore(struct mtl_main_impl* impl, unsigned int* lcore) {
+int mt_sch_get_lcore(struct mtl_main_impl* impl, unsigned int* lcore,
+                     enum mt_lcore_type type) {
   struct mt_sch_mgr* mgr = mt_sch_get_mgr(impl);
   struct mt_user_info* info = &impl->u_info;
   unsigned int cur_lcore = 0;
@@ -586,8 +587,11 @@ int mt_sch_get_lcore(struct mtl_main_impl* impl, unsigned int* lcore) {
       if (!shm_entry->active) {
         *lcore = cur_lcore;
         shm_entry->active = true;
-        strncpy(shm_entry->hostname, info->hostname, sizeof(shm_entry->hostname));
-        strncpy(shm_entry->user, info->user, sizeof(shm_entry->user));
+        struct mt_user_info* u_info = &shm_entry->u_info;
+        strncpy(u_info->hostname, info->hostname, sizeof(u_info->hostname));
+        strncpy(u_info->user, info->user, sizeof(u_info->user));
+        strncpy(u_info->comm, info->comm, sizeof(u_info->comm));
+        shm_entry->type = type;
         shm_entry->pid = info->pid;
         lcore_shm->used++;
         rte_atomic32_inc(&impl->lcore_cnt);
@@ -1029,6 +1033,20 @@ int mt_sch_stop_all(struct mtl_main_impl* impl) {
   return 0;
 }
 
+static const char* lcore_type_names[MT_LCORE_TYPE_MAX] = {
+    "lib_sch",
+    "lib_tap",
+    "lib_rxv_ring",
+    "app_allocated",
+};
+
+static const char* lcore_type_name(enum mt_lcore_type type) {
+  if (type >= MT_LCORE_TYPE_SCH && type < MT_LCORE_TYPE_MAX)
+    return lcore_type_names[type];
+  else
+    return "unknown";
+}
+
 int mtl_lcore_shm_print(void) {
   struct mt_lcore_mgr lcore_mgr;
   struct mt_lcore_shm_entry* shm_entry;
@@ -1039,12 +1057,44 @@ int mtl_lcore_shm_print(void) {
   struct mt_lcore_shm* lcore_shm = lcore_mgr.lcore_shm;
   info("%s, MTL used lcores %d\n", __func__, lcore_shm->used);
 
+  int cpu_ids[RTE_MAX_LCORE];
+  int found = 0;
+
   for (int i = 0; i < RTE_MAX_LCORE; i++) {
     shm_entry = &lcore_shm->lcores_info[i];
 
     if (!shm_entry->active) continue;
-    info("%s, lcore %d active on %s@%s with pid: %d\n", __func__, i, shm_entry->user,
-         shm_entry->hostname, (int)shm_entry->pid);
+    struct mt_user_info* u_info = &shm_entry->u_info;
+    info("%s, lcore %d active by %s@%s, pid: %d(comm: %s) type: %s\n", __func__, i,
+         u_info->user, u_info->hostname, (int)shm_entry->pid, u_info->comm,
+         lcore_type_name(shm_entry->type));
+    cpu_ids[found] = i;
+    found++;
+  }
+
+  if (found > 0) { /* read the cpu usage */
+    struct mt_cpu_usage prev[found];
+    struct mt_cpu_usage cur[found];
+
+    info("%s, collecting cpu usage...\n", __func__);
+    ret = mt_read_cpu_usage(prev, cpu_ids, found);
+    if (ret != found) {
+      err("%s, read cpu prev usage fail, expect %d but only %d get\n", __func__, found,
+          ret);
+    } else {
+      mt_sleep_ms(1000 * 1);
+      ret = mt_read_cpu_usage(cur, cpu_ids, found);
+      if (ret != found) {
+        err("%s, read cpu curr usage fail, expect %d but only %d get\n", __func__, found,
+            ret);
+      } else {
+        /* print the result */
+        for (int i = 0; i < found; i++) {
+          double usage = mt_calculate_cpu_usage(&prev[i], &cur[i]);
+          info("%s, lcore %d cpu usage %.2f%%\n", __func__, cpu_ids[i], usage);
+        }
+      }
+    }
   }
 
   sch_lcore_shm_uinit(&lcore_mgr);
@@ -1060,21 +1110,21 @@ static int lcore_shm_clean_auto_pid(struct mt_lcore_mgr* lcore_mgr) {
 #else
 
 static int lcore_shm_clean_auto_pid(struct mt_lcore_mgr* lcore_mgr) {
-  struct mt_user_info u_info;
+  struct mt_user_info info;
   int clean = 0;
 
-  memset(&u_info, 0, sizeof(u_info));
-  mt_user_info_init(&u_info);
+  memset(&info, 0, sizeof(info));
+  mt_user_info_init(&info);
 
   struct mt_lcore_shm* lcore_shm = lcore_mgr->lcore_shm;
   struct mt_lcore_shm_entry* shm_entry;
   for (int i = 0; i < RTE_MAX_LCORE; i++) {
     shm_entry = &lcore_shm->lcores_info[i];
 
     if (!shm_entry->active) continue;
-    if (0 != strncmp(shm_entry->hostname, u_info.hostname, sizeof(shm_entry->hostname)))
-      continue;
-    if (0 != strncmp(shm_entry->user, u_info.user, sizeof(shm_entry->user))) continue;
+    struct mt_user_info* u_info = &shm_entry->u_info;
+    if (0 != strncmp(u_info->hostname, info.hostname, sizeof(u_info->hostname))) continue;
+    if (0 != strncmp(u_info->user, info.user, sizeof(u_info->user))) continue;
     /* now check if PID is active with zero signal */
     int result = kill(shm_entry->pid, 0);
     if (0 == result) continue;

diff --git a/lib/src/mt_sch.h b/lib/src/mt_sch.h
@@ -66,7 +66,8 @@ static inline void mt_sch_set_cpu_busy(struct mt_sch_impl* sch, bool busy) {
 }
 
 int mt_sch_put_lcore(struct mtl_main_impl* impl, unsigned int lcore);
-int mt_sch_get_lcore(struct mtl_main_impl* impl, unsigned int* lcore);
+int mt_sch_get_lcore(struct mtl_main_impl* impl, unsigned int* lcore,
+                     enum mt_lcore_type type);
 bool mt_sch_lcore_valid(struct mtl_main_impl* impl, unsigned int lcore);
 
 #endif
diff --git a/lib/src/mt_tap.c b/lib/src/mt_tap.c
@@ -900,7 +900,7 @@ int mt_tap_init(struct mtl_main_impl* impl) {
 
   rte_atomic32_set(&cni->stop_tap, 0);
   tap_ctx->has_lcore = false;
-  ret = mt_sch_get_lcore(impl, &lcore);
+  ret = mt_sch_get_lcore(impl, &lcore, MT_LCORE_TYPE_TAP);
   if (ret < 0) {
     err("%s, get lcore fail %d\n", __func__, ret);
     mt_tap_uinit(impl);

diff --git a/lib/src/mt_util.c b/lib/src/mt_util.c
@@ -887,10 +887,13 @@ const char* mt_native_afxdp_port2if(const char* port) {
 int mt_user_info_init(struct mt_user_info* info) {
   int ret = -EIO;
 
+  info->pid = getpid();
+
 #ifdef WINDOWSENV /* todo */
   MTL_MAY_UNUSED(ret);
   snprintf(info->hostname, sizeof(info->hostname), "%s", "unknow");
   snprintf(info->user, sizeof(info->user), "%s", "unknow");
+  snprintf(info->comm, sizeof(info->comm), "%s", "unknow");
 #else
   ret = gethostname(info->hostname, sizeof(info->hostname));
   if (ret < 0) {
@@ -901,9 +904,77 @@ int mt_user_info_init(struct mt_user_info* info) {
   struct passwd* user_info = getpwuid(uid);
   snprintf(info->user, sizeof(info->user), "%s",
            user_info ? user_info->pw_name : "unknow");
+  char comm_path[128];
+  snprintf(comm_path, sizeof(comm_path), "/proc/%d/comm", info->pid);
+  int fd = open(comm_path, O_RDONLY);
+  if (fd >= 0) {
+    ssize_t bytes = read(fd, info->comm, sizeof(info->comm) - 1);
+    if (bytes <= 0)
+      snprintf(info->comm, sizeof(info->comm), "%s", "unknow");
+    else
+      info->comm[strcspn(info->comm, "\n")] = '\0';
+    close(fd);
+  } else {
+    snprintf(info->comm, sizeof(info->comm), "%s", "unknow");
+  }
+  dbg("%s, comm %s\n", __func__, info->comm);
 #endif
 
-  info->pid = getpid();
-
   return 0;
 }
+
+int mt_read_cpu_usage(struct mt_cpu_usage* usages, int* cpu_ids, int num_cpus) {
+#ifdef WINDOWSENV /* todo */
+  MTL_MAY_UNUSED(usages);
+  MTL_MAY_UNUSED(cpu_ids);
+  MTL_MAY_UNUSED(num_cpus);
+  err("%s, not support on windows\n", __func__);
+  return -ENOTSUP;
+#else
+  FILE* file;
+  char line[256];
+  int found = 0;
+
+  file = fopen("/proc/stat", "r");
+  if (!file) {
+    err("%s, open /proc/stat fail\n", __func__);
+    return -EIO;
+  }
+
+  while (fgets(line, sizeof(line) - 1, file)) {
+    struct mt_cpu_usage cur;
+    int cpu;
+    int parsed = sscanf(line,
+                        "cpu%d %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64
+                        " %" PRIu64 " %" PRIu64 " %" PRIu64 "",
+                        &cpu, &cur.user, &cur.nice, &cur.system, &cur.idle, &cur.iowait,
+                        &cur.irq, &cur.softirq, &cur.steal);
+    if (parsed != 9) continue;
+    /* check if match with any input cpus */
+    for (int i = 0; i < num_cpus; i++) {
+      if (cpu == cpu_ids[i]) {
+        found++;
+        usages[i] = cur;
+        dbg("%s, get succ for cpu %d at %d\n", __func__, cpu, i);
+        break;
+      }
+    }
+  }
+
+  fclose(file);
+  return found;
+#endif
+}
+
+double mt_calculate_cpu_usage(struct mt_cpu_usage* prev, struct mt_cpu_usage* curr) {
+  uint64_t prev_idle = prev->idle + prev->iowait;
+  uint64_t curr_idle = curr->idle + curr->iowait;
+  uint64_t prev_total = prev->user + prev->nice + prev->system + prev->idle +
+                        prev->iowait + prev->irq + prev->softirq + prev->steal;
+  uint64_t curr_total = curr->user + curr->nice + curr->system + curr->idle +
+                        curr->iowait + curr->irq + curr->softirq + curr->steal;
+  uint64_t totald = curr_total - prev_total;
+  uint64_t idled = curr_idle - prev_idle;
+
+  return 100.0 * (totald - idled) / totald;
+}
diff --git a/lib/src/mt_util.h b/lib/src/mt_util.h
@@ -253,4 +253,19 @@ const char* mt_native_afxdp_port2if(const char* port);
 
 int mt_user_info_init(struct mt_user_info* info);
 
+struct mt_cpu_usage {
+  uint64_t user;
+  uint64_t nice;
+  uint64_t system;
+  uint64_t idle;
+  uint64_t iowait;
+  uint64_t irq;
+  uint64_t softirq;
+  uint64_t steal;
+};
+
+int mt_read_cpu_usage(struct mt_cpu_usage* usages, int* cpu_ids, int num_cpus);
+
+double mt_calculate_cpu_usage(struct mt_cpu_usage* prev, struct mt_cpu_usage* curr);
+
 #endif
diff --git a/lib/src/st2110/st_rx_video_session.c b/lib/src/st2110/st_rx_video_session.c
@@ -2538,7 +2538,7 @@ static int rv_init_pkt_lcore(struct mtl_main_impl* impl,
   }
   s->pkt_lcore_ring = ring;
 
-  ret = mt_sch_get_lcore(impl, &lcore);
+  ret = mt_sch_get_lcore(impl, &lcore, MT_LCORE_TYPE_RXV_RING_LCORE);
   if (ret < 0) {
     err("%s(%d,%d), get lcore fail %d\n", __func__, mgr_idx, idx, ret);
     rv_uinit_pkt_lcore(impl, s);