From 5a564cb8233039ef67f80b948f324b6cfca3d7e6 Mon Sep 17 00:00:00 2001 From: Ric Li Date: Thu, 16 Nov 2023 16:21:13 +0800 Subject: [PATCH] manager: add daemon server for privileged controls (#582) Added lcore management; work in progress for other features. --------- Signed-off-by: Ric Li --- build.sh | 17 ++-- lib/src/meson.build | 1 + lib/src/mt_instance.c | 153 +++++++++++++++++++++++++++++ lib/src/mt_instance.h | 16 +++ lib/src/mt_main.c | 5 + lib/src/mt_main.h | 7 ++ lib/src/mt_sch.c | 176 +++++++++++++++++++-------------- manager/README.md | 31 ++++++ manager/logging.hpp | 56 +++++++++++ manager/manager-design.svg | 4 + manager/meson.build | 41 ++++++++ manager/meson_options.txt | 4 + manager/mtl_instance.hpp | 146 ++++++++++++++++++++++++++++ manager/mtl_interface.hpp | 41 ++++++++ manager/mtl_lcore.hpp | 42 ++++++++ manager/mtl_manager.cpp | 193 +++++++++++++++++++++++++++++++++++++ manager/mtl_mproto.h | 94 ++++++++++++++++++ 17 files changed, 947 insertions(+), 80 deletions(-) create mode 100644 lib/src/mt_instance.c create mode 100644 lib/src/mt_instance.h create mode 100644 manager/README.md create mode 100644 manager/logging.hpp create mode 100644 manager/manager-design.svg create mode 100644 manager/meson.build create mode 100644 manager/meson_options.txt create mode 100644 manager/mtl_instance.hpp create mode 100644 manager/mtl_interface.hpp create mode 100644 manager/mtl_lcore.hpp create mode 100644 manager/mtl_manager.cpp create mode 100644 manager/mtl_mproto.h diff --git a/build.sh b/build.sh index 480789851..ba9ae0868 100755 --- a/build.sh +++ b/build.sh @@ -70,6 +70,7 @@ APP_BUILD_DIR=${WORKSPACE}/build/app TEST_BUILD_DIR=${WORKSPACE}/build/tests PLUGINS_BUILD_DIR=${WORKSPACE}/build/plugins LD_PRELOAD_BUILD_DIR=${WORKSPACE}/build/ld_preload +MANAGER_BUILD_DIR=${WORKSPACE}/build/manager # build lib meson setup "${LIB_BUILD_DIR}" -Dbuildtype="$buildtype" -Ddisable_pcapng="$disable_pcapng" -Denable_asan="$enable_asan" -Denable_tap="$enable_tap" @@ -85,17 +86,13 @@ popd # build app pushd app/ meson setup "${APP_BUILD_DIR}" -Dbuildtype="$buildtype" -Denable_asan="$enable_asan" -popd -pushd "${APP_BUILD_DIR}" -ninja +meson compile -C "${APP_BUILD_DIR}" popd # build tests pushd tests/ meson setup "${TEST_BUILD_DIR}" -Dbuildtype="$buildtype" -Denable_asan="$enable_asan" -popd -pushd "${TEST_BUILD_DIR}" -ninja +meson compile -C "${TEST_BUILD_DIR}" popd # build plugins @@ -124,4 +121,12 @@ else sudo ninja install fi popd +fi + +# build mtl_manager +if [ "$OS" != "Windows_NT" ]; then +pushd manager/ +meson setup "${MANAGER_BUILD_DIR}" -Dbuildtype="$buildtype" -Denable_asan="$enable_asan" +meson compile -C "${MANAGER_BUILD_DIR}" +popd fi \ No newline at end of file diff --git a/lib/src/meson.build b/lib/src/meson.build index bd336456a..a7d5a4167 100644 --- a/lib/src/meson.build +++ b/lib/src/meson.build @@ -17,6 +17,7 @@ sources = files( 'mt_stat.c', 'mt_rtcp.c', 'mt_flow.c', + 'mt_instance.c', ) if is_windows diff --git a/lib/src/mt_instance.c b/lib/src/mt_instance.c new file mode 100644 index 000000000..438ee3eec --- /dev/null +++ b/lib/src/mt_instance.c @@ -0,0 +1,153 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2023 Intel Corporation + */ + +#include "mt_instance.h" + +#ifndef WINDOWSENV + +#include + +#include "../manager/mtl_mproto.h" +#include "mt_log.h" + +int mt_instance_put_lcore(struct mtl_main_impl* impl, unsigned int lcore_id) { + int ret; + int sock = impl->instance_fd; + + mtl_message_t msg; + msg.header.magic = htonl(MTL_MANAGER_MAGIC); + msg.header.type = htonl(MTL_MSG_TYPE_PUT_LCORE); + msg.body.lcore_msg.lcore = htons(lcore_id); + msg.header.body_len = sizeof(msg.body.lcore_msg); + + ret = send(sock, &msg, sizeof(msg), 0); + if (ret < 0) { + err("%s, send message fail\n", __func__); + return ret; + } + + return 0; +} + +int mt_instance_get_lcore(struct mtl_main_impl* impl, unsigned int lcore_id) { + int ret; + int sock = impl->instance_fd; + + mtl_message_t msg; + msg.header.magic = htonl(MTL_MANAGER_MAGIC); + msg.header.type = htonl(MTL_MSG_TYPE_GET_LCORE); + msg.body.lcore_msg.lcore = htons(lcore_id); + msg.header.body_len = htonl(sizeof(mtl_lcore_message_t)); + + ret = send(sock, &msg, sizeof(mtl_message_t), 0); + if (ret < 0) { + err("%s, send message fail\n", __func__); + return ret; + } + + ret = recv(sock, &msg, sizeof(mtl_message_t), 0); + + if (ret < 0 || ntohl(msg.header.magic) != MTL_MANAGER_MAGIC || + ntohl(msg.header.type) != MTL_MSG_TYPE_RESPONSE) { + err("%s, recv response fail\n", __func__); + return -EIO; + } + + int response = msg.body.response_msg.response; + + /* return negative value incase user check with < 0 */ + return -response; +} + +int mt_instance_init(struct mtl_main_impl* impl) { + impl->instance_fd = -1; + int sock = socket(AF_UNIX, SOCK_STREAM, 0); + if (sock < 0) { + err("%s, create socket fail %d\n", __func__, sock); + return sock; + } + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, MTL_MANAGER_SOCK_PATH, sizeof(addr.sun_path) - 1); + int ret = connect(sock, (struct sockaddr*)&addr, sizeof(addr)); + if (ret < 0) { + err("%s, connect to manager fail\n", __func__); + close(sock); + return ret; + } + + struct mt_user_info* u_info = &impl->u_info; + + mtl_message_t msg; + msg.header.magic = htonl(MTL_MANAGER_MAGIC); + msg.header.type = htonl(MTL_MSG_TYPE_REGISTER); + msg.header.body_len = sizeof(mtl_register_message_t); + + mtl_register_message_t* reg_msg = &msg.body.register_msg; + reg_msg->pid = htonl(u_info->pid); + reg_msg->uid = htonl(getuid()); + strncpy(reg_msg->hostname, u_info->hostname, sizeof(reg_msg->hostname)); + + ret = send(sock, &msg, sizeof(msg), 0); + if (ret < 0) { + err("%s, send message fail\n", __func__); + close(sock); + return ret; + } + + ret = recv(sock, &msg, sizeof(msg), 0); + if (ret < 0 || ntohl(msg.header.magic) != MTL_MANAGER_MAGIC || + ntohl(msg.header.type) != MTL_MSG_TYPE_RESPONSE) { + err("%s, recv response fail\n", __func__); + close(sock); + return ret; + } + + int response = msg.body.response_msg.response; + if (response != 0) { + err("%s, register fail\n", __func__); + close(sock); + return -EIO; + } + + impl->instance_fd = sock; + + info("%s, succ\n", __func__); + + return 0; +} + +int mt_instance_uinit(struct mtl_main_impl* impl) { + int sock = impl->instance_fd; + if (sock <= 0) return -EIO; + + return close(sock); +} + +#else /* not supported on Windows */ + +int mt_instance_init(struct mtl_main_impl* impl) { + impl->instance_fd = -1; + return -ENOTSUP; +} + +int mt_instance_uinit(struct mtl_main_impl* impl) { + MTL_MAY_UNUSED(impl); + return -ENOTSUP; +} + +int mt_instance_get_lcore(struct mtl_main_impl* impl, unsigned int lcore_id) { + MTL_MAY_UNUSED(impl); + MTL_MAY_UNUSED(lcore_id); + return -ENOTSUP; +} + +int mt_instance_put_lcore(struct mtl_main_impl* impl, unsigned int lcore_id) { + MTL_MAY_UNUSED(impl); + MTL_MAY_UNUSED(lcore_id); + return -ENOTSUP; +} + +#endif \ No newline at end of file diff --git a/lib/src/mt_instance.h b/lib/src/mt_instance.h new file mode 100644 index 000000000..b482f7092 --- /dev/null +++ b/lib/src/mt_instance.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2023 Intel Corporation + */ + +#ifndef _MT_LIB_INSTANCE_HEAD_H_ +#define _MT_LIB_INSTANCE_HEAD_H_ + +#include "mt_main.h" + +int mt_instance_init(struct mtl_main_impl* impl); +int mt_instance_uinit(struct mtl_main_impl* impl); + +int mt_instance_get_lcore(struct mtl_main_impl* impl, unsigned int lcore_id); +int mt_instance_put_lcore(struct mtl_main_impl* impl, unsigned int lcore_id); + +#endif \ No newline at end of file diff --git a/lib/src/mt_main.c b/lib/src/mt_main.c index 76a21a487..477d56c61 100644 --- a/lib/src/mt_main.c +++ b/lib/src/mt_main.c @@ -13,6 +13,7 @@ #include "mt_dhcp.h" #include "mt_dma.h" #include "mt_flow.h" +#include "mt_instance.h" #include "mt_log.h" #include "mt_mcast.h" #include "mt_ptp.h" @@ -475,6 +476,8 @@ mtl_handle mtl_init(struct mtl_init_params* p) { impl->privileged = true; #endif + mt_instance_init(impl); + rte_memcpy(&impl->user_para, p, sizeof(*p)); impl->var_para.sch_default_sleep_us = 1 * US_PER_MS; /* default 1ms */ /* use sleep zero if sleep us is smaller than this thresh */ @@ -630,6 +633,8 @@ int mtl_uninit(mtl_handle mt) { mt_stat_uinit(impl); + mt_instance_uinit(impl); + mt_rte_free(impl); mt_dev_uinit(p); diff --git a/lib/src/mt_main.h b/lib/src/mt_main.h index 4d2f1e409..57cf7f90d 100644 --- a/lib/src/mt_main.h +++ b/lib/src/mt_main.h @@ -1197,6 +1197,9 @@ struct mtl_main_impl { int arp_timeout_ms; bool privileged; /* if app running with root privilege */ + + /* connect to mtl manager */ + int instance_fd; }; static inline struct mtl_init_params* mt_get_user_params(struct mtl_main_impl* impl) { @@ -1207,6 +1210,10 @@ static inline bool mt_is_privileged(struct mtl_main_impl* impl) { return impl->privileged; } +static inline bool mt_is_manager_connected(struct mtl_main_impl* impl) { + return impl->instance_fd > 0; +} + static inline struct mt_interface* mt_if(struct mtl_main_impl* impl, enum mtl_port port) { return &impl->inf[port]; } diff --git a/lib/src/mt_sch.c b/lib/src/mt_sch.c index bd58edf4e..bdef65d13 100644 --- a/lib/src/mt_sch.c +++ b/lib/src/mt_sch.c @@ -6,6 +6,7 @@ #include +#include "mt_instance.h" #include "mt_log.h" #include "mt_stat.h" #include "mtl_lcore_shm_api.h" @@ -565,92 +566,114 @@ static int sch_init_lcores(struct mt_sch_mgr* mgr) { int mt_sch_get_lcore(struct mtl_main_impl* impl, unsigned int* lcore, enum mt_lcore_type type) { - struct mt_sch_mgr* mgr = mt_sch_get_mgr(impl); - struct mt_user_info* info = &impl->u_info; unsigned int cur_lcore = 0; int ret; - struct mt_lcore_shm* lcore_shm = mgr->lcore_mgr.lcore_shm; - struct mt_lcore_shm_entry* shm_entry; + if (mt_is_manager_connected(impl)) { + do { + cur_lcore = rte_get_next_lcore(cur_lcore, 1, 0); + if ((cur_lcore < RTE_MAX_LCORE) && + mt_socket_match(rte_lcore_to_socket_id(cur_lcore), + mt_socket_id(impl, MTL_PORT_P))) { + ret = mt_instance_get_lcore(impl, cur_lcore); + if (ret == 0) { + *lcore = cur_lcore; + return 0; + } + } + } while (cur_lcore < RTE_MAX_LCORE); + } else { + struct mt_sch_mgr* mgr = mt_sch_get_mgr(impl); + struct mt_user_info* info = &impl->u_info; - ret = sch_filelock_lock(mgr); - if (ret < 0) { - err("%s, sch_filelock_lock fail\n", __func__); - return ret; - } + struct mt_lcore_shm* lcore_shm = mgr->lcore_mgr.lcore_shm; + struct mt_lcore_shm_entry* shm_entry; - do { - cur_lcore = rte_get_next_lcore(cur_lcore, 1, 0); - shm_entry = &lcore_shm->lcores_info[cur_lcore]; - - if ((cur_lcore < RTE_MAX_LCORE) && mt_socket_match(rte_lcore_to_socket_id(cur_lcore), - mt_socket_id(impl, MTL_PORT_P))) { - if (!shm_entry->active) { - *lcore = cur_lcore; - shm_entry->active = true; - struct mt_user_info* u_info = &shm_entry->u_info; - strncpy(u_info->hostname, info->hostname, sizeof(u_info->hostname)); - strncpy(u_info->user, info->user, sizeof(u_info->user)); - strncpy(u_info->comm, info->comm, sizeof(u_info->comm)); - shm_entry->type = type; - shm_entry->pid = info->pid; - lcore_shm->used++; - rte_atomic32_inc(&impl->lcore_cnt); - mgr->local_lcores_active[cur_lcore] = true; - ret = sch_filelock_unlock(mgr); - info("%s, available lcore %d\n", __func__, cur_lcore); - if (ret < 0) { - err("%s, sch_filelock_unlock fail\n", __func__); - return ret; + ret = sch_filelock_lock(mgr); + if (ret < 0) { + err("%s, sch_filelock_lock fail\n", __func__); + return ret; + } + + do { + cur_lcore = rte_get_next_lcore(cur_lcore, 1, 0); + shm_entry = &lcore_shm->lcores_info[cur_lcore]; + + if ((cur_lcore < RTE_MAX_LCORE) && + mt_socket_match(rte_lcore_to_socket_id(cur_lcore), + mt_socket_id(impl, MTL_PORT_P))) { + if (!shm_entry->active) { + *lcore = cur_lcore; + shm_entry->active = true; + struct mt_user_info* u_info = &shm_entry->u_info; + strncpy(u_info->hostname, info->hostname, sizeof(u_info->hostname)); + strncpy(u_info->user, info->user, sizeof(u_info->user)); + strncpy(u_info->comm, info->comm, sizeof(u_info->comm)); + shm_entry->type = type; + shm_entry->pid = info->pid; + lcore_shm->used++; + rte_atomic32_inc(&impl->lcore_cnt); + mgr->local_lcores_active[cur_lcore] = true; + ret = sch_filelock_unlock(mgr); + info("%s, available lcore %d\n", __func__, cur_lcore); + if (ret < 0) { + err("%s, sch_filelock_unlock fail\n", __func__); + return ret; + } + return 0; } - return 0; } - } - } while (cur_lcore < RTE_MAX_LCORE); + } while (cur_lcore < RTE_MAX_LCORE); + + sch_filelock_unlock(mgr); + } - sch_filelock_unlock(mgr); err("%s, fail to find lcore\n", __func__); return -EIO; } int mt_sch_put_lcore(struct mtl_main_impl* impl, unsigned int lcore) { - int ret; - struct mt_sch_mgr* mgr = mt_sch_get_mgr(impl); - struct mt_lcore_shm* lcore_shm = mgr->lcore_mgr.lcore_shm; + if (mt_is_manager_connected(impl)) { + return mt_instance_put_lcore(impl, lcore); + } else { + int ret; + struct mt_sch_mgr* mgr = mt_sch_get_mgr(impl); + struct mt_lcore_shm* lcore_shm = mgr->lcore_mgr.lcore_shm; - if (lcore >= RTE_MAX_LCORE) { - err("%s, invalid lcore %d\n", __func__, lcore); - return -EIO; - } - if (!lcore_shm) { - err("%s, no lcore shm attached\n", __func__); - return -EIO; - } - ret = sch_filelock_lock(mgr); - if (ret < 0) { - err("%s, sch_filelock_lock fail\n", __func__); - return ret; - } - if (!lcore_shm->lcores_info[lcore].active) { - err("%s, lcore %d not active\n", __func__, lcore); - ret = -EIO; - goto err_unlock; - } + if (lcore >= RTE_MAX_LCORE) { + err("%s, invalid lcore %d\n", __func__, lcore); + return -EIO; + } + if (!lcore_shm) { + err("%s, no lcore shm attached\n", __func__); + return -EIO; + } + ret = sch_filelock_lock(mgr); + if (ret < 0) { + err("%s, sch_filelock_lock fail\n", __func__); + return ret; + } + if (!lcore_shm->lcores_info[lcore].active) { + err("%s, lcore %d not active\n", __func__, lcore); + ret = -EIO; + goto err_unlock; + } - lcore_shm->lcores_info[lcore].active = false; - lcore_shm->used--; - rte_atomic32_dec(&impl->lcore_cnt); - mgr->local_lcores_active[lcore] = false; - ret = sch_filelock_unlock(mgr); - info("%s, lcore %d\n", __func__, lcore); - if (ret < 0) { - err("%s, sch_filelock_unlock fail\n", __func__); + lcore_shm->lcores_info[lcore].active = false; + lcore_shm->used--; + rte_atomic32_dec(&impl->lcore_cnt); + mgr->local_lcores_active[lcore] = false; + ret = sch_filelock_unlock(mgr); + info("%s, lcore %d\n", __func__, lcore); + if (ret < 0) { + err("%s, sch_filelock_unlock fail\n", __func__); + return ret; + } + return 0; + + err_unlock: + sch_filelock_unlock(mgr); return ret; } - return 0; - -err_unlock: - sch_filelock_unlock(mgr); - return ret; } bool mt_sch_lcore_valid(struct mtl_main_impl* impl, unsigned int lcore) { @@ -661,6 +684,9 @@ bool mt_sch_lcore_valid(struct mtl_main_impl* impl, unsigned int lcore) { err("%s, invalid lcore %d\n", __func__, lcore); return -EIO; } + + if (mt_is_manager_connected(impl)) return true; + if (!lcore_shm) { err("%s, no lcore shm attached\n", __func__); return -EIO; @@ -773,9 +799,11 @@ int mt_sch_mrg_init(struct mtl_main_impl* impl, int data_quota_mbs_limit) { mt_pthread_mutex_init(&mgr->mgr_mutex, NULL); - mgr->lcore_lock_fd = -1; - ret = sch_init_lcores(mgr); - if (ret < 0) return ret; + if (!mt_is_manager_connected(impl)) { + mgr->lcore_lock_fd = -1; + ret = sch_init_lcores(mgr); + if (ret < 0) return ret; + } for (int sch_idx = 0; sch_idx < MT_MAX_SCH_NUM; sch_idx++) { sch = mt_sch_instance(impl, sch_idx); @@ -834,7 +862,7 @@ int mt_sch_mrg_uinit(struct mtl_main_impl* impl) { struct mt_sch_impl* sch; struct mt_sch_mgr* mgr = mt_sch_get_mgr(impl); - sch_uinit_lcores(impl, mgr); + if (!mt_is_manager_connected(impl)) sch_uinit_lcores(impl, mgr); for (int sch_idx = 0; sch_idx < MT_MAX_SCH_NUM; sch_idx++) { sch = mt_sch_instance(impl, sch_idx); diff --git a/manager/README.md b/manager/README.md new file mode 100644 index 000000000..4a32f79b7 --- /dev/null +++ b/manager/README.md @@ -0,0 +1,31 @@ +# MTL Manager Documentation + +## Overview + +![design](manager-design.svg) + +MTL Manager is a daemon server designed to operate with root privileges. Its primary role is to manage the lifecycle and configurations of MTL instances. It addresses a variety of administrative tasks, including: + +- Lcore Management: Ensures that MTL instances are aware of the lcores used by others, optimizing resource allocation. +- eBPF/XDP Loader: Dynamically loads and manages eBPF/XDP programs for advanced packet processing and performance tuning. +- NIC Configuration: Configures Network Interface Cards (NICs) with capabilities like adding or deleting flow and queue settings. +- Instances Monitor: Continuously monitors MTL instances, providing status reporting and clearing mechanisms. + +## Build + +To compile the MTL Manager, use the following commands: + +```bash +meson setup build +meson compile -C build +``` + +## Run + +To run the MTL Manager, execute: + +```bash +sudo ./build/MtlManager +``` + +This command will start the MTL Manager with root privileges, which are necessary for the advanced eBPF and network configurations and management tasks it performs. diff --git a/manager/logging.hpp b/manager/logging.hpp new file mode 100644 index 000000000..70c791d4a --- /dev/null +++ b/manager/logging.hpp @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2023 Intel Corporation + */ + +#ifndef LOGGING_HPP +#define LOGGING_HPP + +#include +#include +#include +#include + +enum class log_level { DEBUG, INFO, WARNING, ERROR }; + +class logger { + public: + static void log(log_level level, const std::string& message) { + if (level >= log_level_min) { + print_log_header(level); + std::cout << message << std::endl; + } + } + + static void set_log_level(log_level level) { log_level_min = level; } + + private: + static log_level log_level_min; + + static void print_log_header(log_level level) { + auto now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + const char* level_str = get_log_level_string(level); + + std::cout << "[" << std::put_time(std::localtime(&now), "%F %T") << "] [" << level_str + << "] "; + } + + static const char* get_log_level_string(log_level level) { + switch (level) { + case log_level::DEBUG: + return "DEBUG"; + case log_level::INFO: + return "INFO"; + case log_level::WARNING: + return "WARNING"; + case log_level::ERROR: + return "ERROR"; + default: + return "UNKNOWN"; + } + } +}; + +/* Initialize the log level threshold */ +log_level logger::log_level_min = log_level::DEBUG; + +#endif diff --git a/manager/manager-design.svg b/manager/manager-design.svg new file mode 100644 index 000000000..7f2920bbe --- /dev/null +++ b/manager/manager-design.svg @@ -0,0 +1,4 @@ + + + +mtl managermtl instancemtl instancemtl instancemtl_manager.socklistenconnectconnectconnectwritewritewriteread        struct message {             header(magic, type),             body(union)        }LcoremanagementXDP loader,send map fdset port filterNIC managementStatus clearon clientcrush/disconnect \ No newline at end of file diff --git a/manager/meson.build b/manager/meson.build new file mode 100644 index 000000000..bde38f874 --- /dev/null +++ b/manager/meson.build @@ -0,0 +1,41 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2023 Intel Corporation + +project('mtl_manager', 'cpp', default_options: ['buildtype=release', 'cpp_std=c++17'], + version: run_command(find_program('cat'), files('../VERSION'), check: true).stdout().strip(),) + +exec_env = host_machine.system() +set_variable('is_windows', exec_env == 'windows') + +if is_windows + build = false + reason = 'not supported on Windows' + subdir_done() +endif + +message('BUILD Environment: ' + exec_env) + +cpp_c = meson.get_compiler('cpp') + +sources = ['mtl_manager.cpp'] + +cpp_args = ['-std=c++17', '-Wall'] +link_cpp_args = ['-lstdc++fs'] + +if get_option('buildtype') != 'debug' + cpp_args += ['-Werror'] +endif + +# default no asan dep +asan_dep = [] +if get_option('enable_asan') == true + message('Enable -fsanitize=address') + cpp_args += ['-fsanitize=address'] + asan_dep = cpp_c.find_library('asan', required : true) +endif + +executable('MtlManager', sources, + cpp_args: cpp_args, + link_args: link_cpp_args, + dependencies: [asan_dep] +) \ No newline at end of file diff --git a/manager/meson_options.txt b/manager/meson_options.txt new file mode 100644 index 000000000..fc829b1d9 --- /dev/null +++ b/manager/meson_options.txt @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright 2023 Intel Corporation + +option('enable_asan', type: 'boolean', value: false, description: 'enable/disable address sanitizer, debug usage only') \ No newline at end of file diff --git a/manager/mtl_instance.hpp b/manager/mtl_instance.hpp new file mode 100644 index 000000000..1c09db1a7 --- /dev/null +++ b/manager/mtl_instance.hpp @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2023 Intel Corporation + */ + +#include +#include + +#include +#include +#include + +#include "logging.hpp" +#include "mtl_lcore.hpp" +#include "mtl_mproto.h" + +class mtl_instance { + private: + int conn_fd; + bool is_registered; + int pid; + int uid; + std::string hostname; + std::vector ifindex; + std::vector lcore_ids; + std::shared_ptr lcore_manager_sp; + + private: + int handle_message_get_lcore(mtl_lcore_message_t* lcore_msg); + int handle_message_put_lcore(mtl_lcore_message_t* lcore_msg); + int handle_message_register(mtl_register_message_t* register_msg); + int send_response(bool success) { + mtl_message_t msg; + msg.header.magic = htonl(MTL_MANAGER_MAGIC); + msg.header.type = (mtl_message_type_t)htonl(MTL_MSG_TYPE_RESPONSE); + msg.header.body_len = htonl(sizeof(mtl_response_message_t)); + msg.body.response_msg.response = success ? 0 : 1; + return send(conn_fd, &msg, sizeof(mtl_message_t), 0); + } + + public: + mtl_instance(int conn_fd, std::shared_ptr lcore_manager) + : conn_fd(conn_fd), lcore_manager_sp(lcore_manager) {} + ~mtl_instance() { + logger::log(log_level::INFO, "Remove client " + hostname + ":" + std::to_string(pid)); + for (const auto& lcore_id : lcore_ids) lcore_manager_sp->put_lcore(lcore_id); + ifindex.clear(); + lcore_ids.clear(); + + close(conn_fd); + } + + int get_conn_fd() { return conn_fd; } + int get_pid() { return pid; } + int get_uid() { return uid; } + std::string get_hostname() { return hostname; } + std::vector get_ifindex() { return ifindex; } + int handle_message(const char* buf, int len); +}; + +int mtl_instance::handle_message(const char* buf, int len) { + if ((size_t)len < sizeof(mtl_message_t)) return -1; + mtl_message_t* msg = (mtl_message_t*)buf; + if (ntohl(msg->header.magic) != MTL_MANAGER_MAGIC) { + logger::log(log_level::INFO, "Invalid magic"); + return -1; + } + + switch (ntohl(msg->header.type)) { + case MTL_MSG_TYPE_REGISTER: + handle_message_register(&msg->body.register_msg); + break; + case MTL_MSG_TYPE_GET_LCORE: + handle_message_get_lcore(&msg->body.lcore_msg); + break; + case MTL_MSG_TYPE_PUT_LCORE: + handle_message_put_lcore(&msg->body.lcore_msg); + break; + case MTL_MSG_TYPE_REQUEST_MAP_FD: + logger::log(log_level::INFO, "MTL_MSG_TYPE_REQUEST_MAP_FD"); + break; + case MTL_MSG_TYPE_UDP_PORT_OPERATION: + logger::log(log_level::INFO, "MTL_MSG_TYPE_UDP_PORT_OPERATION"); + break; + default: + logger::log(log_level::INFO, "Unknown message type"); + break; + } + + return 0; +} + +int mtl_instance::handle_message_get_lcore(mtl_lcore_message_t* lcore_msg) { + if (!is_registered) { + logger::log(log_level::INFO, "Instance is not registered"); + return -1; + } + uint16_t lcore_id = ntohs(lcore_msg->lcore); + int ret = lcore_manager_sp->get_lcore(lcore_id); + if (ret < 0) { + send_response(false); + return -1; + } + lcore_ids.push_back(lcore_id); + logger::log(log_level::INFO, "Add lcore " + std::to_string(lcore_id) + " to instance " + + hostname + ":" + std::to_string(pid)); + send_response(true); + return 0; +} + +int mtl_instance::handle_message_put_lcore(mtl_lcore_message_t* lcore_msg) { + if (!is_registered) { + logger::log(log_level::INFO, "Instance is not registered"); + return -1; + } + uint16_t lcore_id = ntohs(lcore_msg->lcore); + lcore_manager_sp->put_lcore(lcore_id); + for (auto it = lcore_ids.begin(); it != lcore_ids.end(); it++) { + if (*it == lcore_id) { + lcore_ids.erase(it); + break; + } + } + + logger::log(log_level::INFO, "Remove lcore " + std::to_string(lcore_id) + + " from instance " + hostname + ":" + + std::to_string(pid)); + return 0; +} + +int mtl_instance::handle_message_register(mtl_register_message_t* register_msg) { + pid = ntohl(register_msg->pid); + uid = ntohl(register_msg->uid); + hostname = std::string(register_msg->hostname, 64); + uint16_t num_if = ntohs(register_msg->num_if); + for (int i = 0; i < num_if; i++) { + ifindex.push_back(ntohs(register_msg->ifindex[i])); + } + + logger::log(log_level::INFO, + "Register instance " + hostname + ":" + std::to_string(pid)); + + is_registered = true; + send_response(true); + + return 0; +} diff --git a/manager/mtl_interface.hpp b/manager/mtl_interface.hpp new file mode 100644 index 000000000..ffdf1d51a --- /dev/null +++ b/manager/mtl_interface.hpp @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2023 Intel Corporation + */ + +#include +#include +#include + +#include +#include + +class mtl_interface { + private: + int ifindex; + struct xdp_program* xdp_prog; + + public: + mtl_interface(int ifindex); + ~mtl_interface(); + + int attach_xdp_prog(std::string xdp_prog_path); + int detach_xdp_prog(); + int get_ifindex(); +}; + +mtl_interface::mtl_interface(int ifindex) { + this->ifindex = ifindex; + this->xdp_prog = nullptr; +} + +mtl_interface::~mtl_interface() { + if (this->xdp_prog != nullptr) { + this->detach_xdp_prog(); + } +} + +int mtl_interface::get_ifindex() { return this->ifindex; } + +int mtl_interface::attach_xdp_prog(std::string xdp_prog_path) { return 0; } + +int mtl_interface::detach_xdp_prog() { return 0; } diff --git a/manager/mtl_lcore.hpp b/manager/mtl_lcore.hpp new file mode 100644 index 000000000..8d60249ac --- /dev/null +++ b/manager/mtl_lcore.hpp @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2023 Intel Corporation + */ + +#include +#include + +#define MTL_MAX_LCORE 128 + +class mtl_lcore { + private: + std::bitset bs; + std::mutex bs_mtx; + + public: + mtl_lcore(); + ~mtl_lcore(); + int get_lcore(uint16_t lcore_id); + int put_lcore(uint16_t lcore_id); +}; + +mtl_lcore::mtl_lcore() { bs.reset(); } + +mtl_lcore::~mtl_lcore() {} + +int mtl_lcore::get_lcore(uint16_t lcore_id) { + std::lock_guard lock(bs_mtx); + if (bs.test(lcore_id)) + return -1; + else + bs.set(lcore_id, true); + return 0; +} + +int mtl_lcore::put_lcore(uint16_t lcore_id) { + std::lock_guard lock(bs_mtx); + if (!bs.test(lcore_id)) + return -1; + else + bs.set(lcore_id, false); + return 0; +} \ No newline at end of file diff --git a/manager/mtl_manager.cpp b/manager/mtl_manager.cpp new file mode 100644 index 000000000..4be74b973 --- /dev/null +++ b/manager/mtl_manager.cpp @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2023 Intel Corporation + */ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "mtl_instance.hpp" +#include "mtl_mproto.h" + +namespace fs = std::filesystem; + +static std::atomic is_running(true); + +int main() { + int ret = 0; + int epfd = -1, sockfd = -1; + std::vector clients; + std::shared_ptr lcore_manager = std::make_shared(); + + logger::set_log_level(log_level::INFO); + + fs::path directory_path(MTL_MANAGER_SOCK_PATH); + directory_path.remove_filename(); + if (!fs::exists(directory_path)) { + fs::create_directory(directory_path); + } + + sigset_t signal_mask; + sigemptyset(&signal_mask); + sigaddset(&signal_mask, SIGINT); + + ret = sigprocmask(SIG_BLOCK, &signal_mask, NULL); + if (ret == -1) { + logger::log(log_level::ERROR, "Failed to set signal mask."); + return ret; + } + + int signal_fd = signalfd(-1, &signal_mask, 0); + if (signal_fd == -1) { + logger::log(log_level::ERROR, "Failed to create signal fd."); + ret = signal_fd; + goto out; + } + + sockfd = socket(AF_UNIX, SOCK_STREAM, 0); + if (sockfd < 0) { + logger::log(log_level::ERROR, "Failed to create socket."); + ret = sockfd; + goto out; + } + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, MTL_MANAGER_SOCK_PATH, sizeof(addr.sun_path) - 1); + + unlink(MTL_MANAGER_SOCK_PATH); + + ret = bind(sockfd, (struct sockaddr*)&addr, sizeof(addr)); + if (ret < 0) { + logger::log(log_level::ERROR, "Failed to bind socket."); + goto out; + } + + /* Allow all users to connect (which might be insecure) */ + fs::permissions(MTL_MANAGER_SOCK_PATH, fs::perms::all, fs::perm_options::replace); + + ret = listen(sockfd, 10); + if (ret < 0) { + logger::log(log_level::ERROR, "Failed to listen on socket."); + goto out; + } + + epfd = epoll_create1(0); + if (epfd < 0) { + logger::log(log_level::ERROR, "Failed to create epoll."); + ret = epfd; + goto out; + } + + struct epoll_event event; + event.data.fd = signal_fd; + event.events = EPOLLIN; + + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, signal_fd, &event); + if (ret == -1) { + logger::log(log_level::ERROR, "Failed to add signal fd to epoll."); + goto out; + } + + struct epoll_event ev; + ev.events = EPOLLIN; + ev.data.fd = sockfd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, sockfd, &ev); + if (ret == -1) { + logger::log(log_level::ERROR, "Failed to add socket fd to epoll."); + goto out; + } + + is_running.store(true); + logger::log(log_level::INFO, "MTL Manager is running. Press Ctrl+C to stop it."); + + while (is_running.load()) { + logger::log(log_level::INFO, + "Listening " + std::to_string(clients.size()) + " clients"); + struct epoll_event events[clients.size() + 2]; + int nfds = epoll_wait(epfd, events, clients.size() + 2, -1); + if (nfds < 0) { + logger::log(log_level::ERROR, "Failed to wait for epoll."); + continue; + } + for (int i = 0; i < nfds; i++) { + if (events[i].data.fd == sockfd) { + int client_sockfd = accept(sockfd, NULL, NULL); + if (client_sockfd < 0) { + logger::log(log_level::ERROR, "Failed to accept client."); + continue; + } + + struct epoll_event ev; + ev.events = EPOLLIN; + ev.data.fd = client_sockfd; + ret = epoll_ctl(epfd, EPOLL_CTL_ADD, client_sockfd, &ev); + if (ret < 0) { + logger::log(log_level::ERROR, "Failed to add client socket fd to epoll."); + close(client_sockfd); + continue; + } + + mtl_instance* client = new mtl_instance(client_sockfd, lcore_manager); + clients.push_back(client); + logger::log(log_level::INFO, + "New client connected. fd: " + std::to_string(client_sockfd)); + } else if (events[i].data.fd == signal_fd) { + struct signalfd_siginfo siginfo; + ssize_t len = read(signal_fd, &siginfo, sizeof(siginfo)); + if (len != sizeof(siginfo)) { + logger::log(log_level::ERROR, "Failed to read signal."); + return 1; + } + + if (siginfo.ssi_signo == SIGINT) { + logger::log(log_level::INFO, "Received SIGINT. Shutting down."); + is_running.store(false); + } + } else { + for (auto it = clients.begin(); it != clients.end();) { + mtl_instance* client = *it; + if (client->get_conn_fd() == events[i].data.fd) { + char buf[256]; + int len = recv(events[i].data.fd, buf, sizeof(buf), 0); + if (len < 0) { + logger::log(log_level::ERROR, "Failed to receive data from client " + + std::to_string(events[i].data.fd)); + ++it; + continue; + } + if (len == 0) { + logger::log(log_level::INFO, "Client " + std::to_string(events[i].data.fd) + + " disconnected."); + ret = epoll_ctl(epfd, EPOLL_CTL_DEL, events[i].data.fd, NULL); + if (ret < 0) + logger::log(log_level::ERROR, "Failed to remove client from epoll."); + delete client; /* close fd in deconstruction */ + it = clients.erase(it); + continue; + } + client->handle_message(buf, len); + } + ++it; + } + } + } + } + +out: + for (auto it = clients.begin(); it != clients.end(); ++it) { + delete *it; + } + if (signal_fd >= 0) close(signal_fd); + if (epfd >= 0) close(epfd); + if (sockfd >= 0) close(sockfd); + + logger::log(log_level::INFO, "MTL Manager exited."); + return ret; +} \ No newline at end of file diff --git a/manager/mtl_mproto.h b/manager/mtl_mproto.h new file mode 100644 index 000000000..2ee90b79c --- /dev/null +++ b/manager/mtl_mproto.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2023 Intel Corporation + */ + +#include +#include +#include + +#ifndef _MTL_MPROTO_HEAD_H_ +#define _MTL_MPROTO_HEAD_H_ + +#if defined(__cplusplus) +extern "C" { +#endif + +#define MTL_MANAGER_SOCK_PATH "/var/run/imtl/mtl_manager.sock" + +#define MTL_MANAGER_MAGIC (0x494D544C) /* ASCII representation of "IMTL" */ + +#pragma pack(push, 1) + +/* message type */ +typedef enum { + /* bidirectional */ + MTL_MSG_TYPE_RAW = 0, + /* client to server */ + MTL_MSG_TYPE_CS = 100, + MTL_MSG_TYPE_REGISTER, + MTL_MSG_TYPE_HEARTBEAT, + MTL_MSG_TYPE_REQUEST_MAP_FD, + MTL_MSG_TYPE_GET_LCORE, + MTL_MSG_TYPE_PUT_LCORE, + MTL_MSG_TYPE_UDP_PORT_OPERATION, + /* server to client */ + MTL_MSG_TYPE_SC = 200, + MTL_MSG_TYPE_RESPONSE, +} mtl_message_type_t; + +/* message header */ +typedef struct { + uint32_t magic; + mtl_message_type_t type; + uint32_t body_len; +} mtl_message_header_t; + +typedef struct { + pid_t pid; + uid_t uid; + char hostname[64]; + uint16_t ifindex[8]; + uint16_t num_if; +} mtl_register_message_t; + +typedef struct { + uint32_t seq; +} mtl_heartbeat_message_t; + +typedef struct { + int ifindex; +} mtl_request_map_fd_message_t; + +typedef struct { + uint16_t lcore; +} mtl_lcore_message_t; + +typedef struct { + int ifindex; + uint16_t port; + uint8_t action; // 1 for add, 0 for remove +} mtl_udp_port_operation_message_t; + +typedef struct { + uint8_t response; // 0 for success, 1 for error +} mtl_response_message_t; + +typedef struct { + mtl_message_header_t header; + union { + mtl_register_message_t register_msg; + mtl_heartbeat_message_t heartbeat_msg; + mtl_request_map_fd_message_t request_map_fd_msg; + mtl_lcore_message_t lcore_msg; + mtl_udp_port_operation_message_t udp_port_op_msg; + mtl_response_message_t response_msg; + } body; +} mtl_message_t; + +#pragma pack(pop) + +#if defined(__cplusplus) +} +#endif + +#endif \ No newline at end of file