Skip to content

Commit

Permalink
Added NUMA socket id array check
Browse files Browse the repository at this point in the history
  • Loading branch information
@PlagueCZ
PlagueCZ committed Sep 19, 2023
1 parent 5e9c11e commit 489fc04
Show file tree
Hide file tree
Showing 6 changed files with 121 additions and 129 deletions.
24 changes: 12 additions & 12 deletions include/dp_lpm.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ struct vm_route {

};

int dp_setup_vm(int port_id, int vni, const int socketid);
int dp_setup_vm6(int port_id, int vni, const int socketid);
int dp_setup_vm(int port_id, int vni, int socketid);
int dp_setup_vm6(int port_id, int vni, int socketid);
int dp_lookup_ip4_route(int port_id, int t_vni, const struct dp_flow *df, int socketid,
struct vm_route *r, uint32_t *route_key, uint64_t *dst_port_id);
int dp_get_ip6_dst_port(int port_id, int t_vni, const struct rte_ipv6_hdr *ipv6_hdr,
Expand All @@ -80,8 +80,8 @@ void dp_del_portid_with_vm_handle(void *key);
uint32_t dp_get_gw_ip4(void);
const uint8_t *dp_get_gw_ip6(void);
uint32_t dp_get_dhcp_range_ip4(uint16_t portid);
uint8_t* dp_get_dhcp_range_ip6(uint16_t portid);
uint8_t* dp_get_vm_ip6(uint16_t portid);
uint8_t *dp_get_dhcp_range_ip6(uint16_t portid);
uint8_t *dp_get_vm_ip6(uint16_t portid);
uint8_t *dp_get_vm_ul_ip6(uint16_t portid);
const uint8_t *dp_get_port_ul_ip6(uint16_t portid);
int dp_add_route(uint16_t portid, uint32_t vni, uint32_t t_vni, uint32_t ip,
Expand All @@ -91,22 +91,22 @@ int dp_add_route6(uint16_t portid, uint32_t vni, uint32_t t_vni, uint8_t *ipv6,
uint8_t *ext_ip6, uint8_t depth, int socketid);
int dp_del_route6(uint16_t portid, uint32_t vni, uint8_t *ipv6, uint8_t depth, int socketid);
int dp_list_routes(int vni, int socketid, uint16_t portid, bool ext_routes, struct dp_grpc_responder *responder);
void dp_set_dhcp_range_ip4(uint16_t portid, uint32_t ip, uint8_t depth, int socketid);
void dp_set_dhcp_range_ip6(uint16_t portid, uint8_t* ipv6, uint8_t depth, int socketid);
void dp_set_vm_ip6(uint16_t portid, uint8_t* ipv6);
void dp_set_dhcp_range_ip4(uint16_t portid, uint32_t ip, uint8_t depth);
void dp_set_dhcp_range_ip6(uint16_t portid, uint8_t *ipv6, uint8_t depth);
void dp_set_vm_ip6(uint16_t portid, uint8_t *ipv6);
void dp_set_vm_ul_ip6(uint16_t portid, uint8_t *ipv6);
void dp_set_mac(uint16_t portid);
struct rte_ether_addr *dp_get_mac(uint16_t portid);
void dp_set_neigh_mac(uint16_t portid, struct rte_ether_addr* neigh);
void dp_set_neigh_mac(uint16_t portid, struct rte_ether_addr *neigh);
struct rte_ether_addr *dp_get_neigh_mac(uint16_t portid);
bool dp_arp_cycle_needed(uint16_t portid);
void dp_del_vm(int portid, int socketid);
int dp_get_active_vm_ports(int* act_ports);
uint8_t* dp_get_vm_machineid(uint16_t portid);
int dp_get_active_vm_ports(int *act_ports);
uint8_t *dp_get_vm_machineid(uint16_t portid);
uint32_t dp_get_vm_vni(uint16_t portid);
uint32_t dp_get_vm_pxe_ip4(uint16_t portid);
void dp_set_vm_pxe_ip4(uint16_t portid, uint32_t ip, int socketid);
char* dp_get_vm_pxe_str(uint16_t portid);
void dp_set_vm_pxe_ip4(uint16_t portid, uint32_t ip);
char *dp_get_vm_pxe_str(uint16_t portid);
void dp_set_vm_pxe_str(uint16_t portid, char *p_str);
int dp_lpm_reset_all_route_tables(int socketid);
int dp_lpm_reset_route_tables(int vni, int socketid);
Expand Down
36 changes: 17 additions & 19 deletions include/dp_vni.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,71 +21,69 @@ extern struct rte_hash *vni_handle_tbl;
#define DP_IP_PROTO_IPV4 DP_IP_PROTO_IPv4_ENCAP
#define DP_IP_PROTO_IPV6 DP_IP_PROTO_IPv6_ENCAP

// Protect array access
// Also, when NUMA is not available, DPDK uses SOCKET_ID_ANY (-1)
#define DP_SOCKETID(SOCKETID) (unlikely((unsigned int)(SOCKETID) >= DP_NB_SOCKETS) ? 0 : (SOCKETID))

// TODO: packing?
struct dp_vni_key {
int vni;
int type;
};

struct dp_vni_value {
struct dp_vni_data {
struct rte_rib *ipv4[DP_NB_SOCKETS];
struct rte_rib6 *ipv6[DP_NB_SOCKETS];
struct dp_ref ref_count;
int socketid;
int vni;
};

static __rte_always_inline struct rte_rib *dp_get_vni_route4_table(int vni, int socketid)
static __rte_always_inline struct rte_rib *dp_get_vni_route4_table(int vni, int socket_id)
{
struct dp_vni_value *temp_val = NULL;
struct dp_vni_data *vni_data;
struct dp_vni_key vni_key = {
.type = DP_IP_PROTO_IPV4,
.vni = vni
};
int ret;

ret = rte_hash_lookup_data(vni_handle_tbl, &vni_key, (void **)&temp_val);
ret = rte_hash_lookup_data(vni_handle_tbl, &vni_key, (void **)&vni_data);
if (DP_FAILED(ret)) {
if (ret != -ENOENT)
DPS_LOG_ERR("VNI lookup error", DP_LOG_VNI(vni), DP_LOG_VNI_TYPE(DP_IP_PROTO_IPV4));
return NULL;
}

if (!temp_val->ipv4[socketid])
return NULL;

return temp_val->ipv4[socketid];
return vni_data->ipv4[DP_SOCKETID(socket_id)];
}

static __rte_always_inline struct rte_rib6 *dp_get_vni_route6_table(int vni, int socketid)
static __rte_always_inline struct rte_rib6 *dp_get_vni_route6_table(int vni, int socket_id)
{
struct dp_vni_value *temp_val = NULL;
struct dp_vni_data *vni_data;
struct dp_vni_key vni_key = {
.type = DP_IP_PROTO_IPV6,
.vni = vni
};
int ret;

ret = rte_hash_lookup_data(vni_handle_tbl, &vni_key, (void **)&temp_val);
ret = rte_hash_lookup_data(vni_handle_tbl, &vni_key, (void **)&vni_data);
if (DP_FAILED(ret)) {
if (ret != -ENOENT)
DPS_LOG_ERR("VNI lookup error", DP_LOG_VNI(vni), DP_LOG_VNI_TYPE(DP_IP_PROTO_IPV6));
return NULL;
}

if (!temp_val->ipv6[socketid])
return NULL;

return temp_val->ipv6[socketid];
return vni_data->ipv6[DP_SOCKETID(socket_id)];
}

int dp_vni_init(int socket_id);
void dp_vni_free(void);
bool dp_is_vni_route_tbl_available(int vni, int type, int socketid);
int dp_create_vni_route_table(int vni, int type, int socketid);
bool dp_is_vni_route_tbl_available(int vni, int type, int socket_id);
int dp_create_vni_route_table(int vni, int type, int socket_id);
int dp_delete_vni_route_table(int vni, int type);
int dp_reset_vni_route_table(int vni, int type, int socketid);
int dp_reset_vni_all_route_tables(int socketid);
int dp_reset_vni_route_table(int vni, int type, int socket_id);
int dp_reset_vni_all_route_tables(int socket_id);

#ifdef __cplusplus
}
Expand Down
23 changes: 5 additions & 18 deletions src/dp_lpm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,6 @@ int dp_add_route(uint16_t portid, uint32_t vni, uint32_t t_vni, uint32_t ip,
struct rte_rib_node *node;
struct rte_rib *root;

RTE_VERIFY(socketid < DP_NB_SOCKETS);
RTE_VERIFY(portid < DP_MAX_PORTS);

root = dp_get_vni_route4_table(vni, socketid);
Expand Down Expand Up @@ -279,7 +278,6 @@ int dp_del_route(uint16_t portid, uint32_t vni, uint32_t ip, uint8_t depth, int
struct rte_rib *root;
uint64_t next_hop;

RTE_VERIFY(socketid < DP_NB_SOCKETS);
RTE_VERIFY(portid < DP_MAX_PORTS);

root = dp_get_vni_route4_table(vni, socketid);
Expand Down Expand Up @@ -352,9 +350,6 @@ int dp_list_routes(int vni, int socketid, uint16_t portid, bool ext_routes,
struct rte_rib *root;
int ret;

// TODO(plague): look into this globally
RTE_VERIFY(socketid < DP_NB_SOCKETS);

root = dp_get_vni_route4_table(vni, socketid);
if (!root)
return DP_GRPC_ERR_NO_VNI;
Expand Down Expand Up @@ -385,7 +380,6 @@ int dp_add_route6(uint16_t portid, uint32_t vni, uint32_t t_vni, uint8_t *ipv6,
struct rte_rib6_node *node;
struct rte_rib6 *root;

RTE_VERIFY(socketid < DP_NB_SOCKETS);
RTE_VERIFY(portid < DP_MAX_PORTS);

root = dp_get_vni_route6_table(vni, socketid);
Expand Down Expand Up @@ -417,7 +411,6 @@ int dp_del_route6(uint16_t portid, uint32_t vni, uint8_t *ipv6, uint8_t depth, i
struct rte_rib6_node *node;
struct rte_rib6 *root;

RTE_VERIFY(socketid < DP_NB_SOCKETS);
RTE_VERIFY(portid < DP_MAX_PORTS);

root = dp_get_vni_route6_table(vni, socketid);
Expand All @@ -432,17 +425,15 @@ int dp_del_route6(uint16_t portid, uint32_t vni, uint8_t *ipv6, uint8_t depth, i
return DP_GRPC_OK;
}

void dp_set_dhcp_range_ip4(uint16_t portid, uint32_t ip, uint8_t depth, int socketid)
void dp_set_dhcp_range_ip4(uint16_t portid, uint32_t ip, uint8_t depth)
{
RTE_VERIFY(socketid < DP_NB_SOCKETS);
RTE_VERIFY(portid < DP_MAX_PORTS);
vm_table[portid].info.own_ip = ip;
vm_table[portid].info.depth = depth;
}

void dp_set_vm_pxe_ip4(uint16_t portid, uint32_t ip, int socketid)
void dp_set_vm_pxe_ip4(uint16_t portid, uint32_t ip)
{
RTE_VERIFY(socketid < DP_NB_SOCKETS);
RTE_VERIFY(portid < DP_MAX_PORTS);
vm_table[portid].info.pxe_ip = ip;
}
Expand All @@ -453,10 +444,9 @@ uint32_t dp_get_vm_pxe_ip4(uint16_t portid)
return vm_table[portid].info.pxe_ip;
}

void dp_set_dhcp_range_ip6(uint16_t portid, uint8_t *ipv6, uint8_t depth, int socketid)
void dp_set_dhcp_range_ip6(uint16_t portid, uint8_t *ipv6, uint8_t depth)
{
RTE_VERIFY(portid < DP_MAX_PORTS);
RTE_VERIFY(socketid < DP_NB_SOCKETS);
rte_memcpy(&vm_table[portid].info.dhcp_ipv6, ipv6, 16);
vm_table[portid].info.depth = depth;
}
Expand Down Expand Up @@ -497,9 +487,8 @@ struct rte_ether_addr *dp_get_neigh_mac(uint16_t portid)
return &vm_table[portid].info.neigh_mac;
}

int dp_setup_vm(int port_id, int vni, const int socketid)
int dp_setup_vm(int port_id, int vni, int socketid)
{
RTE_VERIFY(socketid < DP_NB_SOCKETS);
RTE_VERIFY(port_id < DP_MAX_PORTS);

if (DP_FAILED(dp_create_vni_route_table(vni, DP_IP_PROTO_IPV4, socketid)))
Expand All @@ -511,9 +500,8 @@ int dp_setup_vm(int port_id, int vni, const int socketid)
return DP_OK;
}

int dp_setup_vm6(int port_id, int vni, const int socketid)
int dp_setup_vm6(int port_id, int vni, int socketid)
{
RTE_VERIFY(socketid < DP_NB_SOCKETS);
RTE_VERIFY(port_id < DP_MAX_PORTS);

if (DP_FAILED(dp_create_vni_route_table(vni, DP_IP_PROTO_IPV6, socketid)))
Expand Down Expand Up @@ -587,7 +575,6 @@ int dp_get_ip6_dst_port(int port_id, int t_vni, const struct rte_ipv6_hdr *ipv6_

void dp_del_vm(int portid, int socketid)
{
RTE_VERIFY(socketid < DP_NB_SOCKETS);
RTE_VERIFY(portid < DP_MAX_PORTS);

dp_del_route(portid, vm_table[portid].vni, vm_table[portid].info.own_ip, 32, socketid);
Expand Down
20 changes: 14 additions & 6 deletions src/dp_service.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,17 +137,25 @@ static int init_interfaces(void)

dp_multipath_init();

if (DP_FAILED(dp_ports_init()))
return DP_ERROR;

// TODO(plague): this is called many times, PR for putting the socket of a port into dp_port.c
// (also socket_id is always passed along with port_id)
pf0_socket = rte_eth_dev_socket_id(dp_port_get_pf0_id());
if (DP_FAILED(pf0_socket)) {
DPS_LOG_ERR("Cannot get numa socket for pf0", DP_LOG_PORTID(dp_port_get_pf0_id()), DP_LOG_RET(pf0_socket));
return pf0_socket;
if (pf0_socket == SOCKET_ID_ANY) {
DPS_LOG_WARNING("Cannot get numa socket for pf0", DP_LOG_PORTID(dp_port_get_pf0_id()));
} else {
DPS_LOG_ERR("Cannot get numa socket for pf0", DP_LOG_PORTID(dp_port_get_pf0_id()), DP_LOG_RET(rte_errno));
return pf0_socket;
}
}

if (DP_FAILED(dp_ports_init())
#ifdef ENABLE_VIRTSVC
|| DP_FAILED(dp_virtsvc_init(pf0_socket))
if (DP_FAILED(dp_virtsvc_init(pf0_socket)))
return DP_ERROR;
#endif
|| DP_FAILED(dp_graph_init())
if (DP_FAILED(dp_graph_init())
|| DP_FAILED(dp_telemetry_init()))
return DP_ERROR;

Expand Down
Loading

0 comments on commit 489fc04

Please sign in to comment.