diff --git a/lib/src/datapath/mt_queue.c b/lib/src/datapath/mt_queue.c index 7fdf727a0..c9b918700 100644 --- a/lib/src/datapath/mt_queue.c +++ b/lib/src/datapath/mt_queue.c @@ -297,7 +297,8 @@ int mt_dp_queue_init(struct mtl_main_impl* impl) { struct mt_txq_flow flow; memset(&flow, 0, sizeof(flow)); flow.flags = MT_TXQ_FLOW_F_SYS_QUEUE; - if (mt_drv_kernel_based(impl, i)) flow.flags = MT_TXQ_FLOW_F_FORCE_SOCKET; + if (mt_drv_kernel_based(impl, i) && !mt_pmd_is_native_af_xdp(impl, i)) + flow.flags = MT_TXQ_FLOW_F_FORCE_SOCKET; dp->txq_sys_entry = mt_txq_get(impl, i, &flow); if (!dp->txq_sys_entry) { err("%s(%d), txq sys entry get fail\n", __func__, i); diff --git a/lib/src/dev/mt_af_xdp.c b/lib/src/dev/mt_af_xdp.c index 5ea3259b3..4c6ea659d 100644 --- a/lib/src/dev/mt_af_xdp.c +++ b/lib/src/dev/mt_af_xdp.c @@ -416,7 +416,7 @@ static void xdp_tx_wakeup(struct mt_xdp_queue* xq) { static uint16_t xdp_tx(struct mtl_main_impl* impl, struct mt_xdp_queue* xq, struct rte_mbuf** tx_pkts, uint16_t nb_pkts) { enum mtl_port port = xq->port; - uint16_t q = xq->q; + // uint16_t q = xq->q; struct rte_mempool* mbuf_pool = xq->mbuf_pool; uint16_t tx = 0; struct xsk_ring_prod* pd = &xq->tx_prod; @@ -427,42 +427,45 @@ static uint16_t xdp_tx(struct mtl_main_impl* impl, struct mt_xdp_queue* xq, for (uint16_t i = 0; i < nb_pkts; i++) { struct rte_mbuf* m = tx_pkts[i]; - - if (m->pool == mbuf_pool) { - warn("%s(%d, %u), same mbuf_pool todo\n", __func__, port, q); + struct rte_mbuf* local = rte_pktmbuf_alloc(mbuf_pool); + if (!local) { + dbg("%s(%d, %u), local mbuf alloc fail\n", __func__, port, q); + xq->stat_tx_mbuf_alloc_fail++; goto exit; - } else { - struct rte_mbuf* local = rte_pktmbuf_alloc(mbuf_pool); - if (!local) { - dbg("%s(%d, %u), local mbuf alloc fail\n", __func__, port, q); - xq->stat_tx_mbuf_alloc_fail++; - goto exit; - } + } - uint32_t idx; - if (!xsk_ring_prod__reserve(pd, 1, &idx)) { - dbg("%s(%d, %u), socket_tx reserve fail\n", __func__, port, q); - xq->stat_tx_prod_reserve_fail++; - rte_pktmbuf_free(local); - goto exit; - } - struct xdp_desc* desc = xsk_ring_prod__tx_desc(pd, idx); - desc->len = m->pkt_len; - uint64_t addr = - (uint64_t)local - (uint64_t)xq->umem_buffer - xq->mbuf_pool->header_size; - uint64_t offset = rte_pktmbuf_mtod(local, uint64_t) - (uint64_t)local + - xq->mbuf_pool->header_size; - void* pkt = xsk_umem__get_data(xq->umem_buffer, addr + offset); - offset = offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT; - desc->addr = addr | offset; - rte_memcpy(pkt, rte_pktmbuf_mtod(m, void*), desc->len); - tx_bytes += m->data_len; - rte_pktmbuf_free(m); - dbg("%s(%d, %u), tx local mbuf %p umem pkt %p addr 0x%" PRIu64 "\n", __func__, port, - q, local, pkt, addr); - xq->stat_tx_copy++; - tx++; + uint32_t idx; + if (!xsk_ring_prod__reserve(pd, 1, &idx)) { + dbg("%s(%d, %u), socket_tx reserve fail\n", __func__, port, q); + xq->stat_tx_prod_reserve_fail++; + rte_pktmbuf_free(local); + goto exit; + } + struct xdp_desc* desc = xsk_ring_prod__tx_desc(pd, idx); + desc->len = m->pkt_len; + uint64_t addr = + (uint64_t)local - (uint64_t)xq->umem_buffer - xq->mbuf_pool->header_size; + uint64_t offset = + rte_pktmbuf_mtod(local, uint64_t) - (uint64_t)local + xq->mbuf_pool->header_size; + void* pkt = xsk_umem__get_data(xq->umem_buffer, addr + offset); + offset = offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT; + desc->addr = addr | offset; + + struct rte_mbuf* n = m; + uint16_t nb_segs = m->nb_segs; + for (uint16_t seg = 0; seg < nb_segs; seg++) { + rte_memcpy(pkt, rte_pktmbuf_mtod(n, void*), n->data_len); + pkt += n->data_len; + /* point to next */ + n = n->next; } + + tx_bytes += desc->len; + rte_pktmbuf_free(m); + dbg("%s(%d, %u), tx local mbuf %p umem pkt %p addr 0x%" PRIu64 "\n", __func__, port, + q, local, pkt, addr); + xq->stat_tx_copy++; + tx++; } exit: @@ -614,6 +617,7 @@ int mt_dev_xdp_init(struct mt_interface* inf) { } inf->xdp = xdp; + inf->feature |= MT_IF_FEATURE_TX_MULTI_SEGS; info("%s(%d), start queue %u cnt %u\n", __func__, port, xdp->start_queue, xdp->queues_cnt); return 0; @@ -681,13 +685,16 @@ int mt_tx_xdp_put(struct mt_tx_xdp_entry* entry) { uint8_t* ip = flow->dip_addr; struct mt_xdp_queue* xq = entry->xq; - /* poll all done buf */ - xdp_tx_poll_done(xq); - xdp_queue_tx_stat(xq); + if (xq) { + /* poll all done buf */ + xdp_tx_poll_done(xq); + xdp_queue_tx_stat(xq); + + xq->tx_entry = NULL; + info("%s(%d), ip %u.%u.%u.%u, port %u, queue %u\n", __func__, port, ip[0], ip[1], + ip[2], ip[3], flow->dst_port, entry->queue_id); + } - xq->tx_entry = NULL; - info("%s(%d), ip %u.%u.%u.%u, port %u, queue %u\n", __func__, port, ip[0], ip[1], ip[2], - ip[3], flow->dst_port, entry->queue_id); mt_rte_free(entry); return 0; } diff --git a/lib/src/dev/mt_dev.c b/lib/src/dev/mt_dev.c index 56bf48b12..69f94ae76 100644 --- a/lib/src/dev/mt_dev.c +++ b/lib/src/dev/mt_dev.c @@ -2095,11 +2095,17 @@ int mt_dev_if_init(struct mtl_main_impl* impl) { inf->nb_rx_q = 1; p->flags |= MTL_FLAG_SHARED_RX_QUEUE; inf->system_rx_queues_end = 0; - } else if (mt_pmd_is_dpdk_af_xdp(impl, i) || mt_pmd_is_native_af_xdp(impl, i)) { + } else if (mt_pmd_is_dpdk_af_xdp(impl, i)) { /* no system queues as no cni */ inf->nb_tx_q = queue_pair_cnt; inf->nb_rx_q = queue_pair_cnt; inf->system_rx_queues_end = 0; + } else if (mt_pmd_is_native_af_xdp(impl, i)) { + /* one more for the sys tx queue */ + queue_pair_cnt = RTE_MAX(p->tx_queues_cnt[i] + 1, p->rx_queues_cnt[i]); + inf->nb_tx_q = queue_pair_cnt; + inf->nb_rx_q = queue_pair_cnt; + inf->system_rx_queues_end = 0; } else { info("%s(%d), deprecated sessions tx %u rx %u\n", __func__, i, p->tx_sessions_cnt_max, p->rx_sessions_cnt_max);