Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dp/xdp: add MT_IF_FEATURE_TX_MULTI_SEGS support #555

Merged
merged 1 commit into from
Oct 30, 2023
Merged

dp/xdp: add MT_IF_FEATURE_TX_MULTI_SEGS support #555

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion lib/src/datapath/mt_queue.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,8 @@ int mt_dp_queue_init(struct mtl_main_impl* impl) {
struct mt_txq_flow flow;
memset(&flow, 0, sizeof(flow));
flow.flags = MT_TXQ_FLOW_F_SYS_QUEUE;
if (mt_drv_kernel_based(impl, i)) flow.flags = MT_TXQ_FLOW_F_FORCE_SOCKET;
if (mt_drv_kernel_based(impl, i) && !mt_pmd_is_native_af_xdp(impl, i))
flow.flags = MT_TXQ_FLOW_F_FORCE_SOCKET;
dp->txq_sys_entry = mt_txq_get(impl, i, &flow);
if (!dp->txq_sys_entry) {
err("%s(%d), txq sys entry get fail\n", __func__, i);
Expand Down
87 changes: 47 additions & 40 deletions lib/src/dev/mt_af_xdp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,7 @@ static void xdp_tx_wakeup(struct mt_xdp_queue* xq) {
static uint16_t xdp_tx(struct mtl_main_impl* impl, struct mt_xdp_queue* xq,
struct rte_mbuf** tx_pkts, uint16_t nb_pkts) {
enum mtl_port port = xq->port;
uint16_t q = xq->q;
// uint16_t q = xq->q;
struct rte_mempool* mbuf_pool = xq->mbuf_pool;
uint16_t tx = 0;
struct xsk_ring_prod* pd = &xq->tx_prod;
Expand All @@ -427,42 +427,45 @@ static uint16_t xdp_tx(struct mtl_main_impl* impl, struct mt_xdp_queue* xq,

for (uint16_t i = 0; i < nb_pkts; i++) {
struct rte_mbuf* m = tx_pkts[i];

if (m->pool == mbuf_pool) {
warn("%s(%d, %u), same mbuf_pool todo\n", __func__, port, q);
struct rte_mbuf* local = rte_pktmbuf_alloc(mbuf_pool);
if (!local) {
dbg("%s(%d, %u), local mbuf alloc fail\n", __func__, port, q);
xq->stat_tx_mbuf_alloc_fail++;
goto exit;
} else {
struct rte_mbuf* local = rte_pktmbuf_alloc(mbuf_pool);
if (!local) {
dbg("%s(%d, %u), local mbuf alloc fail\n", __func__, port, q);
xq->stat_tx_mbuf_alloc_fail++;
goto exit;
}
}

uint32_t idx;
if (!xsk_ring_prod__reserve(pd, 1, &idx)) {
dbg("%s(%d, %u), socket_tx reserve fail\n", __func__, port, q);
xq->stat_tx_prod_reserve_fail++;
rte_pktmbuf_free(local);
goto exit;
}
struct xdp_desc* desc = xsk_ring_prod__tx_desc(pd, idx);
desc->len = m->pkt_len;
uint64_t addr =
(uint64_t)local - (uint64_t)xq->umem_buffer - xq->mbuf_pool->header_size;
uint64_t offset = rte_pktmbuf_mtod(local, uint64_t) - (uint64_t)local +
xq->mbuf_pool->header_size;
void* pkt = xsk_umem__get_data(xq->umem_buffer, addr + offset);
offset = offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT;
desc->addr = addr | offset;
rte_memcpy(pkt, rte_pktmbuf_mtod(m, void*), desc->len);
tx_bytes += m->data_len;
rte_pktmbuf_free(m);
dbg("%s(%d, %u), tx local mbuf %p umem pkt %p addr 0x%" PRIu64 "\n", __func__, port,
q, local, pkt, addr);
xq->stat_tx_copy++;
tx++;
uint32_t idx;
if (!xsk_ring_prod__reserve(pd, 1, &idx)) {
dbg("%s(%d, %u), socket_tx reserve fail\n", __func__, port, q);
xq->stat_tx_prod_reserve_fail++;
rte_pktmbuf_free(local);
goto exit;
}
struct xdp_desc* desc = xsk_ring_prod__tx_desc(pd, idx);
desc->len = m->pkt_len;
uint64_t addr =
(uint64_t)local - (uint64_t)xq->umem_buffer - xq->mbuf_pool->header_size;
uint64_t offset =
rte_pktmbuf_mtod(local, uint64_t) - (uint64_t)local + xq->mbuf_pool->header_size;
void* pkt = xsk_umem__get_data(xq->umem_buffer, addr + offset);
offset = offset << XSK_UNALIGNED_BUF_OFFSET_SHIFT;
desc->addr = addr | offset;

struct rte_mbuf* n = m;
uint16_t nb_segs = m->nb_segs;
for (uint16_t seg = 0; seg < nb_segs; seg++) {
rte_memcpy(pkt, rte_pktmbuf_mtod(n, void*), n->data_len);
pkt += n->data_len;
/* point to next */
n = n->next;
}

tx_bytes += desc->len;
rte_pktmbuf_free(m);
dbg("%s(%d, %u), tx local mbuf %p umem pkt %p addr 0x%" PRIu64 "\n", __func__, port,
q, local, pkt, addr);
xq->stat_tx_copy++;
tx++;
}

exit:
Expand Down Expand Up @@ -614,6 +617,7 @@ int mt_dev_xdp_init(struct mt_interface* inf) {
}

inf->xdp = xdp;
inf->feature |= MT_IF_FEATURE_TX_MULTI_SEGS;
info("%s(%d), start queue %u cnt %u\n", __func__, port, xdp->start_queue,
xdp->queues_cnt);
return 0;
Expand Down Expand Up @@ -681,13 +685,16 @@ int mt_tx_xdp_put(struct mt_tx_xdp_entry* entry) {
uint8_t* ip = flow->dip_addr;
struct mt_xdp_queue* xq = entry->xq;

/* poll all done buf */
xdp_tx_poll_done(xq);
xdp_queue_tx_stat(xq);
if (xq) {
/* poll all done buf */
xdp_tx_poll_done(xq);
xdp_queue_tx_stat(xq);

xq->tx_entry = NULL;
info("%s(%d), ip %u.%u.%u.%u, port %u, queue %u\n", __func__, port, ip[0], ip[1],
ip[2], ip[3], flow->dst_port, entry->queue_id);
}

xq->tx_entry = NULL;
info("%s(%d), ip %u.%u.%u.%u, port %u, queue %u\n", __func__, port, ip[0], ip[1], ip[2],
ip[3], flow->dst_port, entry->queue_id);
mt_rte_free(entry);
return 0;
}
Expand Down
8 changes: 7 additions & 1 deletion lib/src/dev/mt_dev.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2095,11 +2095,17 @@ int mt_dev_if_init(struct mtl_main_impl* impl) {
inf->nb_rx_q = 1;
p->flags |= MTL_FLAG_SHARED_RX_QUEUE;
inf->system_rx_queues_end = 0;
} else if (mt_pmd_is_dpdk_af_xdp(impl, i) || mt_pmd_is_native_af_xdp(impl, i)) {
} else if (mt_pmd_is_dpdk_af_xdp(impl, i)) {
/* no system queues as no cni */
inf->nb_tx_q = queue_pair_cnt;
inf->nb_rx_q = queue_pair_cnt;
inf->system_rx_queues_end = 0;
} else if (mt_pmd_is_native_af_xdp(impl, i)) {
/* one more for the sys tx queue */
queue_pair_cnt = RTE_MAX(p->tx_queues_cnt[i] + 1, p->rx_queues_cnt[i]);
inf->nb_tx_q = queue_pair_cnt;
inf->nb_rx_q = queue_pair_cnt;
inf->system_rx_queues_end = 0;
} else {
info("%s(%d), deprecated sessions tx %u rx %u\n", __func__, i,
p->tx_sessions_cnt_max, p->rx_sessions_cnt_max);
Expand Down