From 46e06ab9b467e7785344b726dac8fe3006a86a75 Mon Sep 17 00:00:00 2001 From: Tao Date: Thu, 31 Aug 2023 15:04:23 +0200 Subject: [PATCH 01/11] static vf offload dumping --- include/dp_log.h | 1 + include/dp_mbuf_dyn.h | 1 + include/monitoring/dp_graphtrace.h | 13 +- include/monitoring/dp_graphtrace_shared.h | 7 ++ include/rte_flow/dp_rte_flow.h | 6 + include/rte_flow/dp_rte_flow_helpers.h | 33 ++++++ include/rte_flow/dp_rte_flow_init.h | 5 + src/dp_cntrack.c | 26 +++- src/dp_port.c | 8 ++ src/monitoring/dp_graphtrace.c | 11 +- src/nodes/rx_node.c | 2 +- src/rte_flow/dp_rte_flow_init.c | 132 +++++++++++++++++++++ src/rte_flow/dp_rte_flow_traffic_forward.c | 36 ++++-- tools/dp_graphtrace.c | 25 +++- 14 files changed, 283 insertions(+), 23 deletions(-) diff --git a/include/dp_log.h b/include/dp_log.h index 961327cc3..6e497f12a 100644 --- a/include/dp_log.h +++ b/include/dp_log.h @@ -48,6 +48,7 @@ extern "C" { #define DP_LOG_SOCKID(VALUE) _DP_LOG_UINT("socket_id", VALUE) #define DP_LOG_IFNAME(VALUE) _DP_LOG_STR("interface_name", VALUE) #define DP_LOG_LCORE(VALUE) _DP_LOG_UINT("lcore_id", VALUE) +#define DP_LOG_RTE_GROUP(VALUE) _DP_LOG_UINT("rte_group", VALUE) // networking stack #define DP_LOG_IPV4(VALUE) _DP_LOG_IPV4("ipv4", VALUE) #define DP_LOG_IPV6(VALUE) _DP_LOG_IPV6("ipv6", VALUE) diff --git a/include/dp_mbuf_dyn.h b/include/dp_mbuf_dyn.h index 5c36b1160..47e1a8e8f 100644 --- a/include/dp_mbuf_dyn.h +++ b/include/dp_mbuf_dyn.h @@ -31,6 +31,7 @@ struct dp_flow { uint16_t offload_ipv6 : 1; // tmp solution to set if we should offload ipv6 pkts uint16_t dir : 2; // store the direction of each packet uint16_t offload_decision: 2; // store the offload status of each packet + uint16_t offload_mark: 1; // store the offload mark of each packet } flags; uint16_t l3_type; //layer-3 for inner packets. it can be crafted or extracted from raw frames union { diff --git a/include/monitoring/dp_graphtrace.h b/include/monitoring/dp_graphtrace.h index 5bc3ff2af..5df0fc97b 100644 --- a/include/monitoring/dp_graphtrace.h +++ b/include/monitoring/dp_graphtrace.h @@ -19,7 +19,7 @@ enum dp_graphtrace_loglevel { int dp_graphtrace_init(void); void dp_graphtrace_free(void); -void _dp_graphtrace_send(struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs); +void _dp_graphtrace_send(struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs, int type); // Logging the trace for debugging #ifdef ENABLE_PYTEST @@ -45,25 +45,30 @@ extern bool _dp_graphtrace_enabled; static __rte_always_inline void dp_graphtrace_next(struct rte_node *node, void *obj, rte_edge_t next_index) { if (_dp_graphtrace_enabled) - _dp_graphtrace_send(node, node->nodes[next_index], &obj, 1); + _dp_graphtrace_send(node, node->nodes[next_index], &obj, 1, DP_GRAPHTRACE_PKT_TYPE_SOFTWARE); _dp_graphtrace_log_next(node, obj, next_index); } static __rte_always_inline void dp_graphtrace_next_burst(struct rte_node *node, void **objs, uint16_t nb_objs, rte_edge_t next_index) { if (_dp_graphtrace_enabled) - _dp_graphtrace_send(node, node->nodes[next_index], objs, nb_objs); + _dp_graphtrace_send(node, node->nodes[next_index], objs, nb_objs, DP_GRAPHTRACE_PKT_TYPE_SOFTWARE); _dp_graphtrace_log_next_burst(node, objs, nb_objs, next_index); } static __rte_always_inline void dp_graphtrace_tx_burst(struct rte_node *node, void **objs, uint16_t nb_objs, uint16_t port_id) { if (_dp_graphtrace_enabled) - _dp_graphtrace_send(node, NULL, objs, nb_objs); + _dp_graphtrace_send(node, NULL, objs, nb_objs, DP_GRAPHTRACE_PKT_TYPE_SOFTWARE); RTE_SET_USED(port_id); _dp_graphtrace_log_tx_burst(node, objs, nb_objs, port_id); } +static __rte_always_inline void dp_graphtrace_capture_offload_pkt(void *obj) +{ + _dp_graphtrace_send(NULL, NULL, &obj, 1, DP_GRAPHTRACE_PKT_TYPE_OFFLOAD); +} + static __rte_always_inline void dp_graphtrace_enable(void) { _dp_graphtrace_enabled = true; diff --git a/include/monitoring/dp_graphtrace_shared.h b/include/monitoring/dp_graphtrace_shared.h index 58c17b997..9bea74a05 100644 --- a/include/monitoring/dp_graphtrace_shared.h +++ b/include/monitoring/dp_graphtrace_shared.h @@ -26,12 +26,19 @@ enum dp_graphtrace_action { DP_GRAPHTRACE_ACTION_STOP, }; +enum { + DP_GRAPHTRACE_PKT_TYPE_UNKNOWN = 0, + DP_GRAPHTRACE_PKT_TYPE_SOFTWARE, + DP_GRAPHTRACE_PKT_TYPE_OFFLOAD, +}; + struct dp_graphtrace { struct rte_mempool *mempool; struct rte_ring *ringbuf; }; struct dp_graphtrace_pktinfo { + uint8_t pkt_type; uint32_t pktid; struct rte_node *node; struct rte_node *next_node; diff --git a/include/rte_flow/dp_rte_flow.h b/include/rte_flow/dp_rte_flow.h index b128866eb..29f208eac 100644 --- a/include/rte_flow/dp_rte_flow.h +++ b/include/rte_flow/dp_rte_flow.h @@ -31,6 +31,12 @@ extern "C" #define DP_IP_ICMP_CODE_DST_PORT_UNREACHABLE 3 #define DP_IP_ICMP_CODE_FRAGMENT_NEEDED 4 +enum { + DP_RTE_FLOW_DEFAULT_GROUP, + DP_RTE_FLOW_MONITORING_GROUP, + DP_RTE_FLOW_VNET_GROUP, +}; + struct dp_icmp_err_ip_info { struct rte_ipv4_hdr *err_ipv4_hdr; rte_be16_t l4_src_port; diff --git a/include/rte_flow/dp_rte_flow_helpers.h b/include/rte_flow/dp_rte_flow_helpers.h index c84ff82db..6e8a8f9cb 100644 --- a/include/rte_flow/dp_rte_flow_helpers.h +++ b/include/rte_flow/dp_rte_flow_helpers.h @@ -12,6 +12,8 @@ extern "C" { #endif +#define DP_PKT_OFFLOAD_MARK 1 + #define DP_TCP_CONTROL_FLAGS \ (RTE_TCP_FIN_FLAG|RTE_TCP_SYN_FLAG|RTE_TCP_RST_FLAG) @@ -91,6 +93,15 @@ static const struct rte_flow_item_icmp6 dp_flow_item_icmp6_mask = { .type = 0xff, }; +static __rte_always_inline +void dp_set_eth_match_all_item(struct rte_flow_item *item) +{ + item->type = RTE_FLOW_ITEM_TYPE_ETH; + item->spec = NULL; + item->mask = NULL; + item->last = NULL; +} + static __rte_always_inline void dp_set_eth_flow_item(struct rte_flow_item *item, struct rte_flow_item_eth *eth_spec, @@ -533,6 +544,28 @@ void dp_set_set_meta_action(struct rte_flow_action *action, action->conf = meta_action; } +static __rte_always_inline +void dp_set_sample_action(struct rte_flow_action *action, + struct rte_flow_action_sample *sample_action, + uint32_t sample_ratio, struct rte_flow_action *sub_action) +{ + sample_action->ratio = sample_ratio; + if (sub_action) + sample_action->actions = sub_action; + action->type = RTE_FLOW_ACTION_TYPE_SAMPLE; + action->conf = sample_action; +} + +static __rte_always_inline +void dp_set_jump_group_action(struct rte_flow_action *action, + struct rte_flow_action_jump *jump_action, + uint32_t group_id) +{ + jump_action->group = group_id; + action->type = RTE_FLOW_ACTION_TYPE_JUMP; + action->conf = jump_action; +} + static __rte_always_inline void dp_set_end_action(struct rte_flow_action *action) { diff --git a/include/rte_flow/dp_rte_flow_init.h b/include/rte_flow/dp_rte_flow_init.h index 864b78150..85a27d387 100644 --- a/include/rte_flow/dp_rte_flow_init.h +++ b/include/rte_flow/dp_rte_flow_init.h @@ -8,6 +8,11 @@ extern "C" { #include int dp_install_isolated_mode_ipip(int port_id, uint8_t proto_id); + +int dp_install_jump_rule_int_default_group(uint16_t port_id, uint32_t group_id); +int dp_install_default_rule_in_monitoring_group(uint16_t port_id); +int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id); + #ifdef ENABLE_VIRTSVC int dp_install_isolated_mode_virtsvc(int port_id, uint8_t proto_id, uint8_t svc_ipv6[16], uint16_t svc_port); #endif diff --git a/src/dp_cntrack.c b/src/dp_cntrack.c index 7ac4b84c4..8ebbcf13a 100644 --- a/src/dp_cntrack.c +++ b/src/dp_cntrack.c @@ -6,6 +6,8 @@ #include "dp_log.h" #include "dp_vnf.h" #include "rte_flow/dp_rte_flow.h" +#include "rte_flow/dp_rte_flow_helpers.h" +#include "monitoring/dp_graphtrace.h" static struct flow_key key_cache[2]; static int key_cache_index = 0; @@ -75,6 +77,21 @@ static __rte_always_inline void dp_cntrack_tcp_state(struct flow_value *flow_val } +static __rte_always_inline int dp_capture_offloaded_pkts(struct rte_mbuf *m, struct flow_value *flow_val, struct dp_flow *df) +{ + + printf("flow_val->offload_flags.orig = %d, flow_val->offload_flags.reply = %d\n", flow_val->offload_flags.orig, flow_val->offload_flags.reply); + if (!offload_mode_enabled || + flow_val->offload_flags.orig == DP_FLOW_NON_OFFLOAD || flow_val->offload_flags.reply == DP_FLOW_NON_OFFLOAD) + return 0; + + dp_graphtrace_capture_offload_pkt(m); + df->flags.offload_mark = DP_PKT_OFFLOAD_MARK; + printf("offloaded pkt captured\n"); + return 1; + +} + static __rte_always_inline void dp_cntrack_init_flow_offload_flags(struct flow_value *flow_val, struct dp_flow *df) { if (!offload_mode_enabled) @@ -247,6 +264,8 @@ static __rte_always_inline int dp_get_flow_val(struct rte_mbuf *m, struct dp_flo ) { // flow is the same as it was for the previous packet *p_flow_val = cached_flow_val; + if (dp_capture_offloaded_pkts(m, *p_flow_val, df)) + return DP_ERROR; // it is not really an error, but we need to stop processing this pkt dp_set_pkt_flow_direction(curr_key, cached_flow_val, df); dp_set_flow_offload_flag(m, cached_flow_val, df); return DP_OK; @@ -269,6 +288,9 @@ static __rte_always_inline int dp_get_flow_val(struct rte_mbuf *m, struct dp_flo return DP_OK; } + if (dp_capture_offloaded_pkts(m, *p_flow_val, df)) + return DP_ERROR; // it is not really an error, but we need to stop processing this pkt + // already established flow found dp_set_pkt_flow_direction(curr_key, *p_flow_val, df); dp_set_flow_offload_flag(m, *p_flow_val, df); @@ -283,7 +305,9 @@ int dp_cntrack_handle(__rte_unused struct rte_node *node, struct rte_mbuf *m, st int ret; ret = dp_get_flow_val(m, df, &flow_val); - if (DP_FAILED(ret)) + if (DP_FAILED(ret)) { + if (!df->flags.offload_mark) + DPNODE_LOG_WARNING(node, "Cannot establish flow value", DP_LOG_RET(ret)); return ret; flow_val->timestamp = rte_rdtsc(); diff --git a/src/dp_port.c b/src/dp_port.c index a4425a763..5ae539550 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -11,6 +11,7 @@ #include "monitoring/dp_event.h" #include "nodes/rx_node.h" #include "rte_flow/dp_rte_flow_init.h" +#include "rte_flow/dp_rte_flow.h" static const struct rte_eth_conf port_conf_default = { .rxmode = { @@ -462,6 +463,13 @@ int dp_port_start(uint16_t port_id) if (DP_FAILED(dp_port_install_isolated_mode(port_id))) return DP_ERROR; } + + + if (port->port_type == DP_PORT_VF) { + dp_install_jump_rule_int_default_group(port_id, DP_RTE_FLOW_MONITORING_GROUP); + dp_install_default_rule_in_monitoring_group(port_id); + dp_install_default_capture_rule_in_vnet_group(port_id); + } } return DP_OK; diff --git a/src/monitoring/dp_graphtrace.c b/src/monitoring/dp_graphtrace.c index 72d3e0e09..428681838 100644 --- a/src/monitoring/dp_graphtrace.c +++ b/src/monitoring/dp_graphtrace.c @@ -126,7 +126,7 @@ void dp_graphtrace_free(void) } -void _dp_graphtrace_send(struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs) +void _dp_graphtrace_send(struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs, int type) { uint16_t nb_dups = 0; struct rte_mbuf *dups[nb_objs]; @@ -144,8 +144,13 @@ void _dp_graphtrace_send(struct rte_node *node, struct rte_node *next_node, void dups[nb_dups++] = dup; pktinfo = dp_get_graphtrace_pktinfo(dup); pktinfo->pktid = dp_get_pkt_mark(objs[i])->id; - pktinfo->node = node; - pktinfo->next_node = next_node; + if (type == DP_GRAPHTRACE_PKT_TYPE_SOFTWARE) { + pktinfo->pkt_type = DP_GRAPHTRACE_PKT_TYPE_SOFTWARE; + pktinfo->node = node; + pktinfo->next_node = next_node; + } else if (type == DP_GRAPHTRACE_PKT_TYPE_OFFLOAD) { + pktinfo->pkt_type = DP_GRAPHTRACE_PKT_TYPE_OFFLOAD; + } } } diff --git a/src/nodes/rx_node.c b/src/nodes/rx_node.c index 664ce753b..fa9346205 100644 --- a/src/nodes/rx_node.c +++ b/src/nodes/rx_node.c @@ -98,7 +98,7 @@ static uint16_t rx_node_process(struct rte_graph *graph, uint16_t cnt) { struct rx_node_ctx *ctx = (struct rx_node_ctx *)node->ctx; - uint16_t n_pkts; + uint16_t n_pkts = 0; RTE_SET_USED(cnt); // this is a source node, input data is not present yet diff --git a/src/rte_flow/dp_rte_flow_init.c b/src/rte_flow/dp_rte_flow_init.c index f6d1ceae4..81209850f 100644 --- a/src/rte_flow/dp_rte_flow_init.c +++ b/src/rte_flow/dp_rte_flow_init.c @@ -12,6 +12,30 @@ static const struct rte_flow_attr dp_flow_attr_prio_ingress = { .transfer = 0, }; +static const struct rte_flow_attr dp_flow_attr_default_jump_ingress = { + .group = DP_RTE_FLOW_DEFAULT_GROUP, + .priority = 1, + .ingress = 1, + .egress = 0, + .transfer = 1, +}; + +static const struct rte_flow_attr dp_flow_attr_default_monitoring_ingress = { + .group = DP_RTE_FLOW_MONITORING_GROUP, + .priority = 3, + .ingress = 1, + .egress = 0, + .transfer = 1, +}; + +static const struct rte_flow_attr dp_flow_attr_default_capture_ingress = { + .group = DP_RTE_FLOW_VNET_GROUP, + .priority = 3, + .ingress = 1, + .egress = 0, + .transfer = 0, +}; + int dp_install_isolated_mode_ipip(int port_id, uint8_t proto_id) { struct rte_flow_item_eth eth_spec; // #1 @@ -38,6 +62,114 @@ int dp_install_isolated_mode_ipip(int port_id, uint8_t proto_id) return DP_OK; } +int dp_install_jump_rule_int_default_group(uint16_t port_id, uint32_t dst_group) +{ + struct rte_flow_item pattern[2]; // first is a NULL ethernet header matching, second is the end + int pattern_cnt = 0; + + // jump action from default group to monitoring group + struct rte_flow_action_jump jump_action; // #1 + struct rte_flow_action action[2]; // + end + int action_cnt = 0; + + struct rte_flow *flow; + + memset(pattern, 0, sizeof(pattern)); + memset(action, 0, sizeof(action)); + + // all ethernet packets + dp_set_eth_match_all_item(&pattern[pattern_cnt++]); + dp_set_end_flow_item(&pattern[pattern_cnt++]); + + // create actions that jump from the default group + // create jump action + dp_set_jump_group_action(&action[action_cnt++], &jump_action, dst_group); + + // end actions + dp_set_end_action(&action[action_cnt++]); + + // validate and install flow rule + flow = dp_install_rte_flow(port_id, &dp_flow_attr_default_jump_ingress, pattern, action); + + if (!flow) + return DP_ERROR; + + DPS_LOG_DEBUG("Installed the default jumping flow rule that destinated to group", DP_LOG_PORTID(port_id), DP_LOG_RTE_GROUP(dst_group)); + return DP_OK; +} + +int dp_install_default_rule_in_monitoring_group(uint16_t port_id) +{ + + struct rte_flow_item pattern[2]; // first is a NULL ethernet header matching, second is the end + int pattern_cnt = 0; + + struct rte_flow_action_sample sample_action; // 1 + struct rte_flow_action_jump jump_action; // 2 + struct rte_flow_action action[3]; // + end + int action_cnt = 0; + + struct rte_flow *flow; + + memset(pattern, 0, sizeof(pattern)); + memset(action, 0, sizeof(action)); + + // all ethernet packets + dp_set_eth_match_all_item(&pattern[pattern_cnt++]); + dp_set_end_flow_item(&pattern[pattern_cnt++]); + + // create actions + // create sampling action + dp_set_sample_action(&action[action_cnt++], &sample_action, 1, NULL); // mirror all packets, without explicite sub sample action + + // create jump group action + dp_set_jump_group_action(&action[action_cnt++], &jump_action, DP_RTE_FLOW_VNET_GROUP); // jump to group DP_RTE_FLOW_VNET_GROUP + + // end actions + dp_set_end_action(&action[action_cnt++]); + + // validate and install flow rule + flow = dp_install_rte_flow(port_id, &dp_flow_attr_default_monitoring_ingress, pattern, action); + + if (!flow) + return DP_ERROR; + + DPS_LOG_DEBUG("Installed the default monitoring flow rule", DP_LOG_PORTID(port_id)); + return DP_OK; + +} + +int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id) +{ + + struct rte_flow_item pattern[2]; // first is a NULL ethernet header matching, second is the end + int pattern_cnt = 0; + + struct rte_flow_action_queue queue_action; // 1 + struct rte_flow_action action[2]; // + end + int action_cnt = 0; + + memset(pattern, 0, sizeof(pattern)); + memset(action, 0, sizeof(action)); + + // all ethernet packets + dp_set_eth_match_all_item(&pattern[pattern_cnt++]); + dp_set_end_flow_item(&pattern[pattern_cnt++]); + + // create actions + // create flow action -- queue, send to default software handling queue + dp_set_redirect_queue_action(&action[action_cnt++], &queue_action, 0); + // create flow action -- end + dp_set_end_action(&action[action_cnt++]); + + + if (!dp_install_rte_flow(port_id, &dp_flow_attr_default_capture_ingress, pattern, action)) + return DP_ERROR; + + DPS_LOG_DEBUG("Installed the default capture flow rule", DP_LOG_PORTID(port_id)); + return DP_OK; +} + #ifdef ENABLE_VIRTSVC int dp_install_isolated_mode_virtsvc(int port_id, uint8_t proto_id, uint8_t svc_ipv6[16], rte_be16_t svc_port) { diff --git a/src/rte_flow/dp_rte_flow_traffic_forward.c b/src/rte_flow/dp_rte_flow_traffic_forward.c index 971b71c89..a84cfc298 100644 --- a/src/rte_flow/dp_rte_flow_traffic_forward.c +++ b/src/rte_flow/dp_rte_flow_traffic_forward.c @@ -8,7 +8,7 @@ #define DP_IPIP_ENCAP_HEADER_SIZE (sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv6_hdr)) -static const struct rte_flow_attr dp_flow_attr_ingress = { +static const struct rte_flow_attr dp_flow_pf_attr_ingress = { .group = 0, .priority = 0, .ingress = 1, @@ -16,6 +16,14 @@ static const struct rte_flow_attr dp_flow_attr_ingress = { .transfer = 0, }; +static const struct rte_flow_attr dp_flow_vf_attr_ingress = { + .group = DP_RTE_FLOW_VNET_GROUP, + .priority = 0, + .ingress = 1, + .egress = 0, + .transfer = 0, +}; + static const struct rte_flow_attr dp_flow_attr_egress = { .group = 0, .priority = 0, @@ -24,7 +32,7 @@ static const struct rte_flow_attr dp_flow_attr_egress = { .transfer = 0, }; -static const struct rte_flow_attr dp_flow_attr_transfer = { +static const struct rte_flow_attr dp_flow_pf_attr_transfer = { .group = 0, .priority = 0, #ifdef ENABLE_DPDK_22_11 @@ -36,6 +44,18 @@ static const struct rte_flow_attr dp_flow_attr_transfer = { .transfer = 1, }; +static const struct rte_flow_attr dp_flow_vf_attr_transfer = { + .group = DP_RTE_FLOW_VNET_GROUP, + .priority = 0, +#ifdef ENABLE_DPDK_22_11 + .ingress = 0, +#else + .ingress = 1, +#endif + .egress = 0, + .transfer = 1, +}; + static __rte_always_inline struct flow_age_ctx *allocate_agectx(void) { struct flow_age_ctx *agectx; @@ -233,7 +253,7 @@ static __rte_always_inline int dp_offload_handle_tunnel_encap_traffic(struct rte dp_set_end_action(&hairpin_actions[hairpin_action_cnt++]); - if (DP_FAILED(dp_install_rte_flow_with_indirect(m->port, &dp_flow_attr_ingress, + if (DP_FAILED(dp_install_rte_flow_with_indirect(m->port, &dp_flow_vf_attr_ingress, hairpin_pattern, hairpin_actions, hairpin_age_action, df, hairpin_agectx)) ) { @@ -280,7 +300,7 @@ static __rte_always_inline int dp_offload_handle_tunnel_encap_traffic(struct rte attr = &dp_flow_attr_egress; t_port_id = dp_port_get_pf1_id(); } else { - attr = &dp_flow_attr_transfer; + attr = &dp_flow_vf_attr_transfer; t_port_id = m->port; } if (DP_FAILED(dp_install_rte_flow_with_indirect(t_port_id, attr, @@ -406,9 +426,9 @@ static __rte_always_inline int dp_offload_handle_tunnel_decap_traffic(struct rte if (DP_FAILED(dp_install_rte_flow_with_indirect(m->port, #ifndef ENABLE_DPDK_22_11 - cross_pf_port ? &dp_flow_attr_ingress : + cross_pf_port ? &dp_flow_pf_attr_ingress : #endif - &dp_flow_attr_transfer, + &dp_flow_pf_attr_transfer, pattern, actions, age_action, df, agectx)) ) { @@ -492,7 +512,7 @@ static __rte_always_inline int dp_offload_handle_local_traffic(struct rte_mbuf * // TODO: this attribute has not been tested with DPDK 22.11, // so maybe 'dp_flow_attr_transfer' should be ifdef'd too - if (DP_FAILED(dp_install_rte_flow_with_indirect(m->port, &dp_flow_attr_transfer, + if (DP_FAILED(dp_install_rte_flow_with_indirect(m->port, &dp_flow_pf_attr_transfer, pattern, actions, age_action, df, agectx)) ) { @@ -576,7 +596,7 @@ static __rte_always_inline int dp_offload_handle_in_network_traffic(struct rte_m // bouncing back rule's expiration is taken care of by the rte flow rule expiration mechanism; // no need to perform query to perform checking on expiration status, thus an indirect action is not needed - if (DP_FAILED(dp_install_rte_flow_with_age(m->port, &dp_flow_attr_ingress, pattern, actions, df->conntrack, agectx))) { + if (DP_FAILED(dp_install_rte_flow_with_age(m->port, &dp_flow_pf_attr_ingress, pattern, actions, df->conntrack, agectx))) { dp_destroy_rte_flow_agectx(agectx); return DP_ERROR; } diff --git a/tools/dp_graphtrace.c b/tools/dp_graphtrace.c index f5ac50877..780a7e0f2 100644 --- a/tools/dp_graphtrace.c +++ b/tools/dp_graphtrace.c @@ -104,12 +104,25 @@ static void print_packet(struct rte_mbuf *pkt) struct dp_graphtrace_pktinfo *pktinfo = dp_get_graphtrace_pktinfo(pkt); dp_graphtrace_sprint(pkt, printbuf, sizeof(printbuf)); - printf("%u: " NODENAME_FMT " %s " NODENAME_FMT ": %s\n", - pktinfo->pktid, - pktinfo->node->name, - pktinfo->next_node ? "->" : " ", - pktinfo->next_node ? pktinfo->next_node->name : "", - printbuf); + + if (pktinfo->pkt_type == DP_GRAPHTRACE_PKT_TYPE_SOFTWARE) { + dp_graphtrace_sprint(pkt, printbuf, sizeof(printbuf)); + printf("%u: " NODENAME_FMT " %s " NODENAME_FMT ": %s\n", + pktinfo->pktid, + pktinfo->node->name, + pktinfo->next_node ? "->" : " ", + pktinfo->next_node ? pktinfo->next_node->name : "", + printbuf); + } else if (pktinfo->pkt_type == DP_GRAPHTRACE_PKT_TYPE_OFFLOAD) { + printf("%u: captured offload packet : %s\n", + pktinfo->pktid, + printbuf); + } else { + printf("%u: " NODENAME_FMT ": unknown packet type %u\n", + pktinfo->pktid, + pktinfo->node->name, + pktinfo->pkt_type); + } } static int dp_graphtrace_dump(struct dp_graphtrace *graphtrace) From 5afd1e0a59c0a210c21841ddd232d899fd54ebdb Mon Sep 17 00:00:00 2001 From: Tao Date: Fri, 1 Sep 2023 15:30:38 +0200 Subject: [PATCH 02/11] auto start/stop vf offload pkt tracing --- include/dp_port.h | 1 + include/monitoring/dp_graphtrace.h | 5 +++ include/rte_flow/dp_rte_flow_init.h | 2 ++ src/dp_cntrack.c | 3 -- src/dp_port.c | 7 +++- src/monitoring/dp_graphtrace.c | 33 +++++++++++++++++ src/rte_flow/dp_rte_flow_init.c | 42 +++++++++++++++++++--- src/rte_flow/dp_rte_flow_traffic_forward.c | 6 ++-- tools/dp_graphtrace.c | 30 +++++++++------- 9 files changed, 105 insertions(+), 24 deletions(-) diff --git a/include/dp_port.h b/include/dp_port.h index 3265e729e..8c784b137 100644 --- a/include/dp_port.h +++ b/include/dp_port.h @@ -37,6 +37,7 @@ struct dp_port { uint8_t peer_pf_hairpin_tx_rx_queue_offset; uint16_t peer_pf_port_id; enum dp_vf_port_attach_status attach_status; + struct rte_flow *default_flow; }; struct dp_ports { diff --git a/include/monitoring/dp_graphtrace.h b/include/monitoring/dp_graphtrace.h index 5df0fc97b..d8ef485e1 100644 --- a/include/monitoring/dp_graphtrace.h +++ b/include/monitoring/dp_graphtrace.h @@ -79,6 +79,11 @@ static __rte_always_inline void dp_graphtrace_disable(void) _dp_graphtrace_enabled = false; } +static __rte_always_inline bool dp_graphtrace_is_enabled(void) +{ + return _dp_graphtrace_enabled; +} + #ifdef __cplusplus } #endif diff --git a/include/rte_flow/dp_rte_flow_init.h b/include/rte_flow/dp_rte_flow_init.h index 85a27d387..c299b0fce 100644 --- a/include/rte_flow/dp_rte_flow_init.h +++ b/include/rte_flow/dp_rte_flow_init.h @@ -13,6 +13,8 @@ int dp_install_jump_rule_int_default_group(uint16_t port_id, uint32_t group_id); int dp_install_default_rule_in_monitoring_group(uint16_t port_id); int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id); +int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group); + #ifdef ENABLE_VIRTSVC int dp_install_isolated_mode_virtsvc(int port_id, uint8_t proto_id, uint8_t svc_ipv6[16], uint16_t svc_port); #endif diff --git a/src/dp_cntrack.c b/src/dp_cntrack.c index 8ebbcf13a..524248c15 100644 --- a/src/dp_cntrack.c +++ b/src/dp_cntrack.c @@ -79,15 +79,12 @@ static __rte_always_inline void dp_cntrack_tcp_state(struct flow_value *flow_val static __rte_always_inline int dp_capture_offloaded_pkts(struct rte_mbuf *m, struct flow_value *flow_val, struct dp_flow *df) { - - printf("flow_val->offload_flags.orig = %d, flow_val->offload_flags.reply = %d\n", flow_val->offload_flags.orig, flow_val->offload_flags.reply); if (!offload_mode_enabled || flow_val->offload_flags.orig == DP_FLOW_NON_OFFLOAD || flow_val->offload_flags.reply == DP_FLOW_NON_OFFLOAD) return 0; dp_graphtrace_capture_offload_pkt(m); df->flags.offload_mark = DP_PKT_OFFLOAD_MARK; - printf("offloaded pkt captured\n"); return 1; } diff --git a/src/dp_port.c b/src/dp_port.c index 5ae539550..746f69691 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -12,6 +12,7 @@ #include "nodes/rx_node.h" #include "rte_flow/dp_rte_flow_init.h" #include "rte_flow/dp_rte_flow.h" +#include "monitoring/dp_graphtrace.h" static const struct rte_eth_conf port_conf_default = { .rxmode = { @@ -433,6 +434,7 @@ int dp_port_start(uint16_t port_id) { struct dp_port *port; int ret; + bool graphtrace_enabled = dp_graphtrace_is_enabled(); port = dp_port_get(port_id); if (!port) @@ -466,7 +468,10 @@ int dp_port_start(uint16_t port_id) if (port->port_type == DP_PORT_VF) { - dp_install_jump_rule_int_default_group(port_id, DP_RTE_FLOW_MONITORING_GROUP); + if (graphtrace_enabled) + dp_install_jump_rule_int_default_group(port_id, DP_RTE_FLOW_MONITORING_GROUP); + else + dp_install_jump_rule_int_default_group(port_id, DP_RTE_FLOW_VNET_GROUP); dp_install_default_rule_in_monitoring_group(port_id); dp_install_default_capture_rule_in_vnet_group(port_id); } diff --git a/src/monitoring/dp_graphtrace.c b/src/monitoring/dp_graphtrace.c index 428681838..64ccfdfbe 100644 --- a/src/monitoring/dp_graphtrace.c +++ b/src/monitoring/dp_graphtrace.c @@ -6,6 +6,8 @@ #include "dp_log.h" #include "dpdk_layer.h" #include "monitoring/dp_graphtrace_shared.h" +#include "rte_flow/dp_rte_flow_init.h" +#include "rte_flow/dp_rte_flow.h" #ifdef ENABLE_PYTEST # include "dp_conf.h" @@ -14,6 +16,7 @@ static enum dp_graphtrace_loglevel graphtrace_loglevel; static struct dp_graphtrace graphtrace; bool _dp_graphtrace_enabled = false; +static bool _offload_enabled; static int dp_graphtrace_init_memzone(void) { @@ -37,6 +40,7 @@ static int dp_graphtrace_init_memzone(void) return DP_ERROR; } + _offload_enabled = dp_conf_is_offload_enabled(); return DP_OK; } @@ -49,10 +53,23 @@ static void dp_graphtrace_free_memzone(void) graphtrace.mempool = NULL; } +static __rte_always_inline +int dp_graphtrace_turn_on_vf_offload_tracing(void) +{ + return dp_change_all_vf_default_jump_rte_flow_group(DP_RTE_FLOW_MONITORING_GROUP); +} + +static __rte_always_inline +int dp_graphtrace_turn_off_vf_offload_tracing(void) +{ + return dp_change_all_vf_default_jump_rte_flow_group(DP_RTE_FLOW_VNET_GROUP); +} + static __rte_always_inline void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_graphtrace_mp_reply *reply) { const struct dp_graphtrace_mp_request *request = (const struct dp_graphtrace_mp_request *)mp_msg->param; + int ret; if (mp_msg->len_param != sizeof(struct dp_graphtrace_mp_request)) { DPS_LOG_WARNING("Invalid graphtrace request message size", DP_LOG_VALUE(mp_msg->len_param)); @@ -63,11 +80,27 @@ void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_gra switch ((enum dp_graphtrace_action)request->action) { case DP_GRAPHTRACE_ACTION_START: dp_graphtrace_enable(); + if (_offload_enabled) { + ret = dp_graphtrace_turn_on_vf_offload_tracing(); + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Cannot turn on offload tracing", DP_LOG_RET(ret)); + reply->error_code = ret; + return; + } + } reply->error_code = DP_OK; DPS_LOG_INFO("Graphtrace enabled"); return; case DP_GRAPHTRACE_ACTION_STOP: dp_graphtrace_disable(); + if (_offload_enabled) { + ret = dp_graphtrace_turn_off_vf_offload_tracing(); + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Cannot turn off offload tracing", DP_LOG_RET(ret)); + reply->error_code = ret; + return; + } + } reply->error_code = DP_OK; DPS_LOG_INFO("Graphtrace disabled"); return; diff --git a/src/rte_flow/dp_rte_flow_init.c b/src/rte_flow/dp_rte_flow_init.c index 81209850f..e91cf7e7d 100644 --- a/src/rte_flow/dp_rte_flow_init.c +++ b/src/rte_flow/dp_rte_flow_init.c @@ -73,6 +73,7 @@ int dp_install_jump_rule_int_default_group(uint16_t port_id, uint32_t dst_group) int action_cnt = 0; struct rte_flow *flow; + struct dp_port *port = dp_port_get_vf(port_id); memset(pattern, 0, sizeof(pattern)); memset(action, 0, sizeof(action)); @@ -94,6 +95,8 @@ int dp_install_jump_rule_int_default_group(uint16_t port_id, uint32_t dst_group) if (!flow) return DP_ERROR; + port->default_flow = flow; + DPS_LOG_DEBUG("Installed the default jumping flow rule that destinated to group", DP_LOG_PORTID(port_id), DP_LOG_RTE_GROUP(dst_group)); return DP_OK; } @@ -108,19 +111,24 @@ int dp_install_default_rule_in_monitoring_group(uint16_t port_id) struct rte_flow_action_jump jump_action; // 2 struct rte_flow_action action[3]; // + end int action_cnt = 0; - + + struct rte_flow_action sub_action[1]; + int sub_action_cnt = 0; + struct rte_flow *flow; memset(pattern, 0, sizeof(pattern)); memset(action, 0, sizeof(action)); + memset(sub_action, 0, sizeof(sub_action)); // all ethernet packets dp_set_eth_match_all_item(&pattern[pattern_cnt++]); dp_set_end_flow_item(&pattern[pattern_cnt++]); - // create actions + // create actions // create sampling action - dp_set_sample_action(&action[action_cnt++], &sample_action, 1, NULL); // mirror all packets, without explicite sub sample action + dp_set_end_action(&sub_action[sub_action_cnt++]); + dp_set_sample_action(&action[action_cnt++], &sample_action, 1, sub_action); // mirror all packets, without explicite sub sample action // create jump group action dp_set_jump_group_action(&action[action_cnt++], &jump_action, DP_RTE_FLOW_VNET_GROUP); // jump to group DP_RTE_FLOW_VNET_GROUP @@ -161,7 +169,6 @@ int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id) dp_set_redirect_queue_action(&action[action_cnt++], &queue_action, 0); // create flow action -- end dp_set_end_action(&action[action_cnt++]); - if (!dp_install_rte_flow(port_id, &dp_flow_attr_default_capture_ingress, pattern, action)) return DP_ERROR; @@ -170,6 +177,33 @@ int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id) return DP_OK; } +int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group) +{ + struct dp_ports *ports = get_dp_ports(); + struct dp_port *port; + struct rte_flow_error error; + int ret; + + DP_FOREACH_PORT(ports, port) { + if (port->port_type == DP_PORT_VF && port->allocated) { + if (port->default_flow) + ret = rte_flow_destroy(port->port_id, port->default_flow, &error); + + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Failed to destroy default flow", DP_LOG_PORTID(port->port_id), DP_LOG_RET(ret)); + return DP_ERROR; + } + + if (DP_FAILED(dp_install_jump_rule_int_default_group(port->port_id, dst_group))) { + DPS_LOG_ERR("Failed to install default jump flow", DP_LOG_PORTID(port->port_id)); + return DP_ERROR; + } + } + } + + return DP_OK; +} + #ifdef ENABLE_VIRTSVC int dp_install_isolated_mode_virtsvc(int port_id, uint8_t proto_id, uint8_t svc_ipv6[16], rte_be16_t svc_port) { diff --git a/src/rte_flow/dp_rte_flow_traffic_forward.c b/src/rte_flow/dp_rte_flow_traffic_forward.c index a84cfc298..2f460314a 100644 --- a/src/rte_flow/dp_rte_flow_traffic_forward.c +++ b/src/rte_flow/dp_rte_flow_traffic_forward.c @@ -9,7 +9,7 @@ #define DP_IPIP_ENCAP_HEADER_SIZE (sizeof(struct rte_ether_hdr) + sizeof(struct rte_ipv6_hdr)) static const struct rte_flow_attr dp_flow_pf_attr_ingress = { - .group = 0, + .group = DP_RTE_FLOW_DEFAULT_GROUP, .priority = 0, .ingress = 1, .egress = 0, @@ -25,7 +25,7 @@ static const struct rte_flow_attr dp_flow_vf_attr_ingress = { }; static const struct rte_flow_attr dp_flow_attr_egress = { - .group = 0, + .group = DP_RTE_FLOW_DEFAULT_GROUP, .priority = 0, .ingress = 0, .egress = 1, @@ -33,7 +33,7 @@ static const struct rte_flow_attr dp_flow_attr_egress = { }; static const struct rte_flow_attr dp_flow_pf_attr_transfer = { - .group = 0, + .group = DP_RTE_FLOW_DEFAULT_GROUP, .priority = 0, #ifdef ENABLE_DPDK_22_11 .ingress = 0, diff --git a/tools/dp_graphtrace.c b/tools/dp_graphtrace.c index 780a7e0f2..c896493ce 100644 --- a/tools/dp_graphtrace.c +++ b/tools/dp_graphtrace.c @@ -105,23 +105,27 @@ static void print_packet(struct rte_mbuf *pkt) dp_graphtrace_sprint(pkt, printbuf, sizeof(printbuf)); - if (pktinfo->pkt_type == DP_GRAPHTRACE_PKT_TYPE_SOFTWARE) { + switch (pktinfo->pkt_type) { + case DP_GRAPHTRACE_PKT_TYPE_SOFTWARE: dp_graphtrace_sprint(pkt, printbuf, sizeof(printbuf)); printf("%u: " NODENAME_FMT " %s " NODENAME_FMT ": %s\n", - pktinfo->pktid, - pktinfo->node->name, - pktinfo->next_node ? "->" : " ", - pktinfo->next_node ? pktinfo->next_node->name : "", - printbuf); - } else if (pktinfo->pkt_type == DP_GRAPHTRACE_PKT_TYPE_OFFLOAD) { + pktinfo->pktid, + pktinfo->node->name, + pktinfo->next_node ? "->" : " ", + pktinfo->next_node ? pktinfo->next_node->name : "", + printbuf); + break; + case DP_GRAPHTRACE_PKT_TYPE_OFFLOAD: printf("%u: captured offload packet : %s\n", - pktinfo->pktid, - printbuf); - } else { + pktinfo->pktid, + printbuf); + break; + default: printf("%u: " NODENAME_FMT ": unknown packet type %u\n", - pktinfo->pktid, - pktinfo->node->name, - pktinfo->pkt_type); + pktinfo->pktid, + pktinfo->node->name, + pktinfo->pkt_type); + break; } } From 8b38e4a8edb5c3ff32330705caa6b8577a1113ed Mon Sep 17 00:00:00 2001 From: Tao Date: Wed, 6 Sep 2023 16:47:01 +0200 Subject: [PATCH 03/11] configure flow rules via monitoring queue to avoid race condition --- include/monitoring/dp_event.h | 5 ++++ include/monitoring/dp_monitoring.h | 18 +++++-------- include/rte_flow/dp_rte_flow_init.h | 3 ++- src/monitoring/dp_event.c | 40 ++++++++++++++++++++++++++--- src/monitoring/dp_graphtrace.c | 26 +++++-------------- src/monitoring/dp_monitoring.c | 10 ++++++-- src/rte_flow/dp_rte_flow_init.c | 12 +++++++++ 7 files changed, 77 insertions(+), 37 deletions(-) diff --git a/include/monitoring/dp_event.h b/include/monitoring/dp_event.h index e797fe017..e137f3df7 100644 --- a/include/monitoring/dp_event.h +++ b/include/monitoring/dp_event.h @@ -16,7 +16,12 @@ int dp_link_status_change_event_callback(uint16_t port_id, void dp_process_event_link_msg(struct rte_mbuf *m); int dp_send_event_flow_aging_msg(void); +int dp_send_event_hardware_capture_start_msg(void); +int dp_send_event_hardware_capture_stop_msg(void); + void dp_process_event_flow_aging_msg(struct rte_mbuf *m); +void dp_process_event_hardware_capture_start_msg(struct rte_mbuf *m); +void dp_process_event_hardware_capture_stop_msg(struct rte_mbuf *m); #ifdef __cplusplus } diff --git a/include/monitoring/dp_monitoring.h b/include/monitoring/dp_monitoring.h index 9f35d8c9d..ae477d43d 100644 --- a/include/monitoring/dp_monitoring.h +++ b/include/monitoring/dp_monitoring.h @@ -8,20 +8,16 @@ extern "C" { #endif -enum dp_status_type { - DP_STATUS_TYPE_UNKNOWN, - DP_STATUS_TYPE_LINK, - DP_STATUS_TYPE_FLOW_AGING, -}; - -enum dp_status_scope { - DP_STATUS_SCOPE_LOCAL, - DP_STATUS_SCOPE_REMOTE, +enum dp_event_type { + DP_EVENT_TYPE_UNKNOWN, + DP_EVENT_TYPE_LINK_STATUS, + DP_EVENT_TYPE_FLOW_AGING, + DP_EVENT_TYPE_HARDWARE_CAPTURE_START, + DP_EVENT_TYPE_HARDWARE_CAPTURE_STOP, }; struct dp_event_msg_head { - enum dp_status_type type; - enum dp_status_scope scope; + enum dp_event_type type; }; struct dp_link_status { diff --git a/include/rte_flow/dp_rte_flow_init.h b/include/rte_flow/dp_rte_flow_init.h index c299b0fce..0eceee36e 100644 --- a/include/rte_flow/dp_rte_flow_init.h +++ b/include/rte_flow/dp_rte_flow_init.h @@ -13,7 +13,8 @@ int dp_install_jump_rule_int_default_group(uint16_t port_id, uint32_t group_id); int dp_install_default_rule_in_monitoring_group(uint16_t port_id); int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id); -int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group); +int dp_turn_on_vf_offload_tracing(void); +int dp_turn_off_vf_offload_tracing(void); #ifdef ENABLE_VIRTSVC int dp_install_isolated_mode_virtsvc(int port_id, uint8_t proto_id, uint8_t svc_ipv6[16], uint16_t svc_port); diff --git a/src/monitoring/dp_event.c b/src/monitoring/dp_event.c index c4c86eb70..07718bce8 100644 --- a/src/monitoring/dp_event.c +++ b/src/monitoring/dp_event.c @@ -5,6 +5,7 @@ #include "dp_log.h" #include "dp_port.h" #include "monitoring/dp_monitoring.h" +#include "rte_flow/dp_rte_flow_init.h" static int dp_send_event_msg(struct dp_event_msg *msg) @@ -38,8 +39,7 @@ static int dp_send_event_link_msg(uint16_t port_id, uint8_t status) { struct dp_event_msg link_status_msg = { .msg_head = { - .type = DP_STATUS_TYPE_LINK, - .scope = DP_STATUS_SCOPE_LOCAL, + .type = DP_EVENT_TYPE_LINK_STATUS, }, .event_entry = { .link_status = { @@ -81,19 +81,51 @@ void dp_process_event_link_msg(struct rte_mbuf *m) DPS_LOG_WARNING("Cannot set link status", DP_LOG_PORTID(port_id), DP_LOG_VALUE(status)); } +void dp_process_event_hardware_capture_start_msg(__rte_unused struct rte_mbuf *m) +{ + if (DP_FAILED(dp_turn_on_vf_offload_tracing())) + DPS_LOG_WARNING("Cannot turn on offload tracing"); +} + +void dp_process_event_hardware_capture_stop_msg(__rte_unused struct rte_mbuf *m) +{ + if (DP_FAILED(dp_turn_off_vf_offload_tracing())) + DPS_LOG_WARNING("Cannot turn off offload tracing"); +} + // Flow-aging message - sent periodically to age-out conntracked flows int dp_send_event_flow_aging_msg(void) { struct dp_event_msg flow_aging_msg = { .msg_head = { - .type = DP_STATUS_TYPE_FLOW_AGING, - .scope = DP_STATUS_SCOPE_LOCAL, + .type = DP_EVENT_TYPE_FLOW_AGING, }, }; return dp_send_event_msg(&flow_aging_msg); } +int dp_send_event_hardware_capture_start_msg(void) +{ + struct dp_event_msg graphtrace_start_msg = { + .msg_head = { + .type = DP_EVENT_TYPE_HARDWARE_CAPTURE_START, + }, + }; + return dp_send_event_msg(&graphtrace_start_msg); +} + +int dp_send_event_hardware_capture_stop_msg(void) +{ + struct dp_event_msg graphtrace_start_msg = { + .msg_head = { + .type = DP_EVENT_TYPE_HARDWARE_CAPTURE_STOP, + }, + }; + return dp_send_event_msg(&graphtrace_start_msg); +} + + void dp_process_event_flow_aging_msg(__rte_unused struct rte_mbuf *m) { if (dp_conf_is_offload_enabled()) { diff --git a/src/monitoring/dp_graphtrace.c b/src/monitoring/dp_graphtrace.c index 64ccfdfbe..b78070412 100644 --- a/src/monitoring/dp_graphtrace.c +++ b/src/monitoring/dp_graphtrace.c @@ -8,6 +8,7 @@ #include "monitoring/dp_graphtrace_shared.h" #include "rte_flow/dp_rte_flow_init.h" #include "rte_flow/dp_rte_flow.h" +#include "monitoring/dp_event.h" #ifdef ENABLE_PYTEST # include "dp_conf.h" @@ -53,17 +54,6 @@ static void dp_graphtrace_free_memzone(void) graphtrace.mempool = NULL; } -static __rte_always_inline -int dp_graphtrace_turn_on_vf_offload_tracing(void) -{ - return dp_change_all_vf_default_jump_rte_flow_group(DP_RTE_FLOW_MONITORING_GROUP); -} - -static __rte_always_inline -int dp_graphtrace_turn_off_vf_offload_tracing(void) -{ - return dp_change_all_vf_default_jump_rte_flow_group(DP_RTE_FLOW_VNET_GROUP); -} static __rte_always_inline void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_graphtrace_mp_reply *reply) @@ -81,10 +71,9 @@ void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_gra case DP_GRAPHTRACE_ACTION_START: dp_graphtrace_enable(); if (_offload_enabled) { - ret = dp_graphtrace_turn_on_vf_offload_tracing(); - if (DP_FAILED(ret)) { - DPS_LOG_ERR("Cannot turn on offload tracing", DP_LOG_RET(ret)); - reply->error_code = ret; + if (DP_FAILED(dp_send_event_hardware_capture_start_msg())) { + DPS_LOG_ERR("Cannot send hardware capture start message"); + reply->error_code = DP_ERROR; return; } } @@ -94,10 +83,9 @@ void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_gra case DP_GRAPHTRACE_ACTION_STOP: dp_graphtrace_disable(); if (_offload_enabled) { - ret = dp_graphtrace_turn_off_vf_offload_tracing(); - if (DP_FAILED(ret)) { - DPS_LOG_ERR("Cannot turn off offload tracing", DP_LOG_RET(ret)); - reply->error_code = ret; + if (DP_FAILED(dp_send_event_hardware_capture_stop_msg())) { + DPS_LOG_ERR("Cannot send hardware capture stop message"); + reply->error_code = DP_ERROR; return; } } diff --git a/src/monitoring/dp_monitoring.c b/src/monitoring/dp_monitoring.c index 19a052886..30d0a0e13 100644 --- a/src/monitoring/dp_monitoring.c +++ b/src/monitoring/dp_monitoring.c @@ -8,12 +8,18 @@ void dp_process_event_msg(struct rte_mbuf *m) struct dp_event_msg *event_msg = rte_pktmbuf_mtod(m, struct dp_event_msg *); switch (event_msg->msg_head.type) { - case DP_STATUS_TYPE_LINK: + case DP_EVENT_TYPE_LINK_STATUS: dp_process_event_link_msg(m); break; - case DP_STATUS_TYPE_FLOW_AGING: + case DP_EVENT_TYPE_FLOW_AGING: dp_process_event_flow_aging_msg(m); break; + case DP_EVENT_TYPE_HARDWARE_CAPTURE_START: + dp_process_event_hardware_capture_start_msg(m); + break; + case DP_EVENT_TYPE_HARDWARE_CAPTURE_STOP: + dp_process_event_hardware_capture_stop_msg(m); + break; default: DPS_LOG_WARNING("Unknown monitoring status message type", DP_LOG_VALUE(event_msg->msg_head.type)); } diff --git a/src/rte_flow/dp_rte_flow_init.c b/src/rte_flow/dp_rte_flow_init.c index e91cf7e7d..7ce336ffd 100644 --- a/src/rte_flow/dp_rte_flow_init.c +++ b/src/rte_flow/dp_rte_flow_init.c @@ -177,6 +177,7 @@ int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id) return DP_OK; } +static __rte_always_inline int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group) { struct dp_ports *ports = get_dp_ports(); @@ -204,6 +205,17 @@ int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group) return DP_OK; } +int dp_turn_on_vf_offload_tracing(void) +{ + return dp_change_all_vf_default_jump_rte_flow_group(DP_RTE_FLOW_MONITORING_GROUP); +} + +int dp_turn_off_vf_offload_tracing(void) +{ + return dp_change_all_vf_default_jump_rte_flow_group(DP_RTE_FLOW_VNET_GROUP); +} + + #ifdef ENABLE_VIRTSVC int dp_install_isolated_mode_virtsvc(int port_id, uint8_t proto_id, uint8_t svc_ipv6[16], rte_be16_t svc_port) { From 898d468ca9c0832d7a02011e7fee0d317602dfd8 Mon Sep 17 00:00:00 2001 From: Tao Date: Thu, 7 Sep 2023 14:13:09 +0200 Subject: [PATCH 04/11] added hw tracing as an option for dp_graphtrace; code optimization --- include/dp_cntrack.h | 2 + include/monitoring/dp_graphtrace.h | 10 +-- include/monitoring/dp_graphtrace_shared.h | 6 +- include/rte_flow/dp_rte_flow_helpers.h | 3 +- src/dp_cntrack.c | 15 ++-- src/dp_port.c | 23 +++++- src/monitoring/dp_graphtrace.c | 36 +++++---- src/nodes/conntrack_node.c | 4 +- src/nodes/rx_node.c | 2 +- src/rte_flow/dp_rte_flow_init.c | 21 ++--- tools/dp_graphtrace.c | 98 ++++++++++++++++++++++- 11 files changed, 167 insertions(+), 53 deletions(-) diff --git a/include/dp_cntrack.h b/include/dp_cntrack.h index 04f77a924..76dc5648d 100644 --- a/include/dp_cntrack.h +++ b/include/dp_cntrack.h @@ -10,6 +10,8 @@ extern "C" { #endif +#define DP_IS_CAPTURED_HW_PKT 5 + void dp_cntrack_init(void); int dp_cntrack_handle(struct rte_node *node, struct rte_mbuf *m, struct dp_flow *df); diff --git a/include/monitoring/dp_graphtrace.h b/include/monitoring/dp_graphtrace.h index d8ef485e1..e47c9170f 100644 --- a/include/monitoring/dp_graphtrace.h +++ b/include/monitoring/dp_graphtrace.h @@ -19,7 +19,7 @@ enum dp_graphtrace_loglevel { int dp_graphtrace_init(void); void dp_graphtrace_free(void); -void _dp_graphtrace_send(struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs, int type); +void _dp_graphtrace_send(int type, struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs); // Logging the trace for debugging #ifdef ENABLE_PYTEST @@ -45,28 +45,28 @@ extern bool _dp_graphtrace_enabled; static __rte_always_inline void dp_graphtrace_next(struct rte_node *node, void *obj, rte_edge_t next_index) { if (_dp_graphtrace_enabled) - _dp_graphtrace_send(node, node->nodes[next_index], &obj, 1, DP_GRAPHTRACE_PKT_TYPE_SOFTWARE); + _dp_graphtrace_send(DP_GRAPHTRACE_PKT_TYPE_SOFTWARE, node, node->nodes[next_index], &obj, 1); _dp_graphtrace_log_next(node, obj, next_index); } static __rte_always_inline void dp_graphtrace_next_burst(struct rte_node *node, void **objs, uint16_t nb_objs, rte_edge_t next_index) { if (_dp_graphtrace_enabled) - _dp_graphtrace_send(node, node->nodes[next_index], objs, nb_objs, DP_GRAPHTRACE_PKT_TYPE_SOFTWARE); + _dp_graphtrace_send(DP_GRAPHTRACE_PKT_TYPE_SOFTWARE, node, node->nodes[next_index], objs, nb_objs); _dp_graphtrace_log_next_burst(node, objs, nb_objs, next_index); } static __rte_always_inline void dp_graphtrace_tx_burst(struct rte_node *node, void **objs, uint16_t nb_objs, uint16_t port_id) { if (_dp_graphtrace_enabled) - _dp_graphtrace_send(node, NULL, objs, nb_objs, DP_GRAPHTRACE_PKT_TYPE_SOFTWARE); + _dp_graphtrace_send(DP_GRAPHTRACE_PKT_TYPE_SOFTWARE, node, NULL, objs, nb_objs); RTE_SET_USED(port_id); _dp_graphtrace_log_tx_burst(node, objs, nb_objs, port_id); } static __rte_always_inline void dp_graphtrace_capture_offload_pkt(void *obj) { - _dp_graphtrace_send(NULL, NULL, &obj, 1, DP_GRAPHTRACE_PKT_TYPE_OFFLOAD); + _dp_graphtrace_send(DP_GRAPHTRACE_PKT_TYPE_OFFLOAD, NULL, NULL, &obj, 1); } static __rte_always_inline void dp_graphtrace_enable(void) diff --git a/include/monitoring/dp_graphtrace_shared.h b/include/monitoring/dp_graphtrace_shared.h index 9bea74a05..375fb3ea9 100644 --- a/include/monitoring/dp_graphtrace_shared.h +++ b/include/monitoring/dp_graphtrace_shared.h @@ -24,9 +24,11 @@ enum dp_graphtrace_action { DP_GRAPHTRACE_ACTION_NULL, DP_GRAPHTRACE_ACTION_START, DP_GRAPHTRACE_ACTION_STOP, + DP_GRAPHTRACE_ACTION_ENABLE_HW_CAPTURE, + DP_GRAPHTRACE_ACTION_DISABLE_HW_CAPTURE, }; -enum { +enum dp_graphtrace_pkt_type { DP_GRAPHTRACE_PKT_TYPE_UNKNOWN = 0, DP_GRAPHTRACE_PKT_TYPE_SOFTWARE, DP_GRAPHTRACE_PKT_TYPE_OFFLOAD, @@ -38,7 +40,7 @@ struct dp_graphtrace { }; struct dp_graphtrace_pktinfo { - uint8_t pkt_type; + enum dp_graphtrace_pkt_type pkt_type; uint32_t pktid; struct rte_node *node; struct rte_node *next_node; diff --git a/include/rte_flow/dp_rte_flow_helpers.h b/include/rte_flow/dp_rte_flow_helpers.h index 6e8a8f9cb..3f295a916 100644 --- a/include/rte_flow/dp_rte_flow_helpers.h +++ b/include/rte_flow/dp_rte_flow_helpers.h @@ -550,8 +550,7 @@ void dp_set_sample_action(struct rte_flow_action *action, uint32_t sample_ratio, struct rte_flow_action *sub_action) { sample_action->ratio = sample_ratio; - if (sub_action) - sample_action->actions = sub_action; + sample_action->actions = sub_action; // it seems that driver does not support null sub action action->type = RTE_FLOW_ACTION_TYPE_SAMPLE; action->conf = sample_action; } diff --git a/src/dp_cntrack.c b/src/dp_cntrack.c index 524248c15..bd303a438 100644 --- a/src/dp_cntrack.c +++ b/src/dp_cntrack.c @@ -77,15 +77,15 @@ static __rte_always_inline void dp_cntrack_tcp_state(struct flow_value *flow_val } -static __rte_always_inline int dp_capture_offloaded_pkts(struct rte_mbuf *m, struct flow_value *flow_val, struct dp_flow *df) +static __rte_always_inline bool dp_capture_offloaded_pkts(struct rte_mbuf *m, struct flow_value *flow_val, struct dp_flow *df) { if (!offload_mode_enabled || flow_val->offload_flags.orig == DP_FLOW_NON_OFFLOAD || flow_val->offload_flags.reply == DP_FLOW_NON_OFFLOAD) - return 0; + return false; dp_graphtrace_capture_offload_pkt(m); df->flags.offload_mark = DP_PKT_OFFLOAD_MARK; - return 1; + return true; } @@ -262,7 +262,7 @@ static __rte_always_inline int dp_get_flow_val(struct rte_mbuf *m, struct dp_flo // flow is the same as it was for the previous packet *p_flow_val = cached_flow_val; if (dp_capture_offloaded_pkts(m, *p_flow_val, df)) - return DP_ERROR; // it is not really an error, but we need to stop processing this pkt + return DP_IS_CAPTURED_HW_PKT; // it is not really an error, but we need to stop processing this pkt dp_set_pkt_flow_direction(curr_key, cached_flow_val, df); dp_set_flow_offload_flag(m, cached_flow_val, df); return DP_OK; @@ -286,7 +286,7 @@ static __rte_always_inline int dp_get_flow_val(struct rte_mbuf *m, struct dp_flo } if (dp_capture_offloaded_pkts(m, *p_flow_val, df)) - return DP_ERROR; // it is not really an error, but we need to stop processing this pkt + return DP_IS_CAPTURED_HW_PKT; // it is not really an error, but we need to stop processing this pkt // already established flow found dp_set_pkt_flow_direction(curr_key, *p_flow_val, df); @@ -303,8 +303,9 @@ int dp_cntrack_handle(__rte_unused struct rte_node *node, struct rte_mbuf *m, st ret = dp_get_flow_val(m, df, &flow_val); if (DP_FAILED(ret)) { - if (!df->flags.offload_mark) - DPNODE_LOG_WARNING(node, "Cannot establish flow value", DP_LOG_RET(ret)); + DPNODE_LOG_WARNING(node, "Cannot establish flow value", DP_LOG_RET(ret)); + return ret; + } else if (ret == DP_IS_CAPTURED_HW_PKT) return ret; flow_val->timestamp = rte_rdtsc(); diff --git a/src/dp_port.c b/src/dp_port.c index 746f69691..ef86b742c 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -469,11 +469,26 @@ int dp_port_start(uint16_t port_id) if (port->port_type == DP_PORT_VF) { if (graphtrace_enabled) - dp_install_jump_rule_int_default_group(port_id, DP_RTE_FLOW_MONITORING_GROUP); + ret = dp_install_jump_rule_int_default_group(port_id, DP_RTE_FLOW_MONITORING_GROUP); else - dp_install_jump_rule_int_default_group(port_id, DP_RTE_FLOW_VNET_GROUP); - dp_install_default_rule_in_monitoring_group(port_id); - dp_install_default_capture_rule_in_vnet_group(port_id); + ret = dp_install_jump_rule_int_default_group(port_id, DP_RTE_FLOW_VNET_GROUP); + + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Cannot install default jump rule", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); + return DP_ERROR; + } + + ret = dp_install_default_rule_in_monitoring_group(port_id); + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Cannot install default rule in monitoring group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); + return DP_ERROR; + } + + ret = dp_install_default_capture_rule_in_vnet_group(port_id); + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Cannot install default capture rule in vnet group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); + return DP_ERROR; + } } } diff --git a/src/monitoring/dp_graphtrace.c b/src/monitoring/dp_graphtrace.c index b78070412..c7c77ffb8 100644 --- a/src/monitoring/dp_graphtrace.c +++ b/src/monitoring/dp_graphtrace.c @@ -17,7 +17,7 @@ static enum dp_graphtrace_loglevel graphtrace_loglevel; static struct dp_graphtrace graphtrace; bool _dp_graphtrace_enabled = false; -static bool _offload_enabled; +static bool offload_enabled; static int dp_graphtrace_init_memzone(void) { @@ -41,7 +41,7 @@ static int dp_graphtrace_init_memzone(void) return DP_ERROR; } - _offload_enabled = dp_conf_is_offload_enabled(); + offload_enabled = dp_conf_is_offload_enabled(); return DP_OK; } @@ -70,7 +70,16 @@ void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_gra switch ((enum dp_graphtrace_action)request->action) { case DP_GRAPHTRACE_ACTION_START: dp_graphtrace_enable(); - if (_offload_enabled) { + reply->error_code = DP_OK; + DPS_LOG_INFO("Graphtrace enabled"); + return; + case DP_GRAPHTRACE_ACTION_STOP: + dp_graphtrace_disable(); + reply->error_code = DP_OK; + DPS_LOG_INFO("Graphtrace disabled"); + return; + case DP_GRAPHTRACE_ACTION_ENABLE_HW_CAPTURE: + if (offload_enabled) { if (DP_FAILED(dp_send_event_hardware_capture_start_msg())) { DPS_LOG_ERR("Cannot send hardware capture start message"); reply->error_code = DP_ERROR; @@ -78,11 +87,10 @@ void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_gra } } reply->error_code = DP_OK; - DPS_LOG_INFO("Graphtrace enabled"); + DPS_LOG_INFO("Hardware capture enabled"); return; - case DP_GRAPHTRACE_ACTION_STOP: - dp_graphtrace_disable(); - if (_offload_enabled) { + case DP_GRAPHTRACE_ACTION_DISABLE_HW_CAPTURE: + if (offload_enabled) { if (DP_FAILED(dp_send_event_hardware_capture_stop_msg())) { DPS_LOG_ERR("Cannot send hardware capture stop message"); reply->error_code = DP_ERROR; @@ -90,7 +98,7 @@ void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_gra } } reply->error_code = DP_OK; - DPS_LOG_INFO("Graphtrace disabled"); + DPS_LOG_INFO("Hardware capture disabled"); return; case DP_GRAPHTRACE_ACTION_NULL: default: @@ -147,7 +155,7 @@ void dp_graphtrace_free(void) } -void _dp_graphtrace_send(struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs, int type) +void _dp_graphtrace_send(int type, struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs) { uint16_t nb_dups = 0; struct rte_mbuf *dups[nb_objs]; @@ -165,13 +173,13 @@ void _dp_graphtrace_send(struct rte_node *node, struct rte_node *next_node, void dups[nb_dups++] = dup; pktinfo = dp_get_graphtrace_pktinfo(dup); pktinfo->pktid = dp_get_pkt_mark(objs[i])->id; - if (type == DP_GRAPHTRACE_PKT_TYPE_SOFTWARE) { + if (type == DP_GRAPHTRACE_PKT_TYPE_SOFTWARE) pktinfo->pkt_type = DP_GRAPHTRACE_PKT_TYPE_SOFTWARE; - pktinfo->node = node; - pktinfo->next_node = next_node; - } else if (type == DP_GRAPHTRACE_PKT_TYPE_OFFLOAD) { + else if (type == DP_GRAPHTRACE_PKT_TYPE_OFFLOAD) pktinfo->pkt_type = DP_GRAPHTRACE_PKT_TYPE_OFFLOAD; - } + + pktinfo->node = node; + pktinfo->next_node = next_node; } } diff --git a/src/nodes/conntrack_node.c b/src/nodes/conntrack_node.c index a7e640e84..a79c2d404 100644 --- a/src/nodes/conntrack_node.c +++ b/src/nodes/conntrack_node.c @@ -50,6 +50,7 @@ static __rte_always_inline rte_edge_t get_next_index(struct rte_node *node, stru { struct rte_ipv4_hdr *ipv4_hdr; struct dp_flow *df; + int ret; df = dp_get_flow_ptr(m); ipv4_hdr = dp_get_ipv4_hdr(m); @@ -70,7 +71,8 @@ static __rte_always_inline rte_edge_t get_next_index(struct rte_node *node, stru || df->l4_type == IPPROTO_UDP || df->l4_type == IPPROTO_ICMP ) { - if (DP_FAILED(dp_cntrack_handle(node, m, df))) + ret = dp_cntrack_handle(node, m, df); + if (DP_FAILED(ret) || (ret == DP_IS_CAPTURED_HW_PKT)) return CONNTRACK_NEXT_DROP; } else { return CONNTRACK_NEXT_DROP; diff --git a/src/nodes/rx_node.c b/src/nodes/rx_node.c index fa9346205..664ce753b 100644 --- a/src/nodes/rx_node.c +++ b/src/nodes/rx_node.c @@ -98,7 +98,7 @@ static uint16_t rx_node_process(struct rte_graph *graph, uint16_t cnt) { struct rx_node_ctx *ctx = (struct rx_node_ctx *)node->ctx; - uint16_t n_pkts = 0; + uint16_t n_pkts; RTE_SET_USED(cnt); // this is a source node, input data is not present yet diff --git a/src/rte_flow/dp_rte_flow_init.c b/src/rte_flow/dp_rte_flow_init.c index 7ce336ffd..1c38d6b66 100644 --- a/src/rte_flow/dp_rte_flow_init.c +++ b/src/rte_flow/dp_rte_flow_init.c @@ -75,9 +75,6 @@ int dp_install_jump_rule_int_default_group(uint16_t port_id, uint32_t dst_group) struct rte_flow *flow; struct dp_port *port = dp_port_get_vf(port_id); - memset(pattern, 0, sizeof(pattern)); - memset(action, 0, sizeof(action)); - // all ethernet packets dp_set_eth_match_all_item(&pattern[pattern_cnt++]); dp_set_end_flow_item(&pattern[pattern_cnt++]); @@ -117,10 +114,6 @@ int dp_install_default_rule_in_monitoring_group(uint16_t port_id) struct rte_flow *flow; - memset(pattern, 0, sizeof(pattern)); - memset(action, 0, sizeof(action)); - memset(sub_action, 0, sizeof(sub_action)); - // all ethernet packets dp_set_eth_match_all_item(&pattern[pattern_cnt++]); dp_set_end_flow_item(&pattern[pattern_cnt++]); @@ -157,9 +150,6 @@ int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id) struct rte_flow_action action[2]; // + end int action_cnt = 0; - memset(pattern, 0, sizeof(pattern)); - memset(action, 0, sizeof(action)); - // all ethernet packets dp_set_eth_match_all_item(&pattern[pattern_cnt++]); dp_set_end_flow_item(&pattern[pattern_cnt++]); @@ -177,8 +167,7 @@ int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id) return DP_OK; } -static __rte_always_inline -int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group) +static int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group) { struct dp_ports *ports = get_dp_ports(); struct dp_port *port; @@ -191,13 +180,13 @@ int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group) ret = rte_flow_destroy(port->port_id, port->default_flow, &error); if (DP_FAILED(ret)) { - DPS_LOG_ERR("Failed to destroy default flow", DP_LOG_PORTID(port->port_id), DP_LOG_RET(ret)); - return DP_ERROR; + DPS_LOG_WARNING("Failed to destroy default flow", DP_LOG_PORTID(port->port_id), DP_LOG_RET(ret)); + continue; } if (DP_FAILED(dp_install_jump_rule_int_default_group(port->port_id, dst_group))) { - DPS_LOG_ERR("Failed to install default jump flow", DP_LOG_PORTID(port->port_id)); - return DP_ERROR; + DPS_LOG_WARNING("Failed to install default jump flow", DP_LOG_PORTID(port->port_id)); + continue; } } } diff --git a/tools/dp_graphtrace.c b/tools/dp_graphtrace.c index c896493ce..facd94e95 100644 --- a/tools/dp_graphtrace.c +++ b/tools/dp_graphtrace.c @@ -4,6 +4,7 @@ #include #include #include +#include #include "dp_error.h" #include "dp_log.h" @@ -31,6 +32,24 @@ static const char *eal_arg_strings[] = { "--no-pci", // do not try to use any hardware "--log-level=6", // hide DPDK's informational messages (level 7) }; + +enum { + DP_GRAPHTRACE_OPT_NONE, + DP_GRAPHTRACE_OPT_HELP, + DP_GRAPHTRACE_OPT_CAPTURE_HW_PKT, +} cmd_opt_type; + +static bool capture_hw_pkt = false; + +#define OPTSTRING \ + "h" /* help */ + +static const struct option longopts[] = { + { "help", 0, 0, DP_GRAPHTRACE_OPT_HELP }, + { "capture-hw-pkt", 0, 0, DP_GRAPHTRACE_OPT_CAPTURE_HW_PKT }, +}; + + static char *eal_args_mem[RTE_DIM(eal_arg_strings)]; static char *eal_args[RTE_DIM(eal_args_mem)]; @@ -194,6 +213,26 @@ static int dp_graphtrace_stop(void) return dp_graphtrace_request(DP_GRAPHTRACE_ACTION_STOP, &reply); } +static int dp_graphtrace_enable_hw_capture(void) +{ + struct dp_graphtrace_mp_reply reply; + + if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_ENABLE_HW_CAPTURE, &reply))) + return DP_ERROR; + + return DP_OK; +} + +static int dp_graphtrace_disable_hw_capture(void) +{ + struct dp_graphtrace_mp_reply reply; + + if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_DISABLE_HW_CAPTURE, &reply))) + return DP_ERROR; + + return DP_OK; +} + static void dp_graphtrace_monitor_primary(void *arg __rte_unused) { int ret; @@ -240,8 +279,23 @@ static int do_graphtrace(struct dp_graphtrace *graphtrace) return DP_ERROR; } + if (capture_hw_pkt) { + if (DP_FAILED(dp_graphtrace_enable_hw_capture())) { + fprintf(stderr, "Failed to enable hardware packet capture\n"); + rte_eal_alarm_cancel(dp_graphtrace_monitor_primary, (void *)-1); + return DP_ERROR; + } + } + ret = dp_graphtrace_dump(graphtrace); + if (capture_hw_pkt) { + if (DP_FAILED(dp_graphtrace_disable_hw_capture())) { + fprintf(stderr, "Failed to disable hardware packet capture\n"); + ret = DP_ERROR; + } + } + if (DP_FAILED(dp_graphtrace_stop())) { fprintf(stderr, "Failed to request graph tracing termination\n"); ret = DP_ERROR; @@ -256,12 +310,54 @@ static void signal_handler(__rte_unused int signum) interrupt = true; } -int main(void) +static void dp_print_usage(const char *prgname) +{ + fprintf(stderr, + " --help show this help message and exit\n" + " --capture-hw-pkt capture pkts after offloading rules are installed (experimental feature, only vf's pkts are supported)\n", + prgname); +} + +static int parse_args(int argc, char **argv) +{ + int opt; + char *prgname = argv[0]; + + while ((opt = getopt_long(argc, argv, OPTSTRING, longopts, NULL)) != -1) { + switch (opt) { + case 'h': + case DP_GRAPHTRACE_OPT_HELP: + dp_print_usage(prgname); + return DP_CONF_RUNMODE_EXIT; + case DP_GRAPHTRACE_OPT_CAPTURE_HW_PKT: + capture_hw_pkt = true; + break; + default: + dp_print_usage(prgname); + return DP_CONF_RUNMODE_ERROR; + } + } + + return DP_CONF_RUNMODE_NORMAL; +} + +int main(int argc, char **argv) { struct dp_graphtrace graphtrace; int retcode; int ret; + ret = parse_args(argc, argv); + switch (ret) { + case DP_CONF_RUNMODE_ERROR: + fprintf(stderr, "Cannot parse cmd options %s\n", dp_strerror_verbose(ret)); + return EXIT_FAILURE; + case DP_CONF_RUNMODE_EXIT: + return EXIT_SUCCESS; + case DP_CONF_RUNMODE_NORMAL: + break; + } + ret = eal_init(); if (DP_FAILED(ret)) { fprintf(stderr, "Cannot init EAL %s\n", dp_strerror_verbose(ret)); From 5aa974348989e13e08c89a0ae9fa9c2eb379a8d0 Mon Sep 17 00:00:00 2001 From: Tao Date: Fri, 8 Sep 2023 17:22:34 +0200 Subject: [PATCH 05/11] roll back when necessary --- include/monitoring/dp_graphtrace.h | 2 +- include/rte_flow/dp_rte_flow_init.h | 2 +- src/dp_port.c | 4 +- src/monitoring/dp_graphtrace.c | 9 ++-- src/rte_flow/dp_rte_flow_init.c | 4 +- tools/dp_graphtrace.c | 65 +++++++++++++++++++---------- 6 files changed, 51 insertions(+), 35 deletions(-) diff --git a/include/monitoring/dp_graphtrace.h b/include/monitoring/dp_graphtrace.h index e47c9170f..263cea999 100644 --- a/include/monitoring/dp_graphtrace.h +++ b/include/monitoring/dp_graphtrace.h @@ -19,7 +19,7 @@ enum dp_graphtrace_loglevel { int dp_graphtrace_init(void); void dp_graphtrace_free(void); -void _dp_graphtrace_send(int type, struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs); +void _dp_graphtrace_send(enum dp_graphtrace_pkt_type type, struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs); // Logging the trace for debugging #ifdef ENABLE_PYTEST diff --git a/include/rte_flow/dp_rte_flow_init.h b/include/rte_flow/dp_rte_flow_init.h index 0eceee36e..9e35610b7 100644 --- a/include/rte_flow/dp_rte_flow_init.h +++ b/include/rte_flow/dp_rte_flow_init.h @@ -9,7 +9,7 @@ extern "C" { int dp_install_isolated_mode_ipip(int port_id, uint8_t proto_id); -int dp_install_jump_rule_int_default_group(uint16_t port_id, uint32_t group_id); +int dp_install_jump_rule_in_default_group(uint16_t port_id, uint32_t group_id); int dp_install_default_rule_in_monitoring_group(uint16_t port_id); int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id); diff --git a/src/dp_port.c b/src/dp_port.c index ef86b742c..4b9ed9004 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -469,9 +469,9 @@ int dp_port_start(uint16_t port_id) if (port->port_type == DP_PORT_VF) { if (graphtrace_enabled) - ret = dp_install_jump_rule_int_default_group(port_id, DP_RTE_FLOW_MONITORING_GROUP); + ret = dp_install_jump_rule_in_default_group(port_id, DP_RTE_FLOW_MONITORING_GROUP); else - ret = dp_install_jump_rule_int_default_group(port_id, DP_RTE_FLOW_VNET_GROUP); + ret = dp_install_jump_rule_in_default_group(port_id, DP_RTE_FLOW_VNET_GROUP); if (DP_FAILED(ret)) { DPS_LOG_ERR("Cannot install default jump rule", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); diff --git a/src/monitoring/dp_graphtrace.c b/src/monitoring/dp_graphtrace.c index c7c77ffb8..efc937a17 100644 --- a/src/monitoring/dp_graphtrace.c +++ b/src/monitoring/dp_graphtrace.c @@ -82,6 +82,7 @@ void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_gra if (offload_enabled) { if (DP_FAILED(dp_send_event_hardware_capture_start_msg())) { DPS_LOG_ERR("Cannot send hardware capture start message"); + dp_graphtrace_disable(); // roll back if failed on this side, and error code cannot be sent back reply->error_code = DP_ERROR; return; } @@ -155,7 +156,7 @@ void dp_graphtrace_free(void) } -void _dp_graphtrace_send(int type, struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs) +void _dp_graphtrace_send(enum dp_graphtrace_pkt_type type, struct rte_node *node, struct rte_node *next_node, void **objs, uint16_t nb_objs) { uint16_t nb_dups = 0; struct rte_mbuf *dups[nb_objs]; @@ -173,11 +174,7 @@ void _dp_graphtrace_send(int type, struct rte_node *node, struct rte_node *next_ dups[nb_dups++] = dup; pktinfo = dp_get_graphtrace_pktinfo(dup); pktinfo->pktid = dp_get_pkt_mark(objs[i])->id; - if (type == DP_GRAPHTRACE_PKT_TYPE_SOFTWARE) - pktinfo->pkt_type = DP_GRAPHTRACE_PKT_TYPE_SOFTWARE; - else if (type == DP_GRAPHTRACE_PKT_TYPE_OFFLOAD) - pktinfo->pkt_type = DP_GRAPHTRACE_PKT_TYPE_OFFLOAD; - + pktinfo->pkt_type = type; pktinfo->node = node; pktinfo->next_node = next_node; } diff --git a/src/rte_flow/dp_rte_flow_init.c b/src/rte_flow/dp_rte_flow_init.c index 1c38d6b66..51b6e6b08 100644 --- a/src/rte_flow/dp_rte_flow_init.c +++ b/src/rte_flow/dp_rte_flow_init.c @@ -62,7 +62,7 @@ int dp_install_isolated_mode_ipip(int port_id, uint8_t proto_id) return DP_OK; } -int dp_install_jump_rule_int_default_group(uint16_t port_id, uint32_t dst_group) +int dp_install_jump_rule_in_default_group(uint16_t port_id, uint32_t dst_group) { struct rte_flow_item pattern[2]; // first is a NULL ethernet header matching, second is the end int pattern_cnt = 0; @@ -184,7 +184,7 @@ static int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group) continue; } - if (DP_FAILED(dp_install_jump_rule_int_default_group(port->port_id, dst_group))) { + if (DP_FAILED(dp_install_jump_rule_in_default_group(port->port_id, dst_group))) { DPS_LOG_WARNING("Failed to install default jump flow", DP_LOG_PORTID(port->port_id)); continue; } diff --git a/tools/dp_graphtrace.c b/tools/dp_graphtrace.c index facd94e95..d2e8bb116 100644 --- a/tools/dp_graphtrace.c +++ b/tools/dp_graphtrace.c @@ -46,7 +46,7 @@ static bool capture_hw_pkt = false; static const struct option longopts[] = { { "help", 0, 0, DP_GRAPHTRACE_OPT_HELP }, - { "capture-hw-pkt", 0, 0, DP_GRAPHTRACE_OPT_CAPTURE_HW_PKT }, + { "hw-packet", 0, 0, DP_GRAPHTRACE_OPT_CAPTURE_HW_PKT }, }; @@ -181,7 +181,7 @@ static int dp_graphtrace_request(enum dp_graphtrace_action action, struct dp_gra if (DP_FAILED(ret)) { fprintf(stderr, "Cannot send graphtrace request %s\n", dp_strerror_verbose(ret)); return DP_ERROR; - } + } if (DP_FAILED(reply->error_code)) { fprintf(stderr, "Graphtrace request failed %s\n", dp_strerror_verbose(reply->error_code)); @@ -195,8 +195,10 @@ static int dp_graphtrace_start(void) { struct dp_graphtrace_mp_reply reply; - if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_START, &reply))) + if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_START, &reply))) { + fprintf(stderr, "Failed to request graph tracing\n"); return DP_ERROR; + } primary_alive = true; return DP_OK; @@ -210,15 +212,22 @@ static int dp_graphtrace_stop(void) if (!primary_alive) return DP_OK; - return dp_graphtrace_request(DP_GRAPHTRACE_ACTION_STOP, &reply); + if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_STOP, &reply))) { + fprintf(stderr, "Failed to request graph tracing termination\n"); + return DP_ERROR; + } + + return DP_OK; } static int dp_graphtrace_enable_hw_capture(void) { struct dp_graphtrace_mp_reply reply; - if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_ENABLE_HW_CAPTURE, &reply))) + if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_ENABLE_HW_CAPTURE, &reply))) { + fprintf(stderr, "Failed to enable hardware packet capture\n"); return DP_ERROR; + } return DP_OK; } @@ -227,8 +236,10 @@ static int dp_graphtrace_disable_hw_capture(void) { struct dp_graphtrace_mp_reply reply; - if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_DISABLE_HW_CAPTURE, &reply))) + if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_DISABLE_HW_CAPTURE, &reply))) { + fprintf(stderr, "Failed to disable hardware packet capture\n"); return DP_ERROR; + } return DP_OK; } @@ -257,6 +268,20 @@ static void dp_graphtrace_monitor_primary(void *arg __rte_unused) fprintf(stderr, "Warning: Cannot re-schedule primary process monitor %s\n", dp_strerror_verbose(ret)); } +static int dp_graphtrace_init_hw_pkt_capture(struct dp_graphtrace *graphtrace) +{ + if (!capture_hw_pkt) + return DP_OK; + + if (DP_FAILED(dp_graphtrace_enable_hw_capture())) { + fprintf(stderr, "Failed to enable hardware packet capture\n"); + rte_eal_alarm_cancel(dp_graphtrace_monitor_primary, (void *)-1); + return DP_ERROR; + } + + return DP_OK; +} + static int do_graphtrace(struct dp_graphtrace *graphtrace) { int ret; @@ -274,34 +299,28 @@ static int do_graphtrace(struct dp_graphtrace *graphtrace) } if (DP_FAILED(dp_graphtrace_start())) { - fprintf(stderr, "Failed to request graph tracing\n"); rte_eal_alarm_cancel(dp_graphtrace_monitor_primary, (void *)-1); return DP_ERROR; } - if (capture_hw_pkt) { - if (DP_FAILED(dp_graphtrace_enable_hw_capture())) { - fprintf(stderr, "Failed to enable hardware packet capture\n"); - rte_eal_alarm_cancel(dp_graphtrace_monitor_primary, (void *)-1); - return DP_ERROR; - } + if (DP_FAILED(dp_graphtrace_init_hw_pkt_capture(graphtrace))) { + if (dp_graphtrace_stop()) // rollback if failed + fprintf(stderr, "Failed to stop graph tracing after failing to init hw pkt capture\n"); + rte_eal_alarm_cancel(dp_graphtrace_monitor_primary, (void *)-1); + return DP_ERROR; } ret = dp_graphtrace_dump(graphtrace); - if (capture_hw_pkt) { - if (DP_FAILED(dp_graphtrace_disable_hw_capture())) { - fprintf(stderr, "Failed to disable hardware packet capture\n"); + if (capture_hw_pkt) + if (DP_FAILED(dp_graphtrace_disable_hw_capture())) ret = DP_ERROR; - } - } - if (DP_FAILED(dp_graphtrace_stop())) { - fprintf(stderr, "Failed to request graph tracing termination\n"); + if (DP_FAILED(dp_graphtrace_stop())) ret = DP_ERROR; - } rte_eal_alarm_cancel(dp_graphtrace_monitor_primary, (void *)-1); + return ret; } @@ -314,7 +333,7 @@ static void dp_print_usage(const char *prgname) { fprintf(stderr, " --help show this help message and exit\n" - " --capture-hw-pkt capture pkts after offloading rules are installed (experimental feature, only vf's pkts are supported)\n", + " --hw-packet capture pkts after offloading rules are installed (experimental feature, only VF's outgoing packets are supported)\n", prgname); } @@ -350,7 +369,7 @@ int main(int argc, char **argv) ret = parse_args(argc, argv); switch (ret) { case DP_CONF_RUNMODE_ERROR: - fprintf(stderr, "Cannot parse cmd options %s\n", dp_strerror_verbose(ret)); + fprintf(stderr, "Cannot parse command-line options %s\n", dp_strerror_verbose(ret)); return EXIT_FAILURE; case DP_CONF_RUNMODE_EXIT: return EXIT_SUCCESS; From 9751749b913f875a1159408926b73a1b194e3f2b Mon Sep 17 00:00:00 2001 From: Tao Date: Fri, 8 Sep 2023 17:39:32 +0200 Subject: [PATCH 06/11] process hw installation failure during vf start --- include/monitoring/dp_graphtrace_shared.h | 14 ++- src/dp_cntrack.c | 11 +- src/dp_port.c | 55 ++++++---- src/monitoring/dp_graphtrace.c | 91 ++++++++++------ src/rte_flow/dp_rte_flow_init.c | 11 +- tools/dp_graphtrace.c | 120 ++++++++-------------- 6 files changed, 154 insertions(+), 148 deletions(-) diff --git a/include/monitoring/dp_graphtrace_shared.h b/include/monitoring/dp_graphtrace_shared.h index 375fb3ea9..62c356dff 100644 --- a/include/monitoring/dp_graphtrace_shared.h +++ b/include/monitoring/dp_graphtrace_shared.h @@ -21,15 +21,16 @@ extern "C" { #endif enum dp_graphtrace_action { - DP_GRAPHTRACE_ACTION_NULL, DP_GRAPHTRACE_ACTION_START, DP_GRAPHTRACE_ACTION_STOP, - DP_GRAPHTRACE_ACTION_ENABLE_HW_CAPTURE, - DP_GRAPHTRACE_ACTION_DISABLE_HW_CAPTURE, +}; + +enum dp_graphtrace_op_type { + DP_GRAPHTRACE_OP_TYPE_SOFTWARE, + DP_GRAPHTRACE_OP_TYPE_OFFLOAD, }; enum dp_graphtrace_pkt_type { - DP_GRAPHTRACE_PKT_TYPE_UNKNOWN = 0, DP_GRAPHTRACE_PKT_TYPE_SOFTWARE, DP_GRAPHTRACE_PKT_TYPE_OFFLOAD, }; @@ -47,7 +48,10 @@ struct dp_graphtrace_pktinfo { }; struct dp_graphtrace_mp_request { - uint8_t action; + enum dp_graphtrace_action action; + union { + enum dp_graphtrace_op_type op_type; + } action_params; }; struct dp_graphtrace_mp_reply { diff --git a/src/dp_cntrack.c b/src/dp_cntrack.c index bd303a438..218918380 100644 --- a/src/dp_cntrack.c +++ b/src/dp_cntrack.c @@ -262,7 +262,7 @@ static __rte_always_inline int dp_get_flow_val(struct rte_mbuf *m, struct dp_flo // flow is the same as it was for the previous packet *p_flow_val = cached_flow_val; if (dp_capture_offloaded_pkts(m, *p_flow_val, df)) - return DP_IS_CAPTURED_HW_PKT; // it is not really an error, but we need to stop processing this pkt + return DP_IS_CAPTURED_HW_PKT; dp_set_pkt_flow_direction(curr_key, cached_flow_val, df); dp_set_flow_offload_flag(m, cached_flow_val, df); return DP_OK; @@ -286,7 +286,7 @@ static __rte_always_inline int dp_get_flow_val(struct rte_mbuf *m, struct dp_flo } if (dp_capture_offloaded_pkts(m, *p_flow_val, df)) - return DP_IS_CAPTURED_HW_PKT; // it is not really an error, but we need to stop processing this pkt + return DP_IS_CAPTURED_HW_PKT; // already established flow found dp_set_pkt_flow_direction(curr_key, *p_flow_val, df); @@ -302,11 +302,8 @@ int dp_cntrack_handle(__rte_unused struct rte_node *node, struct rte_mbuf *m, st int ret; ret = dp_get_flow_val(m, df, &flow_val); - if (DP_FAILED(ret)) { - DPNODE_LOG_WARNING(node, "Cannot establish flow value", DP_LOG_RET(ret)); - return ret; - } else if (ret == DP_IS_CAPTURED_HW_PKT) - return ret; + if (DP_FAILED(ret) || ret == DP_IS_CAPTURED_HW_PKT) + return ret; // it is not really an error when ret == DP_IS_CAPTURED_HW_PKT, but we need to stop processing this pkt flow_val->timestamp = rte_rdtsc(); diff --git a/src/dp_port.c b/src/dp_port.c index 4b9ed9004..f941a0f9e 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -430,11 +430,40 @@ static int dp_port_bind_port_hairpins(struct dp_port *port) return DP_OK; } +static int dp_install_vf_init_rte_rules(uint32_t port_id) +{ + bool graphtrace_enabled = dp_graphtrace_is_enabled(); + int ret; + + if (graphtrace_enabled) + ret = dp_install_jump_rule_in_default_group(port_id, DP_RTE_FLOW_MONITORING_GROUP); + else + ret = dp_install_jump_rule_in_default_group(port_id, DP_RTE_FLOW_VNET_GROUP); + + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Cannot install default jump rule", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); + return DP_ERROR; + } + + ret = dp_install_default_rule_in_monitoring_group(port_id); + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Cannot install default rule in monitoring group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); + return DP_ERROR; + } + + ret = dp_install_default_capture_rule_in_vnet_group(port_id); + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Cannot install default capture rule in vnet group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); + return DP_ERROR; + } + + return DP_OK; +} + int dp_port_start(uint16_t port_id) { struct dp_port *port; int ret; - bool graphtrace_enabled = dp_graphtrace_is_enabled(); port = dp_port_get(port_id); if (!port) @@ -468,27 +497,9 @@ int dp_port_start(uint16_t port_id) if (port->port_type == DP_PORT_VF) { - if (graphtrace_enabled) - ret = dp_install_jump_rule_in_default_group(port_id, DP_RTE_FLOW_MONITORING_GROUP); - else - ret = dp_install_jump_rule_in_default_group(port_id, DP_RTE_FLOW_VNET_GROUP); - - if (DP_FAILED(ret)) { - DPS_LOG_ERR("Cannot install default jump rule", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); - return DP_ERROR; - } - - ret = dp_install_default_rule_in_monitoring_group(port_id); - if (DP_FAILED(ret)) { - DPS_LOG_ERR("Cannot install default rule in monitoring group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); - return DP_ERROR; - } - - ret = dp_install_default_capture_rule_in_vnet_group(port_id); - if (DP_FAILED(ret)) { - DPS_LOG_ERR("Cannot install default capture rule in vnet group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); - return DP_ERROR; - } + ret = dp_install_vf_init_rte_rules(port_id); + if (DP_FAILED(ret)) + assert(0); // if any flow rule failed, stop process running due to possible hw/driver failure } } diff --git a/src/monitoring/dp_graphtrace.c b/src/monitoring/dp_graphtrace.c index efc937a17..910ea80ab 100644 --- a/src/monitoring/dp_graphtrace.c +++ b/src/monitoring/dp_graphtrace.c @@ -17,6 +17,7 @@ static enum dp_graphtrace_loglevel graphtrace_loglevel; static struct dp_graphtrace graphtrace; bool _dp_graphtrace_enabled = false; +static bool _dp_graphtrace_hw_enabled = false; static bool offload_enabled; static int dp_graphtrace_init_memzone(void) @@ -54,6 +55,62 @@ static void dp_graphtrace_free_memzone(void) graphtrace.mempool = NULL; } +static void dp_graphtrace_set_hw_enabled(bool enabled) +{ + _dp_graphtrace_hw_enabled = enabled; +} + +static int dp_handle_graphtrace_start(const struct dp_graphtrace_mp_request *request) +{ + int ret; + + switch ((enum dp_graphtrace_op_type)request->action_params.op_type) { + case DP_GRAPHTRACE_OP_TYPE_SOFTWARE: + dp_graphtrace_enable(); + DPS_LOG_INFO("Graphtrace enabled only for software path"); + return DP_OK; + case DP_GRAPHTRACE_OP_TYPE_OFFLOAD: + if (!offload_enabled) + return -EPERM; + + ret = dp_send_event_hardware_capture_start_msg(); + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Cannot send hardware capture start message"); + return ret; + } + dp_graphtrace_enable(); + dp_graphtrace_set_hw_enabled(true); + DPS_LOG_INFO("Graphtrace enabled for software and offload paths"); + return DP_OK; + } + +} + +static int dp_handle_graphtrace_stop(const struct dp_graphtrace_mp_request *request) +{ + int ret; + + switch ((enum dp_graphtrace_op_type)request->action_params.op_type) { + case DP_GRAPHTRACE_OP_TYPE_SOFTWARE: + dp_graphtrace_disable(); + DPS_LOG_INFO("Graphtrace disabled only for software path"); + return DP_OK; + case DP_GRAPHTRACE_OP_TYPE_OFFLOAD: + if (_dp_graphtrace_hw_enabled) { + dp_graphtrace_disable(); + ret = dp_send_event_hardware_capture_stop_msg(); + if (DP_FAILED(ret)) { + DPS_LOG_ERR("Cannot send hardware capture stop message"); + return ret; + } + dp_graphtrace_set_hw_enabled(false); + DPS_LOG_INFO("Graphtrace disabled for software and offload paths"); + return DP_OK; + } else { + return -EPERM; + } + } +} static __rte_always_inline void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_graphtrace_mp_reply *reply) @@ -69,39 +126,13 @@ void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_gra switch ((enum dp_graphtrace_action)request->action) { case DP_GRAPHTRACE_ACTION_START: - dp_graphtrace_enable(); - reply->error_code = DP_OK; - DPS_LOG_INFO("Graphtrace enabled"); + ret = dp_handle_graphtrace_start(request); + reply->error_code = ret; return; case DP_GRAPHTRACE_ACTION_STOP: - dp_graphtrace_disable(); - reply->error_code = DP_OK; - DPS_LOG_INFO("Graphtrace disabled"); - return; - case DP_GRAPHTRACE_ACTION_ENABLE_HW_CAPTURE: - if (offload_enabled) { - if (DP_FAILED(dp_send_event_hardware_capture_start_msg())) { - DPS_LOG_ERR("Cannot send hardware capture start message"); - dp_graphtrace_disable(); // roll back if failed on this side, and error code cannot be sent back - reply->error_code = DP_ERROR; - return; - } - } - reply->error_code = DP_OK; - DPS_LOG_INFO("Hardware capture enabled"); - return; - case DP_GRAPHTRACE_ACTION_DISABLE_HW_CAPTURE: - if (offload_enabled) { - if (DP_FAILED(dp_send_event_hardware_capture_stop_msg())) { - DPS_LOG_ERR("Cannot send hardware capture stop message"); - reply->error_code = DP_ERROR; - return; - } - } - reply->error_code = DP_OK; - DPS_LOG_INFO("Hardware capture disabled"); + ret = dp_handle_graphtrace_stop(request); + reply->error_code = ret; return; - case DP_GRAPHTRACE_ACTION_NULL: default: DPS_LOG_WARNING("Unknown graphtrace request action", DP_LOG_VALUE(request->action)); reply->error_code = -EINVAL; diff --git a/src/rte_flow/dp_rte_flow_init.c b/src/rte_flow/dp_rte_flow_init.c index 51b6e6b08..ee83596b7 100644 --- a/src/rte_flow/dp_rte_flow_init.c +++ b/src/rte_flow/dp_rte_flow_init.c @@ -170,18 +170,19 @@ int dp_install_default_capture_rule_in_vnet_group(uint16_t port_id) static int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group) { struct dp_ports *ports = get_dp_ports(); - struct dp_port *port; struct rte_flow_error error; int ret; + DP_FOREACH_PORT(ports, port) { if (port->port_type == DP_PORT_VF && port->allocated) { - if (port->default_flow) + if (port->default_flow) { ret = rte_flow_destroy(port->port_id, port->default_flow, &error); - if (DP_FAILED(ret)) { - DPS_LOG_WARNING("Failed to destroy default flow", DP_LOG_PORTID(port->port_id), DP_LOG_RET(ret)); - continue; + if (DP_FAILED(ret)) { + DPS_LOG_WARNING("Failed to destroy default flow", DP_LOG_PORTID(port->port_id), DP_LOG_RET(ret)); + continue; + } } if (DP_FAILED(dp_install_jump_rule_in_default_group(port->port_id, dst_group))) { diff --git a/tools/dp_graphtrace.c b/tools/dp_graphtrace.c index d2e8bb116..a4961f2fc 100644 --- a/tools/dp_graphtrace.c +++ b/tools/dp_graphtrace.c @@ -33,11 +33,11 @@ static const char *eal_arg_strings[] = { "--log-level=6", // hide DPDK's informational messages (level 7) }; -enum { - DP_GRAPHTRACE_OPT_NONE, +enum cmd_opt_type { + DP_GRAPHTRACE_OPT_MIN = 256, DP_GRAPHTRACE_OPT_HELP, DP_GRAPHTRACE_OPT_CAPTURE_HW_PKT, -} cmd_opt_type; +}; static bool capture_hw_pkt = false; @@ -47,6 +47,7 @@ static bool capture_hw_pkt = false; static const struct option longopts[] = { { "help", 0, 0, DP_GRAPHTRACE_OPT_HELP }, { "hw-packet", 0, 0, DP_GRAPHTRACE_OPT_CAPTURE_HW_PKT }, + { NULL, 0, 0, 0 }, }; @@ -90,6 +91,15 @@ static int dp_graphtrace_connect(struct dp_graphtrace *graphtrace) return DP_OK; } +static void dp_graphtrace_config_start_stop_parameters(struct dp_graphtrace_mp_request *graphtrace_request) +{ + if (capture_hw_pkt) + graphtrace_request->action_params.op_type = DP_GRAPHTRACE_OP_TYPE_OFFLOAD; + else + graphtrace_request->action_params.op_type = DP_GRAPHTRACE_OP_TYPE_SOFTWARE; +} + + static int dp_graphtrace_send_request(enum dp_graphtrace_action action, struct dp_graphtrace_mp_reply *reply) { struct rte_mp_msg mp_request; @@ -104,8 +114,17 @@ static int dp_graphtrace_send_request(enum dp_graphtrace_action action, struct d graphtrace_request = (struct dp_graphtrace_mp_request *)mp_request.param; graphtrace_request->action = action; + switch (action) { + case DP_GRAPHTRACE_ACTION_START: + dp_graphtrace_config_start_stop_parameters(graphtrace_request); + break; + case DP_GRAPHTRACE_ACTION_STOP: + dp_graphtrace_config_start_stop_parameters(graphtrace_request); + break; + } + if (DP_FAILED(rte_mp_request_sync(&mp_request, &mp_reply, &connect_timeout))) { - fprintf(stderr, "Cannot request graphtrace action %s\n", dp_strerror_verbose(rte_errno)); + fprintf(stderr, "Cannot request graphtrace %s\n", dp_strerror_verbose(rte_errno)); return -rte_errno; } @@ -122,8 +141,6 @@ static void print_packet(struct rte_mbuf *pkt) char printbuf[512]; struct dp_graphtrace_pktinfo *pktinfo = dp_get_graphtrace_pktinfo(pkt); - dp_graphtrace_sprint(pkt, printbuf, sizeof(printbuf)); - switch (pktinfo->pkt_type) { case DP_GRAPHTRACE_PKT_TYPE_SOFTWARE: dp_graphtrace_sprint(pkt, printbuf, sizeof(printbuf)); @@ -135,16 +152,11 @@ static void print_packet(struct rte_mbuf *pkt) printbuf); break; case DP_GRAPHTRACE_PKT_TYPE_OFFLOAD: - printf("%u: captured offload packet : %s\n", + printf("%u: " NODENAME_FMT " " NODENAME_FMT ": %s\n", pktinfo->pktid, + "Offloaded", "", printbuf); break; - default: - printf("%u: " NODENAME_FMT ": unknown packet type %u\n", - pktinfo->pktid, - pktinfo->node->name, - pktinfo->pkt_type); - break; } } @@ -179,12 +191,12 @@ static int dp_graphtrace_request(enum dp_graphtrace_action action, struct dp_gra ret = dp_graphtrace_send_request(action, reply); if (DP_FAILED(ret)) { - fprintf(stderr, "Cannot send graphtrace request %s\n", dp_strerror_verbose(ret)); - return DP_ERROR; - } + fprintf(stderr, "Cannot send graphtrace request, action=%d %s\n", action, dp_strerror_verbose(ret)); + return ret; + } if (DP_FAILED(reply->error_code)) { - fprintf(stderr, "Graphtrace request failed %s\n", dp_strerror_verbose(reply->error_code)); + fprintf(stderr, "Graphtrace action request failed, action=%d %s\n", action, dp_strerror_verbose(reply->error_code)); return DP_ERROR; } @@ -193,12 +205,12 @@ static int dp_graphtrace_request(enum dp_graphtrace_action action, struct dp_gra static int dp_graphtrace_start(void) { + int ret; struct dp_graphtrace_mp_reply reply; - if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_START, &reply))) { - fprintf(stderr, "Failed to request graph tracing\n"); - return DP_ERROR; - } + ret = dp_graphtrace_request(DP_GRAPHTRACE_ACTION_START, &reply); + if (DP_FAILED(ret)) + return ret; primary_alive = true; return DP_OK; @@ -212,36 +224,7 @@ static int dp_graphtrace_stop(void) if (!primary_alive) return DP_OK; - if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_STOP, &reply))) { - fprintf(stderr, "Failed to request graph tracing termination\n"); - return DP_ERROR; - } - - return DP_OK; -} - -static int dp_graphtrace_enable_hw_capture(void) -{ - struct dp_graphtrace_mp_reply reply; - - if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_ENABLE_HW_CAPTURE, &reply))) { - fprintf(stderr, "Failed to enable hardware packet capture\n"); - return DP_ERROR; - } - - return DP_OK; -} - -static int dp_graphtrace_disable_hw_capture(void) -{ - struct dp_graphtrace_mp_reply reply; - - if (DP_FAILED(dp_graphtrace_request(DP_GRAPHTRACE_ACTION_DISABLE_HW_CAPTURE, &reply))) { - fprintf(stderr, "Failed to disable hardware packet capture\n"); - return DP_ERROR; - } - - return DP_OK; + return dp_graphtrace_request(DP_GRAPHTRACE_ACTION_STOP, &reply); } static void dp_graphtrace_monitor_primary(void *arg __rte_unused) @@ -268,20 +251,6 @@ static void dp_graphtrace_monitor_primary(void *arg __rte_unused) fprintf(stderr, "Warning: Cannot re-schedule primary process monitor %s\n", dp_strerror_verbose(ret)); } -static int dp_graphtrace_init_hw_pkt_capture(struct dp_graphtrace *graphtrace) -{ - if (!capture_hw_pkt) - return DP_OK; - - if (DP_FAILED(dp_graphtrace_enable_hw_capture())) { - fprintf(stderr, "Failed to enable hardware packet capture\n"); - rte_eal_alarm_cancel(dp_graphtrace_monitor_primary, (void *)-1); - return DP_ERROR; - } - - return DP_OK; -} - static int do_graphtrace(struct dp_graphtrace *graphtrace) { int ret; @@ -299,28 +268,20 @@ static int do_graphtrace(struct dp_graphtrace *graphtrace) } if (DP_FAILED(dp_graphtrace_start())) { - rte_eal_alarm_cancel(dp_graphtrace_monitor_primary, (void *)-1); - return DP_ERROR; - } - - if (DP_FAILED(dp_graphtrace_init_hw_pkt_capture(graphtrace))) { - if (dp_graphtrace_stop()) // rollback if failed - fprintf(stderr, "Failed to stop graph tracing after failing to init hw pkt capture\n"); + fprintf(stderr, "Failed to request graph tracing\n"); rte_eal_alarm_cancel(dp_graphtrace_monitor_primary, (void *)-1); return DP_ERROR; } ret = dp_graphtrace_dump(graphtrace); - if (capture_hw_pkt) - if (DP_FAILED(dp_graphtrace_disable_hw_capture())) - ret = DP_ERROR; - - if (DP_FAILED(dp_graphtrace_stop())) + if (DP_FAILED(dp_graphtrace_stop())) { + fprintf(stderr, "Failed to request graph tracing termination\n"); ret = DP_ERROR; + } rte_eal_alarm_cancel(dp_graphtrace_monitor_primary, (void *)-1); - + return ret; } @@ -332,7 +293,8 @@ static void signal_handler(__rte_unused int signum) static void dp_print_usage(const char *prgname) { fprintf(stderr, - " --help show this help message and exit\n" + "%s: dp-service graph-tracing utility\n" + " --help show this help message and exit\n" " --hw-packet capture pkts after offloading rules are installed (experimental feature, only VF's outgoing packets are supported)\n", prgname); } From 3ed39244ee6f375398cad0457a6d7abef53ac7c8 Mon Sep 17 00:00:00 2001 From: Tao Date: Mon, 18 Sep 2023 09:32:13 +0000 Subject: [PATCH 07/11] handle rollback when installing monitoring rte flow rules during vf starting --- src/dp_port.c | 40 ++++++++++++++++++++++++++++----- src/monitoring/dp_graphtrace.c | 6 ++--- src/rte_flow/dp_rte_flow_init.c | 1 - 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/dp_port.c b/src/dp_port.c index f941a0f9e..c6e898fbc 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -430,11 +430,33 @@ static int dp_port_bind_port_hairpins(struct dp_port *port) return DP_OK; } +static int dp_vf_init_monitoring_rule_rollback(uint32_t port_id) +{ + struct dp_port *port = dp_port_get_vf(port_id); + struct rte_flow_error error; + + if (DP_FAILED(rte_flow_destroy(port->port_id, port->default_flow, &error))) { + DPS_LOG_ERR("Failed to destroy default flow while rollback from vf init monitoring rule installation", \ + DP_LOG_PORTID(port->port_id), DP_LOG_FLOW_ERROR(error.message)); + return DP_ERROR; + } + + if (DP_FAILED(dp_install_jump_rule_in_default_group(port->port_id, DP_RTE_FLOW_VNET_GROUP))) { + DPS_LOG_ERR("Failed to install default jump flow rule while rollback from vf init monitoring rule installation", \ + DP_LOG_PORTID(port->port_id)); + return DP_ERROR; + } + + return DP_OK; +} + static int dp_install_vf_init_rte_rules(uint32_t port_id) { bool graphtrace_enabled = dp_graphtrace_is_enabled(); int ret; + // at least one rule must be there, otherwise new packets cannot be delivered to software path + // same as the isolation rule on pf if (graphtrace_enabled) ret = dp_install_jump_rule_in_default_group(port_id, DP_RTE_FLOW_MONITORING_GROUP); else @@ -445,18 +467,24 @@ static int dp_install_vf_init_rte_rules(uint32_t port_id) return DP_ERROR; } - ret = dp_install_default_rule_in_monitoring_group(port_id); - if (DP_FAILED(ret)) { - DPS_LOG_ERR("Cannot install default rule in monitoring group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); - return DP_ERROR; - } - + // this rule must be there, otherwise new packets cannot be delivered to software path, making communication failure ret = dp_install_default_capture_rule_in_vnet_group(port_id); if (DP_FAILED(ret)) { DPS_LOG_ERR("Cannot install default capture rule in vnet group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); return DP_ERROR; } + if (graphtrace_enabled) { + ret = dp_install_default_rule_in_monitoring_group(port_id); + if (DP_FAILED(ret)) { + DPS_LOG_WARNING("Cannot install default rule in monitoring group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); + if (DP_FAILED(dp_vf_init_monitoring_rule_rollback(port_id))) { + DPS_LOG_ERR("Cannot rollback from the monitoring rule installation on vf", DP_LOG_PORTID(port_id)); + return DP_ERROR; + } + } + } + return DP_OK; } diff --git a/src/monitoring/dp_graphtrace.c b/src/monitoring/dp_graphtrace.c index 910ea80ab..8febcfa22 100644 --- a/src/monitoring/dp_graphtrace.c +++ b/src/monitoring/dp_graphtrace.c @@ -64,7 +64,7 @@ static int dp_handle_graphtrace_start(const struct dp_graphtrace_mp_request *req { int ret; - switch ((enum dp_graphtrace_op_type)request->action_params.op_type) { + switch (request->action_params.op_type) { case DP_GRAPHTRACE_OP_TYPE_SOFTWARE: dp_graphtrace_enable(); DPS_LOG_INFO("Graphtrace enabled only for software path"); @@ -90,7 +90,7 @@ static int dp_handle_graphtrace_stop(const struct dp_graphtrace_mp_request *requ { int ret; - switch ((enum dp_graphtrace_op_type)request->action_params.op_type) { + switch (request->action_params.op_type) { case DP_GRAPHTRACE_OP_TYPE_SOFTWARE: dp_graphtrace_disable(); DPS_LOG_INFO("Graphtrace disabled only for software path"); @@ -124,7 +124,7 @@ void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_gra return; } - switch ((enum dp_graphtrace_action)request->action) { + switch (request->action) { case DP_GRAPHTRACE_ACTION_START: ret = dp_handle_graphtrace_start(request); reply->error_code = ret; diff --git a/src/rte_flow/dp_rte_flow_init.c b/src/rte_flow/dp_rte_flow_init.c index ee83596b7..8f0f83ce1 100644 --- a/src/rte_flow/dp_rte_flow_init.c +++ b/src/rte_flow/dp_rte_flow_init.c @@ -173,7 +173,6 @@ static int dp_change_all_vf_default_jump_rte_flow_group(uint32_t dst_group) struct rte_flow_error error; int ret; - DP_FOREACH_PORT(ports, port) { if (port->port_type == DP_PORT_VF && port->allocated) { if (port->default_flow) { From db0af4514fd9403f3109f5bc29bc87ba3b9e6e7c Mon Sep 17 00:00:00 2001 From: Tao Date: Mon, 18 Sep 2023 09:41:00 +0000 Subject: [PATCH 08/11] compress stop msg handling to stop graphtrace --- src/dp_port.c | 2 +- src/monitoring/dp_graphtrace.c | 32 ++++++++++++-------------------- tools/dp_graphtrace.c | 2 -- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/dp_port.c b/src/dp_port.c index c6e898fbc..c4630e42c 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -446,7 +446,7 @@ static int dp_vf_init_monitoring_rule_rollback(uint32_t port_id) DP_LOG_PORTID(port->port_id)); return DP_ERROR; } - + return DP_OK; } diff --git a/src/monitoring/dp_graphtrace.c b/src/monitoring/dp_graphtrace.c index 8febcfa22..55db17e9d 100644 --- a/src/monitoring/dp_graphtrace.c +++ b/src/monitoring/dp_graphtrace.c @@ -86,30 +86,22 @@ static int dp_handle_graphtrace_start(const struct dp_graphtrace_mp_request *req } -static int dp_handle_graphtrace_stop(const struct dp_graphtrace_mp_request *request) +static int dp_handle_graphtrace_stop() { int ret; - switch (request->action_params.op_type) { - case DP_GRAPHTRACE_OP_TYPE_SOFTWARE: - dp_graphtrace_disable(); - DPS_LOG_INFO("Graphtrace disabled only for software path"); - return DP_OK; - case DP_GRAPHTRACE_OP_TYPE_OFFLOAD: - if (_dp_graphtrace_hw_enabled) { - dp_graphtrace_disable(); - ret = dp_send_event_hardware_capture_stop_msg(); - if (DP_FAILED(ret)) { - DPS_LOG_ERR("Cannot send hardware capture stop message"); - return ret; - } - dp_graphtrace_set_hw_enabled(false); - DPS_LOG_INFO("Graphtrace disabled for software and offload paths"); - return DP_OK; - } else { - return -EPERM; + dp_graphtrace_disable(); + if (_dp_graphtrace_hw_enabled) { + ret = dp_send_event_hardware_capture_stop_msg(); + if (DP_FAILED(dp_send_event_hardware_capture_stop_msg())) { + DPS_LOG_ERR("Cannot send hardware capture stop message"); + return ret; } + dp_graphtrace_set_hw_enabled(false); + DPS_LOG_INFO("Graphtrace reconfigured to stop hw capturing"); } + DPS_LOG_INFO("Graphtrace disabled"); + return DP_OK; } static __rte_always_inline @@ -130,7 +122,7 @@ void dp_handle_graphtrace_request(const struct rte_mp_msg *mp_msg, struct dp_gra reply->error_code = ret; return; case DP_GRAPHTRACE_ACTION_STOP: - ret = dp_handle_graphtrace_stop(request); + ret = dp_handle_graphtrace_stop(); reply->error_code = ret; return; default: diff --git a/tools/dp_graphtrace.c b/tools/dp_graphtrace.c index a4961f2fc..31924b414 100644 --- a/tools/dp_graphtrace.c +++ b/tools/dp_graphtrace.c @@ -99,7 +99,6 @@ static void dp_graphtrace_config_start_stop_parameters(struct dp_graphtrace_mp_r graphtrace_request->action_params.op_type = DP_GRAPHTRACE_OP_TYPE_SOFTWARE; } - static int dp_graphtrace_send_request(enum dp_graphtrace_action action, struct dp_graphtrace_mp_reply *reply) { struct rte_mp_msg mp_request; @@ -119,7 +118,6 @@ static int dp_graphtrace_send_request(enum dp_graphtrace_action action, struct d dp_graphtrace_config_start_stop_parameters(graphtrace_request); break; case DP_GRAPHTRACE_ACTION_STOP: - dp_graphtrace_config_start_stop_parameters(graphtrace_request); break; } From d1c1d4dd6056da6774e3459b05fdd2501f00f666 Mon Sep 17 00:00:00 2001 From: Tao Date: Mon, 18 Sep 2023 09:48:16 +0000 Subject: [PATCH 09/11] monitoring rule shall be installed by default, if failed, rollback to directly jump to vnet group --- src/dp_port.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/dp_port.c b/src/dp_port.c index c4630e42c..89c375e03 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -474,14 +474,12 @@ static int dp_install_vf_init_rte_rules(uint32_t port_id) return DP_ERROR; } - if (graphtrace_enabled) { - ret = dp_install_default_rule_in_monitoring_group(port_id); - if (DP_FAILED(ret)) { - DPS_LOG_WARNING("Cannot install default rule in monitoring group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); - if (DP_FAILED(dp_vf_init_monitoring_rule_rollback(port_id))) { - DPS_LOG_ERR("Cannot rollback from the monitoring rule installation on vf", DP_LOG_PORTID(port_id)); - return DP_ERROR; - } + ret = dp_install_default_rule_in_monitoring_group(port_id); + if (DP_FAILED(ret)) { + DPS_LOG_WARNING("Cannot install default rule in monitoring group", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); + if (DP_FAILED(dp_vf_init_monitoring_rule_rollback(port_id))) { + DPS_LOG_ERR("Cannot rollback from the monitoring rule installation on vf", DP_LOG_PORTID(port_id)); + return DP_ERROR; } } From e2db975709cfa6cda7b2e38eb802bf6b5c70c32a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Thu, 21 Sep 2023 15:04:56 +0200 Subject: [PATCH 10/11] Fix compilation errors --- src/monitoring/dp_graphtrace.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/monitoring/dp_graphtrace.c b/src/monitoring/dp_graphtrace.c index 55db17e9d..8de474ff6 100644 --- a/src/monitoring/dp_graphtrace.c +++ b/src/monitoring/dp_graphtrace.c @@ -84,9 +84,10 @@ static int dp_handle_graphtrace_start(const struct dp_graphtrace_mp_request *req return DP_OK; } + return -EINVAL; } -static int dp_handle_graphtrace_stop() +static int dp_handle_graphtrace_stop(void) { int ret; From 66c24ce87f7c0766604ac30d608fad3f53a57927 Mon Sep 17 00:00:00 2001 From: Tao Date: Fri, 22 Sep 2023 11:19:58 +0200 Subject: [PATCH 11/11] remove vf init rules when offloading is disabled --- src/dp_port.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dp_port.c b/src/dp_port.c index 89c375e03..b27caa7f9 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -522,7 +522,7 @@ int dp_port_start(uint16_t port_id) } - if (port->port_type == DP_PORT_VF) { + if (port->port_type == DP_PORT_VF && dp_conf_is_offload_enabled()) { ret = dp_install_vf_init_rte_rules(port_id); if (DP_FAILED(ret)) assert(0); // if any flow rule failed, stop process running due to possible hw/driver failure