From ca11cab3ad914dda5e9a54ea7f1c255d87398398 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Tue, 5 Sep 2023 13:57:34 +0200 Subject: [PATCH 01/10] Updated Dockerfile to Debian 12 --- Dockerfile | 21 ++++++++++----------- hack/dpdk_21_11_gcc12.patch | 26 ++++++++++++++++++++++++++ src/nodes/dhcpv6_node.c | 10 +++++----- 3 files changed, 41 insertions(+), 16 deletions(-) create mode 100644 hack/dpdk_21_11_gcc12.patch diff --git a/Dockerfile b/Dockerfile index 21bf8235e..c12277fec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:11-slim as builder +FROM debian:12-slim as builder ARG DPDK_VER=21.11 ARG DPSERVICE_FEATURES="" @@ -6,7 +6,7 @@ ARG DPSERVICE_FEATURES="" WORKDIR /workspace # Install prerequisite packages -RUN apt-get update && apt-get install -y --no-install-recommends ON \ +RUN apt-get update && apt-get upgrade && apt-get install -y --no-install-recommends ON \ libibverbs-dev \ libmnl-dev \ libnuma-dev \ @@ -37,7 +37,7 @@ curl \ build-essential \ pkg-config \ protobuf-compiler-grpc \ -libgrpc++1 \ +libgrpc++1.51 \ libgrpc++-dev \ linux-headers-${OSARCH} \ && rm -rf /var/lib/apt/lists/* @@ -50,13 +50,14 @@ ENV DPDK_DIR=/workspace/dpdk-${DPDK_VER} # Copy DPDK patches COPY hack/*.patch hack/ +RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_21_11_gcc12.patch RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_21_11_clang.patch RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_21_11_log.patch RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_21_11_xstats_mem_leak.patch RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_21_11_graph_alloc.patch # Compile DPDK -RUN cd $DPDK_DIR && meson -Dmax_ethports=132 -Dplatform=generic -Ddisable_drivers=common/dpaax,\ +RUN cd $DPDK_DIR && meson setup -Dmax_ethports=132 -Dplatform=generic -Ddisable_drivers=common/dpaax,\ common/cpt,common/iavf,\ common/octeontx,common/octeontx2,common/cnxk,common/qat,regex/octeontx2,net/cnxk,dma/cnxk,\ common/sfc_efx,common/auxiliary,common/dpaa,common/fslmc,common/ifpga,common/vdev,common/vmbus,\ @@ -97,7 +98,7 @@ FROM builder AS testbuilder RUN rm -rf build && meson setup build $DPSERVICE_FEATURES --buildtype=release && cd ./build && ninja RUN rm -rf build && CC=clang CXX=clang++ meson setup build $DPSERVICE_FEATURES && cd ./build && ninja -FROM debian:11-slim as tester +FROM debian:12-slim as tester RUN apt-get update && apt-get install -y --no-install-recommends ON \ libibverbs-dev \ @@ -106,7 +107,7 @@ libnuma1 \ pciutils \ procps \ libuuid1 \ -libgrpc++1 \ +libgrpc++1.51 \ iproute2 \ udev \ gawk \ @@ -119,23 +120,22 @@ WORKDIR / COPY --from=testbuilder /workspace/test ./test COPY --from=testbuilder /workspace/build/src/dp_service ./build/src/dp_service COPY --from=testbuilder /usr/local/lib /usr/local/lib -COPY --from=testbuilder /lib/* /lib/ COPY --from=testbuilder /workspace/client/github.com/onmetal/* ./build RUN ldconfig WORKDIR /test ENTRYPOINT ["pytest-3", "-x", "-v"] -FROM debian:11-slim as production +FROM debian:12-slim as production -RUN apt-get update && apt-get install -y --no-install-recommends ON \ +RUN apt-get update && apt-get upgrade && apt-get install -y --no-install-recommends ON \ libibverbs-dev \ numactl \ libnuma1 \ pciutils \ procps \ libuuid1 \ -libgrpc++1 \ +libgrpc++1.51 \ iproute2 \ udev \ gawk \ @@ -146,7 +146,6 @@ COPY --from=builder /workspace/build/src/dp_service . COPY --from=builder /workspace/build/tools/dp_grpc_client /workspace/build/tools/dump/dpservice-dump . COPY --from=builder /workspace/hack/prepare.sh . COPY --from=builder /usr/local/lib /usr/local/lib -COPY --from=builder /lib/* /lib/ COPY --from=builder /workspace/exporter/* /workspace/client/github.com/onmetal/* . RUN ldconfig diff --git a/hack/dpdk_21_11_gcc12.patch b/hack/dpdk_21_11_gcc12.patch new file mode 100644 index 000000000..01abfdd09 --- /dev/null +++ b/hack/dpdk_21_11_gcc12.patch @@ -0,0 +1,26 @@ +diff --git a/lib/ring/rte_ring_elem_pvt.h b/lib/ring/rte_ring_elem_pvt.h +index 275ec55393..04b55c34a4 100644 +--- a/lib/ring/rte_ring_elem_pvt.h ++++ b/lib/ring/rte_ring_elem_pvt.h +@@ -10,6 +10,12 @@ + #ifndef _RTE_RING_ELEM_PVT_H_ + #define _RTE_RING_ELEM_PVT_H_ + ++#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 120000) ++#pragma GCC diagnostic push ++#pragma GCC diagnostic ignored "-Wstringop-overflow" ++#pragma GCC diagnostic ignored "-Wstringop-overread" ++#endif ++ + static __rte_always_inline void + __rte_ring_enqueue_elems_32(struct rte_ring *r, const uint32_t size, + uint32_t idx, const void *obj_table, uint32_t n) +@@ -382,4 +388,8 @@ end: + return n; + } + ++#if defined(RTE_TOOLCHAIN_GCC) && (GCC_VERSION >= 120000) ++#pragma GCC diagnostic pop ++#endif ++ + #endif /* _RTE_RING_ELEM_PVT_H_ */ diff --git a/src/nodes/dhcpv6_node.c b/src/nodes/dhcpv6_node.c index eeb1ad1f7..47eb1fa9a 100644 --- a/src/nodes/dhcpv6_node.c +++ b/src/nodes/dhcpv6_node.c @@ -170,19 +170,19 @@ static int generate_reply_options(struct rte_mbuf *m, uint8_t *options, int opti if (DP_FAILED(resize_packet(m, reply_options_len - options_len))) return DP_ERROR; - rte_memcpy(options, &opt_sid, sizeof(opt_sid)); + // had to use memcpy() here, because GCC's array-bounds check fails for rte_memcpy (using XMM optimization) + memcpy(options, &opt_sid, sizeof(opt_sid)); options += sizeof(opt_sid); if (reply_options.opt_cid_len) { - rte_memcpy(options, &reply_options.opt_cid, reply_options.opt_cid_len); + memcpy(options, (void *)&reply_options.opt_cid, reply_options.opt_cid_len); options += reply_options.opt_cid_len; } if (reply_options.opt_iana_len) { - rte_memcpy(options, &reply_options.opt_iana, reply_options.opt_iana_len); + memcpy(options, (void *)&reply_options.opt_iana, reply_options.opt_iana_len); options += reply_options.opt_iana_len; } if (reply_options.opt_rapid_len) - // had to use the direct value here, because GCC's array-bounds check fails here (granted, the structure is hard to parse) - rte_memcpy(options, &reply_options.opt_rapid, sizeof(struct dhcpv6_option)); + memcpy(options, &reply_options.opt_rapid, reply_options.opt_rapid_len); return reply_options_len; } From 195f2a6e2218b29c13e9156970207b1324bdce53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Mon, 11 Sep 2023 16:53:25 +0200 Subject: [PATCH 02/10] Improved Docker image for operations --- Dockerfile | 21 ++++++++++++++------- config/default/dp-service.yaml | 10 ++++++---- hack/rel_download.sh | 7 ++++++- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index c12277fec..ceb19b5bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,7 +78,7 @@ COPY hack/rel_download.sh hack/rel_download.sh RUN --mount=type=secret,id=github_token,dst=/run/secrets/github_token \ sh -c 'GITHUB_TOKEN=$(if [ -f /run/secrets/github_token ]; then cat /run/secrets/github_token; else echo ""; fi) \ && ./hack/rel_download.sh -dir=exporter -owner=onmetal -repo=prometheus-dpdk-exporter -pat=$GITHUB_TOKEN \ -&& ./hack/rel_download.sh -dir=client -owner=onmetal -repo=dpservice-cli -pat=$GITHUB_TOKEN \' +&& ./hack/rel_download.sh -dir=client -owner=onmetal -repo=dpservice-cli -strip=2 -pat=$GITHUB_TOKEN' # Now copy the rest to enable DPDK layer caching COPY meson.build meson.build @@ -119,8 +119,8 @@ python3-scapy \ WORKDIR / COPY --from=testbuilder /workspace/test ./test COPY --from=testbuilder /workspace/build/src/dp_service ./build/src/dp_service +COPY --from=testbuilder /workspace/client/* ./build COPY --from=testbuilder /usr/local/lib /usr/local/lib -COPY --from=testbuilder /workspace/client/github.com/onmetal/* ./build RUN ldconfig WORKDIR /test @@ -139,14 +139,21 @@ libgrpc++1.51 \ iproute2 \ udev \ gawk \ +bash-completion \ && rm -rf /var/lib/apt/lists/* WORKDIR / -COPY --from=builder /workspace/build/src/dp_service . -COPY --from=builder /workspace/build/tools/dp_grpc_client /workspace/build/tools/dump/dpservice-dump . -COPY --from=builder /workspace/hack/prepare.sh . +COPY --from=builder /workspace/build/src/dp_service \ + /workspace/build/tools/dp_grpc_client \ + /workspace/build/tools/dump/dpservice-dump \ + /workspace/client/* \ + /workspace/exporter/* \ + /workspace/hack/prepare.sh \ + /usr/local/bin COPY --from=builder /usr/local/lib /usr/local/lib -COPY --from=builder /workspace/exporter/* /workspace/client/github.com/onmetal/* . RUN ldconfig -ENTRYPOINT ["/dp_service"] +# Ensure bash-completion is working in operations +RUN echo 'PATH=${PATH}:/\nsource /etc/bash_completion\nsource <(dpservice-cli completion bash)' >> /root/.bashrc + +ENTRYPOINT ["dp_service"] diff --git a/config/default/dp-service.yaml b/config/default/dp-service.yaml index e3b473e9c..943d37f57 100644 --- a/config/default/dp-service.yaml +++ b/config/default/dp-service.yaml @@ -22,7 +22,8 @@ spec: - name: init-dp-service image: dp-service imagePullPolicy: IfNotPresent - command: ['/prepare.sh'] + command: + - prepare.sh volumeMounts: - mountPath: /dev name: hugepages @@ -58,8 +59,9 @@ spec: livenessProbe: exec: command: - - /dp_grpc_client - - --is_initialized + - dpservice-cli + - get + - init initialDelaySeconds: 10 periodSeconds: 10 - name: prometheus-agent @@ -78,7 +80,7 @@ spec: name: https protocol: TCP command: - - /prometheus-dpdk-exporter + - prometheus-dpdk-exporter resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File diff --git a/hack/rel_download.sh b/hack/rel_download.sh index 0280cf4cd..1a32d8942 100755 --- a/hack/rel_download.sh +++ b/hack/rel_download.sh @@ -17,6 +17,7 @@ if [ "$#" -lt 4 ]; then fi # Process parameters +STRIPTAR="0" for i in "$@" do case $i in @@ -40,6 +41,10 @@ case $i in RELEASE="${i#*=}" shift ;; + -strip=*) + STRIPTAR="${i#*=}" + shift + ;; *) # unknown option ;; @@ -69,7 +74,7 @@ fi # Use curl to download the asset using the asset ID. curl -s -L -H "Accept: application/octet-stream" -H "Authorization: Bearer $PAT" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/repos/$REPO_OWNER/$REPO_NAME/releases/assets/$ASSET_ID -o $DIRECTORY/$DIRECTORY.tar.gz -tar -xzf $DIRECTORY/$DIRECTORY.tar.gz -C $DIRECTORY +tar -xzf $DIRECTORY/$DIRECTORY.tar.gz -C $DIRECTORY --strip-components $STRIPTAR rm $DIRECTORY/$DIRECTORY.tar.gz rm -rf $DIRECTORY/LICENSE* rm -rf $DIRECTORY/README.md From 61cf7a0f94fe65119525d110963e505396dea101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Tue, 12 Sep 2023 17:18:58 +0200 Subject: [PATCH 03/10] Running ports are now stopped on cleanup --- src/dp_port.c | 6 ++++++ src/dp_service.c | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/dp_port.c b/src/dp_port.c index 71b389072..afc22854a 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -402,6 +402,12 @@ int dp_ports_init(void) void dp_ports_free(void) { + // without stopping started ports, DPDK complains + DP_FOREACH_PORT(&dp_ports, port) { + // TODO(plague): PR to create proper rollback in dp_port_start() + if (port->allocated) + rte_eth_dev_stop(port->port_id); + } free(dp_ports.ports); } diff --git a/src/dp_service.c b/src/dp_service.c index 0e2edd96a..912281c4e 100644 --- a/src/dp_service.c +++ b/src/dp_service.c @@ -191,7 +191,6 @@ static void free_interfaces(void) dp_lb_free(); dp_nat_free(); dp_flow_free(); - // ports seem to be stopped by DPDK at the end dp_telemetry_free(); dp_graph_free(); #ifdef ENABLE_VIRTSVC From 9de2a8c5d9441c69ce6c338d9ecd89ac1f6428e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Thu, 14 Sep 2023 14:08:43 +0200 Subject: [PATCH 04/10] Updated patches for DPDK 22.11 --- Dockerfile | 9 +++----- docs/README.md | 2 +- docs/deployment/telemetry.md | 2 +- docs/development/building.md | 11 +++++----- hack/dpdk_21_11_clang.patch | 13 ----------- hack/dpdk_21_11_graph_alloc.patch | 22 ------------------- hack/dpdk_21_11_xstats_mem_leak.patch | 11 ---------- ..._11_gcc12.patch => dpdk_22_11_gcc12.patch} | 0 ...k_21_11_log.patch => dpdk_22_11_log.patch} | 10 ++++----- 9 files changed, 16 insertions(+), 64 deletions(-) delete mode 100644 hack/dpdk_21_11_clang.patch delete mode 100644 hack/dpdk_21_11_graph_alloc.patch delete mode 100644 hack/dpdk_21_11_xstats_mem_leak.patch rename hack/{dpdk_21_11_gcc12.patch => dpdk_22_11_gcc12.patch} (100%) rename hack/{dpdk_21_11_log.patch => dpdk_22_11_log.patch} (90%) diff --git a/Dockerfile b/Dockerfile index ceb19b5bd..be6250273 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM debian:12-slim as builder -ARG DPDK_VER=21.11 +ARG DPDK_VER=22.11 ARG DPSERVICE_FEATURES="" WORKDIR /workspace @@ -50,11 +50,8 @@ ENV DPDK_DIR=/workspace/dpdk-${DPDK_VER} # Copy DPDK patches COPY hack/*.patch hack/ -RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_21_11_gcc12.patch -RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_21_11_clang.patch -RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_21_11_log.patch -RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_21_11_xstats_mem_leak.patch -RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_21_11_graph_alloc.patch +RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_22_11_gcc12.patch +RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_22_11_log.patch # Compile DPDK RUN cd $DPDK_DIR && meson setup -Dmax_ethports=132 -Dplatform=generic -Ddisable_drivers=common/dpaax,\ diff --git a/docs/README.md b/docs/README.md index ae2e08847..732c99e68 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,7 +2,7 @@ ## Overview This is a beta version which -- Uses [DPDK Graph Framework](https://doc.dpdk.org/guides/prog_guide/graph_lib.html) for the data plane. DPDK version 21.11 LTS or higher needed. +- Uses [DPDK Graph Framework](https://doc.dpdk.org/guides/prog_guide/graph_lib.html) for the data plane. DPDK version 22.11 LTS or higher needed. - [rte_flow](https://doc.dpdk.org/guides/prog_guide/rte_flow.html) offloading between the virtual network interfaces on a single heypervisor. - Uses GRPC to add virtual interfaces, loadbalancers, NAT Gateways and routes. There is a golang based GRPC test client (CLI) which can connect to the GRPC server diff --git a/docs/deployment/telemetry.md b/docs/deployment/telemetry.md index b2183b303..b452c6794 100644 --- a/docs/deployment/telemetry.md +++ b/docs/deployment/telemetry.md @@ -9,7 +9,7 @@ The socket to use is of type `SOCK_SEQPACKET` (5), the file path is `/var/run/dp To send telemetry requests, a node name with a parameter (must be zero if not present) is sent and a JSON responses can be read: ``` # echo "/dp_service/nat/used_port_count,0" | socat - UNIX:/var/run/dpdk/rte/dpdk_telemetry.v2,socktype=5 -{"version":"DPDK 21.11.0","pid":1369006,"max_output_len":16384}{"/dp_service/nat/used_port_count":{"vm1":1,"vm2":0,"vm3":0}} +{"version":"DPDK 22.11.0","pid":1369006,"max_output_len":16384}{"/dp_service/nat/used_port_count":{"vm1":1,"vm2":0,"vm3":0}} ``` For the complete list of commands (telemetry nodes), send `/,0`. diff --git a/docs/development/building.md b/docs/development/building.md index 707fb7e77..ce302c5a2 100644 --- a/docs/development/building.md +++ b/docs/development/building.md @@ -26,13 +26,14 @@ Dp-service currently only supports x86 architecture (amd64 instruction set actua ### DPDK -The dataplane service is built upon the [DPDK library](https://dpdk.org). Currently, the only supported version is 21.x. Unfortunately, some patching is needed, thus it needs to be built from source. Building from source also has the advantage of easier debugging later. +The dataplane service is built upon the [DPDK library](https://dpdk.org). Currently, the only supported version is 22.11 LTS. Unfortunately, some patching is needed, thus it needs to be built from source. Building from source also has the advantage of easier debugging later. ```bash git clone https://github.com/onmetal/net-dpservice.git -wget http://fast.dpdk.org/rel/dpdk-21.11.2.tar.xz -tar xf dpdk-21.11.2.tar.xz -cd dpdk-stable-21.11.2 -patch -p1 < ../net-dpservice/hack/dpdk_21_11_log.patch +wget https://fast.dpdk.org/rel/dpdk-22.11.3.tar.xz +tar xf dpdk-22.11.3.tar.xz +cd dpdk-stable-22.11.3 +patch -p1 < ../net-dpservice/hack/dpdk_22_11_gcc12.patch +patch -p1 < ../net-dpservice/hack/dpdk_22_11_log.patch meson setup build ninja -C build sudo ninja -C build install diff --git a/hack/dpdk_21_11_clang.patch b/hack/dpdk_21_11_clang.patch deleted file mode 100644 index c83061aef..000000000 --- a/hack/dpdk_21_11_clang.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/lib/graph/rte_graph_worker.h b/lib/graph/rte_graph_worker.h -index eef77f732a..0c0b9c095a 100644 ---- a/lib/graph/rte_graph_worker.h -+++ b/lib/graph/rte_graph_worker.h -@@ -155,7 +155,7 @@ rte_graph_walk(struct rte_graph *graph) - * +-----+ <= cir_start + mask - */ - while (likely(head != graph->tail)) { -- node = RTE_PTR_ADD(graph, cir_start[(int32_t)head++]); -+ node = (struct rte_node *)RTE_PTR_ADD(graph, cir_start[(int32_t)head++]); - RTE_ASSERT(node->fence == RTE_GRAPH_FENCE); - objs = node->objs; - rte_prefetch0(objs); diff --git a/hack/dpdk_21_11_graph_alloc.patch b/hack/dpdk_21_11_graph_alloc.patch deleted file mode 100644 index de7135d75..000000000 --- a/hack/dpdk_21_11_graph_alloc.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/lib/graph/rte_graph_worker.h b/lib/graph/rte_graph_worker.h -index 0c0b9c095ad..6dc74616597 100644 ---- a/lib/graph/rte_graph_worker.h -+++ b/lib/graph/rte_graph_worker.h -@@ -224,7 +224,7 @@ __rte_node_enqueue_prologue(struct rte_graph *graph, struct rte_node *node, - __rte_node_enqueue_tail_update(graph, node); - - if (unlikely(node->size < (idx + space))) -- __rte_node_stream_alloc(graph, node); -+ __rte_node_stream_alloc_size(graph, node, node->size + space); - } - - /** -@@ -432,7 +432,7 @@ rte_node_next_stream_get(struct rte_graph *graph, struct rte_node *node, - uint16_t free_space = node->size - idx; - - if (unlikely(free_space < nb_objs)) -- __rte_node_stream_alloc_size(graph, node, nb_objs); -+ __rte_node_stream_alloc_size(graph, node, node->size + nb_objs); - - return &node->objs[idx]; - } diff --git a/hack/dpdk_21_11_xstats_mem_leak.patch b/hack/dpdk_21_11_xstats_mem_leak.patch deleted file mode 100644 index bcbad378c..000000000 --- a/hack/dpdk_21_11_xstats_mem_leak.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff --git a/lib/ethdev/rte_ethdev.c b/lib/ethdev/rte_ethdev.c -index a1d475a292..1f5d3c1b16 100644 ---- a/lib/ethdev/rte_ethdev.c -+++ b/lib/ethdev/rte_ethdev.c -@@ -6249,6 +6249,7 @@ eth_dev_handle_port_xstats(const char *cmd __rte_unused, - for (i = 0; i < num_xstats; i++) - rte_tel_data_add_dict_u64(d, xstat_names[i].name, - eth_xstats[i].value); -+ free(eth_xstats); - return 0; - } diff --git a/hack/dpdk_21_11_gcc12.patch b/hack/dpdk_22_11_gcc12.patch similarity index 100% rename from hack/dpdk_21_11_gcc12.patch rename to hack/dpdk_22_11_gcc12.patch diff --git a/hack/dpdk_21_11_log.patch b/hack/dpdk_22_11_log.patch similarity index 90% rename from hack/dpdk_21_11_log.patch rename to hack/dpdk_22_11_log.patch index 847a96823..2522e8d72 100644 --- a/hack/dpdk_21_11_log.patch +++ b/hack/dpdk_22_11_log.patch @@ -2,7 +2,7 @@ diff --git a/lib/eal/common/eal_common_log.c b/lib/eal/common/eal_common_log.c index 1be35f5397..159beb63ab 100644 --- a/lib/eal/common/eal_common_log.c +++ b/lib/eal/common/eal_common_log.c -@@ -29,11 +29,14 @@ static struct rte_logs { +@@ -28,11 +28,14 @@ static struct rte_logs { uint32_t type; /**< Bitfield with enabled logs. */ uint32_t level; /**< Log level. */ FILE *file; /**< Output file set by rte_openlog_stream, or NULL. */ @@ -17,7 +17,7 @@ index 1be35f5397..159beb63ab 100644 }; struct rte_eal_opt_loglevel { -@@ -78,6 +81,13 @@ rte_openlog_stream(FILE *f) +@@ -77,6 +80,13 @@ rte_openlog_stream(FILE *f) return 0; } @@ -31,7 +31,7 @@ index 1be35f5397..159beb63ab 100644 FILE * rte_log_get_stream(void) { -@@ -500,7 +510,7 @@ rte_vlog(uint32_t level, uint32_t logtype, const char *format, va_list ap) +@@ -499,7 +509,7 @@ rte_vlog(uint32_t level, uint32_t logtype, const char *format, va_list ap) RTE_PER_LCORE(log_cur_msg).loglevel = level; RTE_PER_LCORE(log_cur_msg).logtype = logtype; @@ -44,7 +44,7 @@ diff --git a/lib/eal/include/rte_log.h b/lib/eal/include/rte_log.h index 319e4044a5..ee17cf93a0 100644 --- a/lib/eal/include/rte_log.h +++ b/lib/eal/include/rte_log.h -@@ -88,6 +88,18 @@ extern "C" { +@@ -87,6 +87,18 @@ extern "C" { */ int rte_openlog_stream(FILE *f); @@ -67,7 +67,7 @@ diff --git a/lib/eal/version.map b/lib/eal/version.map index ab28c22791..72c8a18111 100644 --- a/lib/eal/version.map +++ b/lib/eal/version.map -@@ -141,6 +141,7 @@ DPDK_22 { +@@ -155,6 +155,7 @@ DPDK_23 { rte_log_set_level; rte_log_set_level_pattern; rte_log_set_level_regexp; From 0f3bcef1d823e297bf9236fc1c8c3bbba1829d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Thu, 14 Sep 2023 16:07:29 +0200 Subject: [PATCH 05/10] Dropped DPDK 21.11 conditional compilation --- meson.build | 3 --- meson_options.txt | 2 -- src/rte_flow/dp_rte_flow_init.c | 8 ------- src/rte_flow/dp_rte_flow_traffic_forward.c | 28 +++++----------------- 4 files changed, 6 insertions(+), 35 deletions(-) diff --git a/meson.build b/meson.build index ac6c7d669..2d5da623f 100644 --- a/meson.build +++ b/meson.build @@ -49,9 +49,6 @@ add_global_arguments('-DDEBUG=true', language: ['c', 'cpp']) if get_option('enable_virtual_services') add_global_arguments('-DENABLE_VIRTSVC', language: ['c', 'cpp']) endif -if get_option('enable_dpdk_22_11_support') - add_global_arguments('-DENABLE_DPDK_22_11', language: 'c') -endif if get_option('enable_static_underlay_ip') add_global_arguments('-DENABLE_STATIC_UNDERLAY_IP', language: 'c') endif diff --git a/meson_options.txt b/meson_options.txt index 7db0fb096..9bd88c0a7 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -2,8 +2,6 @@ option('enable_usermode', type: 'boolean', value: false, description: 'Make the resulting binary runnable without root (requires sudo during build)') option('enable_tests', type: 'boolean', value: false, description: 'Make sure automated testing has all prerequisities') -option('enable_dpdk_22_11_support', type: 'boolean', value: false, description: - 'Enable compatibility code for dpdk version 22.11') option('enable_virtual_services', type: 'boolean', value: false, description: 'Build with virtual services functionality') option('enable_static_underlay_ip', type: 'boolean', value: false, description: diff --git a/src/rte_flow/dp_rte_flow_init.c b/src/rte_flow/dp_rte_flow_init.c index c6af43825..d965ca4a7 100644 --- a/src/rte_flow/dp_rte_flow_init.c +++ b/src/rte_flow/dp_rte_flow_init.c @@ -15,11 +15,7 @@ static const struct rte_flow_attr dp_flow_attr_prio_ingress = { static const struct rte_flow_attr dp_flow_attr_default_jump_ingress = { .group = DP_RTE_FLOW_DEFAULT_GROUP, .priority = 1, -#ifdef ENABLE_DPDK_22_11 .ingress = 0, -#else - .ingress = 1, -#endif .egress = 0, .transfer = 1, }; @@ -27,11 +23,7 @@ static const struct rte_flow_attr dp_flow_attr_default_jump_ingress = { static const struct rte_flow_attr dp_flow_attr_default_monitoring_ingress = { .group = DP_RTE_FLOW_MONITORING_GROUP, .priority = 3, -#ifdef ENABLE_DPDK_22_11 .ingress = 0, -#else - .ingress = 1, -#endif .egress = 0, .transfer = 1, }; diff --git a/src/rte_flow/dp_rte_flow_traffic_forward.c b/src/rte_flow/dp_rte_flow_traffic_forward.c index ebfff95e5..05948c663 100644 --- a/src/rte_flow/dp_rte_flow_traffic_forward.c +++ b/src/rte_flow/dp_rte_flow_traffic_forward.c @@ -36,11 +36,7 @@ static const struct rte_flow_attr dp_flow_attr_egress = { static const struct rte_flow_attr dp_flow_pf_attr_transfer = { .group = DP_RTE_FLOW_DEFAULT_GROUP, .priority = 0, -#ifdef ENABLE_DPDK_22_11 .ingress = 0, -#else - .ingress = 1, -#endif .egress = 0, .transfer = 1, }; @@ -48,11 +44,7 @@ static const struct rte_flow_attr dp_flow_pf_attr_transfer = { static const struct rte_flow_attr dp_flow_vf_attr_transfer = { .group = DP_RTE_FLOW_VNET_GROUP, .priority = 0, -#ifdef ENABLE_DPDK_22_11 .ingress = 0, -#else - .ingress = 1, -#endif .egress = 0, .transfer = 1, }; @@ -334,19 +326,15 @@ static __rte_always_inline int dp_offload_handle_tunnel_decap_traffic(struct rte struct rte_flow_action_set_ipv4 set_ipv4; // #3 (optional) struct rte_flow_action_set_tp set_tp; // #4 (optional) struct rte_flow_action_age flow_age; // #5 -#ifndef ENABLE_DPDK_22_11 - struct rte_flow_action_queue redirect_queue; // #6 (replaces send_to_port) -#endif - struct rte_flow_action_port_id send_to_port; // #6 + struct rte_flow_action_queue redirect_queue; // #6 (choose one) + struct rte_flow_action_port_id send_to_port; // #6 (choose one) struct rte_flow_action actions[7]; // + end int action_cnt = 0; // misc variables needed to create the flow struct flow_age_ctx *agectx; struct rte_flow_action *age_action; -#ifndef ENABLE_DPDK_22_11 struct dp_port *port; -#endif struct rte_ether_hdr new_eth_hdr; rte_be32_t actual_ol_ipv4_addr; bool cross_pf_port; @@ -407,7 +395,7 @@ static __rte_always_inline int dp_offload_handle_tunnel_decap_traffic(struct rte age_action = &actions[action_cnt++]; dp_set_flow_age_action(age_action, &flow_age, df->conntrack->timeout_value, agectx); -#ifndef ENABLE_DPDK_22_11 + // TODO: this branch has not been tested with DPDK 22.11 if (cross_pf_port) { // move this packet to the right hairpin rx queue of pf, so as to be moved to vf port = dp_port_get_vf((uint16_t)df->nxt_hop); @@ -420,16 +408,14 @@ static __rte_always_inline int dp_offload_handle_tunnel_decap_traffic(struct rte dp_set_redirect_queue_action(&actions[action_cnt++], &redirect_queue, DP_NR_RESERVED_RX_QUEUES - 1 + port->peer_pf_hairpin_tx_rx_queue_offset); } else -#endif dp_set_send_to_port_action(&actions[action_cnt++], &send_to_port, df->nxt_hop); dp_set_end_action(&actions[action_cnt++]); if (DP_FAILED(dp_install_rte_flow_with_indirect(m->port, -#ifndef ENABLE_DPDK_22_11 - cross_pf_port ? &dp_flow_pf_attr_ingress : -#endif - &dp_flow_pf_attr_transfer, + cross_pf_port + ? &dp_flow_pf_attr_ingress + : &dp_flow_pf_attr_transfer, pattern, actions, age_action, df, agectx)) ) { @@ -511,8 +497,6 @@ static __rte_always_inline int dp_offload_handle_local_traffic(struct rte_mbuf * dp_set_end_action(&actions[action_cnt++]); - // TODO: this attribute has not been tested with DPDK 22.11, - // so maybe 'dp_flow_attr_transfer' should be ifdef'd too if (DP_FAILED(dp_install_rte_flow_with_indirect(m->port, &dp_flow_pf_attr_transfer, pattern, actions, age_action, df, agectx)) From ee2d7f15832f4b8a6cd434dcaf60899e0a054eaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Wed, 20 Sep 2023 15:57:13 +0200 Subject: [PATCH 06/10] Improved port start/stop --- src/dp_port.c | 96 +++++++++++++++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 42 deletions(-) diff --git a/src/dp_port.c b/src/dp_port.c index afc22854a..6bc84de85 100644 --- a/src/dp_port.c +++ b/src/dp_port.c @@ -400,13 +400,30 @@ int dp_ports_init(void) return DP_OK; } +static int dp_stop_eth_port(uint16_t port_id) +{ + int ret, ret2; + + // TODO(Tao): look into tearing down hairpins + + // error already logged + ret = rx_node_set_enabled(port_id, false); + + ret2 = rte_eth_dev_stop(port_id); + if (DP_FAILED(ret2)) { + DPS_LOG_ERR("Cannot stop ethernet port", DP_LOG_PORTID(port_id), DP_LOG_RET(ret2)); + ret = ret2; + } + + return ret; +} + void dp_ports_free(void) { // without stopping started ports, DPDK complains DP_FOREACH_PORT(&dp_ports, port) { - // TODO(plague): PR to create proper rollback in dp_port_start() if (port->allocated) - rte_eth_dev_stop(port->port_id); + dp_stop_eth_port(port->port_id); } free(dp_ports.ports); } @@ -492,6 +509,34 @@ static int dp_install_vf_init_rte_rules(uint32_t port_id) return DP_OK; } +static int dp_init_port(struct dp_port *port) +{ + if (DP_FAILED(rx_node_set_enabled(port->port_id, true))) + return DP_ERROR; + + // TAP devices do not support offloading/isolation + if (dp_conf_get_nic_type() == DP_CONF_NIC_TYPE_TAP) + return DP_OK; + + if (port->port_type == DP_PORT_PF) + if (DP_FAILED(dp_port_install_isolated_mode(port->port_id))) + return DP_ERROR; + + if (dp_conf_is_offload_enabled()) { +#ifdef ENABLE_PYTEST + if (port->peer_pf_port_id != dp_port_get_pf1_id()) +#endif + if (DP_FAILED(dp_port_bind_port_hairpins(port))) + return DP_ERROR; + + if (port->port_type == DP_PORT_VF) + if (DP_FAILED(dp_install_vf_init_rte_rules(port->port_id))) + assert(false); // if any flow rule failed, stop process running due to possible hw/driver failure + } + + return DP_OK; +} + int dp_port_start(uint16_t port_id) { struct dp_port *port; @@ -504,64 +549,31 @@ int dp_port_start(uint16_t port_id) ret = rte_eth_dev_start(port_id); if (DP_FAILED(ret)) { DPS_LOG_ERR("Cannot start ethernet port", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); - return DP_ERROR; + return ret; } - if (DP_FAILED(rx_node_set_enabled(port_id, true))) - return DP_ERROR; + ret = dp_init_port(port); + if (DP_FAILED(ret)) { + dp_stop_eth_port(port->port_id); + return ret; + } port->link_status = RTE_ETH_LINK_UP; port->allocated = true; - - if (dp_conf_get_nic_type() != DP_CONF_NIC_TYPE_TAP) { - if (dp_conf_is_offload_enabled()) { -#ifdef ENABLE_PYTEST - if (port->peer_pf_port_id != dp_port_get_pf1_id()) -#endif - if (DP_FAILED(dp_port_bind_port_hairpins(port))) - return DP_ERROR; - } - - if (port->port_type == DP_PORT_PF) { - if (DP_FAILED(dp_port_install_isolated_mode(port_id))) - return DP_ERROR; - } - - - if (port->port_type == DP_PORT_VF && dp_conf_is_offload_enabled()) { - ret = dp_install_vf_init_rte_rules(port_id); - if (DP_FAILED(ret)) - assert(0); // if any flow rule failed, stop process running due to possible hw/driver failure - } - } - return DP_OK; } int dp_port_stop(uint16_t port_id) { struct dp_port *port; - int ret; port = dp_port_get(port_id); if (!port) return DP_ERROR; - // TODO(plague): research - no need to tear down hairpins? - - if (DP_FAILED(rx_node_set_enabled(port_id, false))) + if (DP_FAILED(dp_stop_eth_port(port_id))) return DP_ERROR; - /* Tap interfaces in test environment can not be stopped */ - /* due to a bug in dpdk tap device library. */ - if (dp_conf_get_nic_type() != DP_CONF_NIC_TYPE_TAP) { - ret = rte_eth_dev_stop(port_id); - if (DP_FAILED(ret)) { - DPS_LOG_ERR("Cannot stop ethernet port", DP_LOG_PORTID(port_id), DP_LOG_RET(ret)); - return DP_ERROR; - } - } - port->allocated = false; return DP_OK; } From a6ef0071eca7a4bec364919adb6bfe8f6335f82b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Thu, 21 Sep 2023 16:26:26 +0200 Subject: [PATCH 07/10] Improve Docker build time --- Dockerfile | 2 +- src/rte_flow/dp_rte_flow_traffic_forward.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index be6250273..5e8049e87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,7 +66,7 @@ net/octeontx_ep,net/pcap,net/pfe,net/qede,net/sfc,net/softnic,net/thunderx,net/t net/vdev_ntsvc,net/vhost,net/virtio,net/vmxnet3,net/bnx2x,net/netsvc,net/vdev_netsvc,\ crypto/dpaa_sec,crypto/bcmfs,crypto/caam_jr,crypto/cnxk,dpaa_sec,crypto/dpaa2_sec,crypto/nitrox,\ crypto/null,crypto/octeontx,crypto/octeontx2,crypto/scheduler,crypto/virtio -Ddisable_libs=power,\ -vhost,gpudev build +vhost,gpudev build -Ddisable_apps="*" -Dtests=false RUN cd $DPDK_DIR/build && ninja RUN cd $DPDK_DIR/build && ninja install diff --git a/src/rte_flow/dp_rte_flow_traffic_forward.c b/src/rte_flow/dp_rte_flow_traffic_forward.c index 05948c663..f10dbdddb 100644 --- a/src/rte_flow/dp_rte_flow_traffic_forward.c +++ b/src/rte_flow/dp_rte_flow_traffic_forward.c @@ -395,7 +395,6 @@ static __rte_always_inline int dp_offload_handle_tunnel_decap_traffic(struct rte age_action = &actions[action_cnt++]; dp_set_flow_age_action(age_action, &flow_age, df->conntrack->timeout_value, agectx); - // TODO: this branch has not been tested with DPDK 22.11 if (cross_pf_port) { // move this packet to the right hairpin rx queue of pf, so as to be moved to vf port = dp_port_get_vf((uint16_t)df->nxt_hop); From 86b4d5f945acbafc20649b8e7ccd4972c051c901 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Wed, 27 Sep 2023 15:52:21 +0200 Subject: [PATCH 08/10] Renamed dp_service to dpservice-bin --- Dockerfile | 7 +++---- docs/deployment/README.md | 7 ++++++- docs/deployment/commandline.md | 4 ++-- docs/development/building.md | 2 +- docs/development/debugging.md | 4 ++-- docs/development/kernel.md | 2 +- docs/development/running.md | 10 +++++----- docs/testing/README.md | 6 +++--- docs/testing/grpc_client.md | 2 +- docs/testing/mellanox.md | 8 ++++---- docs/testing/performance.md | 10 +++++----- src/dp_error.c | 2 +- src/dp_service.c | 2 +- src/meson.build | 6 +++--- test/dp_service.py | 2 +- 15 files changed, 39 insertions(+), 35 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5e8049e87..0524b076c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,7 +115,7 @@ python3-scapy \ WORKDIR / COPY --from=testbuilder /workspace/test ./test -COPY --from=testbuilder /workspace/build/src/dp_service ./build/src/dp_service +COPY --from=testbuilder /workspace/build/src/dpservice-bin ./build/src/dpservice-bin COPY --from=testbuilder /workspace/client/* ./build COPY --from=testbuilder /usr/local/lib /usr/local/lib RUN ldconfig @@ -140,8 +140,7 @@ bash-completion \ && rm -rf /var/lib/apt/lists/* WORKDIR / -COPY --from=builder /workspace/build/src/dp_service \ - /workspace/build/tools/dp_grpc_client \ +COPY --from=builder /workspace/build/src/dpservice-bin \ /workspace/build/tools/dump/dpservice-dump \ /workspace/client/* \ /workspace/exporter/* \ @@ -153,4 +152,4 @@ RUN ldconfig # Ensure bash-completion is working in operations RUN echo 'PATH=${PATH}:/\nsource /etc/bash_completion\nsource <(dpservice-cli completion bash)' >> /root/.bashrc -ENTRYPOINT ["dp_service"] +ENTRYPOINT ["dpservice-bin"] diff --git a/docs/deployment/README.md b/docs/deployment/README.md index 6bfb1fb0f..41a232950 100644 --- a/docs/deployment/README.md +++ b/docs/deployment/README.md @@ -1,6 +1,11 @@ # Dataplane Service deployment -This repository only provides a compiled binary of `dp_service`. To properly use it, you need to use the provided Docker file to generate an image and run it using the [metalnet project](https://github.com/onmetal/metalnet). +This repository only provides a compiled binary of `dpservice-bin`. To properly use it, you need to use the provided Docker file to generate an image and run it using the [metalnet project](https://github.com/onmetal/metalnet). All command-line arguments and configuration files are already handled by the Docker image. For development, direct use of dp-service is covered by the [development section](../development/). + +## Command-line tools +All tool binaries are designed to be prefixed with `dpservice-` to enable the operator to simply type `dps` for list of possible tools. + +The provided Docker image contains `dpservice-bin` as the main process (already started by being the entrypoint), `dpservice-cli` to gRPC communication with the main process, and `dpservice-dump` to provide a way to see the actual traffic handled by dp-service. diff --git a/docs/deployment/commandline.md b/docs/deployment/commandline.md index 5dcf8e029..3bf7c91f1 100644 --- a/docs/deployment/commandline.md +++ b/docs/deployment/commandline.md @@ -1,5 +1,5 @@ # Dataplane Service Command-line Options -`dp_service` accepts two sets of options separated by `--`. The first set contains DPDK options, the second `dp_service` options proper. Both sets support `--help` +`dpservice-bin` accepts two sets of options separated by `--`. The first set contains DPDK options, the second `dpservice-bin` options proper. Both sets support `--help` ## EAL Options For more information on EAL options, please see [the official docs](https://doc.dpdk.org/guides/linux_gsg/linux_eal_parameters.html) @@ -8,5 +8,5 @@ For more information on EAL options, please see [the official docs](https://doc. All options are described in `dpservice-bin --help`, see [the markdown version of it](help_dpservice-bin.md) ## Configuration file -Unless an environment variable `DP_CONF` is set to override the path, `dp_service` uses `/tmp/dp_service.conf` to read configuration before processing any arguments. +Unless an environment variable `DP_CONF` is set to override the path, `dpservice-bin` uses `/tmp/dp_service.conf` to read configuration before processing any arguments. This way you can provide any arguments via such file and simplify the commandline use. The helper script `prepare.sh` generates such a file for Mellanox users. diff --git a/docs/development/building.md b/docs/development/building.md index ce302c5a2..f4590b509 100644 --- a/docs/development/building.md +++ b/docs/development/building.md @@ -57,7 +57,7 @@ ninja -C build Now you can try [running the service](running.md). ### Usermode dpservice -For [easier debugging](debugging.md) you can configure meson to build additional `dp_service_user` binary that does not require root privileges to run. +For [easier debugging](debugging.md) you can configure meson to build additional `dpservice-user` binary that does not require root privileges to run. ```bash meson setup --reconfigure -Denable_usermode=true build ninja -C build diff --git a/docs/development/debugging.md b/docs/development/debugging.md index 1308c4ce5..f83ca1a9f 100644 --- a/docs/development/debugging.md +++ b/docs/development/debugging.md @@ -14,9 +14,9 @@ You can use a pattern, like `*:8` for all debug messages (including DPDK). To on ## Run the service as a user -There is an option in the [build system](building#usermode-dpservice) to generate an additional binary `dp_service_user` that can be run as user instead of root. This is not only a safer way to debug, but enables you to easily use any GUI on top of `gdb` just like any other program. +There is an option in the [build system](building#usermode-dpservice) to generate an additional binary `dpservice-user` that can be run as user instead of root. This is not only a safer way to debug, but enables you to easily use any GUI on top of `gdb` just like any other program. -The drawback here is that you need to put the same capabilites to the `gdb` process as the `dp_service_user` binary has. For the most current list of capabilities please see the `hack/set_cap.sh` helper script. +The drawback here is that you need to put the same capabilites to the `gdb` process as the `dpservice-user` binary has. For the most current list of capabilities please see the `hack/set_cap.sh` helper script. ## Debugging the test-suite diff --git a/docs/development/kernel.md b/docs/development/kernel.md index a6cfc48f2..4a26cd825 100644 --- a/docs/development/kernel.md +++ b/docs/development/kernel.md @@ -37,7 +37,7 @@ If you want to use this timer in your DPDK-based programs, enable `CONFIG_HPET` ## FS capabilities -To run `dp_service` as a user, [linux capabilites](https://man7.org/linux/man-pages/man7/capabilities.7.html) are employed. For that to happen, your filesystem needs to support security labels. For Ext4, the right option is `CONFIG_EXT4_FS_SECURITY`. Other filesystems should have a similar option available. +To run `dpservice-bin` as a user, [linux capabilites](https://man7.org/linux/man-pages/man7/capabilities.7.html) are employed. For that to happen, your filesystem needs to support security labels. For Ext4, the right option is `CONFIG_EXT4_FS_SECURITY`. Other filesystems should have a similar option available. ## Caveats diff --git a/docs/development/running.md b/docs/development/running.md index 02f6dcf45..a8e6d9594 100644 --- a/docs/development/running.md +++ b/docs/development/running.md @@ -36,14 +36,14 @@ For VMs to be able to connect to virtual functions, you need to make them access ## Command-line arguments -Unless run via `dp_service_user`, all `dp_service` commands need to be run as root. +Unless run via `dpservice-user`, all `dpservice-bin` commands need to be run as root. -For details about command-line arguments [see here](../deployment/commandline.md) or use `dp_service --help` for EAL options and `dp_service -- --help` for the actual service options. +For details about command-line arguments [see here](../deployment/commandline.md) or use `dpservice-bin --help` for EAL options and `dpservice-bin -- --help` for the actual service options. Currently, dp-service only supports Mellanoc NICs and virtual interfaces for testing. Thus most arguments are not needed and are being provided another way. ### Mellanox cards -To prepare Mellanox cards to work with `dp_service` a `prepare.sh` script in `hack/` is provided. It should create appropriate number of virtual interfaces and put them into *switchdev mode*. For more information and/or troubleshooting, please see [Mellanox specific documentation](mellanox.md) +To prepare Mellanox cards to work with `dpservice-bin` a `prepare.sh` script in `hack/` is provided. It should create appropriate number of virtual interfaces and put them into *switchdev mode*. For more information and/or troubleshooting, please see [Mellanox specific documentation](mellanox.md) In order to have this preparation happen automatically on every boot, you can use the `hack/preparedp.service` systemd unit. ```bash @@ -54,7 +54,7 @@ systemctl daemon-reload systemctl enable preparedp.service ``` -Then you can simply run `dp_service -l0,1 -- --no-stats --no-offload` for the most basic setup. See help output for more info. +Then you can simply run `dpservice-bin -l0,1 -- --no-stats --no-offload` for the most basic setup. See help output for more info. ### Virtual interfaces For automated testing and some working setups, virtual interfaces are used and thus no more setup is needed. The whole command-line is generated by [test-scripts](../testing/). @@ -64,7 +64,7 @@ If you want to run the service using the same arguments as the automated tests a ### Manual invocation Without the help of scripts or config files, you can run the service directly (adjust the interface names according to your machine): ```bash -./dp_service -a 0000:3b:00.0,representor=0-5 -a 0000:3b:00.1 -l 0,1 -- --pf0=enp59s0f1 --pf1=enp59s0f1 --vf-pattern=enp59s0f0_ --ipv6=2a10:afc0:e01f:209:: --no-stats --no-offload +./dpservice-bin -a 0000:3b:00.0,representor=0-5 -a 0000:3b:00.1 -l 0,1 -- --pf0=enp59s0f1 --pf1=enp59s0f1 --vf-pattern=enp59s0f0_ --ipv6=2a10:afc0:e01f:209:: --no-stats --no-offload ``` `-a` arguments set the PCI addresses of the smartnic's ports along with the VF range specification (6 VFs in this example). diff --git a/docs/testing/README.md b/docs/testing/README.md index 2e1427f2b..7fa681687 100644 --- a/docs/testing/README.md +++ b/docs/testing/README.md @@ -23,13 +23,13 @@ Running that command will show you full test output for reporting or fixing the ### Pytest Developers might consider running `pytest` directly in the `test/` directory to see more detailed information (via `-v` or even full output (via `-s`). With no positional arguments, all tests will run, specify individual unit-test files for making the testing set smaller. When multiple tests are broken, consider using the stop-at-first-error argument `-x`. -Since one test-run only applies to a specific set of options for `dp_service`, argument specifying the type of underlay tunnel (`--tun-opt`, default `ipip`) and whether or not port redundancy should be utilized (`--port-redundancy`) are needed to fully test all code paths. +Since one test-run only applies to a specific set of options for `dpservice-bin`, argument specifying the type of underlay tunnel (`--tun-opt`, default `ipip`) and whether or not port redundancy should be utilized (`--port-redundancy`) are needed to fully test all code paths. -If one should want to instead run your own `dp_service` instance (e.g. for running under a debugger), the `--attach` argument connects to an already running service instead of starting its own. This comes with the caveat of ensuring the right arguments are passed to the service at startup. +If one should want to instead run your own `dpservice-bin` instance (e.g. for running under a debugger), the `--attach` argument connects to an already running service instead of starting its own. This comes with the caveat of ensuring the right arguments are passed to the service at startup. ## GRPC test client -If you want to do some manual testing, you need to use GRPC to communicate with the `dp_service` process. A simple client is provided by this repository. For more information about the client, see [this section](grpc_client.md). +If you want to do some manual testing, you need to use GRPC to communicate with the `dpservice-bin` process. A simple client is provided by this repository. For more information about the client, see [this section](grpc_client.md). ## Mellanox test setup diff --git a/docs/testing/grpc_client.md b/docs/testing/grpc_client.md index d9bd5cd08..d8dfa0106 100644 --- a/docs/testing/grpc_client.md +++ b/docs/testing/grpc_client.md @@ -1,6 +1,6 @@ # Dataplane Service GRPC Client -To communicate with the `dp_service` process, you need to use GRPC (running on localhost at default port 1337). +To communicate with the `dpservice-bin` process, you need to use GRPC (running on localhost at default port 1337). There is a golang command-line client [dpservice-cli](https://github.com/onmetal/dpservice-cli/), with full [command-line documentation](https://github.com/onmetal/dpservice-cli/tree/osc-main/docs/commands#dpservice-cli-commands). diff --git a/docs/testing/mellanox.md b/docs/testing/mellanox.md index 9ea2c4b6e..ee4dff817 100644 --- a/docs/testing/mellanox.md +++ b/docs/testing/mellanox.md @@ -4,7 +4,7 @@ As dp-service is intended as a virtual router for hosting VMs, testing on real h It is also possible to run the whole set of tests that `meson test` runs using `pytest`. See below for details. ## Hardware setup -To simplify starting `dp_service`, it accepts a configuration file (currently hardcoded `/tmp/dp_service.conf`) so that most command-line arguments concerning hardware options are not needed. +To simplify starting `dpservice-bin`, it accepts a configuration file (currently hardcoded `/tmp/dp_service.conf`) so that most command-line arguments concerning hardware options are not needed. This file is generated for you by calling a shell script, `hack/prepare.sh`. @@ -14,12 +14,12 @@ This script also sets up hugepages, Mellanox's virtual functions (VFs), and eswi ## Virtual machines For a VM to use a VF, that VF must use VFIO driver. Let the address of an available VF be `01:00.2`. Then the easiest way to do this is via `dpdk-devbind -b vfio-pci 01:00.2`. KVM should then be able to bind to a VF using vfio: `-net none -device vfio-pci,host=01:00.02`. -If running `dp_service_user`, don't forget to adjust privileges for the appropriate `/dev/vfio/?` devices. +If running `dpservice-user`, don't forget to adjust privileges for the appropriate `/dev/vfio/?` devices. You can then register the VM in the running service via [grpc](grpc_client.md): ```bash -./dp_grpc_client --init -./dp_grpc_client --addmachine test10 --vm_pci 0000:01:00.0_representor_vf0 --vni 123 --ipv4 192.168.123.10 --ipv6 2001::10 +dpservice-cli init +dpservice-cli add interface --id test10 --device 0000:01:00.0_representor_vf0 --vni 123 --ipv4 192.168.123.10 --ipv6 2001::10 ``` If you set two VMs like this, they should be able to connect to each other (ping, netcat, ...). diff --git a/docs/testing/performance.md b/docs/testing/performance.md index 68cde5c82..26afb7605 100644 --- a/docs/testing/performance.md +++ b/docs/testing/performance.md @@ -87,15 +87,15 @@ Isolating CPU core that is used by DPDK application and removing it from Linux s This approach does not bring observable performance enhancement. It is possibly due to it is experimented on a machine with few tasks. ## Compilation optimization -By default, DPDK library is configured to compile as the release mode. dp-service needs to be configured in the release mode as well using `meson build --buildtype=release`. +By default, DPDK library is configured to compile as the release mode. dp-service needs to be configured in the release mode as well using `meson setup --buildtype=release build`. This brings observable performance enhancement. -Additional flags (-march=native -mcpu=native -mtune=native) to compile source code for the native platform can be also added into meson build file. +Additional flags (`-march=native`, `-mcpu=native`, `-mtune=native`) to compile source code for the native platform can be also added into meson build file. ``` -project('dp_service', 'c', 'cpp', - default_options: ['c_args=-Wno-deprecated-declarations -march=native -mcpu=native -mtune=native -Werror -Wno-format-truncation', 'cpp_args=-fpermissive'], - ... +perfflags = [ '-march=native', '-mcpu=native', '-mtune=native' ] +cflags += perfflags +cxxflags += perfflags ``` This does not bring observable performance enhancement. diff --git a/src/dp_error.c b/src/dp_error.c index c4fbcd6b9..e5c56de5b 100644 --- a/src/dp_error.c +++ b/src/dp_error.c @@ -44,7 +44,7 @@ const char *dp_strerror(int error) else if (error >= -_DP_GRPC_ERRCODES) return dp_grpc_strerror(error+_DP_GRPC_ERRCODES); else - return "General dp_service error"; + return "General dpservice error"; } const char *dp_strerror_verbose(int error) diff --git a/src/dp_service.c b/src/dp_service.c index 912281c4e..83a833553 100644 --- a/src/dp_service.c +++ b/src/dp_service.c @@ -63,7 +63,7 @@ static int dp_args_add_mellanox(int *orig_argc, char ***orig_argv) return DP_ERROR; } - // add original dp_service args + // add original dpservice args if (argend >= 0) { for (int j = argend; j < argc; ++j) dp_argv[curarg++] = argv[j]; diff --git a/src/meson.build b/src/meson.build index 1b75b7bf2..a0d42fbf0 100644 --- a/src/meson.build +++ b/src/meson.build @@ -67,17 +67,17 @@ if get_option('enable_virtual_services') ] endif -exe = executable('dp_service', +exe = executable('dpservice-bin', sources: [ dp_sources, grpc_generated ], include_directories: [ includes ], dependencies: [ dpdk_dep, proto_dep, grpc_dep, grpccpp_dep, thread_dep, libuuid_dep ] ) if get_option('enable_usermode') - custom_target('dp_service_user', + custom_target('dpservice-user', depends: exe, input: exe, - output: 'dp_service_user', + output: 'dpservice-user', command: [ '../hack/set_cap.sh', '@INPUT@', '@OUTPUT@' ], build_by_default: true, console: true diff --git a/test/dp_service.py b/test/dp_service.py index 76e07f694..467104a4f 100755 --- a/test/dp_service.py +++ b/test/dp_service.py @@ -33,7 +33,7 @@ def __init__(self, build_path, port_redundancy, fast_flow_timeout, script_path = os.path.dirname(os.path.abspath(__file__)) self.cmd = f"gdb -x {script_path}/gdbinit --args " - self.cmd += f'{self.build_path}/src/dp_service -l 0,1 --log-level=user*:8' + self.cmd += f'{self.build_path}/src/dpservice-bin -l 0,1 --log-level=user*:8' if not self.hardware: self.cmd += (f' --no-pci' f' --vdev={PF0.pci},iface={PF0.tap},mac="{PF0.mac}"' From c6ef0585bc25c76b56dfafb2aff8a623172cf4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Fri, 6 Oct 2023 16:44:41 +0200 Subject: [PATCH 09/10] Fix telemetry for DPDK 22 --- Dockerfile | 1 + docs/development/building.md | 1 + hack/dpdk_22_11_telemetry_key.patch | 17 +++++++++ include/grpc/dp_grpc_conv.h | 2 ++ src/dp_telemetry.c | 21 ++--------- src/grpc/dp_async_grpc.cpp | 2 ++ src/grpc/dp_grpc_conv.cpp | 11 ++++++ test/test_telemetry.py | 56 +++++++++++++++++++++++++++++ 8 files changed, 93 insertions(+), 18 deletions(-) create mode 100644 hack/dpdk_22_11_telemetry_key.patch create mode 100644 test/test_telemetry.py diff --git a/Dockerfile b/Dockerfile index 0524b076c..bd72d82dc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,6 +52,7 @@ ENV DPDK_DIR=/workspace/dpdk-${DPDK_VER} COPY hack/*.patch hack/ RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_22_11_gcc12.patch RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_22_11_log.patch +RUN cd $DPDK_DIR && patch -p1 < ../hack/dpdk_22_11_telemetry_key.patch # Compile DPDK RUN cd $DPDK_DIR && meson setup -Dmax_ethports=132 -Dplatform=generic -Ddisable_drivers=common/dpaax,\ diff --git a/docs/development/building.md b/docs/development/building.md index f4590b509..934e21a1a 100644 --- a/docs/development/building.md +++ b/docs/development/building.md @@ -34,6 +34,7 @@ tar xf dpdk-22.11.3.tar.xz cd dpdk-stable-22.11.3 patch -p1 < ../net-dpservice/hack/dpdk_22_11_gcc12.patch patch -p1 < ../net-dpservice/hack/dpdk_22_11_log.patch +patch -p1 < ../net-dpservice/hack/dpdk_22_11_telemetry_key.patch meson setup build ninja -C build sudo ninja -C build install diff --git a/hack/dpdk_22_11_telemetry_key.patch b/hack/dpdk_22_11_telemetry_key.patch new file mode 100644 index 000000000..219354284 --- /dev/null +++ b/hack/dpdk_22_11_telemetry_key.patch @@ -0,0 +1,17 @@ +diff --git a/lib/telemetry/telemetry_data.c b/lib/telemetry/telemetry_data.c +index 34366ecee3..71fd48376f 100644 +--- a/lib/telemetry/telemetry_data.c ++++ b/lib/telemetry/telemetry_data.c +@@ -101,11 +101,10 @@ static bool + valid_name(const char *name) + { + char allowed[128] = { +- ['0' ... '9'] = 1, ++ ['-' ... ':'] = 1, // - . / 0-9 : + ['A' ... 'Z'] = 1, + ['a' ... 'z'] = 1, + ['_'] = 1, +- ['/'] = 1, + }; + while (*name != '\0') { + if ((size_t)*name >= RTE_DIM(allowed) || allowed[(int)*name] == 0) diff --git a/include/grpc/dp_grpc_conv.h b/include/grpc/dp_grpc_conv.h index adb8a9421..c1cba1a07 100644 --- a/include/grpc/dp_grpc_conv.h +++ b/include/grpc/dp_grpc_conv.h @@ -11,6 +11,8 @@ namespace GrpcConv { Status* CreateStatus(uint32_t grpc_errcode); + bool IsInterfaceIdValid(const std::string& id); + bool StrToIpv4(const std::string& str, uint32_t *dst); bool StrToIpv6(const std::string& str, uint8_t *dst); diff --git a/src/dp_telemetry.c b/src/dp_telemetry.c index 14a15950d..f9efc16fe 100644 --- a/src/dp_telemetry.c +++ b/src/dp_telemetry.c @@ -161,25 +161,10 @@ static int dp_telemetry_handle_nat_used_port_count(const char *cmd, __rte_unused const char *params, struct rte_tel_data *data) { - - int ret; - - ret = dp_telemetry_start_dict(data, cmd); - if (DP_FAILED(ret)) { - DPS_LOG_WARNING("Failed to init telemetry data to get interface's used nat port cnt", - DP_LOG_TELEMETRY_CMD(cmd), DP_LOG_RET(ret)); - return DP_ERROR; - } - - ret = dp_nat_get_used_ports_telemetry(data); - if (DP_FAILED(ret)) { - DPS_LOG_WARNING("Failed to get used nat ports' telemetry data", - DP_LOG_TELEMETRY_CMD(cmd), DP_LOG_RET(ret)); + if (DP_FAILED(dp_telemetry_start_dict(data, cmd)) + || DP_FAILED(dp_nat_get_used_ports_telemetry(data))) return DP_ERROR; - } - return DP_OK; - } // @@ -218,7 +203,7 @@ int dp_telemetry_init(void) } #endif - for (uint i = 0; i < RTE_DIM(commands); ++i) { + for (size_t i = 0; i < RTE_DIM(commands); ++i) { ret = rte_telemetry_register_cmd(commands[i].command, commands[i].callback, commands[i].description); if (DP_FAILED(ret)) { DPS_LOG_ERR("Failed to register telemetry command", DP_LOG_TELEMETRY_CMD(commands[i].command), DP_LOG_RET(ret)); diff --git a/src/grpc/dp_async_grpc.cpp b/src/grpc/dp_async_grpc.cpp index 41dbe49f6..5f424176c 100644 --- a/src/grpc/dp_async_grpc.cpp +++ b/src/grpc/dp_async_grpc.cpp @@ -223,6 +223,8 @@ const char* CreateInterfaceCall::FillRequest(struct dpgrpc_request* request) DP_LOG_IPV4STR(request_.ipv4_config().primary_address().c_str()), DP_LOG_IPV6STR(request_.ipv6_config().primary_address().c_str()), DP_LOG_PCI(request_.device_name().c_str())); + if (!GrpcConv::IsInterfaceIdValid(request_.interface_id())) + return "Invalid interface_id"; request->add_iface.vni = request_.vni(); if (!GrpcConv::StrToIpv4(request_.ipv4_config().primary_address(), &request->add_iface.ip4_addr)) return "Invalid ipv4_config.primary_address"; diff --git a/src/grpc/dp_grpc_conv.cpp b/src/grpc/dp_grpc_conv.cpp index 09d0b950c..c2f4713c1 100644 --- a/src/grpc/dp_grpc_conv.cpp +++ b/src/grpc/dp_grpc_conv.cpp @@ -12,6 +12,17 @@ Status* CreateStatus(uint32_t grpc_errcode) return err_status; } +bool IsInterfaceIdValid(const std::string& id) +{ + for (std::string::const_iterator i = id.begin(); i != id.end(); ++i) { + char c = *i; + // alphanumeric and underscore (allowed by standard DPDK) and '-' for GUID (dp-service extension) + if (!(c == '-' || c == '_' || (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'))) + return false; + } + return true; +} + bool StrToIpv4(const std::string& str, uint32_t *dst) { struct in_addr addr; diff --git a/test/test_telemetry.py b/test/test_telemetry.py new file mode 100644 index 000000000..2630322cc --- /dev/null +++ b/test/test_telemetry.py @@ -0,0 +1,56 @@ +import json +import pytest +from helpers import * + + +BUFSIZE = 10240 + +def get_telemetry(output, client, node): + client.send(f"{node},0\n".encode()) + output[node] = json.loads(client.recv(BURSTSIZE).decode())[node] + json.dumps(output[node]) + +def get_telemetry(request): + with socket.socket(socket.AF_UNIX, socket.SOCK_SEQPACKET) as client: + client.connect("/var/run/dpdk/rte/dpdk_telemetry.v2") + client.recv(BUFSIZE) + client.send(f"{request},0\n".encode()) + response = json.loads(client.recv(BUFSIZE).decode())[request] + client.close() + return response + + +def check_tel_graph(key): + tel = get_telemetry(f"/dp_service/graph/{key}") + assert tel is not None, \ + "Missing graph telemetry" + assert "Node_0_to_255" in tel, \ + f"Missing nodes in {key} graph telemetry" + assert "rx-0-0" in tel["Node_0_to_255"], \ + f"Missing PF0 Rx node in {key} graph telemetry" + +def test_telemetry_graph(prepare_ifaces): + check_tel_graph("obj_count") + check_tel_graph("call_count") + check_tel_graph("cycle_count") + check_tel_graph("realloc_count") + + +def test_telemetry_nat(prepare_ifaces): + tel = get_telemetry("/dp_service/nat/used_port_count") + assert tel is not None, \ + "Missing NAT telemetry" + assert VM1.name in tel and VM2.name in tel and VM3.name in tel, \ + "Running VMs not present in NAT telemetry" + + +def test_telemetry_virtsvc(request, prepare_ifaces): + if not request.config.getoption("--virtsvc"): + pytest.skip("Virtual services not enabled") + tel = get_telemetry("/dp_service/virtsvc/used_port_count") + assert tel is not None, \ + "Missing UDP virtual services telemetry" + assert f"UDP:{virtsvc_udp_virtual_ip}:{virtsvc_udp_virtual_port}" in tel, \ + "Missing UDP virtual service port count" + assert f"TCP:{virtsvc_tcp_virtual_ip}:{virtsvc_tcp_virtual_port}" in tel, \ + "Missing UDP virtual service port count" From 9b55b4c10afb076af5b74e9cd552b843822889f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jarom=C3=ADr=20Smr=C4=8Dek?= <4plague@gmail.com> Date: Mon, 9 Oct 2023 14:35:27 +0200 Subject: [PATCH 10/10] Fix Dockerfile's upgrade step --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bd72d82dc..dc9d1516c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG DPSERVICE_FEATURES="" WORKDIR /workspace # Install prerequisite packages -RUN apt-get update && apt-get upgrade && apt-get install -y --no-install-recommends ON \ +RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends ON \ libibverbs-dev \ libmnl-dev \ libnuma-dev \ @@ -126,7 +126,7 @@ ENTRYPOINT ["pytest-3", "-x", "-v"] FROM debian:12-slim as production -RUN apt-get update && apt-get upgrade && apt-get install -y --no-install-recommends ON \ +RUN apt-get update && apt-get upgrade -y && apt-get install -y --no-install-recommends ON \ libibverbs-dev \ numactl \ libnuma1 \