Upgrade DPDK to 20.11

- address memseg and mbuf changes
- move to meson/ninja build system

Signed-off-by: Saikrishna Edupuganti <[email protected]>

bessctl/conf/port/vhost/launch_vm.py

@@ -175,7 +175,7 @@ def run_forward(vm_id, num_nics):
         nics += ' 00:1{nic}.0'.format(nic=i)
 
     scp(vm_id, os.path.join(bess_dir, 'bin/dpdk-devbind.py'), '')
-    scp(vm_id, os.path.join(bess_dir, 'deps/dpdk-19.11.4/build/app/testpmd'), '')
+    scp(vm_id, os.path.join(bess_dir, 'deps/dpdk-20.11/build/app/dpdk-testpmd'), '')
 
     # virtio-pci devices should not be bound to any driver
     cmd = ssh_cmd(vm_id, 'sudo ./dpdk-devbind.py -u %s' % nics)

build.py

@@ -89,14 +89,13 @@ def cmd(cmd, quiet=False, shell=False):
 DEPS_DIR = '%s/deps' % BESS_DIR
 
 DPDK_URL = 'https://fast.dpdk.org/rel'
-DPDK_VER = 'dpdk-19.11.4'
+DPDK_VER = 'dpdk-20.11'
 DPDK_TARGET = 'x86_64-native-linuxapp-gcc'
 
 kernel_release = cmd('uname -r', quiet=True).strip()
 
 DPDK_DIR = '%s/%s' % (DEPS_DIR, DPDK_VER)
-DPDK_CFLAGS = '"-g -w"'
-DPDK_CONFIG = '%s/build/.config' % DPDK_DIR
+DPDK_BUILD = '%s/build' % DPDK_DIR
 
 extra_libs = set()
 cxx_flags = []
@@ -213,46 +212,6 @@ def is_kernel_header_installed():
     return os.path.isdir("/lib/modules/%s/build" % kernel_release)
 
 
-def check_kernel_headers():
-    # If kernel header is not available, do not attempt to build
-    # any components that require kernel.
-    if not is_kernel_header_installed():
-        set_config(DPDK_CONFIG, 'CONFIG_RTE_EAL_IGB_UIO', 'n')
-        set_config(DPDK_CONFIG, 'CONFIG_RTE_KNI_KMOD', 'n')
-        set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_KNI', 'n')
-        set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_PMD_KNI', 'n')
-
-
-def check_bnx():
-    if check_header('zlib.h', 'gcc') and check_c_lib('z'):
-        extra_libs.add('z')
-    else:
-        print(' - "zlib1g-dev" is not available. Disabling BNX2X PMD...')
-        set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_BNX2X_PMD', 'n')
-
-
-def check_mlx():
-    if check_header('infiniband/ib.h', 'gcc') and check_c_lib('mlx4') and \
-            check_c_lib('mlx5'):
-        extra_libs.add('ibverbs')
-        extra_libs.add('mlx4')
-        extra_libs.add('mlx5')
-    else:
-        print(' - "Mellanox OFED" is not available. '
-              'Disabling MLX4 and MLX5 PMDs...')
-        if check_header('infiniband/verbs.h', 'gcc'):
-            print('   NOTE: "libibverbs-dev" does exist, but it does not '
-                  'work with MLX PMDs. Instead download OFED from '
-                  'http://www.melloanox.com')
-        set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_MLX4_PMD', 'n')
-        set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_MLX5_PMD', 'n')
-
-
-def generate_dpdk_extra_mk():
-    with open('core/extra.dpdk.mk', 'w') as fp:
-        fp.write('LIBS += %s\n' % ' '.join(['-l' + lib for lib in extra_libs]))
-
-
 def find_current_plugins():
     "return list of existing plugins"
     result = []
@@ -293,16 +252,14 @@ def download_dpdk(quiet=False):
 
 def configure_dpdk():
     print('Configuring DPDK...')
-    cmd('make -C %s config T=%s' % (DPDK_DIR, DPDK_TARGET))
-
-    check_kernel_headers()
-    check_mlx()
-    generate_dpdk_extra_mk()
+    meson_opts = '--buildtype=debugoptimized'
 
     arch = os.getenv('CPU')
     if arch:
         print(' - Building DPDK with -march=%s' % arch)
-        set_config(DPDK_CONFIG, "CONFIG_RTE_MACHINE", arch)
+        meson_opts += ' -Dmachine=%s' % arch
+
+    cmd('meson %s %s %s' % (meson_opts, DPDK_BUILD, DPDK_DIR))
 
 
 def makeflags():
@@ -342,10 +299,7 @@ def build_dpdk():
         cmd('patch -d %s -N -p1 < %s || true' % (DPDK_DIR, f), shell=True)
 
     print('Building DPDK...')
-    nproc = int(cmd('nproc', quiet=True))
-    cmd('make -C %s EXTRA_CFLAGS=%s %s' % (DPDK_DIR,
-                                           DPDK_CFLAGS,
-                                           makeflags()))
+    cmd('ninja -C %s install' % DPDK_BUILD)
 
 
 def generate_protobuf_files():

container_build.py

@@ -39,7 +39,7 @@
 import re
 import argparse
 
-IMAGE = 'nefelinetworks/bess_build:latest' + os.getenv('TAG_SUFFIX', '')
+IMAGE = 'nefelinetworks/bess_build:' + os.getenv('TAG_SUFFIX', 'latest')
 BESS_DIR_HOST = os.path.dirname(os.path.abspath(__file__))
 BESS_DIR_CONTAINER = '/build/bess'
 BUILD_SCRIPT = './build.py'

core/Makefile

@@ -61,28 +61,13 @@ endif
 
 HAS_PKG_CONFIG := $(shell command -v $(PKG_CONFIG) 2>&1 >/dev/null && echo yes || echo no)
 
-RTE_SDK ?= $(abspath ../deps/dpdk-19.11.4)
-RTE_TARGET ?= $(shell uname -m)-native-linuxapp-gcc
-DPDK_LIB ?= dpdk
-
-ifneq ($(wildcard $(RTE_SDK)/$(RTE_TARGET)/*),)
-  DPDK_INC_DIR := $(RTE_SDK)/$(RTE_TARGET)/include
-  DPDK_LIB_DIR := $(RTE_SDK)/$(RTE_TARGET)/build/lib
-else ifneq ($(wildcard $(RTE_SDK)/build/*),)
-  # if the user didn't do "make install" for DPDK
-  DPDK_INC_DIR := $(RTE_SDK)/build/include
-  DPDK_LIB_DIR := $(RTE_SDK)/build/lib
-else ifneq ($(MAKECMDGOALS),clean)
-  $(error DPDK is not available. Make sure $(abspath $(RTE_SDK)) is available and built)
-endif
-
 # We always want these libraries to be dynamically linked even when the
 # user requests a static build.
 ALWAYS_DYN_LIBS := -lpthread -ldl
 # These libraries are not supported by pkg-config.
 ALWAYS_LIBS := -lpcap -lgflags -lnuma
 # If pkg-config is available, we just need a list of the dependecies.
-PKG_CONFIG_DEPS := libglog protobuf grpc++ libunwind zlib
+PKG_CONFIG_DEPS := libdpdk libglog protobuf grpc++ libunwind zlib
 # If pkg-config is not available, we need to list the libs we depend on.
 NO_PKG_CONFIG_LIBS := -lglog -lgflags -lprotobuf -lgrpc++ -lunwind -lz
 # If pkg-config is not available and we're static linking, we also need
@@ -110,7 +95,7 @@ endif
 COREDIR := $(abspath .)
 CPU ?= native
 CXXFLAGS += -std=c++17 -g3 -ggdb3 -march=$(CPU) \
-            -isystem $(DPDK_INC_DIR) -isystem $(COREDIR) \
+            -isystem $(COREDIR) \
             -isystem $(dir $<).. -isystem $(COREDIR)/modules \
             -D_GNU_SOURCE \
             -Werror -Wall -Wextra -Wcast-align -Wno-error=deprecated-declarations \
@@ -133,7 +118,7 @@ ifeq "$(shell expr $(CXXCOMPILER) = g++)" "1"
   CXXFLAGS += -fno-gnu-unique
 endif
 
-LDFLAGS += -rdynamic -L$(DPDK_LIB_DIR) -Wl,-rpath=$(DPDK_LIB_DIR) -pthread
+LDFLAGS += -rdynamic
 ifdef BESS_LINK_DYNAMIC
   LIBS_ALL_SHARED = -Wl,-call_shared
   LIBS_DL_SHARED =
@@ -158,7 +143,6 @@ else # Used static libraries
 endif
 
 LIBS += -Wl,-non_shared \
-        -Wl,--whole-archive -l$(DPDK_LIB) -Wl,--no-whole-archive \
         $(LIBS_ALL_SHARED) \
         $(PKG_LIBS) $(ALWAYS_LIBS) \
         $(LIBS_DL_SHARED) \

core/packet.h

@@ -261,20 +261,14 @@ class alignas(64) Packet {
       const uint16_t buf_len_;
 
       // offset 56:
-      uint64_t _dummy6_;  // rte_mbuf.timestamp
-
-      // 2nd cacheline - fields only used in slow path or on TX --------------
-      // offset 64:
-      uint64_t _dummy7_;  // rte_mbuf.userdata
-
-      // offset 72:
       struct rte_mempool *pool_;  // Pool from which mbuf was allocated.
 
-      // offset 80:
+      // offset 60:
       Packet *next_;  // Next segment. nullptr if not scattered.
 
       // offset 88:
       uint64_t _dummy8;   // rte_mbuf.tx_offload
+      // TODO: Add struct rte_mbuf_ext_shared_info *shinfo;
       uint16_t _dummy9;   // rte_mbuf.priv_size
       uint16_t _dummy10;  // rte_mbuf.timesync
       uint32_t _dummy11;  // rte_mbuf.seqn

core/packet_pool.cc

@@ -241,14 +241,14 @@ DpdkPacketPool::DpdkPacketPool(size_t capacity, int socket_id)
 
 static Packet *paddr_to_snb_memchunk(struct rte_mempool_memhdr *chunk,
                                      phys_addr_t paddr) {
-  if (chunk->phys_addr == RTE_BAD_IOVA) {
+  if (chunk->iova == RTE_BAD_IOVA) {
     return nullptr;
   }
 
-  if (chunk->phys_addr <= paddr && paddr < chunk->phys_addr + chunk->len) {
+  if (chunk->iova <= paddr && paddr < chunk->iova + chunk->len) {
     uintptr_t vaddr;
 
-    vaddr = (uintptr_t)chunk->addr + paddr - chunk->phys_addr;
+    vaddr = (uintptr_t)chunk->addr + paddr - chunk->iova;
     return reinterpret_cast<Packet *>(vaddr);
   }
 

env/Dockerfile

@@ -29,7 +29,7 @@ ARG BESS_DPDK_BRANCH=master
 RUN cd /build/bess && \
 	curl -s -L https://github.com/NetSys/bess/archive/${BESS_DPDK_BRANCH}.tar.gz | tar zx --strip-components=1 && \
 	./build.py dpdk && \
-	cp /build/bess/deps/dpdk-19.11.4/build/app/testpmd /usr/local/bin/ && \
+	cp /build/bess/deps/dpdk-20.11/build/app/dpdk-testpmd /usr/local/bin/ && \
 	rm -rf /build/bess
 
 ENV CCACHE_DIR=/tmp/ccache

env/build-dep.yml

@@ -23,9 +23,19 @@
         - libgflags-dev
         - libgoogle-glog-dev
         - libgtest-dev
-        - python
+        - python3
+        - python3-pip
+        - python3-setuptools
         - pkg-config
 
+    - name: Install DPDK build system
+      become: true
+      pip:
+        name:
+          - meson
+          - ninja
+        executable: pip3
+
     # pre-packaged meat for Bionic Beaver or higher
     - name: Install gRPC and its requirements (apt)
       apt: name={{item}}