Upgrade DPDK to 20.11

- address memseg and mbuf changes
- move to meson/ninja build system

Signed-off-by: Saikrishna Edupuganti <[email protected]>

bessctl/conf/port/vhost/launch_vm.py

@@ -175,7 +175,7 @@ def run_forward(vm_id, num_nics):
  nics += ' 00:1{nic}.0'.format(nic=i)
 
  scp(vm_id, os.path.join(bess_dir, 'bin/dpdk-devbind.py'), '')
- scp(vm_id, os.path.join(bess_dir, 'deps/dpdk-19.11.4/build/app/testpmd'), '')
+ scp(vm_id, os.path.join(bess_dir, 'deps/dpdk-20.11/build/app/dpdk-testpmd'), '')
 
  # virtio-pci devices should not be bound to any driver
  cmd = ssh_cmd(vm_id, 'sudo ./dpdk-devbind.py -u %s' % nics)

build.py

@@ -89,14 +89,13 @@ def cmd(cmd, quiet=False, shell=False):
 DEPS_DIR = '%s/deps' % BESS_DIR
 
 DPDK_URL = 'https://fast.dpdk.org/rel'
-DPDK_VER = 'dpdk-19.11.4'
+DPDK_VER = 'dpdk-20.11'
 DPDK_TARGET = 'x86_64-native-linuxapp-gcc'
 
 kernel_release = cmd('uname -r', quiet=True).strip()
 
 DPDK_DIR = '%s/%s' % (DEPS_DIR, DPDK_VER)
-DPDK_CFLAGS = '"-g -w"'
-DPDK_CONFIG = '%s/build/.config' % DPDK_DIR
+DPDK_BUILD = '%s/build' % DPDK_DIR
 
 extra_libs = set()
 cxx_flags = []
@@ -213,46 +212,6 @@ def is_kernel_header_installed():
  return os.path.isdir("/lib/modules/%s/build" % kernel_release)
 
 
-def check_kernel_headers():
- # If kernel header is not available, do not attempt to build
- # any components that require kernel.
- if not is_kernel_header_installed():
- set_config(DPDK_CONFIG, 'CONFIG_RTE_EAL_IGB_UIO', 'n')
- set_config(DPDK_CONFIG, 'CONFIG_RTE_KNI_KMOD', 'n')
- set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_KNI', 'n')
- set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_PMD_KNI', 'n')
-
-
-def check_bnx():
- if check_header('zlib.h', 'gcc') and check_c_lib('z'):
- extra_libs.add('z')
- else:
- print(' - "zlib1g-dev" is not available. Disabling BNX2X PMD...')
- set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_BNX2X_PMD', 'n')
-
-
-def check_mlx():
- if check_header('infiniband/ib.h', 'gcc') and check_c_lib('mlx4') and \
- check_c_lib('mlx5'):
- extra_libs.add('ibverbs')
- extra_libs.add('mlx4')
- extra_libs.add('mlx5')
- else:
- print(' - "Mellanox OFED" is not available. '
- 'Disabling MLX4 and MLX5 PMDs...')
- if check_header('infiniband/verbs.h', 'gcc'):
- print(' NOTE: "libibverbs-dev" does exist, but it does not '
- 'work with MLX PMDs. Instead download OFED from '
- 'http://www.melloanox.com')
- set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_MLX4_PMD', 'n')
- set_config(DPDK_CONFIG, 'CONFIG_RTE_LIBRTE_MLX5_PMD', 'n')
-
-
-def generate_dpdk_extra_mk():
- with open('core/extra.dpdk.mk', 'w') as fp:
- fp.write('LIBS += %s\n' % ' '.join(['-l' + lib for lib in extra_libs]))
-
-
 def find_current_plugins():
  "return list of existing plugins"
  result = []
@@ -293,16 +252,14 @@ def download_dpdk(quiet=False):
 
 def configure_dpdk():
  print('Configuring DPDK...')
- cmd('make -C %s config T=%s' % (DPDK_DIR, DPDK_TARGET))
-
- check_kernel_headers()
- check_mlx()
- generate_dpdk_extra_mk()
+ meson_opts = '--buildtype=debugoptimized'
 
  arch = os.getenv('CPU')
  if arch:
  print(' - Building DPDK with -march=%s' % arch)
- set_config(DPDK_CONFIG, "CONFIG_RTE_MACHINE", arch)
+ meson_opts += ' -Dmachine=%s' % arch
+
+ cmd('meson %s %s %s' % (meson_opts, DPDK_BUILD, DPDK_DIR))
 
 
 def makeflags():
@@ -342,10 +299,7 @@ def build_dpdk():
  cmd('patch -d %s -N -p1 < %s || true' % (DPDK_DIR, f), shell=True)
 
  print('Building DPDK...')
- nproc = int(cmd('nproc', quiet=True))
- cmd('make -C %s EXTRA_CFLAGS=%s %s' % (DPDK_DIR,
- DPDK_CFLAGS,
- makeflags()))
+ cmd('ninja -C %s install' % DPDK_BUILD)
 
 
 def generate_protobuf_files():

core/Makefile

@@ -61,28 +61,13 @@ endif
 
 HAS_PKG_CONFIG := $(shell command -v $(PKG_CONFIG) 2>&1 >/dev/null && echo yes || echo no)
 
-RTE_SDK ?= $(abspath ../deps/dpdk-19.11.4)
-RTE_TARGET ?= $(shell uname -m)-native-linuxapp-gcc
-DPDK_LIB ?= dpdk
-
-ifneq ($(wildcard $(RTE_SDK)/$(RTE_TARGET)/*),)
- DPDK_INC_DIR := $(RTE_SDK)/$(RTE_TARGET)/include
- DPDK_LIB_DIR := $(RTE_SDK)/$(RTE_TARGET)/build/lib
-else ifneq ($(wildcard $(RTE_SDK)/build/*),)
- # if the user didn't do "make install" for DPDK
- DPDK_INC_DIR := $(RTE_SDK)/build/include
- DPDK_LIB_DIR := $(RTE_SDK)/build/lib
-else ifneq ($(MAKECMDGOALS),clean)
- $(error DPDK is not available. Make sure $(abspath $(RTE_SDK)) is available and built)
-endif
-
 # We always want these libraries to be dynamically linked even when the
 # user requests a static build.
 ALWAYS_DYN_LIBS := -lpthread -ldl
 # These libraries are not supported by pkg-config.
 ALWAYS_LIBS := -lpcap -lgflags -lnuma
 # If pkg-config is available, we just need a list of the dependecies.
-PKG_CONFIG_DEPS := libglog protobuf grpc++ libunwind zlib
+PKG_CONFIG_DEPS := libdpdk libglog protobuf grpc++ libunwind zlib
 # If pkg-config is not available, we need to list the libs we depend on.
 NO_PKG_CONFIG_LIBS := -lglog -lgflags -lprotobuf -lgrpc++ -lunwind -lz
 # If pkg-config is not available and we're static linking, we also need
@@ -110,7 +95,7 @@ endif
 COREDIR := $(abspath .)
 CPU ?= native
 CXXFLAGS += -std=c++17 -g3 -ggdb3 -march=$(CPU) \
- -isystem $(DPDK_INC_DIR) -isystem $(COREDIR) \
+ -isystem $(COREDIR) \
  -isystem $(dir $<).. -isystem $(COREDIR)/modules \
  -D_GNU_SOURCE \
  -Werror -Wall -Wextra -Wcast-align -Wno-error=deprecated-declarations \
@@ -133,7 +118,6 @@ ifeq "$(shell expr $(CXXCOMPILER) = g++)" "1"
  CXXFLAGS += -fno-gnu-unique
 endif
 
-LDFLAGS += -rdynamic -L$(DPDK_LIB_DIR) -Wl,-rpath=$(DPDK_LIB_DIR) -pthread
 ifdef BESS_LINK_DYNAMIC
  LIBS_ALL_SHARED = -Wl,-call_shared
  LIBS_DL_SHARED =
@@ -158,7 +142,6 @@ else # Used static libraries
 endif
 
 LIBS += -Wl,-non_shared \
- -Wl,--whole-archive -l$(DPDK_LIB) -Wl,--no-whole-archive \
  $(LIBS_ALL_SHARED) \
  $(PKG_LIBS) $(ALWAYS_LIBS) \
  $(LIBS_DL_SHARED) \

core/packet.h

@@ -261,20 +261,14 @@ class alignas(64) Packet {
  const uint16_t buf_len_;
 
  // offset 56:
- uint64_t _dummy6_; // rte_mbuf.timestamp
-
- // 2nd cacheline - fields only used in slow path or on TX --------------
- // offset 64:
- uint64_t _dummy7_; // rte_mbuf.userdata
-
- // offset 72:
  struct rte_mempool *pool_; // Pool from which mbuf was allocated.
 
- // offset 80:
+ // offset 60:
  Packet *next_; // Next segment. nullptr if not scattered.
 
  // offset 88:
  uint64_t _dummy8; // rte_mbuf.tx_offload
+ // TODO: Add struct rte_mbuf_ext_shared_info *shinfo;
  uint16_t _dummy9; // rte_mbuf.priv_size
  uint16_t _dummy10; // rte_mbuf.timesync
  uint32_t _dummy11; // rte_mbuf.seqn

core/packet_pool.cc

@@ -241,14 +241,14 @@ DpdkPacketPool::DpdkPacketPool(size_t capacity, int socket_id)
 
 static Packet *paddr_to_snb_memchunk(struct rte_mempool_memhdr *chunk,
  phys_addr_t paddr) {
- if (chunk->phys_addr == RTE_BAD_IOVA) {
+ if (chunk->iova == RTE_BAD_IOVA) {
  return nullptr;
  }
 
- if (chunk->phys_addr <= paddr && paddr < chunk->phys_addr + chunk->len) {
+ if (chunk->iova <= paddr && paddr < chunk->iova + chunk->len) {
  uintptr_t vaddr;
 
- vaddr = (uintptr_t)chunk->addr + paddr - chunk->phys_addr;
+ vaddr = (uintptr_t)chunk->addr + paddr - chunk->iova;
  return reinterpret_cast<Packet *>(vaddr);
  }
 

env/Dockerfile

@@ -29,7 +29,7 @@ ARG BESS_DPDK_BRANCH=master
 RUN cd /build/bess && \
  curl -s -L https://github.com/NetSys/bess/archive/${BESS_DPDK_BRANCH}.tar.gz | tar zx --strip-components=1 && \
  ./build.py dpdk && \
- cp /build/bess/deps/dpdk-19.11.4/build/app/testpmd /usr/local/bin/ && \
+ cp /build/bess/deps/dpdk-20.11/build/app/dpdk-testpmd /usr/local/bin/ && \
  rm -rf /build/bess
 
 ENV CCACHE_DIR=/tmp/ccache

env/build-dep.yml

@@ -23,9 +23,19 @@
  - libgflags-dev
  - libgoogle-glog-dev
  - libgtest-dev
- - python
+ - python3
+ - python3-pip
+ - python3-setuptools
  - pkg-config
 
+ - name: Install DPDK build system
+ become: true
+ pip:
+ name:
+ - meson
+ - ninja
+ executable: pip3
+
  # pre-packaged meat for Bionic Beaver or higher
  - name: Install gRPC and its requirements (apt)
  apt: name={{item}}