diff --git a/CMakeLists.txt b/CMakeLists.txt index 79504346..42c140e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,7 +31,7 @@ set(COMPILE_ERPC_LIB ON) # Options exposed to the user set(TRANSPORT "dpdk" CACHE STRING "Datapath transport (infiniband/raw/dpdk)") option(ROCE "Use RoCE if TRANSPORT is infiniband" OFF) -option(PERF "Compile for performance" OFF) +option(PERF "Compile for performance" ON) set(PGO "none" CACHE STRING "Profile-guided optimization (generate/use/none)") set(LOG_LEVEL "warn" CACHE STRING "Logging level (none/error/warn/info/reorder/trace/cc)") cmake_dependent_option(LTO "Use link time optimization" ON "PERF" OFF) diff --git a/README.md b/README.md index 18636d8a..7bec9aaa 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ Some highlights: ## eRPC quickstart * Build and run the test suite: `cmake . -DPERF=OFF -DTRANSPORT=infiniband; make -j; sudo ctest`. + * `DPERF=OFF` enables debugging, which greatly reduces performance. Set + `DPERF=ON` for performance measurements. * Here, `infiniband` should be replaced with `raw` for Mellanox Ethernet NICs, or `dpdk` for Intel Ethernet NICs. * A machine with two ports is needed to run the unit tests if DPDK is chosen. @@ -76,7 +78,7 @@ Some highlights: * To build an application, create `scripts/autorun_app_file` and change its contents to one of the available directory names in `apps/`. See `scripts/example_autorun_app_file` for an example. Then generate a - Makefile using `cmake . -DPERF=ON/OFF -DTRANSPORT=raw/infiniband/dpdk`. + Makefile using `cmake . -DPERF=ON -DTRANSPORT=raw/infiniband/dpdk`. * Each application directory in `apps/` contains a config file that must specify all flags defined in `apps/apps_common.h`. For example, `num_processes` specifies the total number of eRPC processes in the cluster. diff --git a/scripts/topo/xl170-topo.sh b/scripts/topo/xl170-topo.sh index 1824016c..3f3d9fde 100755 --- a/scripts/topo/xl170-topo.sh +++ b/scripts/topo/xl170-topo.sh @@ -1,13 +1,24 @@ #!/usr/bin/env bash -# Prints the nodes under each switch in CloudLab's xl170 cluster -source $(dirname $0)/utils.sh +# +# * Prints the nodes under each switch in CloudLab's xl170 cluster +# +# * This script assumes that node n is named +# akalianode-n.RDMA.ron-PG0.utah.cloudlab.us +# +# * Usage: In a cluster with N experiment nodes, run ./xl170-topo.sh N +# + if [ "$#" -ne 1 ]; then echo "Illegal number of parameters" echo "Usage: ./xl170-topo.sh " exit fi -bad_nodes="hp097" +# Fill this up with nodes that you don't want. Example: +# bad_nodes = "hp097" +bad_nodes="" + +# We'll place the nodes under switch n in topodir/switch_n topodir="/tmp/xl170_topo" rm -rf $topodir mkdir $topodir @@ -18,12 +29,12 @@ for ((i = 1; i <= $1; i++)); do hostname="akalianode-$i.RDMA.ron-PG0.utah.cloudlab.us" # Get the HP node ID (e.g., hp012 from hp012.utah.cloudlab.us) - hpnode_id=`ssh -oStrictHostKeyChecking=no $hostname \ + hpnode_id=`ssh -oStrictHostKeyChecking=no $hostname \ "hostname -A | cut -d '.' -f 1"` # Ignore if node is bad if [[ $bad_nodes == *"$hpnode_id"* ]]; then - blue "Ignoring bad node $hpnode_id ($hostname)" + echo "Ignoring bad node $hpnode_id ($hostname)" else hpnode_id=`echo $hpnode_id | sed 's/hp0*//g'` # Trim leading hp and zeros switch_id=$(( ($hpnode_id - 1) / 40 )) @@ -34,13 +45,13 @@ for ((i = 1; i <= $1; i++)); do done wait -# Here, temp contains +# Here, file temp contains # Print out the nodes under each switch echo "" for ((switch_i = 0; switch_i < 5; switch_i++)); do nodes_file=$topodir/switch_$switch_i count=`cat temp | grep " $switch_i" | wc -l` - blue "Under switch $switch_i ($count nodes):" + echo "Under switch $switch_i ($count nodes):" cat temp | grep " $switch_i" | cut -d' ' -f 1 | sort -n > $nodes_file sed -e 's/$/ 31850 0/' -i $nodes_file # Append UDP port and NUMA node cat $nodes_file diff --git a/src/rpc.h b/src/rpc.h index ec69d993..cb41e118 100644 --- a/src/rpc.h +++ b/src/rpc.h @@ -95,8 +95,11 @@ class Rpc { * \param sm_handler The session management callback that is invoked when * sessions are successfully created or destroyed. * - * @param phy_port An Rpc object uses one physical port. This is the - * zero-based index of that port among active ports. + * @param phy_port An Rpc object uses one physical port on the NIC. phy_port + * is the zero-based index of that port among active ports, as listed by + * `ibv_devinfo` for Raw, InfiniBand, and RoCE transports; or by + * `dpdk-devbind` for DPDK transport. + * * @throw runtime_error if construction fails */ Rpc(Nexus *nexus, void *context, uint8_t rpc_id, sm_handler_t sm_handler, diff --git a/src/transport_impl/verbs_common.h b/src/transport_impl/verbs_common.h index 8eeacfe5..4fa5d565 100644 --- a/src/transport_impl/verbs_common.h +++ b/src/transport_impl/verbs_common.h @@ -228,7 +228,7 @@ static void common_resolve_phy_port(uint8_t phy_port, size_t mtu, // If we are here, port resolution has failed assert(resolve.ib_ctx == nullptr); - xmsg << "Failed to resolve RoCE port index " << std::to_string(phy_port); + xmsg << "Failed to resolve verbs port index " << std::to_string(phy_port); throw std::runtime_error(xmsg.str()); } } // namespace erpc