refs #15: Add "hack" for IPsec latency measurements

elements/ipsec/IPsecESPencap.cc

@@ -1,5 +1,6 @@
 #include "IPsecESPencap.hh"
 #include <nba/element/annotation.hh>
+#include <nba/framework/threadcontext.hh>
 #include <random>
 #include <nba/core/checksum.hh>
 #include <xmmintrin.h>
@@ -103,6 +104,14 @@ int IPsecESPencap::process(int input_port, Packet *pkt)
     uint8_t *encaped_iph = (uint8_t *) esph + sizeof(*esph);
     uint8_t *esp_trail = encaped_iph + ip_len;
 
+    // Hack for latency measurement experiments.
+    uintptr_t payload = 0;
+    __m128i timestamp;  // actual size: uin16 + uint64
+    if (ctx->preserve_latency) {
+        payload   = (uintptr_t) pkt->data() + sizeof(struct ether_hdr) + sizeof(struct iphdr);
+        timestamp = _mm_loadu_si128((__m128i *) payload);
+    }
+
     memmove(encaped_iph, iph, ip_len);          // copy the IP header and payload.
     memset(esp_trail, 0, pad_len);              // clear the padding.
     esp_trail[pad_len] = (uint8_t) pad_len;     // store pad_len at the second byte from last.
@@ -112,6 +121,11 @@ int IPsecESPencap::process(int input_port, Packet *pkt)
     esph->esp_spi = sa_entry->spi;
     esph->esp_rpl = sa_entry->rpl;
 
+    // Hack for latency measurement experiments.
+    if (ctx->preserve_latency) {
+        _mm_storeu_si128((__m128i *) payload, timestamp);
+    }
+
     // Generate random IV.
     uint64_t iv_first_half = rand();
     uint64_t iv_second_half = rand();

include/nba/framework/threadcontext.hh

@@ -158,6 +158,7 @@ public:
     unsigned num_batchpool_size;
     unsigned num_taskpool_size;
     unsigned task_completion_queue_size;
+    bool preserve_latency;
 
     struct rte_mempool *batch_pool;
     struct rte_mempool *dbstate_pool;

scripts/exprlib/env.py

@@ -233,26 +233,21 @@ def get_cpu_mask_with_nics(only_phys_cores=True):
         return core_bits
 
     @staticmethod
-    def mangle_main_args(config_name, click_name, emulate_opts=None, extra_args=None):
+    def mangle_main_args(config_name, click_name,
+                         emulate_opts=None,
+                         extra_args=None, extra_dpdk_args=None):
         args = [
             '-c', hex(ExperimentEnv.get_cpu_mask_with_nics()),
             '-n', os.environ.get('NBA_MEM_CHANNELS', '4'),
         ]
+        # TODO: translate emulate_opts to void_pmd options
+        if extra_dpdk_args:
+            args.extend(extra_dpdk_args)
+        args.append('--')
         if extra_args:
             args.extend(extra_args)
-        args.append('--')
         args.append(config_name)
         args.append(click_name)
-        if emulate_opts:
-            emulate_args = {
-                '--emulated-ipversion': 4,
-                '--emulated-pktsize': 64,
-                '--emulated-fixed-flows': 0,
-            }
-            emulate_args.update(emulate_opts)
-            args.append('--emulate-io')
-            for k, v in emulate_args.items():
-                args.append('{0}={1}'.format(k, v))
         return args
 
     @staticmethod
@@ -275,15 +270,16 @@ def chdir_to_root():
     def execute_main(self, config_name, click_name,
                      running_time=30.0,
                      emulate_opts=None,
-                     extra_args=None,
+                     extra_args=None, extra_dpdk_args=None,
                      custom_stdout_coro=None):
         '''
         Executes the main program asynchronosuly.
         '''
         self.chdir_to_root()
         config_path = os.path.normpath(os.path.join('configs', config_name))
         click_path = os.path.normpath(os.path.join('configs', click_name))
-        args = self.mangle_main_args(config_path, click_path, emulate_opts, extra_args)
+        args = self.mangle_main_args(config_path, click_path,
+                                     emulate_opts, extra_args, extra_dpdk_args)
 
         # Reset/initialize events.
         self._singalled = False

scripts/run_app_perf.py

@@ -40,6 +40,8 @@ async def do_experiment(loop, env, args, conds, thruput_reader):
     env.envvars['NBA_COMP_BATCH_SIZE'] = str(comp_batchsz)
     env.envvars['NBA_COPROC_PPDEPTH'] = str(coproc_ppdepth)
 
+    extra_nba_args = []
+
     if 'ipv6' in args.element_config_to_use:
         # All random ipv6 pkts
         pktgen.args = ['-i', 'all', '-f', '0', '-v', '6', '-p', str(pktsz)]
@@ -48,6 +50,7 @@ async def do_experiment(loop, env, args, conds, thruput_reader):
     elif 'ipsec' in args.element_config_to_use:
         # ipv4 pkts with fixed 1K flows
         pktgen.args = ['-i', 'all', '-f', '1024', '-r', '0', '-v', '4', '-p', str(pktsz)]
+        extra_nba_args.append('--preserve-latency')
         if args.latency:
             pktgen.args += ['-g', '3', '-l', '--latency-histogram']
     else:
@@ -56,8 +59,6 @@ async def do_experiment(loop, env, args, conds, thruput_reader):
         if args.latency:
             pktgen.args += ['-g', '10', '-l', '--latency-histogram']
 
-    #cpu_records     = env.measure_cpu_usage(interval=2, begin_after=26.0, repeat=True)
-
     # Clear data.
     env.reset_readers()
     thruput_reader.pktsize_hint = pktsz
@@ -81,7 +82,10 @@ async def do_experiment(loop, env, args, conds, thruput_reader):
             main_cmdargs = ['bin/main'] + env.mangle_main_args(config_path, click_path)
             retcode = await execute_async_simple(main_cmdargs, timeout=args.timeout)
         else:
-            retcode = await env.execute_main(args.sys_config_to_use, conf_name + '.click', running_time=32.0)
+            retcode = await env.execute_main(args.sys_config_to_use,
+                                             conf_name + '.click',
+                                             extra_args=extra_nba_args,
+                                             running_time=32.0)
 
     if args.latency:
         hist_task.cancel()

src/lib/io.cc

@@ -328,12 +328,12 @@ static void io_local_stat_timer_cb(struct ev_loop *loop, struct ev_timer *watche
         memset(&ctx->port_stats[j], 0, sizeof(struct io_port_stat));
     }
  #ifdef NBA_CPU_MICROBENCH
-	char buf[2048];
+    char buf[2048];
     char *bufp = &buf[0];
     for (int e = 0; e < 5; e++) {
         bufp += sprintf(bufp, "[worker:%02u].%d %'12lld, %'12lld, %'12lld\n", ctx->loc.core_id, e, ctx->papi_ctr_rx[e], ctx->papi_ctr_tx[e], ctx->papi_ctr_comp[e]);
     }
-	printf("%s", buf);
+    printf("%s", buf);
     memset(ctx->papi_ctr_rx, 0, sizeof(long long) * 5);
     memset(ctx->papi_ctr_tx, 0, sizeof(long long) * 5);
     memset(ctx->papi_ctr_comp, 0, sizeof(long long) * 5);

src/main.cc

@@ -159,11 +159,13 @@ int main(int argc, char **argv)
 
     /* Parse command-line arguments. */
     dummy_device = false;
+    bool preserve_latency = false;
     char *system_config = new char[PATH_MAX];
     char *pipeline_config = new char[PATH_MAX];
 
     struct option long_opts[] = {
-        {"dummy-device", optional_argument, NULL, 0},
+        {"dummy-device", no_argument, NULL, 0},
+        {"preserve-latency", no_argument, NULL, 0},
         {"loglevel", required_argument, NULL, 'l'},
         {0, 0, 0, 0}
     };
@@ -173,11 +175,11 @@ int main(int argc, char **argv)
         if (c == -1) break;
         switch (c) {
         case 0:
-            /* Skip flag-only options. */
-            if (long_opts[optidx].flag != NULL) break;
             /* Process {long_opts[optidx].name}:{optarg} kv pairs. */
             if (!strcmp("dummy-device", long_opts[optidx].name)) {
                 dummy_device = true;
+            } else if (!strcmp("preserve-latency", long_opts[optidx].name)) {
+                preserve_latency = true;
             }
             break;
         case 'l':
@@ -647,6 +649,7 @@ int main(int argc, char **argv)
             ctx->task_completion_queue_size = system_params["COPROC_COMPLETIONQ_LENGTH"];
             ctx->num_tx_ports = num_ports;
             ctx->num_nodes = num_nodes;
+            ctx->preserve_latency = preserve_latency;
 
             ctx->io_ctx = nullptr;
             ctx->coproc_ctx = nullptr;