Add metrohash to nflowcache

src/inline/Makefile.am

@@ -1,3 +1,3 @@
 
 
-EXTRA_DIST = inline.c nfdump_inline.c nffile_inline.c
+EXTRA_DIST = inline.c nfdump_inline.c nffile_inline.c metrohash.c

src/nfpcapd/metrohash.c → src/inline/metrohash.c

@@ -25,15 +25,6 @@
 
 #include <stdint.h>
 
-// remember the last SlotSize packets with len and hash
-// for duplicate check
-#define SlotSize 8
-static struct {
-    uint32_t len;
-    uint64_t hash;
-} lastPacketStat[SlotSize] = {0};
-static uint32_t packetSlot = 0;
-
 /* rotate right idiom recognized by compiler*/
 inline static uint64_t rotate_right(uint64_t v, unsigned k) { return (v >> k) | (v << (64 - k)); }
 
@@ -126,7 +117,7 @@ static uint64_t metrohash64_1(const uint8_t *key, uint64_t len, uint32_t seed) {
     hash ^= rotate_right(hash, 33);
 
     return hash;
-}
+}  // End of metrohash64_1
 
 static uint64_t __attribute__((__unused__)) metrohash64_2(const uint8_t *key, uint64_t len, uint32_t seed) {
     static const uint64_t k0 = 0xD6D018F5;
@@ -208,18 +199,4 @@ static uint64_t __attribute__((__unused__)) metrohash64_2(const uint8_t *key, ui
     hash ^= rotate_right(hash, 29);
 
     return hash;
-}
-
-static int is_duplicate(const uint8_t *data_ptr, const uint32_t len) {
-    uint64_t hash = metrohash64_1(data_ptr, len, 0);
-
-    for (int i = 0; i < SlotSize; i++) {
-        if (lastPacketStat[i].len == len && lastPacketStat[i].hash == hash) return 1;
-    }
-
-    // not found - add to next slot round robin
-    lastPacketStat[packetSlot].len = len;
-    lastPacketStat[packetSlot].hash = hash;
-    packetSlot = (packetSlot + 1) & (SlotSize - 1);
-    return 0;
-}  // End of is_duplicate
+}  // End of metrohash64_2

src/nfdump/nflowcache.c

@@ -273,60 +273,8 @@ typedef struct FlowKeyV4_s {
 
 static inline int New_HashKey(void *keymem, recordHandle_t *recordHandle, int swap_flow);
 
-/*
- * hash definition and implementation
- * hash inspired by https://hackmd.io/@heyfey/SJZ-3jbs5 "Designing a Fast, Efficient, Cache-friendly Hash Table"
- * implementation optimized for flow data, without SSE registers (ARM compatibility)
- */
-
-// hash function to generate 32bit hash value from var length input hash key
-static inline uint32_t SuperFastHash(const char *data, int len) {
-    uint32_t hash = len;
-
-    if (hash == 0 || data == NULL) return 0;
-
-    int rem = len & 3;
-    len >>= 2;
-
-    // Main loop
-    uint32_t tmp;
-    for (; len > 0; len--) {
-        hash += *((uint16_t *)data);
-        tmp = (*((uint16_t *)(data + 2)) << 11) ^ hash;
-        hash = (hash << 16) ^ tmp;
-        data += 2 * sizeof(uint16_t);
-        hash += hash >> 11;
-    }
-
-    // Handle end cases
-    switch (rem) {
-        case 3:
-            hash += *((uint16_t *)data);
-            hash ^= hash << 16;
-            hash ^= data[sizeof(uint16_t)] << 18;
-            hash += hash >> 11;
-            break;
-        case 2:
-            hash += *((uint16_t *)data);
-            hash ^= hash << 11;
-            hash += hash >> 17;
-            break;
-        case 1:
-            hash += *data;
-            hash ^= hash << 10;
-            hash += hash >> 1;
-    }
-
-    // Force "avalanching" of final 127 bits
-    hash ^= hash << 3;
-    hash += hash >> 5;
-    hash ^= hash << 4;
-    hash += hash >> 17;
-    hash ^= hash << 25;
-    hash += hash >> 6;
-
-    return hash;
-}  // End of SuperFastHash
+// include hash function in same compiler unit
+#include "metrohash.c"
 
 // cell index calculation from 32bit hash, depending of hash bit size 'shift'
 #define ___fib_hash(hash, shift) ((hash) * 2654435769U) >> (shift)
@@ -353,8 +301,8 @@ typedef struct hashValue_s {
         uint64_t val[2];  // 16 byte static hash value
         void *valPtr;     // value pointer if size > 16bytes
     };
-    uint32_t allign;   // unused - 64bit alignment
-    uint32_t hash;     // calculated 32bit hash
+    uint32_t hash;     // calculated 32bit metrohash
+    uint32_t align;    // calculated 64bit metrohash
     uint32_t ptrSize;  // if > 0, valPtr points to value
     uint32_t index;    // index into record array for statistics values
 } hashValue_t;
@@ -1376,7 +1324,7 @@ static void AddBidirFlow(recordHandle_t *recordHandle) {
         keymem = (void *)hashValue.val;
     }
 
-    hashValue.hash = SuperFastHash(keymem, keyLen);
+    hashValue.hash = metrohash64_1(keymem, keyLen, 0);
 
     int index = flowHash_get(flowHash, hashValue);
     if (index >= 0) {
@@ -1422,7 +1370,7 @@ static void AddBidirFlow(recordHandle_t *recordHandle) {
 
         // generate reverse hash key to search for bidir flow
         New_HashKey(keymem, recordHandle, 1);
-        hashValue.hash = SuperFastHash(keymem, keyLen);
+        hashValue.hash = metrohash64_1(keymem, keyLen, 0);
 
         index = flowHash_get(flowHash, hashValue);
         if (index >= 0) {
@@ -1445,7 +1393,7 @@ static void AddBidirFlow(recordHandle_t *recordHandle) {
             // no bidir flow found
             // insert original flow into the cache
             New_HashKey(keymem, recordHandle, 0);
-            hashValue.hash = SuperFastHash(keymem, keyLen);
+            hashValue.hash = metrohash64_1(keymem, keyLen, 0);
 
             int insert;
             index = flowHash_add(flowHash, hashValue, &insert);
@@ -1530,7 +1478,7 @@ void AddFlowCache(recordHandle_t *recordHandle) {
         keymem = (void *)hashValue.val;
     }
 
-    hashValue.hash = SuperFastHash(keymem, keyLen);
+    hashValue.hash = metrohash64_1(keymem, keyLen, 0);
 
     int insert;
     int index = flowHash_add(flowHash, hashValue, &insert);

src/nfpcapd/pcaproc.c

@@ -93,6 +93,15 @@ typedef struct vlan_hdr_s {
     uint16_t type;
 } vlan_hdr_t;
 
+// remember the last SlotSize packets with len and hash
+// for duplicate check
+#define SlotSize 8
+static struct {
+    uint32_t len;
+    uint64_t hash;
+} lastPacketStat[SlotSize] = {0};
+static uint32_t packetSlot = 0;
+
 static time_t lastRun = 0;  // remember last run to idle cache
 
 static inline void SetServer_latency(struct FlowNode *node);
@@ -111,6 +120,20 @@ static inline void ProcessOtherFlow(packetParam_t *packetParam, struct FlowNode
 
 #include "metrohash.c"
 
+static int is_duplicate(const uint8_t *data_ptr, const uint32_t len) {
+    uint64_t hash = metrohash64_1(data_ptr, len, 0);
+
+    for (int i = 0; i < SlotSize; i++) {
+        if (lastPacketStat[i].len == len && lastPacketStat[i].hash == hash) return 1;
+    }
+
+    // not found - add to next slot round robin
+    lastPacketStat[packetSlot].len = len;
+    lastPacketStat[packetSlot].hash = hash;
+    packetSlot = (packetSlot + 1) & (SlotSize - 1);
+    return 0;
+}  // End of is_duplicate
+
 pcapfile_t *OpenNewPcapFile(pcap_t *p, char *filename, pcapfile_t *pcapfile) {
     if (!pcapfile) {
         // Create struct