Implement #543 and moved fragment flags

man/nfdump.1

@@ -832,6 +832,9 @@ True if source and destination IP of a record are IPv4 IPs.
 .It Cm ipv6
 True if source and destination IP of a record are IPv6 IPs.
 .Pp
+.It Cm ttl Ar comp num
+True if IP ttl matches comparison.
+.Pp
 .It Cm proto Ar protocol
 True if the record protocol field matches
 .Ar protocol. protocol
@@ -1684,6 +1687,8 @@ Destination Address(country code):Port
 Source Port
 .It Cm %dp
 Destination Port
+.It Cm %ttl
+IP ttl
 .It Cm %it
 ICMP-type
 .It Cm %ic

src/collector/pcap_reader.c

@@ -301,7 +301,7 @@ static ssize_t decode_packet(struct pcap_pkthdr *hdr, u_char *pcap_pkgdata, void
     /* check header length */
     if (ip->ip_hl < 5) {
         LogError("bad-hlen %d", ip->ip_hl);
-        return 0;
+        return -1;
     }
 
     // add IP header length

src/libnfdump/filter/grammar.y

@@ -91,6 +91,8 @@ static int AddFlagsString(direction_t direction, char *flags);
 
 static int AddTosNumber(direction_t direction, uint16_t comp, uint64_t tos);
 
+static int AddIPttl(uint16_t comp, uint64_t ttl);
+
 static int AddPackets(direction_t direction, uint16_t comp, uint64_t packets);
 
 static int AddBytes(direction_t direction, uint16_t comp, uint64_t bytes);
@@ -172,7 +174,7 @@ static int AddASList(direction_t direction, void *U64List);
 
 %token EQ LT GT LE GE
 %token ANY NOT IDENT COUNT
-%token IP IPV4 IPV6 NET
+%token IP IPV4 IPV6 IPTTL NET
 %token SRC DST IN OUT PREV NEXT BGP ROUTER INGRESS EGRESS
 %token NAT XLATE TUN
 %token ENGINE ENGINETYPE ENGINEID EXPORTER
@@ -275,13 +277,17 @@ term:	ANY { /* this is an unconditionally true expression, as a filter applies i
 	  $$.self = AddTosNumber($1.direction, $3.comp, $4); if ( $$.self < 0 ) YYABORT;
 	}
 
+	| IPTTL comp NUMBER {
+	  $$.self = AddIPttl($2.comp, $3); if ( $$.self < 0 ) YYABORT;
+	}
+
 	| FWDSTAT comp NUMBER {
 	  $$.self = AddFwdStatNum($2.comp, $3); if ( $$.self < 0 ) YYABORT;
 	}
 
 	| FWDSTAT STRING {
 	  $$.self = AddFwdStatString($2); if ( $$.self < 0 ) YYABORT;
-        }
+  }
 
 	| DURATION comp NUMBER {
 		$$.self = NewElement(EXgenericFlowID, 0, SIZEmsecLast, $3, $2.comp, FUNC_DURATION, NULLPtr); 
@@ -893,6 +899,15 @@ static int AddTosNumber(direction_t direction, uint16_t comp, uint64_t tos) {
 	return ret;
 } // End of AddTosNumber
 
+static int AddIPttl(uint16_t comp, uint64_t ttl) {
+	if ( ttl > 255 ) {
+		yyerror("TTL number out of range");
+		return -1;
+  }
+
+	return NewElement(EXipInfoID, OFFipTTL, SIZEipTTL, ttl, comp, FUNC_NONE, NULLPtr);
+} // End of AddIPttl
+
 static int AddPackets(direction_t direction, uint16_t comp, uint64_t packets) {
 
 	int ret = -1;

src/libnfdump/filter/scanner.l

@@ -121,6 +121,7 @@ H				(0X|0x)[0-9A-Fa-f]+
 any				{ return ANY;  }
 ipv4			{ return IPV4; }
 ipv6			{ return IPV6; }
+ttl			  { return IPTTL; }
 ident			{ return IDENT; }
 count			{ return COUNT; }
 cnt			  { return COUNT; }

src/libnffile/nfxV3.h

@@ -239,10 +239,7 @@ typedef struct EXflowMisc_s {
     uint8_t flowEndReason;
 #define OFFflowEndReason offsetof(EXflowMisc_t, flowEndReason)
 #define SIZEflowEndReason MemberSize(EXflowMisc_t, flowEndReason)
-    uint8_t revTcpFlags;
-    uint8_t fragmentFlags;
-#define OFFfragmentFlags offsetof(EXflowMisc_t, fragmentFlags)
-#define SIZEfragmentFlags MemberSize(EXflowMisc_t, fragmentFlags)
+    uint16_t align;
 } EXflowMisc_t;
 #define EXflowMiscSize (sizeof(EXflowMisc_t) + sizeof(elementHeader_t))
 
@@ -686,8 +683,22 @@ typedef struct EXnokiaNatString_s {
 } EXnokiaNatString_t;
 #define EXnokiaNatStringSize (sizeof(EXnokiaNatString_t) + sizeof(elementHeader_t))
 
+#define EXipInfoID 42
+typedef struct EXipInfo_s {
+    uint8_t ttl;
+#define OFFipTTL offsetof(EXipInfo_t, ttl)
+#define SIZEipTTL MemberSize(EXipInfo_t, ttl)
+#define flagMF 0x20
+#define flagDF 0x40
+    uint8_t fragmentFlags;
+#define OFFfragmentFlags offsetof(EXipInfo_t, fragmentFlags)
+#define SIZEfragmentFlags MemberSize(EXipInfo_t, fragmentFlags)
+    uint16_t align;  // unused for nfpcapd
+} EXipInfo_t;
+#define EXipInfoSize (sizeof(EXipInfo_t) + sizeof(elementHeader_t))
+
 // max possible elements
-#define MAXEXTENSIONS 42
+#define MAXEXTENSIONS 43
 
 // push a fixed length extension to the v3 record
 // h v3 record header
@@ -747,16 +758,17 @@ static const struct extensionTable_s {
     uint32_t id;    // id number
     uint32_t size;  // number of bytes incl. header, 0xFFFF for dyn length
     char *name;     // name of extension
-} extensionTable[] = {
-    {0, 0, "EXnull"},          EXTENSION(EXgenericFlow),   EXTENSION(EXipv4Flow),     EXTENSION(EXipv6Flow),     EXTENSION(EXflowMisc),
-    EXTENSION(EXcntFlow),      EXTENSION(EXvLan),          EXTENSION(EXasRouting),    EXTENSION(EXbgpNextHopV4), EXTENSION(EXbgpNextHopV6),
-    EXTENSION(EXipNextHopV4),  EXTENSION(EXipNextHopV6),   EXTENSION(EXipReceivedV4), EXTENSION(EXipReceivedV6), EXTENSION(EXmplsLabel),
-    EXTENSION(EXmacAddr),      EXTENSION(EXasAdjacent),    EXTENSION(EXlatency),      EXTENSION(EXsamplerInfo),  EXTENSION(EXnselCommon),
-    EXTENSION(EXnatXlateIPv4), EXTENSION(EXnatXlateIPv6),  EXTENSION(EXnatXlatePort), EXTENSION(EXnselAcl),      EXTENSION(EXnselUser),
-    EXTENSION(EXnatCommon),    EXTENSION(EXnatPortBlock),  EXTENSION(EXnbarApp),      EXTENSION(EXlabel),        EXTENSION(EXinPayload),
-    EXTENSION(EXoutPayload),   EXTENSION(EXtunIPv4),       EXTENSION(EXtunIPv6),      EXTENSION(EXobservation),  EXTENSION(EXinmonMeta),
-    EXTENSION(EXinmonFrame),   EXTENSION(EXvrf),           EXTENSION(EXpfinfo),       EXTENSION(EXlayer2),       EXTENSION(EXflowId),
-    EXTENSION(EXnokiaNat),     EXTENSION(EXnokiaNatString)};
+} extensionTable[] = {{0, 0, "EXnull"},          EXTENSION(EXgenericFlow),    EXTENSION(EXipv4Flow),     EXTENSION(EXipv6Flow),
+                      EXTENSION(EXflowMisc),     EXTENSION(EXcntFlow),        EXTENSION(EXvLan),         EXTENSION(EXasRouting),
+                      EXTENSION(EXbgpNextHopV4), EXTENSION(EXbgpNextHopV6),   EXTENSION(EXipNextHopV4),  EXTENSION(EXipNextHopV6),
+                      EXTENSION(EXipReceivedV4), EXTENSION(EXipReceivedV6),   EXTENSION(EXmplsLabel),    EXTENSION(EXmacAddr),
+                      EXTENSION(EXasAdjacent),   EXTENSION(EXlatency),        EXTENSION(EXsamplerInfo),  EXTENSION(EXnselCommon),
+                      EXTENSION(EXnatXlateIPv4), EXTENSION(EXnatXlateIPv6),   EXTENSION(EXnatXlatePort), EXTENSION(EXnselAcl),
+                      EXTENSION(EXnselUser),     EXTENSION(EXnatCommon),      EXTENSION(EXnatPortBlock), EXTENSION(EXnbarApp),
+                      EXTENSION(EXlabel),        EXTENSION(EXinPayload),      EXTENSION(EXoutPayload),   EXTENSION(EXtunIPv4),
+                      EXTENSION(EXtunIPv6),      EXTENSION(EXobservation),    EXTENSION(EXinmonMeta),    EXTENSION(EXinmonFrame),
+                      EXTENSION(EXvrf),          EXTENSION(EXpfinfo),         EXTENSION(EXlayer2),       EXTENSION(EXflowId),
+                      EXTENSION(EXnokiaNat),     EXTENSION(EXnokiaNatString), EXTENSION(EXipInfo)};
 
 typedef struct record_map_s {
     recordHeaderV3_t *recordHeader;

src/netflow/ipfix.c

@@ -183,7 +183,8 @@ static const struct ipfixTranslationMap_s {
     {IPFIX_flowDirection, SIZEdir, NumberCopy, EXflowMiscID, OFFdir, STACK_NONE, "flow direction"},
     {IPFIX_biflowDirection, SIZEbiFlowDir, NumberCopy, EXflowMiscID, OFFbiFlowDir, STACK_NONE, "biFlow direction"},
     {IPFIX_flowEndReason, SIZEflowEndReason, NumberCopy, EXflowMiscID, OFFflowEndReason, STACK_NONE, "Flow end reason"},
-    {IPFIX_fragmentFlags, SIZEfragmentFlags, NumberCopy, EXflowMiscID, OFFfragmentFlags, STACK_NONE, "IP fragment flags"},
+    {IPFIX_ipTTL, SIZEipTTL, NumberCopy, EXipInfoID, OFFipTTL, STACK_NONE, "IP ttl"},
+    {IPFIX_fragmentFlags, SIZEfragmentFlags, NumberCopy, EXipInfoID, OFFfragmentFlags, STACK_NONE, "IP fragment flags"},
     {IPFIX_ipNextHopIPv6Address, SIZENextHopV6IP, NumberCopy, EXipNextHopV6ID, OFFNextHopV6IP, STACK_NONE, "IPv6 next hop IP"},
     {IPFIX_bgpNextHopIPv6Address, SIZEbgp6NextIP, NumberCopy, EXbgpNextHopV6ID, OFFbgp6NextIP, STACK_NONE, "IPv6 bgp next hop IP"},
     {IPFIX_mplsTopLabelStackSection, SIZEmplsLabel1, NumberCopy, EXmplsLabelID, OFFmplsLabel1, STACK_NONE, "mpls label 1"},

src/netflow/ipfix.h

@@ -294,6 +294,7 @@ typedef struct ipfix_template_elements_e_s {
 #define IPFIX_icmpTypeV6 178
 #define IPFIX_icmpCodeV6 179
 
+#define IPFIX_ipTTL 192
 #define IPFIX_fragmentFlags 197
 
 #define IPFIX_postNATSourceIPv4Address 225

src/netflow/netflow_v9.c

@@ -167,7 +167,9 @@ static const struct v9TranslationMap_s {
     {NF9_IPV6_DST_MASK, SIZEdstMask, NumberCopy, EXflowMiscID, OFFdstMask, STACK_NONE, "dst mask bits"},
     {NF9_ICMP, Stack_ONLY, NumberCopy, EXgenericFlowID, OFFdstPort, STACK_ICMP, "icmp type/code"},
     {NF9_DST_TOS, SIZEdstTos, NumberCopy, EXflowMiscID, OFFdstTos, STACK_NONE, "post IP class of Service"},
-    {NF_F_fragmentFlags, SIZEfragmentFlags, NumberCopy, EXflowMiscID, OFFfragmentFlags, STACK_NONE, "IP fragment flags"},
+    {NF_F_flowEndReason, SIZEflowEndReason, NumberCopy, EXflowMiscID, OFFflowEndReason, STACK_NONE, "Flow end reason"},
+    {NF_F_ipTTL, SIZEipTTL, NumberCopy, EXipInfoID, OFFipTTL, STACK_NONE, "IP ttl"},
+    {NF_F_fragmentFlags, SIZEfragmentFlags, NumberCopy, EXipInfoID, OFFfragmentFlags, STACK_NONE, "IP fragment flags"},
     {NF9_IN_SRC_MAC, SIZEinSrcMac, NumberCopy, EXmacAddrID, OFFinSrcMac, STACK_NONE, "in src MAC addr"},
     {NF9_OUT_DST_MAC, SIZEoutDstMac, NumberCopy, EXmacAddrID, OFFoutDstMac, STACK_NONE, "out dst MAC addr"},
     {NF9_SRC_VLAN, SIZEvlanID, NumberCopy, EXvLanID, OFFvlanID, STACK_NONE, "src VLAN ID"},

src/netflow/netflow_v9.h

@@ -226,6 +226,7 @@ typedef struct common_header_s {
 // IPFIX elements in v9 ( # > 127 )
 #define NF_F_BGP_ADJ_NEXT_AS 128
 #define NF_F_BGP_ADJ_PREV_AS 129
+#define NF_F_flowEndReason 136
 #define NF_F_dot1qVlanId 243
 #define NF_F_postDot1qVlanId 254
 
@@ -245,6 +246,7 @@ typedef struct common_header_s {
 #define NF_F_ICMP_TYPE_IPV6 178
 #define NF_F_ICMP_CODE_IPV6 179
 
+#define NF_F_ipTTL 192
 #define NF_F_fragmentFlags 197
 
 #define NF_F_XLATE_SRC_ADDR_IPV4 225

src/nfcapd/nfcapd.c

@@ -328,6 +328,11 @@ static void run(packet_function_t receive_packet, int socket, int pfd, int rfd,
 
             // in case of reading from file EOF => -2
             if (cnt == -2) done = 1;
+            if (cnt == 0) {
+                ignored_packets++;
+                packets++;
+                continue;
+            }
 #else
             cnt = recvfrom(socket, in_buff, NETWORK_INPUT_BUFF_SIZE, 0, (struct sockaddr *)&nf_sender, &nf_sender_size);
 #endif

src/nfdump/nflowcache.c

@@ -148,6 +148,7 @@ static struct aggregationElement_s {
                         {"srcgeo", {EXlocal, OFFgeoSrcIP, SizeGEOloc, 0}, 0, SRC_GEO, 0, 0, "%sc"},
                         {"dstgeo", {EXlocal, OFFgeoDstIP, SizeGEOloc, 0}, 0, DST_GEO, 0, 0, "%dc"},
                         {"ethertype", {EXlayer2ID, OFFetherType, SIZEetherType, 0}, 0, NOPREPROCESS, 0, 0, "%eth"},
+                        {"ttl", {EXipInfoID, OFFipTTL, SIZEipTTL, 0}, 0, NOPREPROCESS, 0, 0, "%ttl"},
                         {NULL, {0, 0, 0}, 0, NOPREPROCESS, 0, 0, NULL}};
 
 // FlowHash stat record, to aggregate flow counters in -A or -s stat/aggregate mode

src/nfdump/nfstat.c

@@ -219,8 +219,10 @@ static struct StatParameter_s {
     // {"exace", "Egress ACL", {EXnselAclID, OFFegressAcl, SIZEegressAcl, 0}, IS_HEX, NULL},
     {"ivrf", "I-vrf ID", {EXvrfID, OFFingressVrf, SIZEingressVrf, 0}, IS_NUMBER, NULL},
     {"evrf", "E-vrf ID", {EXvrfID, OFFegressVrf, SIZEegressVrf, 0}, IS_NUMBER, NULL},
+    {"ttl", "TTL", {EXipInfoID, OFFipTTL, SIZEipTTL, 0}, IS_NUMBER, NULL},
 
-    {NULL, NULL, {0, 0, 0, 0}, 0, NULL}};
+    // header info != NULL
+    {NULL, "DONE", {0, 0, 0, 0}, 0, NULL}};
 
 // key record in hash for element stat
 typedef struct hashkey_s {

src/nfpcapd/flowdump.c

@@ -134,6 +134,11 @@ static int StorePcapFlow(flowParam_t *flowParam, struct FlowNode *Node) {
         }
 
         if (flowParam->extendedFlow) {
+            UpdateRecordSize(EXipInfoSize);
+            PushExtension(recordHeader, EXipInfo, ipInfo);
+            ipInfo->ttl = Node->ttl;
+            ipInfo->fragmentFlags = Node->fragmentFlags;
+
             if (Node->vlanID) {
                 UpdateRecordSize(EXvLanSize);
                 PushExtension(recordHeader, EXvLan, vlan);

src/nfpcapd/flowtree.h

@@ -117,7 +117,9 @@ struct FlowNode {
     void *pflog;
     void *payload;         // payload
     uint32_t payloadSize;  // Size of payload
-    uint32_t fragmentFlags;
+    uint8_t ttl;
+    uint8_t fragmentFlags;
+    uint16_t align;
     uint32_t mpls[10];
     uint64_t srcMac;
     uint64_t dstMac;

src/nfpcapd/pcaproc.c

@@ -845,6 +845,8 @@ int ProcessPacket(packetParam_t *packetParam, const struct pcap_pkthdr *hdr, con
         Node->t_last.tv_sec = hdr->ts.tv_sec;
         Node->t_last.tv_usec = hdr->ts.tv_usec;
         Node->bytes = ntohs(ip6->ip6_plen) + size_ip;
+        Node->ttl = ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim;
+        Node->fragmentFlags = 0;
 
         // keep compiler happy - gets optimized out anyway
         void *p = (void *)&ip6->ip6_src;
@@ -907,7 +909,7 @@ int ProcessPacket(packetParam_t *packetParam, const struct pcap_pkthdr *hdr, con
             eodata = dataptr + Node->payloadSize;
             Node->payload = NULL;
             Node->payloadSize = 0;
-            Node->fragmentFlags |= IP_MF;
+            Node->fragmentFlags |= flagMF;
         } else {
             if (!Node) Node = New_Node();
             Node->flowKey.version = AF_INET;
@@ -916,11 +918,12 @@ int ProcessPacket(packetParam_t *packetParam, const struct pcap_pkthdr *hdr, con
             Node->t_last.tv_sec = hdr->ts.tv_sec;
             Node->t_last.tv_usec = hdr->ts.tv_usec;
             Node->bytes = ntohs(ip->ip_len);
-            Node->fragmentFlags |= (ip_off & IP_DF);
+            if (ip_off & IP_DF) Node->fragmentFlags |= flagDF;
 
             Node->flowKey.src_addr.v4 = ntohl(ip->ip_src.s_addr);
             Node->flowKey.dst_addr.v4 = ntohl(ip->ip_dst.s_addr);
         }
+        Node->ttl = ip->ip_ttl;
     } else {
         dbg_printf("ProcessPacket() Unsupported protocol version: %i\n", version);
         packetParam->proc_stat.unknown++;

src/nfreplay/send_v9.c

@@ -357,21 +357,6 @@ static outTemplate_t *GetOutputTemplate(recordHandle_t *recordHandle) {
                 count++;
                 data_length += 4;
                 break;
-            case EXlayer2ID:
-                flowset->field[count].type = htons(NF_F_dot1qVlanId);
-                flowset->field[count].length = htons(2);
-                count++;
-                flowset->field[count].type = htons(NF_F_postDot1qVlanId);
-                flowset->field[count].length = htons(2);
-                count++;
-                flowset->field[count].type = htons(NF_F_dot1qCustomerVlanId);
-                flowset->field[count].length = htons(2);
-                count++;
-                flowset->field[count].type = htons(NF_F_postDot1qCustomerVlanId);
-                flowset->field[count].length = htons(2);
-                count++;
-                data_length += 8;
-                break;
             case EXasRoutingID:
                 flowset->field[count].type = htons(NF9_SRC_AS);
                 flowset->field[count].length = htons(4);
@@ -462,6 +447,21 @@ static outTemplate_t *GetOutputTemplate(recordHandle_t *recordHandle) {
                 count++;
                 data_length += 8;
                 break;
+            case EXlayer2ID:
+                flowset->field[count].type = htons(NF_F_dot1qVlanId);
+                flowset->field[count].length = htons(2);
+                count++;
+                flowset->field[count].type = htons(NF_F_postDot1qVlanId);
+                flowset->field[count].length = htons(2);
+                count++;
+                flowset->field[count].type = htons(NF_F_dot1qCustomerVlanId);
+                flowset->field[count].length = htons(2);
+                count++;
+                flowset->field[count].type = htons(NF_F_postDot1qCustomerVlanId);
+                flowset->field[count].length = htons(2);
+                count++;
+                data_length += 8;
+                break;
         }
     }
 

src/output/output_csv.c

@@ -183,6 +183,10 @@ static void String_BiFlowDir(FILE *stream, recordHandle_t *recordHandle);
 
 static void String_FlowEndReason(FILE *stream, recordHandle_t *recordHandle);
 
+static void String_ipTTL(FILE *stream, recordHandle_t *recordHandle);
+
+static void String_ipFrag(FILE *stream, recordHandle_t *recordHandle);
+
 static void String_Flags(FILE *stream, recordHandle_t *recordHandle);
 
 static void String_InSrcMac(FILE *stream, recordHandle_t *recordHandle);
@@ -375,6 +379,7 @@ static struct format_entry_s {
     {"%dtos", 0, "dstTos", String_DstTos},           // Tos - Dst tos
     {"%bfd", 0, "biDirection", String_BiFlowDir},    // BiFlow Direction
     {"%end", 0, "endReason", String_FlowEndReason},  // Flow End Reason
+
     //
     {"%sn", 1, "srcNet", String_SrcNet},           // Source Address applied source netmask
     {"%dn", 1, "dstNet", String_DstNet},           // Destination Address applied source netmask
@@ -520,6 +525,10 @@ static struct format_entry_s {
     {"%dtor", 0, "dstTor", String_DstTor},             // dst IP 2 letter tor node info
     {"%lbl", 0, "label", String_Label},                // Flow Label
 
+    // EXipInfo
+    {"%ttl", 0, "TTL", String_ipTTL},     // Flow ip ttl
+    {"%frag", 0, "Frag", String_ipFrag},  // IP fragment flags
+
     {NULL, 0, NULL, NULL}};
 
 /* each of the tokens above must not generate output strings larger than this */
@@ -1484,6 +1493,23 @@ static void String_FlowEndReason(FILE *stream, recordHandle_t *recordHandle) {
     fprintf(stream, "%u", flowEndReason);
 }  // End of String_FlowEndReason
 
+static void String_ipTTL(FILE *stream, recordHandle_t *recordHandle) {
+    EXipInfo_t *ipInfo = (EXipInfo_t *)recordHandle->extensionList[EXipInfoID];
+    uint8_t ttl = ipInfo ? ipInfo->ttl : 0;
+
+    fprintf(stream, "%u", ttl);
+}  // End of String_ipTTL
+
+static void String_ipFrag(FILE *stream, recordHandle_t *recordHandle) {
+    EXipInfo_t *ipInfo = (EXipInfo_t *)recordHandle->extensionList[EXipInfoID];
+    EXipInfo_t localIpInfo = {0};
+    if (ipInfo == NULL) ipInfo = &localIpInfo;
+
+    char *DF = ipInfo->fragmentFlags & flagDF ? "DF" : "--";
+    char *MF = ipInfo->fragmentFlags & flagMF ? "MF" : "--";
+    fprintf(stream, "%s%s", DF, MF);
+}  // End of String_ipFrag
+
 static void String_Flags(FILE *stream, recordHandle_t *recordHandle) {
     EXgenericFlow_t *genericFlow = (EXgenericFlow_t *)recordHandle->extensionList[EXgenericFlowID];
     uint32_t flags = genericFlow && genericFlow->proto == IPPROTO_TCP ? genericFlow->tcpFlags : 0;