Merge pull request #99 from kdrenard/netmask_function

Netmask function

FUNCTIONS.md

@@ -42,6 +42,13 @@ Converts double precision number to integer.
 Evaluates `condition` and executes `false_op` if the result is 0 (false)
 otherwise `true_op` is executed.
 
+### NETMASK(address [, v4_mask_length [, v6_mask_length]])
+
+Masks the specified address using the v4 and v6 mask lengths specified
+in number of bits.
+
+Defaults to 24 for IPv4 and 48 for IPv6 (/24 and /48 respectively)
+
 ## String operations
 
 ### RSPLIT(string, n [, char])

src/dns.h

@@ -31,8 +31,6 @@
 #include "packet_handler.h"
 #include "tcp.h"
 
-#define IPPROTO_ICMP 1
-
 namespace packetq {
 
 extern char visible_char_map[256];

src/packet_handler.cpp

@@ -277,6 +277,8 @@ Packet::ParseResult Packet::parse(Packet_handler* handler, const std::vector<int
  bool base_layers_parsed;
  if (m_link_layer_type == 1)
  base_layers_parsed = parse_ethernet();
+ else if (m_link_layer_type == 113)
+ base_layers_parsed = parse_sll();
  else
  base_layers_parsed = parse_ip(m_data, m_len, 0);
 
@@ -291,12 +293,13 @@ bool Packet::parse_ethernet()
 {
  unsigned char* data = m_data;
  int len = m_len;
- if (len < 14 + 5 * 4)
- return false; // check for etherframe size + ipv4 header
+ if (len < 14)
+ return false; // check for etherframe size
 
  int ethertype = data[13] | (data[12] << 8);
  if (ethertype == 0x8100) {
- // VLAN-tagged
+ if (len < 18)
+ return false; // check for etherframe size + VLAN tag
  ethertype = data[17] | (data[16] << 8);
  data += 18;
  len -= 18;
@@ -308,6 +311,28 @@ bool Packet::parse_ethernet()
  return parse_ip(data, len, ethertype);
 }
 
+bool Packet::parse_sll()
+{
+ unsigned char* data = m_data;
+ int len = m_len;
+ if (len < 16)
+ return false; // check for LINUX_SLL size
+
+ int ethertype = data[15] | (data[14] << 8);
+ if (ethertype == 0x8100) {
+ if (len < 20)
+ return false; // check for etherframe size + VLAN tag
+ ethertype = data[19] | (data[18] << 8);
+ data += 20;
+ len -= 20;
+ } else {
+ data += 16;
+ len -= 16;
+ }
+
+ return parse_ip(data, len, ethertype);
+}
+
 bool Packet::parse_ip(unsigned char* data, int len, int ethertype)
 {
  if (len < 5 * 4)

src/packet_handler.h

@@ -33,8 +33,6 @@
 #include "sql.h"
 #include "tcp.h"
 
-#define IPPROTO_ICMP 1
-
 namespace packetq {
 
 class Table;
@@ -177,6 +175,7 @@ class Packet {
 
  ParseResult parse(Packet_handler* handler, const std::vector<int>& columns, Row& destination_row, bool sample);
  bool parse_ethernet();
+ bool parse_sll();
  bool parse_ip(unsigned char* data, int len, int ether_type);
  bool parse_transport(unsigned char* data, int len);
 

src/pcap.cpp

@@ -61,7 +61,7 @@ bool Pcap_file::get_header()
  m_snapshot_length = get_int32();
  // check for ethernet packets
  m_link_layer_type = get_int32();
- if (m_link_layer_type != 1 && m_link_layer_type != 101) {
+ if (m_link_layer_type != 1 && m_link_layer_type != 101 && m_link_layer_type != 113) {
  fprintf(stderr, "PCAP file unsupported linklayer (%d)\n", m_link_layer_type);
  return false;
  }

src/sql.cpp

@@ -1881,6 +1881,9 @@ OP* OP::compile(const std::vector<Table*>& tables, const std::vector<int>& searc
  } else if (cmpi(get_token(), "rsplit") && m_param[1]) {
  m_t = Coltype::_text;
  ret = new Rsplit_func(*this);
+ } else if (cmpi(get_token(), "netmask")) {
+ m_t = Coltype::_text;
+ ret = new Netmask_func(*this);
  } else if (cmpi(get_token(), "count")) {
  m_t = Coltype::_int;
  ret = new Count_func(*this, dest_table);

src/sql.h

@@ -38,6 +38,12 @@
 #include <string>
 #include <sys/types.h>
 #include <vector>
+#include <arpa/inet.h>
+#include <netinet/in.h>
+#ifndef s6_addr32 // For *BSD
+#define s6_addr32 __u6_addr.__u6_addr32
+#endif
+#include <sys/socket.h>
 
 #include "refcountstring.h"
 #include "variant.h"
@@ -796,6 +802,89 @@ class Static_text : public OP {
 };
 
 ///////////////// Functions
+
+class Netmask_func : public OP {
+public:
+ Netmask_func(const OP& op)
+ : OP(op)
+ {
+ }
+ void evaluate(Row** rows, Variant& v)
+ {
+ Variant orig_ip;
+ m_param[0]->evaluate(rows, orig_ip);
+
+ if (!valid_masks)
+ set_masks(rows);
+
+ RefCountStringHandle src(orig_ip.get_text());
+ RefCountStringHandle dest(RefCountString::allocate(INET6_ADDRSTRLEN + 1));
+
+ if (strchr((*src)->data, ':')) {
+ struct in6_addr a6;
+ if (inet_pton(AF_INET6, (*src)->data, &a6) == 1) {
+ a6.s6_addr32[0] &= v6_mask[0];
+ a6.s6_addr32[1] &= v6_mask[1];
+ a6.s6_addr32[2] &= v6_mask[2];
+ a6.s6_addr32[3] &= v6_mask[3];
+ if (inet_ntop(AF_INET6, &a6, (*dest)->data, INET6_ADDRSTRLEN)) {
+ v = *dest;
+ return;
+ }
+ }
+ } else {
+ struct in_addr a4;
+ if (inet_pton(AF_INET, (*src)->data, &a4) == 1) {
+ a4.s_addr &= v4_mask;
+ if (inet_ntop(AF_INET, &a4, (*dest)->data, INET6_ADDRSTRLEN)) {
+ v = *dest;
+ return;
+ }
+ }
+ }
+
+ // Operation on non-IP address text
+ RefCountStringHandle empty(RefCountString::construct(""));
+ v = *empty;
+ }
+
+private:
+ void set_masks(Row** rows)
+ {
+ if (m_param[1]) {
+ Variant v4cidr;
+ m_param[1]->evaluate(rows, v4cidr);
+ int v4size = v4cidr.get_int();
+ if (v4size > -1 && v4size < 33) {
+ v4_mask = htonl(0xffffffff << (32 - v4size));
+ }
+ }
+ if (m_param[2]) {
+ Variant v6cidr;
+ m_param[2]->evaluate(rows, v6cidr);
+ int v6size = v6cidr.get_int();
+ if (v6size > -1 && v6size < 129) {
+ for (int i = 0; i < 4; i++) {
+ if (v6size >= 32) {
+ v6_mask[i] = 0xffffffff;
+ v6size -= 32;
+ } else if (v6size) {
+ v6_mask[i] = htonl(0xffffffff << (32 - v6size));
+ v6size = 0;
+ } else {
+ v6_mask[i] = 0;
+ }
+ }
+ }
+ }
+ valid_masks = true;
+ }
+
+ uint32_t v4_mask = htonl(0xffffff00);
+ uint32_t v6_mask[4] = { 0xffffffff, htonl(0xffff0000), 0, 0 };
+ bool valid_masks = false;
+};
+
 class Truncate_func : public OP {
 public:
  Truncate_func(const OP& op)

src/tcp.h

@@ -23,25 +23,19 @@
 #define __packetq_tcp_h
 
 #include <stdio.h>
-
-// Hack for Linux which does not include this in ethernet.h/ethertypes.h
-#ifndef ETHERTYPE_IPV6
-#define ETHERTYPE_IPV6 0x86dd
-#endif
-#define IPPROTO_TCP 6
-#define IPPROTO_UDP 17
+#include <stdint.h>
 
 namespace packetq {
 
-struct in6_addr {
+struct _in6_addr {
  union {
- unsigned char __u6_addr8[16];
- unsigned short __u6_addr16[8];
- unsigned int  __u6_addr32[4];
+ uint8_t __u6_addr8[16];
+ uint16_t __u6_addr16[8];
+ uint32_t __u6_addr32[4];
  } __in6_u; /* 128-bit IP6 address */
 };
 
-typedef struct in6_addr in6addr_t;
+typedef struct _in6_addr in6addr_t;
 
 class Payload;
 

src/test/Makefile.am

@@ -28,4 +28,5 @@ TESTS = test1.sh test2.sh test3.sh test4.sh test5.sh test6.sh test7.sh \
 
 EXTRA_DIST = $(TESTS) \
  test1.gold test2.gold test3.gold test4.gold test5.gold test6.gold \
- test7.gold sql.txt test8.gold
+ test7.gold sql.txt test8.gold \
+ dns.pcap dns6.pcap

src/test/test1.gold

@@ -35,3 +35,111 @@
  ]
  }
 ]
+[
+ {
+ "table_name": "result-0",
+ "query": "select netmask(src_addr), netmask(dst_addr, 8, 16) from dns",
+ "head": [
+ { "name": "netmask(src_addr)","type": "text" },
+ { "name": "netmask(dst_addr,8,16)","type": "text" }
+ ],
+ "data": [
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"],
+ ["172.17.0.0","8.0.0.0"],
+ ["8.8.8.0","172.0.0.0"]
+ ]
+ }
+]
+[
+ {
+ "table_name": "result-0",
+ "query": "select netmask(src_addr), netmask(dst_addr, 8, 16) from dns",
+ "head": [
+ { "name": "netmask(src_addr)","type": "text" },
+ { "name": "netmask(dst_addr,8,16)","type": "text" }
+ ],
+ "data": [
+ ["2a01:3f0::","2001::"],
+ ["2001:4860:4860::","2a01::"]
+ ]
+ }
+]

src/test/test1.sh

@@ -20,4 +20,8 @@
 
 ../packetq -j -s "select s, dst_addr as Dst_addr, qtype as questiontype, lower(src_addr) as lower_src, if(1 and s < 1 or s <= 1 or s > 1 or s >= 1, 't', 'f'), trim(trim('foofoo' || rsplit(src_addr, 1) || 'foofoo', 'foo'), 'bar'), count(*), len(src_addr), sum(msg_size + -1 - 2 % 4 << 3 >> 2 | 3 & ~4) + 1, min(msg_size), max(msg_size), truncate(1.1) as integer, 1.1 as float, sum(src_port + 1.0 - 2.0 / 1.5 * -2.5) + 1.0, max(src_port + 1.0), min(src_port + 1.0), avg(src_port), stdev(src_port), name('rcode', 0) from dns where src_addr like '%' and (qr or not qr) group by src_addr, s having s >= 0 order by s, lower_src, integer, float" "$srcdir/../../pcap/sample.pcap.gz" > test1.out
 
+../packetq -j -s "select netmask(src_addr), netmask(dst_addr, 8, 16) from dns" "$srcdir/dns.pcap" >>test1.out
+
+../packetq -j -s "select netmask(src_addr), netmask(dst_addr, 8, 16) from dns" "$srcdir/dns6.pcap" >>test1.out
+
 diff -uw "$srcdir/test1.gold" test1.out