Add torlookup for IPs

Makefile.am

@@ -1,6 +1,6 @@
 ACLOCAL_AMFLAGS = -I m4
 
-SUBDIRS = src/lib src/output src/netflow src/collector src/decode src/maxmind src/nfdump src/nfcapd  
+SUBDIRS = src/lib src/output src/netflow src/collector src/decode src/maxmind src/tor src/nfdump src/nfcapd  
 SUBDIRS += src/nfanon src/nfexpire src/nfreplay . src src/test src/nfreader src/inline src/include
 
 if SFLOW

configure.ac

@@ -168,11 +168,23 @@ build_ftconv="yes" , build_ftconv="no"
 AM_CONDITIONAL([FT2NFDUMP], [test "x$build_ftconv" = "xyes"])
 
 AC_ARG_ENABLE(maxmind,
-[  --enable-maxmind        Build geolookup for MaxMind GeoDB; default is NO])
+[  --enable-maxmind        Build geolookup for MaxMind GeoDB; default is NO],
+build_maxmind="yes"
+AM_CONDITIONAL([MAXMIND], true)
+,
+build_maxmind="no"
+AM_CONDITIONAL([MAXMIND], false)
+)
 
-AS_IF([test "x$enable_maxmind" = xyes], 
-build_maxmind="$enable_maxmind", build_maxmind="no")
-AM_CONDITIONAL([MAXMIND], [test "x$build_maxmind" = "xyes"])
+AC_ARG_ENABLE(tor,
+[  --enable-tor        Build torlookup for tor onion lookups; default is NO],
+build_tor="yes"
+AM_CONDITIONAL(TORLOOKUP, true)
+CFLAGS="$CFLAGS -DBUILDTOR"
+,
+build_tor="no"
+AM_CONDITIONAL(TORLOOKUP, false)
+)
 
 AC_ARG_ENABLE(ja4,
 [  --enable-ja4        Build with ja4 fingerprinting code; May require a license; default is NO])
@@ -654,23 +666,23 @@ echo ""
 echo "----------------------------------"
 echo " Build Settings for ${PACKAGE_TARNAME} v${PACKAGE_VERSION}"
 echo "----------------------------------"
-echo "  host type            = $host_os"
-echo "  install dir          = $prefix"
-echo "  CC                   = $CC"
-echo "  CFLAGS               = $AM_CFLAGS $CFLAGS"
-echo "  CPPFLAGS             = $AM_CPPFLAGS $CPPFLAGS"
-echo "  LDFLAGS              = $AM_LDFLAGS $LDFLAGS"
-echo "  LIBS                 = $LIBS"
-echo "  Enable liblz4        = $use_lz4"
-echo "  Enable libbz2        = $use_bzip2"
-echo "  Enable libzstd       = $use_zstd"
-echo "  Enable ja4           = $build_ja4"
-echo "  Build geolookup      = $build_maxmind"
-echo "  Build sflow          = $build_sflow"
-echo "  Build nfpcapd        = $build_nfpcapd"
-echo "  Build flowtools conv = $build_ftconv"
-echo "  Build nfprofile      = $build_nfprofile"
-echo "  Build ft2nfdump      = $build_ftconv"
+echo "  host type          = $host_os"
+echo "  install dir        = $prefix"
+echo "  CC                 = $CC"
+echo "  CFLAGS             = $AM_CFLAGS $CFLAGS"
+echo "  CPPFLAGS           = $AM_CPPFLAGS $CPPFLAGS"
+echo "  LDFLAGS            = $AM_LDFLAGS $LDFLAGS"
+echo "  LIBS               = $LIBS"
+echo "  Enable liblz4      = $use_lz4"
+echo "  Enable libbz2      = $use_bzip2"
+echo "  Enable libzstd     = $use_zstd"
+echo "  Enable ja4         = $build_ja4"
+echo "  Build geolookup    = $build_maxmind"
+echo "  Build torlookup    = $build_tor"
+echo "  Build sflow        = $build_sflow"
+echo "  Build nfpcapd      = $build_nfpcapd"
+echo "  Build nfprofile    = $build_nfprofile"
+echo "  Build ft2nfdump    = $build_ftconv"
 echo "----------------------------------"
 echo ""
 echo " You can run ./make now." 

man/geolookup.1

@@ -1,4 +1,4 @@
-.\" Copyright (c) 2022, Peter Haag
+.\" Copyright (c) 2024, Peter Haag
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -33,28 +33,28 @@
 .Nd geo location lookup for IP addresses
 .Sh SYNOPSIS
 .Nm 
-.Op Fl G Ar geofile
+.Op Fl G Ar geoDBfile
 .Ar iplist
 .Nm
 .Fl d Ar directory
-.Fl w Ar geofile
+.Fl w Ar geoDBfile
 .Sh DESCRIPTION
 .Nm 
 is a tool to lookup AS and geo location information of one or more IP addresses. You need
-to create a lookup database first, in order to use
+to create the nfdump specific lookup database first, before using
 .Nm
 .Pp
 .Nm
 is also used to create the nfdump formatted lookup database file from the maxmind csv files.
 You need to have a maxmind account at https://maxmind.com, in order to download the relevant
-csv file. See below for building instructions.
+csv file.  See the section below for the building instructions.
 .Pp
 .Nm
 accepts a list of IP addresses either on the command line, separated by spaces
 or on
 .Ar stdin
 line by line. The IP address on each line can be embedded in a string separated be
-spaces on the left and right, therefore it can read the piped output from another tool.
+spaces on the left and right, therefore it can read the piped output from other tools.
 .Pp
 The options are as follows:
 .Bl -tag -width Ds
@@ -64,10 +64,10 @@ Use the csv files in
 to build the binary lookup database file. With this argument
 .Nm
 creates a new binary lookup database file.
-.It Fl w Ar geofile
+.It Fl w Ar geoDBfile
 Name of the new lookup database file.
-.It Fl G Ar geofile
-Use the binary geofile as lookup database for the current AS and location lookups.
+.It Fl G Ar geoDBfile
+Use the binary geoDBfile as lookup database for the current AS and location lookups.
 .El
 .Pp
 To specify the geo lookup database 

man/nfdump.1

@@ -1,4 +1,4 @@
-\" Copyright (c) 2023, Peter Haag
+\" Copyright (c) 2024, Peter Haag
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -49,6 +49,7 @@
 .Op Fl I
 .Op Fl D Ar nameserver
 .Op Fl G Ar geoDB
+.Op Fl H Ar torDB
 .Op Fl s Ar statistic
 .Op Fl n Ar num
 .Op Fl o Ar format
@@ -370,12 +371,42 @@ for the path of
 The option
 .Fl G
 overwrites
-.Ar NFGEODB.
+.Ar NFGEODB
+or
+.Ar geodb.path 
+in
+.Ar nfdump.conf.
 In order to prevent reading any
 .Ar geoDB
 file, even if it would exist set
 .Fl G
 .Sy none.
+See also
+.Ar geolookup(1)
+.It Fl H Ar torDB
+Use
+.Ar torDB
+as tor lookup DB for tor exit node lookups.
+.Nm
+tries to read the environment variable
+.Ar NFTORDB
+for the path of
+.Ar torDB.
+The option
+.Fl H
+overwrites
+.Ar NFTORDB
+or
+.Ar tordb.path
+in
+.Ar nfdump.conf.
+In order to prevent reading any
+.Ar torDB
+file, even if it would exist set
+.Fl G
+.Sy none.
+See also
+.Ar torlookup(1)
 .It Fl s Ar statistic Op Ar :p Op Ar /orderby
 Generate the Top N flow record or flow element statistic. By optionally adding
 .Sy :p
@@ -940,15 +971,15 @@ or
 .Cm dst
 the source or destination port may match.
 .Pp
-.It Cm icmp-type Ar num
-.It Cm icmp-code Ar num
+.It Cm icmp type Ar num
+.It Cm icmp code Ar num
 True if the respective icmp field of the record matches
 .Ar num.
 This automatically implies
 .Cm proto icmp.
 .Pp
-.It Cm engine-type Ar num
-.It Cm engine-id Ar num
+.It Cm engine type Ar num
+.It Cm engine id Ar num
 .It Cm sysid Ar num
 True if the respective fields of the record matches
 .Ar num
@@ -1688,6 +1719,10 @@ dst IP geo location info
 src AS organisation name
 .It Cm %dasn
 dst AS organisation name
+.It Cm %stor
+src IP 2 letter tor exit info: TX tor exit node
+.It Cm %dtor
+dst IP 2 letter tor exit info: TX tor exit node
 .It Cm %n
 new line char \\n
 .It Cm %ipl

src/lib/nffile.c

@@ -89,7 +89,7 @@
     (a)->flags = 0;      \
     (a)->type = DATA_BLOCK_TYPE_3;
 
-static const char *nf_creator[MAX_CREATOR] = {"unknown", "nfcapd", "nfpcapd", "sfcapd", "nfdump", "nfanon", "nfprofile", "geolookup", "ft2nfdump"};
+static const char *nf_creator[MAX_CREATOR] = {"unknown", "nfcapd", "nfpcapd", "sfcapd", "nfdump", "nfanon", "nfprofile", "geolookup", "ft2nfdump", "torlookup"};
 
 static unsigned NumWorkers = DEFAULTWORKERS;
 

src/lib/nffileV2.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2023, Peter Haag
+ *  Copyright (c) 2024, Peter Haag
  *  All rights reserved.
  *
  *  Redistribution and use in source and binary forms, with or without
@@ -87,7 +87,8 @@ typedef struct fileHeaderV2_s {
 #define CREATOR_NFPROFILE 6
 #define CREATOR_LOOKUP 7
 #define CREATOR_FT2NFDUMP 8
-#define MAX_CREATOR 9
+#define CREATOR_TORLOOKUP 9
+#define MAX_CREATOR 10
     off_t offAppendix;  // offset in file for appendix blocks with additional data
 
     uint32_t BlockSize;  // max block size of data blocks

src/nfdump/Makefile.am

@@ -1,7 +1,7 @@
 
 bin_PROGRAMS = nfdump
 
-AM_CPPFLAGS = -I.. -Icompat_1_6_x -I../include -I../lib -I../output -I../maxmind -I../netflow -I../collector -I../lib/conf -I../lib/filter -I../decode -I../inline $(DEPS_CFLAGS)
+AM_CPPFLAGS = -I.. -Icompat_1_6_x -I../include -I../lib -I../output -I../maxmind -I../tor -I../netflow -I../collector -I../lib/conf -I../lib/filter -I../decode -I../inline $(DEPS_CFLAGS)
 AM_LDFLAGS  = -L../lib
 
 EXTRA_DIST = nffile_compat.c memhandle.c heapsort_inline.c 
@@ -19,6 +19,6 @@ compat = compat_1_6_x/nfx.h compat_1_6_x/nfx.c
 
 nfdump_SOURCES = nfdump.c spin_lock.h \
 	$(exporter) $(nbar) $(ifvrf) $(nfstat) $(nflowcache) $(nfprof) $(sort) $(compat)
-nfdump_LDADD = ../output/liboutput.a  ../lib/libnfdump.la ../maxmind/libmaxmind.a ../decode/libnfdecode.a
+nfdump_LDADD = ../output/liboutput.a  ../lib/libnfdump.la ../maxmind/libmaxmind.a  ../tor/libtorlookup.a ../decode/libnfdecode.a
 
 CLEANFILES = *.gch

src/nfdump/nfdump.c

@@ -69,6 +69,7 @@
 #include "nfx.h"
 #include "nfxV3.h"
 #include "output.h"
+#include "tor.h"
 #include "util.h"
 #include "version.h"
 
@@ -119,6 +120,7 @@ static void usage(char *name) {
         "-c\t\tLimit number of matching records\n"
         "-D <dns>\tUse nameserver <dns> for host lookup.\n"
         "-G <geoDB>\tUse this nfdump geoDB to lookup country/location.\n"
+        "-H <torDB>\tUse nfdump torDB to lookup tor info.\n"
         "-N\t\tPrint plain numbers\n"
         "-s <expr>[/<order>]\tGenerate statistics for <expr> any valid record element.\n"
         "\t\tand ordered by <order>: packets, bytes, flows, bps pps and bpp.\n"
@@ -180,8 +182,8 @@ static void PrintSummary(stat_record_t *stat_record, outputParams_t *outputParam
         duration = 0;
     }
     if (duration > 0 && stat_record->lastseen > 0) {
-        bps = (stat_record->numbytes << 3) / duration;  // bits per second. ( >> 3 ) -> * 8 to convert octets into bits
-        pps = stat_record->numpackets / duration;       // packets per second
+        bps = (stat_record->numbytes << 3) / duration;                                        // bits per second. ( >> 3 ) -> * 8 to convert octets into bits
+        pps = stat_record->numpackets / duration;                                             // packets per second
         bpp = stat_record->numpackets ? stat_record->numbytes / stat_record->numpackets : 0;  // Bytes per Packet
     }
     if (outputParams->mode == MODE_CSV) {
@@ -518,7 +520,7 @@ int main(int argc, char **argv) {
     nfprof_t profile_data;
     char *wfile, *ffile, *filter, *tstring, *stat_type;
     char *print_format;
-    char *print_order, *query_file, *geo_file, *configFile, *nameserver, *aggr_fmt;
+    char *print_order, *query_file, *configFile, *nameserver, *aggr_fmt;
     int ffd, element_stat, fdump;
     int flow_stat, aggregate, aggregate_mask, bidir;
     int print_stat, gnuplot_stat, syntax_only, compress, worker;
@@ -555,7 +557,8 @@ int main(int argc, char **argv) {
     aggr_fmt = NULL;
 
     configFile = NULL;
-    geo_file = getenv("NFGEODB");
+    char *geo_file = getenv("NFGEODB");
+    char *tor_file = getenv("NFTORDB");
 
     outputParams = (outputParams_t *)calloc(1, sizeof(outputParams_t));
     if (!outputParams) {
@@ -566,7 +569,7 @@ int main(int argc, char **argv) {
 
     Ident[0] = '\0';
     int c;
-    while ((c = getopt(argc, argv, "6aA:Bbc:C:D:E:G:s:ghn:i:jf:qyz::r:v:w:J:M:NImO:R:XZt:TVv:W:x:o:")) != EOF) {
+    while ((c = getopt(argc, argv, "6aA:Bbc:C:D:E:G:s:gH:hn:i:jf:qyz::r:v:w:J:M:NImO:R:XZt:TVv:W:x:o:")) != EOF) {
         switch (c) {
             case 'h':
                 usage(argv[0]);
@@ -632,6 +635,12 @@ int main(int argc, char **argv) {
                 if (strcmp(optarg, "none") != 0 && !CheckPath(optarg, S_IFREG)) exit(EXIT_FAILURE);
                 geo_file = strdup(optarg);
                 break;
+            case 'H':
+                CheckArgLen(optarg, MAXPATHLEN);
+                if (strcmp(optarg, "none") != 0 && !CheckPath(optarg, S_IFREG)) exit(EXIT_FAILURE);
+                tor_file = strdup(optarg);
+                // outputParams->doTag = 1;
+                break;
             case 'X':
                 fdump = 1;
                 break;
@@ -957,6 +966,19 @@ int main(int argc, char **argv) {
         outputParams->hasGeoDB = true;
     }
 
+    if (tor_file == NULL) {
+        tor_file = ConfGetString("tordb.path");
+    }
+    if (tor_file && strcmp(tor_file, "none") == 0) {
+        tor_file = NULL;
+    }
+    if (tor_file) {
+        if (!CheckPath(tor_file, S_IFREG) || !Init_TorLookup() || !LoadTorTree(tor_file)) {
+            LogError("Error reading tor info DB file %s", tor_file);
+            exit(EXIT_FAILURE);
+        }
+        outputParams->hasTorDB = true;
+    }
     if ((aggregate || flow_stat || print_order) && !Init_FlowCache()) exit(250);
 
     if (aggregate && (flow_stat || element_stat)) {

src/output/Makefile.am

@@ -1,5 +1,5 @@
 
-AM_CPPFLAGS = -I.. -I../include -I../lib -I../decode -I../lib/conf  -I../maxmind -I../inline $(DEPS_CFLAGS)
+AM_CPPFLAGS = -I.. -I../include -I../lib -I../decode -I../lib/conf  -I../maxmind -I../tor -I../inline $(DEPS_CFLAGS)
 
 noinst_LIBRARIES = liboutput.a
 

src/output/output.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2022, Peter Haag
+ *  Copyright (c) 2024, Peter Haag
  *  All rights reserved.
  *
  *  Redistribution and use in source and binary forms, with or without
@@ -40,12 +40,15 @@ typedef void (*RecordPrinter_t)(FILE *, recordHandle_t *, int);
 typedef void (*PrologPrinter_t)(void);
 typedef void (*EpilogPrinter_t)(void);
 
-enum { MODE_PLAIN = 0, MODE_JSON, MODE_CSV };
+enum { MODE_PLAIN = 0,
+       MODE_JSON,
+       MODE_CSV };
 typedef struct outputParams_s {
     bool printPlain;
     bool doTag;
     bool quiet;
     bool hasGeoDB;
+    bool hasTorDB;
     int mode;
     int topN;
 } outputParams_t;