Refactoring, adjusted config params.

cmd/peng/main.go → cmd/main.go

@@ -1,6 +1,8 @@
 package main
 
+//TODO add support for ipv6, netflow5 and IPFX
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"github.com/google/gopacket/pcap"
@@ -26,9 +28,9 @@ var (
 		NetworkInterface:   "",
 		Ja3BlackListFile:   "",
 		GeoIpDb:            "",
+		OfflinePcap:        "",
 	}
-
-	timeFrame = "1m"
+	timeFrame = "15s"
 
 	showInterfaceNames bool
 	versionFlag        bool
@@ -37,6 +39,14 @@ var (
 )
 
 func init() {
+	//NetFlow
+	flag.StringVar(&stanislav.FlowPath, "flowPath", "", "dir path to load flows of nProbe")
+	flag.Float64Var(&stanislav.Tolerance, "tolerance", 10, "maximum % tolerance before flag possible periodic flow.")
+	flag.IntVar(&stanislav.NTwToCompare, "nCompare", 3, "number o time windows to compare to evaluate a possible periodicity")
+	flag.StringVar(&stanislav.IpAddrNF, "ip", "", "ip of netflow collector")
+	flag.StringVar(&stanislav.PortNF, "port", "2055", "port of netflow collector")
+	flag.IntVar(&stanislav.Verbose, "verbose", 0, "verbosity level. (1=low,2=medium,3=high")
+
 	//Bitmap
 	flag.UintVar(&config.NumberOfBin, "bin", 16, "number of bin in your bitmap")
 	flag.UintVar(&config.SizeBitmap, "size", 1024, "size of your bitmap")
@@ -51,24 +61,27 @@ func init() {
 	//other
 	flag.BoolVar(&versionFlag, "version", false, "output version")
 	flag.StringVar(&config.SaveFilePath, "export", "", "file path to save the peng result as csv")
-	flag.StringVar(&timeFrame, "timeFrame", "1m", "interval time to detect scans. Number + (s = seconds, m = minutes, h = hours)")
-	flag.UintVar(&config.Verbose, "verbose", 1, "set verbose level (1-3)")
+	flag.StringVar(&timeFrame, "timeFrame", "15s", "interval time to detect scans. Number + (s = seconds, m = minutes, h = hours)")
 	flag.StringVar(&config.NetworkInterface, "network", "", "name of your network interface")
 	flag.BoolVar(&showInterfaceNames, "interfaces", false, "show the list of all your network interfaces")
 	flag.StringVar(&config.Ja3BlackListFile, "ja3", "", "file path of malicious ja3 fingerprints")
 	flag.StringVar(&config.GeoIpDb, "geoip", "", "file path of geoip db")
+	flag.StringVar(&config.OfflinePcap, "pcap", "", "pcap file to read")
 }
 
 func flagConfig() {
 	appString := fmt.Sprintf("________                     \n___  __ \\__________________ _\n__  /_/ /  _ \\_  __ \\_  __ `/\n_  ____//  __/  / / /  /_/ / \n/_/     \\___//_/ /_/_\\__, /  \n                    /____/   \n"+
 		"version %s %s", version, commit)
 
 	flag.Usage = func() { //help flag
-		fmt.Fprintf(flag.CommandLine.Output(), "%s\n\nUsage: sys-status [options]\n", appString)
+		fmt.Fprintf(flag.CommandLine.Output(), "\n\nUsage: flow-periodicity [options]\n")
 		flag.PrintDefaults()
 	}
 
 	flag.Parse()
+	stanislav.PercentageDeviation = stanislav.Tolerance //TODO refactor this
+	//TODO add check for ip and port
+	//TODO add check for tolerance 0 <= tolerance <= 100
 
 	if versionFlag { //version flag
 		fmt.Fprintf(flag.CommandLine.Output(), "%s\n", appString)
@@ -123,21 +136,36 @@ func flagConfig() {
 	fmt.Printf("%s\n", appString)
 }
 
-//var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
-
 func main() {
 	flagConfig()
-	/*
-		if *cpuprofile != "" {
-			f, err := os.Create(*cpuprofile)
-			if err != nil {
-				log.Fatal(err)
-			}
-			pprof.StartCPUProfile(f)
-			defer pprof.StopCPUProfile()
-		}*/
-
-	//peng := stanislav.New(&config)
-	//peng.run()
 
+	if stanislav.FlowPath != "" {
+		stanislav.OfflineMode()
+		FlowStats()
+		stanislav.WriteObjToJSONFile(time.Now().Format(time.RFC3339)+"_report.json", stanislav.PeriodiFlows)
+		return
+	}
+
+	stanislav.Conf = &config //TODO handle nil
+	stanislav.LiveMode()
+
+	ThreatStats()
+	FlowStats()
+
+	stanislav.WriteObjToJSONFile(time.Now().Format(time.RFC3339)+"_report.json", stanislav.PeriodiFlows) //TODO change this like peng that every X sec dump
+}
+
+func ThreatStats() {
+	fmt.Println("\nTHREAT")
+	threatJson, _ := json.Marshal(stanislav.PossibleThreat)
+	fmt.Println(string(threatJson))
+}
+
+func FlowStats() {
+	fmt.Println("\nPeriodic flows")
+	json, err := stanislav.PeriodiFlows.Marshal()
+	if err != nil {
+		return
+	}
+	fmt.Printf("%s", string(json))
 }

inspection.go

@@ -57,6 +57,7 @@ func (p *Peng) inspect(packet gopacket.Packet) {
 	}
 
 	GeoIpSearch(externalIp, p.Config.GeoIpDb)
+	externalIp = ipv4.SrcIP.String() + "/" + ipv4.DstIP.String() //TODO
 
 	if len(ja3BlackList) != 0 {
 		ja3.Security = 0
@@ -73,21 +74,28 @@ func (p *Peng) inspect(packet gopacket.Packet) {
 		}
 
 		if name, ok := ja3BlackList[ja3md5]; ok {
+			AddPossibleThreat(externalIp, "ja3 blocklist "+name)
 			fmt.Printf("[%s] %s appears in the blocked Ja3 list as %s!\n", externalIp, ja3md5, name)
 		}
 		if name, ok := ja3BlackList[ja3smd5]; ok {
+			AddPossibleThreat(externalIp, "ja3s blocklist "+name)
 			fmt.Printf("[%s] %s appears in the blocked Ja3 list as %s!\n", externalIp, ja3smd5, name)
 		}
 
 		//TODO add TLS version check
 		//TLS cipher security check
 		switch ja3.Security {
 		case 1:
+			AddPossibleThreat(externalIp, "Weak tls cipher")
 			fmt.Println("Weak tls cipher")
 		case 2:
+			AddPossibleThreat(externalIp, "Insecure tls cipher")
 			fmt.Println("Insecure tls cipher")
 		}
 	}
+
+	//TODO add http numeric ip
+
 	//				switch hello.CipherSuite {
 	//				case 49169: fallthrough /* TLS_ECDHE_RSA_WITH_RC4_128_SHA */
 	//				case 5: fallthrough /* TLS_RSA_WITH_RC4_128_SHA */

live_mode.go

@@ -6,36 +6,14 @@ import (
 	"runtime"
 	"sync"
 	"syscall"
-	"time"
 )
 
 type proto interface {
 	run()
 	shutdown()
 }
 
-var (
-	config = Config{
-		NumberOfBin:        128,
-		SizeBitmap:         1024,
-		InfluxUrl:          "http://localhost",
-		InfluxPort:         9999,
-		InfluxBucket:       "",
-		InfluxOrganization: "",
-		InfluxAuthToken:    "",
-		SaveFilePath:       "peng_result.csv",
-		UseInflux:          false,
-		Verbose:            uint(1),
-		NetworkInterface:   "eno1",
-		Ja3BlackListFile:   "/media/ale/DatiD/Progetti/Progetti2019/GoPrj/stanislav/resources/ja3/ja3_fingerprints.csv",
-		GeoIpDb:            "/media/ale/DatiD/Progetti/Progetti2019/GoPrj/stanislav/resources/GeoLite2-City.mmdb",
-		TimeFrame:          time.Second * 15,
-	}
-
-	showInterfaceNames bool
-	versionFlag        bool
-	commit             = "commithash"
-)
+var PossibleThreat = make(map[string][]string)
 
 func LiveMode() {
 	var (
@@ -48,7 +26,7 @@ func LiveMode() {
 	logger = opts.Logger
 
 	netflow9 := NewNetflowV9()
-	peng := New(&config)
+	peng := New(Conf)
 	protos := []proto{netflow9, peng}
 
 	for _, p := range protos {

logic.go

@@ -134,6 +134,7 @@ func ChangePeriodicStatus(key string, fi *FlowInfo, v bool) {
 	}
 
 	if v && !fi.CurrentlyPeriodic {
+		AddPossibleThreat(key, fmt.Sprintf(" periodic frequency: %.2fs seen %d times.", fi.TWDuration, fi.PeriodicityCounter))
 		log.Printf("%s \tbecame periodic! Seen %d times. Frequency: %.2fs ", key, fi.PeriodicityCounter, fi.TWDuration)
 	} else {
 		log.Printf("%s \tnot periodic anymore! Seen %d times. Frequency: %.2fs ", key, fi.PeriodicityCounter, fi.TWDuration)

options.go

@@ -18,6 +18,7 @@ var (
 	analisi             = AllFlows{}
 	PeriodiFlows        = PeriodicFlows{}
 	opts                *Options
+	Conf                *Config
 	logger              *log.Logger
 	PercentageDeviation = 5.0
 	Verbose             = 0
@@ -32,7 +33,6 @@ var (
 type Options struct {
 	// global options
 	Verbosity bool
-	LogFile   string `yaml:"log-file"`
 	CPUCap    string `yaml:"cpu-cap"`
 	Logger    *log.Logger
 	version   bool
@@ -55,7 +55,7 @@ func init() {
 // NewOptions constructs new options
 func NewOptions() *Options {
 	return &Options{
-		Verbosity: true,
+		Verbosity: false,
 		version:   false,
 		CPUCap:    "100%",
 		Logger:    log.New(os.Stderr, "[LOG] ", log.Ldate|log.Ltime),
@@ -73,22 +73,13 @@ func NewOptions() *Options {
 func GetOptions() *Options {
 	opts := NewOptions()
 
-	opts.flagSet()
+	//opts.flagSet()
 
 	if opts.Verbosity {
 		opts.Logger.Printf("the full logging enabled")
 		opts.Logger.SetFlags(log.LstdFlags | log.Lshortfile)
 	}
 
-	if opts.LogFile != "" {
-		f, err := os.OpenFile(opts.LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
-		if err != nil {
-			opts.Logger.Println(err)
-		} else {
-			opts.Logger.SetOutput(f)
-		}
-	}
-
 	return opts
 }
 
@@ -135,14 +126,12 @@ func (opts Options) getCPU() int {
 }
 
 func (opts *Options) flagSet() {
-
 	var config string
 	flag.StringVar(&config, "config", "/etc/vflow/vflow.conf", "path to config file")
 
 	// global options
 	flag.BoolVar(&opts.Verbosity, "verbosity", opts.Verbosity, "enable/disable verbose logging")
 	flag.BoolVar(&opts.version, "version", opts.version, "show version")
-	flag.StringVar(&opts.LogFile, "log-file", opts.LogFile, "log file name")
 	flag.StringVar(&opts.CPUCap, "cpu-cap", opts.CPUCap, "Maximum amount of CPU [percent / number]")
 
 	// netflow version 9

peng.go

@@ -2,6 +2,7 @@ package stanislav
 
 import (
 	"encoding/csv"
+	"encoding/json"
 	"fmt"
 	"github.com/google/gopacket"
 	_ "github.com/google/gopacket/layers" //Used to init internal struct
@@ -34,6 +35,7 @@ type Config struct {
 	TimeFrame          time.Duration
 	Ja3BlackListFile   string
 	GeoIpDb            string
+	OfflinePcap        string
 }
 
 var ja3BlackList map[string]string
@@ -57,12 +59,18 @@ func New(cfg *Config) *Peng {
 func (p *Peng) run() {
 	getMyIp()
 	p.LoadBlackListJa3InMemory()
-
-	pHandle, err := pcap.OpenLive(
-		p.Config.NetworkInterface,
-		int32(65535),
-		false,
-		pcap.BlockForever)
+	var pHandle *pcap.Handle
+	var err error
+
+	if p.Config.OfflinePcap == "" {
+		pHandle, err = pcap.OpenLive(
+			p.Config.NetworkInterface,
+			int32(65535),
+			false,
+			pcap.BlockForever)
+	} else {
+		pHandle, err = pcap.OpenOffline(p.Config.OfflinePcap)
+	}
 
 	if err != nil {
 		log.Fatal(err)
@@ -96,10 +104,11 @@ func (p *Peng) shutdown() {
 	p.stop = true
 	logger.Println("stopping peng module...")
 	time.Sleep(1 * time.Second)
+
 	//TODO
-	for k, v := range topCountryVisit {
-		fmt.Printf("[%s] %d visit.\n", k, v)
-	}
+	fmt.Println("\nTOP COUNTRY VISIT")
+	threatJson, _ := json.Marshal(topCountryVisit)
+	fmt.Println(string(threatJson))
 }
 
 func (p *Peng) LoadBlackListJa3InMemory() {
@@ -148,8 +157,13 @@ func (p *Peng) PrintAllInfo() {
 		if p.Config.Verbose >= 2 {
 			fmt.Printf("entropy of each bin: %f\n", v.EntropyOfEachBin())
 		}
+
+		totalEntroy := v.EntropyTotal()
+		if totalEntroy >= 0.5 {
+			AddPossibleThreat("general", fmt.Sprintf("probably a port scan. Total entropy: %.2f", totalEntroy))
+		}
 		if p.Config.Verbose >= 1 {
-			fmt.Printf("total entropy: %f\n", v.EntropyTotal())
+			fmt.Printf("total entropy: %f\n", totalEntroy)
 		}
 	}
 }