diff --git a/cmd/peng/main.go b/cmd/peng/main.go index d4982d4..e83feb7 100644 --- a/cmd/peng/main.go +++ b/cmd/peng/main.go @@ -3,11 +3,11 @@ package main import ( "flag" "fmt" - p "github.com/alessio-perugini/peng" "github.com/google/gopacket/pcap" "log" "net/url" "os" + p "stanislav/pkg/peng" "time" ) diff --git a/doc/img/entropy.svg b/doc/img/entropy.svg index 2d9fa88..1e9e55f 100644 --- a/doc/img/entropy.svg +++ b/doc/img/entropy.svg @@ -1,87 +1,109 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/go.mod b/go.mod index b4488d1..38606fd 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.15 require ( github.com/VerizonDigital/vflow v0.8.0 github.com/alessio-perugini/peng v0.1.3 + github.com/dreadl0ck/ja3 v1.0.0 github.com/google/gopacket v1.1.18 github.com/influxdata/influxdb-client-go v1.4.0 ) diff --git a/go.sum b/go.sum index 148fbad..ec3de2a 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,10 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/deepmap/oapi-codegen v1.3.6 h1:Wj44p9A0V0PJ+AUg0BWdyGcsS1LY18U+0rCuPQgK0+o= github.com/deepmap/oapi-codegen v1.3.6/go.mod h1:aBozjEveG+33xPiP55Iw/XbVkhtZHEGLq3nxlX0+hfU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dreadl0ck/ja3 v1.0.0 h1:p0CXWPmgaR6iUCO5SlS/IPKM1daJvW6Wk3Ih9rWwIDM= +github.com/dreadl0ck/ja3 v1.0.0/go.mod h1:oFgkk9u6FDv/Cwyg2peYv9eFotbQTcyDkWChn8kQJAc= +github.com/dreadl0ck/tlsx v0.0.0-20200303221230-8cb859306e07 h1:ZPbZ1CELNcwKr0D/MHZK0oPRrb4hH28AKldTWknV6lg= +github.com/dreadl0ck/tlsx v0.0.0-20200303221230-8cb859306e07/go.mod h1:amAb73WEEgPHWniMfwro6UpN6St3e5ypgq2tXM89IOo= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -135,6 +139,8 @@ golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/pkg/bitmap/bitmap.go b/pkg/bitmap/bitmap.go index f03bb55..d407012 100644 --- a/pkg/bitmap/bitmap.go +++ b/pkg/bitmap/bitmap.go @@ -59,15 +59,15 @@ func (b *Bitmap) Size() uint64 { func (b *Bitmap) GetBitSets() uint64 { var total uint64 - for i:=0; i < len(b.data); i++ { + for i := 0; i < len(b.data); i++ { total += uint64(bits.OnesCount(uint(b.data[i]))) } return total } -func (b *Bitmap) ResetAllBits(){ +func (b *Bitmap) ResetAllBits() { for i := 0; i < len(b.data); i++ { b.data[i] &= ^b.data[i] } -} \ No newline at end of file +} diff --git a/pkg/peng/inspection.go b/pkg/peng/inspection.go index 1927049..f33b091 100644 --- a/pkg/peng/inspection.go +++ b/pkg/peng/inspection.go @@ -3,6 +3,7 @@ package peng import ( "fmt" "github.com/alessio-perugini/peng/pkg/portbitmap" + "github.com/dreadl0ck/ja3" "github.com/google/gopacket" "github.com/google/gopacket/layers" _ "github.com/google/gopacket/layers" //Used to init internal struct @@ -34,11 +35,7 @@ func (p *Peng) inspect(packet gopacket.Packet) { if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil { tcp, _ := tcpLayer.(*layers.TCP) if tcp.SYN && !tcp.ACK { - if packetDestToMyPc { - addPortToBitmap(uint16(tcp.DstPort), p.ServerTraffic) - } else { - addPortToBitmap(uint16(tcp.DstPort), p.ClientTraffic) - } + p.PortScanningHandler(uint16(tcp.DstPort), packetDestToMyPc) if p.Config.Verbose == 3 { if packetDestToMyPc { @@ -49,6 +46,35 @@ func (p *Peng) inspect(packet gopacket.Packet) { } } } + /* + if udpLayer := packet.Layer(layers.LayerTypeUDP); udpLayer != nil { + udp, _ := udpLayer.(*layers.UDP) + + totalSize := udp.Length + payloadSize := len(udp.Payload) + goodput := totalSize - uint16(payloadSize) + + if + }*/ + //TODO add blacklist check + ja3string := ja3.DigestHexPacket(packet) + ja3sstring := ja3.DigestHexPacketJa3s(packet) + if p.Config.Verbose == 2 { + if ja3string != "" { + fmt.Printf("J1: %s\n", ja3string) + } + if ja3sstring != "" { + fmt.Printf("J2: %s\n", ja3sstring) + } + } +} + +func (p *Peng) PortScanningHandler(port uint16, incomingPck bool) { + if incomingPck { + addPortToBitmap(port, p.ServerTraffic) + } else { + addPortToBitmap(port, p.ClientTraffic) + } } func addPortToBitmap(port uint16, pBitmap *portbitmap.PortBitmap) { diff --git a/pkg/periodicity/live_mode.go b/pkg/periodicity/live_mode.go index d51ffc6..9004adc 100644 --- a/pkg/periodicity/live_mode.go +++ b/pkg/periodicity/live_mode.go @@ -23,9 +23,8 @@ func LiveMode() { signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM) logger = opts.Logger - //netflow5 := NewNetflowV5() netflow9 := NewNetflowV9() - protos := []proto{ /*netflow5,*/ netflow9} + protos := []proto{netflow9} for _, p := range protos { wg.Add(1) diff --git a/pkg/periodicity/logic.go b/pkg/periodicity/logic.go index 6090c8c..2b42acf 100644 --- a/pkg/periodicity/logic.go +++ b/pkg/periodicity/logic.go @@ -121,172 +121,6 @@ func InspectFlow(rf RawFlow) { } } -func InspectFlowUsingJitter(rf RawFlow) { - if !(rf.Ipv4DstAddr != "" && rf.Ipv4SrcAddr != "0.0.0.0" && rf.Ipv4DstAddr != "0.0.0.0" && - !ExcludeMultiAndBroadcast(rf.Ipv4SrcAddr) && !ExcludeMultiAndBroadcast(rf.Ipv4DstAddr)) { - return - } - - //https://tools.ietf.org/html/rfc5102#section-5 - if rf.EndReason == 2 { - return - } - - key := fmt.Sprintf("%s/%s/%d", rf.Ipv4SrcAddr, rf.Ipv4DstAddr, rf.PortDst) - if flowInfo, flowSeen := analisi[key]; flowSeen { - if flowInfo.TimeWindowsExpiresAt.IsZero() { //compute new TimeWindow - SetTwDuration(flowInfo, rf) - - //add to list the current flow - tw := TimeWindow{LastFlowTime: rf.FirstSwitched, NumberSeenFlows: 1, Duration: ConvFloatToDuration(flowInfo.TWDuration)} - flowInfo.TimeWindows = append(flowInfo.TimeWindows, tw) - } else { - maxTime := flowInfo.TimeWindowsExpiresAt.Add(flowInfo.Deviation) - minTime := flowInfo.TimeWindowsExpiresAt.Add(-flowInfo.Deviation) - - if flowInfo.TimeWindowsExpiresAt.After(rf.FirstSwitched) { // timewindow not exceed - indexLastTW := len(flowInfo.TimeWindows) - 1 - flowInfo.LastSwitched = rf.LastSwitched - lastTw := flowInfo.TimeWindows[indexLastTW] - - lastTw.TotalJitter += math.Abs(rf.FirstSwitched.Sub(lastTw.LastFlowTime).Seconds()) - lastTw.NumberSeenFlows++ - lastTw.LastFlowTime = rf.FirstSwitched - - flowInfo.TimeWindows[indexLastTW] = lastTw - } else { //time window expired - timeElapsedAfterLastInsert := rf.FirstSwitched.Sub(maxTime).Seconds() - nTWSkipped := timeElapsedAfterLastInsert / flowInfo.TWDuration - - if nTWSkipped <= 1.0 { //Compute jitter for last TW - indexLastTW := len(flowInfo.TimeWindows) - 1 - lastTw := flowInfo.TimeWindows[indexLastTW] - lastTw.Jitter = ComputeJitter(lastTw) - flowInfo.TimeWindows[indexLastTW] = lastTw //Update jitter TODO add pointer - - //Check if i can analyze last X TW. Used to mark possible periodicity - if len(flowInfo.TimeWindows) == NTwToCompare { - if IsPeriodic(*flowInfo, indexLastTW, NTwToCompare) { - ChangePeriodicStatus(key, flowInfo, true) - - flowInfo.PeriodicityCounter += NTwToCompare - flowInfo.TimeWindows = flowInfo.TimeWindows[:0] //clear last X TW - - if flowInfo.PeriodicityCounter > 2 { - PeriodiFlows[key] = flowInfo //TODO mode in the if? - } - } - } - } else { - ChangePeriodicStatus(key, flowInfo, false) - ResetCurrentTW(key, flowInfo, rf.LastSwitched) - return - } - - if minTime.Before(rf.FirstSwitched) && maxTime.After(rf.FirstSwitched) { - SetTwDuration(flowInfo, rf) - - flowInfo.LastSwitched = rf.LastSwitched - tw := TimeWindow{LastFlowTime: rf.FirstSwitched, NumberSeenFlows: 1, Duration: ConvFloatToDuration(flowInfo.TWDuration)} - flowInfo.TimeWindows = append(flowInfo.TimeWindows, tw) - } else { - ResetCurrentTW(key, flowInfo, rf.LastSwitched) - } - } - } - } else { //If the new key isn't in the map prepare and add new FlowInfo - analisi[key] = &FlowInfo{ - Client: rf.Ipv4SrcAddr, - Server: rf.Ipv4DstAddr, - ServerPort: rf.PortDst, - LastSwitched: rf.LastSwitched, - TimeWindows: make([]TimeWindow, 0, NTwToCompare), - } - } -} - -func InspectFlowOLD(rf RawFlow) { - if !(rf.Ipv4DstAddr != "" && rf.Ipv4SrcAddr != "0.0.0.0" && rf.Ipv4DstAddr != "0.0.0.0" && - !ExcludeMultiAndBroadcast(rf.Ipv4SrcAddr) && !ExcludeMultiAndBroadcast(rf.Ipv4DstAddr)) { - return - } - - key := fmt.Sprintf("%s/%s/%d", rf.Ipv4SrcAddr, rf.Ipv4DstAddr, rf.PortDst) - if flowInfo, flowSeen := analisi[key]; flowSeen { - //https://tools.ietf.org/html/rfc5102#section-5 - if rf.EndReason == 2 { //if is 2 the flow is not end - return - } - - if flowInfo.TimeWindowsExpiresAt.IsZero() { //compute new TimeWindow - SetTwDuration(flowInfo, rf) - - //add to list the current flow - tw := TimeWindow{LastFlowTime: rf.FirstSwitched, NumberSeenFlows: 1, Duration: ConvFloatToDuration(flowInfo.TWDuration)} - flowInfo.TimeWindows = append(flowInfo.TimeWindows, tw) - } else { - //maxTime := flowInfo.TimeWindowsExpiresAt.Add(flowInfo.Deviation) - //minTime := flowInfo.TimeWindowsExpiresAt.Add(-flowInfo.Deviation) - - if flowInfo.TimeWindowsExpiresAt.After(rf.FirstSwitched) { // timewindow not exceed - indexLastTW := len(flowInfo.TimeWindows) - 1 - flowInfo.LastSwitched = rf.LastSwitched - lastTw := flowInfo.TimeWindows[indexLastTW] - - lastTw.TotalJitter += math.Abs(rf.FirstSwitched.Sub(lastTw.LastFlowTime).Seconds()) - lastTw.NumberSeenFlows++ - lastTw.LastFlowTime = rf.FirstSwitched - - flowInfo.TimeWindows[indexLastTW] = lastTw - } else { //time window expired - timeElapsedAfterLastInsert := rf.FirstSwitched.Sub(flowInfo.TimeWindowsExpiresAt).Seconds() - nTWSkipped := timeElapsedAfterLastInsert / flowInfo.TWDuration - - if nTWSkipped < 1.2 { //Compute jitter for last TW - indexLastTW := len(flowInfo.TimeWindows) - 1 - lastTw := flowInfo.TimeWindows[indexLastTW] - lastTw.Jitter = ComputeJitter(lastTw) - flowInfo.TimeWindows[indexLastTW] = lastTw //Update jitter TODO add pointer - - //Check if i can analyze last X TW. Used to mark possible periodicity - if len(flowInfo.TimeWindows) == NTwToCompare { - if IsPeriodic(*flowInfo, indexLastTW, NTwToCompare) { - ChangePeriodicStatus(key, flowInfo, true) - - flowInfo.PeriodicityCounter += NTwToCompare - flowInfo.TimeWindows = flowInfo.TimeWindows[:0] //clear last X TW - PeriodiFlows[key] = flowInfo //TODO mode in the if? - } else { - ChangePeriodicStatus(key, flowInfo, false) - ResetCurrentTW(key, flowInfo, rf.LastSwitched) - return - } - } - } else { - ChangePeriodicStatus(key, flowInfo, false) - ResetCurrentTW(key, flowInfo, rf.LastSwitched) - return - } - - //TODO refactor this - SetTwDuration(flowInfo, rf) - - //add to list the current flow - tw := TimeWindow{LastFlowTime: rf.FirstSwitched, NumberSeenFlows: 1, Duration: ConvFloatToDuration(flowInfo.TWDuration)} - flowInfo.TimeWindows = append(flowInfo.TimeWindows, tw) - } - } - } else { //If the new key isn't in the map prepare and add new FlowInfo - analisi[key] = &FlowInfo{ - Client: rf.Ipv4SrcAddr, - Server: rf.Ipv4DstAddr, - ServerPort: rf.PortDst, - LastSwitched: rf.LastSwitched, - TimeWindows: make([]TimeWindow, 0, NTwToCompare), - } - } -} - func ResetCurrentTW(key string, fi *FlowInfo, lastSwitched time.Time) { fi.TimeWindows = fi.TimeWindows[:0] //clear last X TW fi.LastSwitched = lastSwitched @@ -308,46 +142,6 @@ func ChangePeriodicStatus(key string, fi *FlowInfo, v bool) { fi.CurrentlyPeriodic = v } -func IsPeriodic(v FlowInfo, twOffset, nWindowToCompare int) bool { - if len(v.TimeWindows) < 1 || twOffset-(nWindowToCompare-1) < 0 || twOffset < 0 || nWindowToCompare <= 0 { - return false - } - maxAllowdJitter := v.TWDuration - ((v.TWDuration * PercentageDeviation) / 100.0) - if len(v.TimeWindows) == 1 { - if v.TimeWindows[0].NumberSeenFlows == 2 { - fmt.Print() - } - return v.TimeWindows[0].Jitter < maxAllowdJitter - } - - jitterTw := make([]float64, nWindowToCompare) - for i := 0; i < nWindowToCompare; i++ { - jitterTw[nWindowToCompare-i-1] = v.TimeWindows[twOffset-i].Jitter - - if jitterTw[nWindowToCompare-i-1] == -1 { - return false - } - } - - jitterDiff, jitterTotal, jitterAvg := 0.0, 0.0, 0.0 - for i := nWindowToCompare - 1; i > 0; i-- { - jitterDiff = math.Abs(jitterTw[i-1] - jitterTw[i]) - jitterTotal += jitterDiff - } - - jitterAvg = jitterTotal / float64(nWindowToCompare-1) - - return jitterAvg < maxAllowdJitter -} - -//jitter = sum(|x(i) - x(i-1)|) / (n-1)) -func ComputeJitter(tw TimeWindow) float64 { - if tw.NumberSeenFlows == 1 { - return tw.TotalJitter - } - return tw.TotalJitter / float64(tw.NumberSeenFlows-1) -} - func ExcludeMultiAndBroadcast(ip string) bool { return IsMulticastAddress(ip) || IsBroadcastAddress(ip) }