From 043f8b9747980750f8db0be9378f5d547cae1933 Mon Sep 17 00:00:00 2001 From: unx Date: Thu, 27 Jan 2022 11:25:43 +0100 Subject: [PATCH 1/2] perf: ~4x RPS boost by using cidranger package --- go.mod | 9 +++++++-- go.sum | 22 ++++++++++++++++++---- pkg/database/database.go | 17 ++++++++++------- pkg/webserver/webserver.go | 2 +- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 4174440..a2ab2c0 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,11 @@ module github.com/BOOMfinity-Developers/GrOxyP go 1.17 require ( - github.com/ip2location/ip2proxy-go v3.0.0+incompatible - github.com/klauspost/compress v1.14.2 + github.com/segmentio/encoding v0.3.3 + github.com/yl2chen/cidranger v1.0.2 +) + +require ( + github.com/segmentio/asm v1.1.3 // indirect + golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect ) diff --git a/go.sum b/go.sum index 0616aa9..c60132e 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,18 @@ -github.com/ip2location/ip2proxy-go v3.0.0+incompatible h1:Huqkp/Lw24CAT4a+UyupNmEoFGmrJAIerqPgMdb5x/A= -github.com/ip2location/ip2proxy-go v3.0.0+incompatible/go.mod h1:ntasiq+RCKmbpZN+0Ng7qlq5Gw/C4urmGeXaV6z2DqA= -github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw= -github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= +github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= +github.com/segmentio/encoding v0.3.3 h1:VG1HceOLwHkSDdkxshlu9pD4FftWkScmHc8RDQ+w9uM= +github.com/segmentio/encoding v0.3.3/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= +github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= +golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ= +golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/database/database.go b/pkg/database/database.go index fb1bbbb..d92b5d3 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "github.com/BOOMfinity-Developers/GrOxyP/pkg/config" + "github.com/yl2chen/cidranger" "io" "net" "net/http" @@ -12,7 +13,7 @@ import ( ) var cfg = config.GetConfig() -var nets []*net.IPNet +var ranger = cidranger.NewPCTrieRanger() func UpdateDatabase(disableUpdate bool) error { //arg for debug if disableUpdate { @@ -56,7 +57,10 @@ func convertDatabase() error { scanner := bufio.NewScanner(file) for scanner.Scan() { if _, currNet, err := net.ParseCIDR(scanner.Text()); err == nil { - nets = append(nets, currNet) + err := ranger.Insert(cidranger.NewBasicRangerEntry(*currNet)) + if err != nil { + fmt.Printf("Error while inserting CIDR to database: %v\n", err.Error()) + } } } @@ -67,11 +71,10 @@ func convertDatabase() error { } func SearchIPInDatabase(query string) (bool, string) { - q := net.ParseIP(query) - for _, currNet := range nets { - if currNet.Contains(q) { - return true, currNet.String() - } + if containingNetworks, err := ranger.ContainingNetworks(net.ParseIP(query)); len(containingNetworks) > 0 && err == nil { + network := containingNetworks[0].Network() + return true, network.String() } + return false, "" } diff --git a/pkg/webserver/webserver.go b/pkg/webserver/webserver.go index abe579a..fb10b3b 100644 --- a/pkg/webserver/webserver.go +++ b/pkg/webserver/webserver.go @@ -1,9 +1,9 @@ package webserver import ( - "encoding/json" "fmt" "github.com/BOOMfinity-Developers/GrOxyP/pkg/database" + "github.com/segmentio/encoding/json" "net/http" ) From 35456b39fe04d03f7973d3ab1a4fb2160a45ed07 Mon Sep 17 00:00:00 2001 From: MrBoombastic Date: Thu, 27 Jan 2022 19:03:38 +0100 Subject: [PATCH 2/2] new benchmarks, code comments, update database in intervals --- README.md | 26 +++++++++++++------------- cmd/groxyp/groxyp.go | 23 +++++++++++++++++++---- config.json.example | 1 + go.sum | 2 ++ pkg/config/config.go | 1 + pkg/config/types.go | 8 +++++--- pkg/database/database.go | 25 ++++++++++++++++++++++--- pkg/webserver/types.go | 3 ++- pkg/webserver/webserver.go | 9 +++++++-- 9 files changed, 72 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index a5cdd45..6ecc4bd 100644 --- a/README.md +++ b/README.md @@ -25,15 +25,15 @@ Run on Windows 11, AMD Ryzen 7 3700X, 32GB RAM 3200MHz. $ go-wrk -c 100 -d 20 http://localhost:5656/ip?q=194.35.232.123 Running 20s test @ http://localhost:5656/ip?q=194.35.232.123 -100 goroutine(s) running concurrently -396026 requests in 17.544609038s, 57.79MB read -Requests/sec: 22572.52 -Transfer/sec: 3.29MB -Avg Req Time: 4.430165ms + 100 goroutine(s) running concurrently +574077 requests in 17.079976921s, 83.76MB read +Requests/sec: 33611.11 +Transfer/sec: 4.90MB +Avg Req Time: 2.975206ms Fastest Request: 0s -Slowest Request: 32.7456ms +Slowest Request: 32.4233ms Number of Errors: 0 -# Stats: ~50% CPU, ~20MB RAM +# Stats: ~20% CPU, ~50MB RAM ``` - 1 connection, 20 seconds: @@ -43,14 +43,14 @@ $ go-wrk -c 1 -d 20 http://localhost:5656/ip?q=194.35.232.123 Running 20s test @ http://localhost:5656/ip?q=194.35.232.123 1 goroutine(s) running concurrently -83425 requests in 19.6422217s, 12.17MB read -Requests/sec: 4247.23 -Transfer/sec: 634.60KB -Avg Req Time: 235.447µs +283966 requests in 18.9446991s, 41.43MB read +Requests/sec: 14989.21 +Transfer/sec: 2.19MB +Avg Req Time: 66.714µs Fastest Request: 0s -Slowest Request: 3.1675ms +Slowest Request: 3.641ms Number of Errors: 0 -# Stats: ~10% CPU, ~14MB RAM +# Stats: ~10% CPU, ~38MB RAM ``` # Installation and usage diff --git a/cmd/groxyp/groxyp.go b/cmd/groxyp/groxyp.go index 67ab270..5534c8e 100644 --- a/cmd/groxyp/groxyp.go +++ b/cmd/groxyp/groxyp.go @@ -5,17 +5,32 @@ import ( "github.com/BOOMfinity-Developers/GrOxyP/pkg/config" "github.com/BOOMfinity-Developers/GrOxyP/pkg/database" "github.com/BOOMfinity-Developers/GrOxyP/pkg/webserver" + "log" + "time" ) func main() { - err := database.UpdateDatabase(false) - if err != nil { - return - } + // Getting config from config.json var cfg = config.GetConfig() + // Downloading fresh database immediately + err := database.UpdateDatabase(false) if err != nil { return } + // Updating database "in background" at given interval + go func() { + // Parsing duration + interval, err := time.ParseDuration(cfg.DatabaseUpdateInterval) + if err != nil { + log.Fatal(err) + } + // Starting interval + err = database.SetUpdateInterval(interval, false) + if err != nil { + fmt.Println(err) + } + }() + // Starting webserver to listen HTTP queries err = webserver.Listen(cfg.WebserverPort) if err != nil { fmt.Println(err) diff --git a/config.json.example b/config.json.example index c6a3eff..638223a 100644 --- a/config.json.example +++ b/config.json.example @@ -1,5 +1,6 @@ { "databaseDownloadURL": "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/ipv4.txt", "databaseFilename": "ips.txt", + "databaseUpdateInterval": "4h0m0s", "webserverPort": 5656 } \ No newline at end of file diff --git a/go.sum b/go.sum index c60132e..f09f12e 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,7 @@ github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= github.com/segmentio/encoding v0.3.3 h1:VG1HceOLwHkSDdkxshlu9pD4FftWkScmHc8RDQ+w9uM= github.com/segmentio/encoding v0.3.3/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -13,6 +14,7 @@ github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+Seva github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ= golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/config/config.go b/pkg/config/config.go index f3354c9..5d0d646 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -6,6 +6,7 @@ import ( "os" ) +// GetConfig reads config.json file and returns parsed settings func GetConfig() Config { //Source: https://github.com/MrBoombastic/GoProdukcji file, err := os.Open("config.json") diff --git a/pkg/config/types.go b/pkg/config/types.go index 5433e33..9c1c098 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -1,7 +1,9 @@ package config +// Config is a structure of config.js file type Config struct { - DatabaseFilename string `json:"databaseFilename"` - DatabaseDownloadURL string `json:"databaseDownloadURL"` - WebserverPort uint16 `json:"webserverPort"` + DatabaseFilename string `json:"databaseFilename"` + DatabaseDownloadURL string `json:"databaseDownloadURL"` + DatabaseUpdateInterval string `json:"databaseUpdateInterval"` + WebserverPort uint16 `json:"webserverPort"` } diff --git a/pkg/database/database.go b/pkg/database/database.go index d92b5d3..107c9ac 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -10,16 +10,22 @@ import ( "net" "net/http" "os" + "time" ) +// Getting config var cfg = config.GetConfig() + +// Defining CIDR checker to check, if given IP is included in given CIDR var ranger = cidranger.NewPCTrieRanger() -func UpdateDatabase(disableUpdate bool) error { //arg for debug +// UpdateDatabase is for downloading database from GitHub to ips.txt and then storing it in memory +func UpdateDatabase(disableUpdate bool) error { + // If disableUpdate is true, application will NOT update its database. Useful for debug or offline mode if disableUpdate { return nil } - //Source: https://golang.cafe/blog/golang-unzip-file-example.html + // Source: https://golang.cafe/blog/golang-unzip-file-example.html fmt.Println("INFO: Downloading database...") response, err := http.Get("https://raw.githubusercontent.com/X4BNet/lists_vpn/main/ipv4.txt") if err != nil { @@ -43,10 +49,23 @@ func UpdateDatabase(disableUpdate bool) error { //arg for debug if err != nil { return err } + return nil +} +// SetUpdateInterval is simple function to run UpdateDatabase at given interval +func SetUpdateInterval(d time.Duration, disableUpdate bool) error { + for range time.Tick(d) { + fmt.Println("INFO: Database update started...") + err := UpdateDatabase(disableUpdate) + if err != nil { + return err + } + fmt.Println("INFO: Database update done") + } return nil } +// convertDatabase converts downloaded ips.txt file to networks in memory func convertDatabase() error { file, err := os.Open(cfg.DatabaseFilename) if err != nil { @@ -70,11 +89,11 @@ func convertDatabase() error { return nil } +// SearchIPInDatabase checks if given IP is on the list. If so, returns "true" and reason. func SearchIPInDatabase(query string) (bool, string) { if containingNetworks, err := ranger.ContainingNetworks(net.ParseIP(query)); len(containingNetworks) > 0 && err == nil { network := containingNetworks[0].Network() return true, network.String() } - return false, "" } diff --git a/pkg/webserver/types.go b/pkg/webserver/types.go index 3949e1a..7658720 100644 --- a/pkg/webserver/types.go +++ b/pkg/webserver/types.go @@ -1,6 +1,7 @@ package webserver -type apiResponseIpType = struct { +// apiResponseIP is a structure of /ip endpoint response +type apiResponseIP = struct { IP string `json:"ip"` Proxy bool `json:"proxy"` Rule string `json:"rule"` diff --git a/pkg/webserver/webserver.go b/pkg/webserver/webserver.go index fb10b3b..86923bb 100644 --- a/pkg/webserver/webserver.go +++ b/pkg/webserver/webserver.go @@ -7,6 +7,7 @@ import ( "net/http" ) +// hello returns "OK" on every non-existing endpoint func hello(w http.ResponseWriter, _ *http.Request) { _, err := fmt.Fprintf(w, "OK\n") if err != nil { @@ -15,12 +16,13 @@ func hello(w http.ResponseWriter, _ *http.Request) { } } +// ip returns queried IP, if queried IP is behind a proxy or VPN and which network has been blocked (reason/rule) func ip(w http.ResponseWriter, req *http.Request) { - //IP for testing: uk2345.nordvpn.com [194.35.232.123] - should be proxy + // IP for testing: uk2345.nordvpn.com [194.35.232.123] - should be proxy ip := req.FormValue("q") proxy, rule := database.SearchIPInDatabase(ip) w.Header().Set("Content-Type", "application/json") - response := apiResponseIpType{ + response := apiResponseIP{ IP: ip, Proxy: proxy, Rule: rule, @@ -32,6 +34,9 @@ func ip(w http.ResponseWriter, req *http.Request) { } } +// Listen starts HTTP server for IP queries. +// Available endpoints: /ip +// Usage is in README func Listen(port uint16) error { //Source: https://gobyexample.com/http-servers http.HandleFunc("/", hello)