diff --git a/README.md b/README.md index a5cdd45..6ecc4bd 100644 --- a/README.md +++ b/README.md @@ -25,15 +25,15 @@ Run on Windows 11, AMD Ryzen 7 3700X, 32GB RAM 3200MHz. $ go-wrk -c 100 -d 20 http://localhost:5656/ip?q=194.35.232.123 Running 20s test @ http://localhost:5656/ip?q=194.35.232.123 -100 goroutine(s) running concurrently -396026 requests in 17.544609038s, 57.79MB read -Requests/sec: 22572.52 -Transfer/sec: 3.29MB -Avg Req Time: 4.430165ms + 100 goroutine(s) running concurrently +574077 requests in 17.079976921s, 83.76MB read +Requests/sec: 33611.11 +Transfer/sec: 4.90MB +Avg Req Time: 2.975206ms Fastest Request: 0s -Slowest Request: 32.7456ms +Slowest Request: 32.4233ms Number of Errors: 0 -# Stats: ~50% CPU, ~20MB RAM +# Stats: ~20% CPU, ~50MB RAM ``` - 1 connection, 20 seconds: @@ -43,14 +43,14 @@ $ go-wrk -c 1 -d 20 http://localhost:5656/ip?q=194.35.232.123 Running 20s test @ http://localhost:5656/ip?q=194.35.232.123 1 goroutine(s) running concurrently -83425 requests in 19.6422217s, 12.17MB read -Requests/sec: 4247.23 -Transfer/sec: 634.60KB -Avg Req Time: 235.447µs +283966 requests in 18.9446991s, 41.43MB read +Requests/sec: 14989.21 +Transfer/sec: 2.19MB +Avg Req Time: 66.714µs Fastest Request: 0s -Slowest Request: 3.1675ms +Slowest Request: 3.641ms Number of Errors: 0 -# Stats: ~10% CPU, ~14MB RAM +# Stats: ~10% CPU, ~38MB RAM ``` # Installation and usage diff --git a/cmd/groxyp/groxyp.go b/cmd/groxyp/groxyp.go index 67ab270..5534c8e 100644 --- a/cmd/groxyp/groxyp.go +++ b/cmd/groxyp/groxyp.go @@ -5,17 +5,32 @@ import ( "github.com/BOOMfinity-Developers/GrOxyP/pkg/config" "github.com/BOOMfinity-Developers/GrOxyP/pkg/database" "github.com/BOOMfinity-Developers/GrOxyP/pkg/webserver" + "log" + "time" ) func main() { - err := database.UpdateDatabase(false) - if err != nil { - return - } + // Getting config from config.json var cfg = config.GetConfig() + // Downloading fresh database immediately + err := database.UpdateDatabase(false) if err != nil { return } + // Updating database "in background" at given interval + go func() { + // Parsing duration + interval, err := time.ParseDuration(cfg.DatabaseUpdateInterval) + if err != nil { + log.Fatal(err) + } + // Starting interval + err = database.SetUpdateInterval(interval, false) + if err != nil { + fmt.Println(err) + } + }() + // Starting webserver to listen HTTP queries err = webserver.Listen(cfg.WebserverPort) if err != nil { fmt.Println(err) diff --git a/config.json.example b/config.json.example index c6a3eff..638223a 100644 --- a/config.json.example +++ b/config.json.example @@ -1,5 +1,6 @@ { "databaseDownloadURL": "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/ipv4.txt", "databaseFilename": "ips.txt", + "databaseUpdateInterval": "4h0m0s", "webserverPort": 5656 } \ No newline at end of file diff --git a/go.mod b/go.mod index 4174440..a2ab2c0 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,11 @@ module github.com/BOOMfinity-Developers/GrOxyP go 1.17 require ( - github.com/ip2location/ip2proxy-go v3.0.0+incompatible - github.com/klauspost/compress v1.14.2 + github.com/segmentio/encoding v0.3.3 + github.com/yl2chen/cidranger v1.0.2 +) + +require ( + github.com/segmentio/asm v1.1.3 // indirect + golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect ) diff --git a/go.sum b/go.sum index 0616aa9..f09f12e 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,20 @@ -github.com/ip2location/ip2proxy-go v3.0.0+incompatible h1:Huqkp/Lw24CAT4a+UyupNmEoFGmrJAIerqPgMdb5x/A= -github.com/ip2location/ip2proxy-go v3.0.0+incompatible/go.mod h1:ntasiq+RCKmbpZN+0Ng7qlq5Gw/C4urmGeXaV6z2DqA= -github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw= -github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= +github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= +github.com/segmentio/encoding v0.3.3 h1:VG1HceOLwHkSDdkxshlu9pD4FftWkScmHc8RDQ+w9uM= +github.com/segmentio/encoding v0.3.3/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU= +github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g= +golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ= +golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/config/config.go b/pkg/config/config.go index f3354c9..5d0d646 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -6,6 +6,7 @@ import ( "os" ) +// GetConfig reads config.json file and returns parsed settings func GetConfig() Config { //Source: https://github.com/MrBoombastic/GoProdukcji file, err := os.Open("config.json") diff --git a/pkg/config/types.go b/pkg/config/types.go index 5433e33..9c1c098 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -1,7 +1,9 @@ package config +// Config is a structure of config.js file type Config struct { - DatabaseFilename string `json:"databaseFilename"` - DatabaseDownloadURL string `json:"databaseDownloadURL"` - WebserverPort uint16 `json:"webserverPort"` + DatabaseFilename string `json:"databaseFilename"` + DatabaseDownloadURL string `json:"databaseDownloadURL"` + DatabaseUpdateInterval string `json:"databaseUpdateInterval"` + WebserverPort uint16 `json:"webserverPort"` } diff --git a/pkg/database/database.go b/pkg/database/database.go index fb1bbbb..107c9ac 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -5,20 +5,27 @@ import ( "errors" "fmt" "github.com/BOOMfinity-Developers/GrOxyP/pkg/config" + "github.com/yl2chen/cidranger" "io" "net" "net/http" "os" + "time" ) +// Getting config var cfg = config.GetConfig() -var nets []*net.IPNet -func UpdateDatabase(disableUpdate bool) error { //arg for debug +// Defining CIDR checker to check, if given IP is included in given CIDR +var ranger = cidranger.NewPCTrieRanger() + +// UpdateDatabase is for downloading database from GitHub to ips.txt and then storing it in memory +func UpdateDatabase(disableUpdate bool) error { + // If disableUpdate is true, application will NOT update its database. Useful for debug or offline mode if disableUpdate { return nil } - //Source: https://golang.cafe/blog/golang-unzip-file-example.html + // Source: https://golang.cafe/blog/golang-unzip-file-example.html fmt.Println("INFO: Downloading database...") response, err := http.Get("https://raw.githubusercontent.com/X4BNet/lists_vpn/main/ipv4.txt") if err != nil { @@ -42,10 +49,23 @@ func UpdateDatabase(disableUpdate bool) error { //arg for debug if err != nil { return err } + return nil +} +// SetUpdateInterval is simple function to run UpdateDatabase at given interval +func SetUpdateInterval(d time.Duration, disableUpdate bool) error { + for range time.Tick(d) { + fmt.Println("INFO: Database update started...") + err := UpdateDatabase(disableUpdate) + if err != nil { + return err + } + fmt.Println("INFO: Database update done") + } return nil } +// convertDatabase converts downloaded ips.txt file to networks in memory func convertDatabase() error { file, err := os.Open(cfg.DatabaseFilename) if err != nil { @@ -56,7 +76,10 @@ func convertDatabase() error { scanner := bufio.NewScanner(file) for scanner.Scan() { if _, currNet, err := net.ParseCIDR(scanner.Text()); err == nil { - nets = append(nets, currNet) + err := ranger.Insert(cidranger.NewBasicRangerEntry(*currNet)) + if err != nil { + fmt.Printf("Error while inserting CIDR to database: %v\n", err.Error()) + } } } @@ -66,12 +89,11 @@ func convertDatabase() error { return nil } +// SearchIPInDatabase checks if given IP is on the list. If so, returns "true" and reason. func SearchIPInDatabase(query string) (bool, string) { - q := net.ParseIP(query) - for _, currNet := range nets { - if currNet.Contains(q) { - return true, currNet.String() - } + if containingNetworks, err := ranger.ContainingNetworks(net.ParseIP(query)); len(containingNetworks) > 0 && err == nil { + network := containingNetworks[0].Network() + return true, network.String() } return false, "" } diff --git a/pkg/webserver/types.go b/pkg/webserver/types.go index 3949e1a..7658720 100644 --- a/pkg/webserver/types.go +++ b/pkg/webserver/types.go @@ -1,6 +1,7 @@ package webserver -type apiResponseIpType = struct { +// apiResponseIP is a structure of /ip endpoint response +type apiResponseIP = struct { IP string `json:"ip"` Proxy bool `json:"proxy"` Rule string `json:"rule"` diff --git a/pkg/webserver/webserver.go b/pkg/webserver/webserver.go index abe579a..86923bb 100644 --- a/pkg/webserver/webserver.go +++ b/pkg/webserver/webserver.go @@ -1,12 +1,13 @@ package webserver import ( - "encoding/json" "fmt" "github.com/BOOMfinity-Developers/GrOxyP/pkg/database" + "github.com/segmentio/encoding/json" "net/http" ) +// hello returns "OK" on every non-existing endpoint func hello(w http.ResponseWriter, _ *http.Request) { _, err := fmt.Fprintf(w, "OK\n") if err != nil { @@ -15,12 +16,13 @@ func hello(w http.ResponseWriter, _ *http.Request) { } } +// ip returns queried IP, if queried IP is behind a proxy or VPN and which network has been blocked (reason/rule) func ip(w http.ResponseWriter, req *http.Request) { - //IP for testing: uk2345.nordvpn.com [194.35.232.123] - should be proxy + // IP for testing: uk2345.nordvpn.com [194.35.232.123] - should be proxy ip := req.FormValue("q") proxy, rule := database.SearchIPInDatabase(ip) w.Header().Set("Content-Type", "application/json") - response := apiResponseIpType{ + response := apiResponseIP{ IP: ip, Proxy: proxy, Rule: rule, @@ -32,6 +34,9 @@ func ip(w http.ResponseWriter, req *http.Request) { } } +// Listen starts HTTP server for IP queries. +// Available endpoints: /ip +// Usage is in README func Listen(port uint16) error { //Source: https://gobyexample.com/http-servers http.HandleFunc("/", hello)