Merge pull request #3 from BOOMfinity-Developers/slave

0.0.1

README.md

@@ -25,15 +25,15 @@ Run on Windows 11, AMD Ryzen 7 3700X, 32GB RAM 3200MHz.
 $ go-wrk -c 100 -d 20 http://localhost:5656/ip?q=194.35.232.123
 
 Running 20s test @ http://localhost:5656/ip?q=194.35.232.123
-100 goroutine(s) running concurrently
-396026 requests in 17.544609038s, 57.79MB read
-Requests/sec:           22572.52
-Transfer/sec:           3.29MB
-Avg Req Time:           4.430165ms
+  100 goroutine(s) running concurrently
+574077 requests in 17.079976921s, 83.76MB read
+Requests/sec:           33611.11
+Transfer/sec:           4.90MB
+Avg Req Time:           2.975206ms
 Fastest Request:        0s
-Slowest Request:        32.7456ms
+Slowest Request:        32.4233ms
 Number of Errors:       0
-# Stats: ~50% CPU, ~20MB RAM
+# Stats: ~20% CPU, ~50MB RAM
 ```
 
 - 1 connection, 20 seconds:
@@ -43,14 +43,14 @@ $ go-wrk -c 1 -d 20 http://localhost:5656/ip?q=194.35.232.123
 
 Running 20s test @ http://localhost:5656/ip?q=194.35.232.123
   1 goroutine(s) running concurrently
-83425 requests in 19.6422217s, 12.17MB read
-Requests/sec:           4247.23
-Transfer/sec:           634.60KB
-Avg Req Time:           235.447µs
+283966 requests in 18.9446991s, 41.43MB read
+Requests/sec:           14989.21
+Transfer/sec:           2.19MB
+Avg Req Time:           66.714µs
 Fastest Request:        0s
-Slowest Request:        3.1675ms
+Slowest Request:        3.641ms
 Number of Errors:       0
-# Stats: ~10% CPU, ~14MB RAM
+# Stats: ~10% CPU, ~38MB RAM
 ```
 
 # Installation and usage

cmd/groxyp/groxyp.go

@@ -5,17 +5,32 @@ import (
 	"github.com/BOOMfinity-Developers/GrOxyP/pkg/config"
 	"github.com/BOOMfinity-Developers/GrOxyP/pkg/database"
 	"github.com/BOOMfinity-Developers/GrOxyP/pkg/webserver"
+	"log"
+	"time"
 )
 
 func main() {
-	err := database.UpdateDatabase(false)
-	if err != nil {
-		return
-	}
+	// Getting config from config.json
 	var cfg = config.GetConfig()
+	// Downloading fresh database immediately
+	err := database.UpdateDatabase(false)
 	if err != nil {
 		return
 	}
+	// Updating database "in background" at given interval
+	go func() {
+		// Parsing duration
+		interval, err := time.ParseDuration(cfg.DatabaseUpdateInterval)
+		if err != nil {
+			log.Fatal(err)
+		}
+		// Starting interval
+		err = database.SetUpdateInterval(interval, false)
+		if err != nil {
+			fmt.Println(err)
+		}
+	}()
+	// Starting webserver to listen HTTP queries
 	err = webserver.Listen(cfg.WebserverPort)
 	if err != nil {
 		fmt.Println(err)

config.json.example

@@ -1,5 +1,6 @@
 {
   "databaseDownloadURL": "https://raw.githubusercontent.com/X4BNet/lists_vpn/main/ipv4.txt",
   "databaseFilename": "ips.txt",
+  "databaseUpdateInterval": "4h0m0s",
   "webserverPort": 5656
 }

go.mod

@@ -3,6 +3,11 @@ module github.com/BOOMfinity-Developers/GrOxyP
 go 1.17
 
 require (
-	github.com/ip2location/ip2proxy-go v3.0.0+incompatible
-	github.com/klauspost/compress v1.14.2
+	github.com/segmentio/encoding v0.3.3
+	github.com/yl2chen/cidranger v1.0.2
+)
+
+require (
+	github.com/segmentio/asm v1.1.3 // indirect
+	golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 // indirect
 )

go.sum

@@ -1,4 +1,20 @@
-github.com/ip2location/ip2proxy-go v3.0.0+incompatible h1:Huqkp/Lw24CAT4a+UyupNmEoFGmrJAIerqPgMdb5x/A=
-github.com/ip2location/ip2proxy-go v3.0.0+incompatible/go.mod h1:ntasiq+RCKmbpZN+0Ng7qlq5Gw/C4urmGeXaV6z2DqA=
-github.com/klauspost/compress v1.14.2 h1:S0OHlFk/Gbon/yauFJ4FfJJF5V0fc5HbBTJazi28pRw=
-github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc=
+github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg=
+github.com/segmentio/encoding v0.3.3 h1:VG1HceOLwHkSDdkxshlu9pD4FftWkScmHc8RDQ+w9uM=
+github.com/segmentio/encoding v0.3.3/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM=
+github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/yl2chen/cidranger v1.0.2 h1:lbOWZVCG1tCRX4u24kuM1Tb4nHqWkDxwLdoS+SevawU=
+github.com/yl2chen/cidranger v1.0.2/go.mod h1:9U1yz7WPYDwf0vpNWFaeRh0bjwz5RVgRy/9UEQfHl0g=
+golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ=
+golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/config/config.go

@@ -6,6 +6,7 @@ import (
 	"os"
 )
 
+// GetConfig reads config.json file and returns parsed settings
 func GetConfig() Config {
 	//Source: https://github.com/MrBoombastic/GoProdukcji
 	file, err := os.Open("config.json")

pkg/config/types.go

@@ -1,7 +1,9 @@
 package config
 
+// Config is a structure of config.js file
 type Config struct {
-	DatabaseFilename    string `json:"databaseFilename"`
-	DatabaseDownloadURL string `json:"databaseDownloadURL"`
-	WebserverPort       uint16 `json:"webserverPort"`
+	DatabaseFilename       string `json:"databaseFilename"`
+	DatabaseDownloadURL    string `json:"databaseDownloadURL"`
+	DatabaseUpdateInterval string `json:"databaseUpdateInterval"`
+	WebserverPort          uint16 `json:"webserverPort"`
 }

pkg/database/database.go

@@ -5,20 +5,27 @@ import (
 	"errors"
 	"fmt"
 	"github.com/BOOMfinity-Developers/GrOxyP/pkg/config"
+	"github.com/yl2chen/cidranger"
 	"io"
 	"net"
 	"net/http"
 	"os"
+	"time"
 )
 
+// Getting config
 var cfg = config.GetConfig()
-var nets []*net.IPNet
 
-func UpdateDatabase(disableUpdate bool) error { //arg for debug
+// Defining CIDR checker to check, if given IP is included in given CIDR
+var ranger = cidranger.NewPCTrieRanger()
+
+// UpdateDatabase is for downloading database from GitHub to ips.txt and then storing it in memory
+func UpdateDatabase(disableUpdate bool) error {
+	// If disableUpdate is true, application will NOT update its database. Useful for debug or offline mode
 	if disableUpdate {
 		return nil
 	}
-	//Source: https://golang.cafe/blog/golang-unzip-file-example.html
+	// Source: https://golang.cafe/blog/golang-unzip-file-example.html
 	fmt.Println("INFO: Downloading database...")
 	response, err := http.Get("https://raw.githubusercontent.com/X4BNet/lists_vpn/main/ipv4.txt")
 	if err != nil {
@@ -42,10 +49,23 @@ func UpdateDatabase(disableUpdate bool) error { //arg for debug
 	if err != nil {
 		return err
 	}
+	return nil
+}
 
+// SetUpdateInterval is simple function to run UpdateDatabase at given interval
+func SetUpdateInterval(d time.Duration, disableUpdate bool) error {
+	for range time.Tick(d) {
+		fmt.Println("INFO: Database update started...")
+		err := UpdateDatabase(disableUpdate)
+		if err != nil {
+			return err
+		}
+		fmt.Println("INFO: Database update done")
+	}
 	return nil
 }
 
+// convertDatabase converts downloaded ips.txt file to networks in memory
 func convertDatabase() error {
 	file, err := os.Open(cfg.DatabaseFilename)
 	if err != nil {
@@ -56,7 +76,10 @@ func convertDatabase() error {
 	scanner := bufio.NewScanner(file)
 	for scanner.Scan() {
 		if _, currNet, err := net.ParseCIDR(scanner.Text()); err == nil {
-			nets = append(nets, currNet)
+			err := ranger.Insert(cidranger.NewBasicRangerEntry(*currNet))
+			if err != nil {
+				fmt.Printf("Error while inserting CIDR to database: %v\n", err.Error())
+			}
 		}
 	}
 
@@ -66,12 +89,11 @@ func convertDatabase() error {
 	return nil
 }
 
+// SearchIPInDatabase checks if given IP is on the list. If so, returns "true" and reason.
 func SearchIPInDatabase(query string) (bool, string) {
-	q := net.ParseIP(query)
-	for _, currNet := range nets {
-		if currNet.Contains(q) {
-			return true, currNet.String()
-		}
+	if containingNetworks, err := ranger.ContainingNetworks(net.ParseIP(query)); len(containingNetworks) > 0 && err == nil {
+		network := containingNetworks[0].Network()
+		return true, network.String()
 	}
 	return false, ""
 }

pkg/webserver/types.go

@@ -1,6 +1,7 @@
 package webserver
 
-type apiResponseIpType = struct {
+// apiResponseIP is a structure of /ip endpoint response
+type apiResponseIP = struct {
 	IP    string `json:"ip"`
 	Proxy bool   `json:"proxy"`
 	Rule  string `json:"rule"`

pkg/webserver/webserver.go

@@ -1,12 +1,13 @@
 package webserver
 
 import (
-	"encoding/json"
 	"fmt"
 	"github.com/BOOMfinity-Developers/GrOxyP/pkg/database"
+	"github.com/segmentio/encoding/json"
 	"net/http"
 )
 
+// hello returns "OK" on every non-existing endpoint
 func hello(w http.ResponseWriter, _ *http.Request) {
 	_, err := fmt.Fprintf(w, "OK\n")
 	if err != nil {
@@ -15,12 +16,13 @@ func hello(w http.ResponseWriter, _ *http.Request) {
 	}
 }
 
+// ip returns queried IP, if queried IP is behind a proxy or VPN and which network has been blocked (reason/rule)
 func ip(w http.ResponseWriter, req *http.Request) {
-	//IP for testing: uk2345.nordvpn.com [194.35.232.123] - should be proxy
+	// IP for testing: uk2345.nordvpn.com [194.35.232.123] - should be proxy
 	ip := req.FormValue("q")
 	proxy, rule := database.SearchIPInDatabase(ip)
 	w.Header().Set("Content-Type", "application/json")
-	response := apiResponseIpType{
+	response := apiResponseIP{
 		IP:    ip,
 		Proxy: proxy,
 		Rule:  rule,
@@ -32,6 +34,9 @@ func ip(w http.ResponseWriter, req *http.Request) {
 	}
 }
 
+// Listen starts HTTP server for IP queries.
+// Available endpoints: /ip
+// Usage is in README
 func Listen(port uint16) error {
 	//Source: https://gobyexample.com/http-servers
 	http.HandleFunc("/", hello)