Merge pull request #6 from Packet-Clearing-House/mysql-customer-db

Mysql customer db

.gitignore

@@ -1,4 +1,5 @@
 # Binaries for programs and plugins
+.DS_Store
 *.exe
 *.dll
 *.so

DNSAuth/config.go

@@ -8,6 +8,7 @@ import (
 type Config struct {
 	BGP *bgp.BGPConf
 	CustomerDB string `cfg:"customer-db; required; "`
+	CustomerRefresh int `cfg:"customer-refresh; 24; "`
 	InfluxDB string `cfg:"influx-db; required; "`
 	WatchDir string `cfg:"watch-dir; required; "`
 }

DNSAuth/customers.go

@@ -2,46 +2,77 @@ package main
 
 import (
 	"database/sql"
-	"log"
-	"github.com/asergeyev/nradix"
+	"sync"
+	_ "github.com/go-sql-driver/mysql"
+	radix "github.com/armon/go-radix"
 	_"github.com/lib/pq"
 )
 
-type Customer struct {
-	Prefix string
-	Name string
-	PrefixMonit bool
-	ASNMonit bool
-}
+var DB_URL = "root:pass@(127.0.0.1)/customers"
 
-var DB_URL = "postgres://[email protected]/pipeline?sslmode=disable"
+// Function that reverse a word (test.com -> moc.tset)
+func reverse(s string) string {
+	r := []rune(s)
+	for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
+		r[i], r[j] = r[j], r[i]
+	}
+	return string(r)
+}
 
-func getCustomerTree() (*nradix.Tree, error) {
+type CustomerDB struct {
+	*sync.Mutex
+	dburl string
+	tree *radix.Tree
+}
 
-	tree := nradix.NewTree(0)
+// Resolve the customer name from DNS qname 
+// Returns Unknown if not found
+func (c *CustomerDB) Resolve(qname string) string {
+
+	name := "Unknown"
+	c.Lock()
+	_, value, found := c.tree.LongestPrefix(reverse(qname))
+	c.Unlock()
+	if found {
+		name = value.(string)
+	}
+	return name
+}
 
-	db, err := sql.Open("postgres", DB_URL)
+func (c *CustomerDB) Refresh() error {
+
+	tree := radix.New()
+	mysql, err := sql.Open("mysql", DB_URL)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
-	rows, err := db.Query("SELECT ip::cidr, name, asn, prefix FROM ns_customers;")
+	rows, err := mysql.Query("SELECT group_name, zone FROM zones;")
 	if err != nil {
-		return nil, err
+		return err
 	}
 	defer rows.Close()
 
-
 	for rows.Next() {
-		c := Customer{}
-		err := rows.Scan(&c.Prefix, &c.Name, &c.ASNMonit, &c.PrefixMonit)
+		var name, zone string
+		err := rows.Scan(&name, &zone)
 		if err != nil {
-			log.Fatal(err)
-		}
-		err = tree.AddCIDR(c.Prefix, &c)
-		if err != nil {
-			log.Println(err)
+			return err
 		}
+		tree.Insert(reverse(zone), name)
+	}
+
+	c.Lock()
+	c.tree = tree
+	c.Unlock()
+	return nil
+}
+
+// Init the customer DB. Connects to mysql to fetch all data and build a radix tree
+func NewCustomerDB(path string) (*CustomerDB) {
+	return &CustomerDB{
+		new(sync.Mutex),
+		path,
+		radix.New(),
 	}
-	return tree, nil
 }

DNSAuth/dnsauth.toml

@@ -1,13 +1,11 @@
-customer-db = "postgres://user:[email protected]/postgres?sslmode=disable"
-influx-db = "http://127.0.0.1:8086/write?db=authdns"
-watch-dir = "/home/user/count"
+# URL for the MySQL instance to retreive customers
+customer-db = "root:pass@(127.0.0.1)/customers"
+
+# Refreshing interval (hours) of the customer database.
+customer-refresh = 10
 
-#[bgp]
-#    router-id = “116.121.4.10"
-#    local-as = 53
-#    peer-addr = “176.53.11.25"
-#    peer-as = 12
-#    connect-port = 179
-#    listen-port = 179 # need sudo to run (privilege port)
-#    remove-as-from-path = [12, 4234]
+# The URL of the influx DB instance.
+influx-db = "http://127.0.0.1:8086/write?db=authdns"
 
+# The directory DNSAuth should watch for new log files coming in.
+watch-dir = "./"

DNSAuth/influx_pusher.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"time"
 	"log"
 	"github.com/Packet-Clearing-House/DNSAuth/libs/metrics"
 	"strings"
@@ -13,6 +14,9 @@ import (
 var INFLUX_URL = "http://127.0.0.1:8086/write?db=authdns"
 
 func push(registry *metrics.Registry) {
+
+	starttime := time.Now()
+
 	str := registry.Encode(&metrics.InfluxEncodeur{})
 	splits := strings.Split(str, "\n")
 
@@ -34,12 +38,15 @@ func push(registry *metrics.Registry) {
 		}
 	}
 	resp, err := http.Post(INFLUX_URL, "application/octet-stream", buffer)
+	proctime := time.Since(starttime)
+
 	if err != nil {
-		log.Println(err)
+		log.Println("[Influx] ERROR: ", err)
 	} else if resp.StatusCode != 204 {
 		buf, _ := ioutil.ReadAll(resp.Body)
 		log.Println(string(buf))
 		resp.Body.Close()
+		log.Println("[Influx] Inserted " + strconv.Itoa(cpt) + " points in " + proctime.String())
 	}
-	log.Println("Influx pusher inserted " + strconv.Itoa(cpt) + " points!")
+
 }

DNSAuth/main.go

@@ -14,7 +14,6 @@ import (
 	"github.com/Packet-Clearing-House/DNSAuth/libs/metrics"
 	"strconv"
 	"github.com/Packet-Clearing-House/DNSAuth/libs/dnsdist"
-	"github.com/asergeyev/nradix"
 	"github.com/Packet-Clearing-House/DNSAuth/DNSAuth/bgp"
 )
 
@@ -36,10 +35,11 @@ var confpath = flag.String("c", "./dnsauth.toml", "Path for the config path (def
 
 
 var dnsqueries = metrics.NewTTLTaggedMetrics("dnsauth_queries", []string{"direction", "pop", "qtype", "rcode", "customer", "protocol", "version", "prefix", "origin_as"}, 500)
-var tree *nradix.Tree
+var customerDB *CustomerDB
 
 var BGP_LOOKUPS = false
 
+
 func main() {
 
 	flag.Parse()
@@ -54,15 +54,27 @@ func main() {
 	DB_URL = config.CustomerDB
 	INFLUX_URL = config.InfluxDB
 
-	log.Println("Getting customer list from postgres...")
-	t, err := getCustomerTree()
-	tree = t
-	if err != nil {
-		log.Fatalln("FAILED: ", err)
-	}
-	log.Println("OK!")
+
+	// Starting the customerDB fetching process
+	log.Println("Initializing customer DB (will be refresh every " + strconv.Itoa(config.CustomerRefresh) + " hours)...")
+	customerDB = NewCustomerDB(DB_URL)
+	go func () {
+		// Refresh function
+		refresh := func () {
+			log.Println("[CustomerDB] Refreshing list from mysql...")
+			if err := customerDB.Refresh(); err != nil {
+				log.Println("[CustomerDB] ERROR: Could not refresh customer list (", err, ")!")
+			}
+		}
+
+		refresh()
+		for _ = range time.Tick(time.Duration(config.CustomerRefresh) * time.Hour) {
+			refresh()
+		}
+	}()
 
 
+	// Checking for BGP conf and initaliazing BGP connection if needed
 	if config.BGP != nil {
 		BGP_LOOKUPS = true
 		log.Println("Starting BGP Resolver...")
@@ -74,17 +86,12 @@ func main() {
 	} else {
 		log.Println("BGP lookups will be ignored, no BGP config provided.")
 	}
-
-
 
+	// Running the metric pushing process
 	metrics.DefaultRegistry.Register(dnsqueries)
 	go func() {
 		for {
-			log.Println("Pushing metrics!!")
-			starttime := time.Now()
 			push(&metrics.DefaultRegistry)
-			proctime := time.Since(starttime)
-			log.Println("Took " + proctime.String() + "seconds")
 			time.Sleep(time.Minute)
 		}
 	}()
@@ -98,13 +105,11 @@ func main() {
 	visit := func (path string, f os.FileInfo, err error) error {
 		if strings.HasSuffix(path, ".dmp.gz") {
 
-			if _, found := files[path]; found {
-				newFiles[path] = true
-			} else {
-				newFiles[path] = true
+			if _, found := files[path]; !found {
 				go aggreagate(path, limiter)
 				limiter <-true
 			}
+			newFiles[path] = true
 		}
 		return nil
 	}
@@ -214,36 +219,16 @@ func handleQuery(time time.Time, pop, line string) {
 
 	fields := strings.Fields(line)
 
-	name := "Unknown"
 	prefix := ""
 	originAs := ""
 	version := "4"
 
 	// Resolving destination address to client
-	c, _ := tree.FindCIDR(fields[NS_IP])
-
-
+	qname := fields[QNAME][:len(fields[QNAME])-1]
+	name := customerDB.Resolve(qname)
 
-	// If we do find a result...
-	if c != nil {
-		customer := c.(*Customer)
-		name = customer.Name
-
-		// ...resolving client ip through BGP
-		if BGP_LOOKUPS && (customer.PrefixMonit || customer.ASNMonit) {
-			entry, err := bgp.Resolve(fields[CLIENT_IP])
-			if err == nil {
-				// I SHOULD DO SOMETHING HERE #DEBUG?
-				originAs = strconv.Itoa(int(entry.Path[len(entry.Path) - 1]))
-				if customer.PrefixMonit {
-					prefix = entry.Prefix
-				}
-			}
-		}
-	}
 
 	if ipv := net.ParseIP(fields[CLIENT_IP]); ipv != nil {
-		log.Println(ipv)
 		if ipv.To4() == nil {
 			version = "6"
 		}
@@ -270,4 +255,4 @@ func handleQuery(time time.Time, pop, line string) {
 	}
 
 	dnsqueries.GetAt(time, fields[DIRECTION], pop, qtypestr, rcodestr, name, protocol, version, originAs, prefix).Inc()
-}
+}

DNSAuth/tests/mon-01.xyz.foonet.net_2017-10-17.17-07.dmp

@@ -0,0 +1,2 @@
+Q 2fd9:4d55:875b:bae4:b46f:51e1:5388:22a6 1501:0121:0800:0000:: 1 0 2 auction.com. 60
+R 103.76.246.187 123.99.248.35 0 0 2 blabla.test.com. 595 0