Merge pull request #43 from rluisr/fix/issue#40

set 0 on route connection, health if MySQL Router is down

.github/workflows/lint.yml

@@ -9,7 +9,7 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-go@v4
         with:
           go-version-file: 'go.mod'

.github/workflows/release.yml

@@ -10,7 +10,7 @@ jobs:
     name: Binary Release
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-go@v4
         with:
           go-version-file: 'go.mod'
@@ -27,7 +27,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v4

go.mod

@@ -4,18 +4,18 @@ go 1.20
 
 require (
 	github.com/jessevdk/go-flags v1.5.0
-	github.com/prometheus/client_golang v1.15.1
-	github.com/rluisr/mysqlrouter-go v1.1.1
+	github.com/prometheus/client_golang v1.17.0
+	github.com/rluisr/mysqlrouter-go v1.2.0
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.9.0 // indirect
-	golang.org/x/sys v0.6.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 )

go.sum

@@ -17,22 +17,36 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI=
 github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk=
+github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
 github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4=
 github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
 github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
 github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
 github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rluisr/mysqlrouter-go v1.1.1 h1:8Rm5op3GDUNhJtVXaO7nWosoK3nr0YIIVmLSXsY/cbo=
 github.com/rluisr/mysqlrouter-go v1.1.1/go.mod h1:TakU6i+cdcQhI4uRiAIpMa51lV/IJiAMHGyDxeFSVuc=
+github.com/rluisr/mysqlrouter-go v1.2.0 h1:2E65PtqEiCgQRE14SSsi8+I6IPPrqKveFcO2JG+pHck=
+github.com/rluisr/mysqlrouter-go v1.2.0/go.mod h1:TakU6i+cdcQhI4uRiAIpMa51lV/IJiAMHGyDxeFSVuc=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
 google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

hack/README.md

@@ -0,0 +1,11 @@
+If you want to run InnoDB Cluster with http and https REST API, run this command.
+
+```shell
+$ git clone [email protected]:rluisr/mysqlrouter-go.git
+$ cd mysqlrouter-go/test
+$ docker compose up --build --force-recreate --always-recreate-deps --renew-anon-volumes -d;
+```
+
+### Basic
+
+root / mysql

main.go

@@ -47,6 +47,12 @@ var args struct {
 	Version bool `short:"v" long:"version" description:"Show version"`
 }
 
+var (
+	lastRouter           *mysqlrouter.Router
+	lastRoutes           []*mysqlrouter.Routes
+	lastRouteConnections []*mysqlrouter.RouteConnections
+)
+
 const (
 	nameSpace = "mysqlrouter"
 )
@@ -127,55 +133,111 @@ func recordMetrics() {
 }
 
 func collectMetrics() {
-	// router
-	router, err := mysqlRouterClient.GetRouterStatus()
+	router, err := collectRouterMetrics()
 	if err != nil {
 		writeError(err)
-		routerUpGauge.Set(float64(0))
-		return
+		router = nil
+	}
+
+	collectMetadataMetrics(router)
+	collectRouteMetrics(router)
+}
+
+func collectRouterMetrics() (*mysqlrouter.Router, error) {
+	router, err := mysqlRouterClient.GetRouterStatus()
+	if err != nil {
+		routerUpGauge.Set(0)
+		return nil, err
 	}
-	routerUpGauge.Set(float64(1))
+
+	routerUpGauge.Set(1)
 	routerStatusGauge.WithLabelValues(strconv.Itoa(router.ProcessID), router.ProductEdition, router.TimeStarted.String(), router.Version, router.Hostname)
 
-	// metadata
-	metadatas, err := mysqlRouterClient.GetAllMetadata()
+	lastRouter = router
+
+	return router, nil
+}
+
+func collectMetadataMetrics(router *mysqlrouter.Router) {
+	// nil means router is down
+	// so we don't need to collect metadata metrics
+	if router == nil {
+		return
+	}
+
+	metadata, err := mysqlRouterClient.GetAllMetadata()
 	if err != nil {
 		writeError(err)
 		return
 	}
-	for _, metadata := range metadatas {
-		metadataGauge.WithLabelValues(metadata.Name)
+
+	for _, m := range metadata {
+		metadataGauge.WithLabelValues(m.Name)
 
 		// config
-		metadataConfig, gmcErr := mysqlRouterClient.GetMetadataConfig(metadata.Name)
+		metadataConfig, gmcErr := mysqlRouterClient.GetMetadataConfig(m.Name)
 		if gmcErr != nil {
 			writeError(gmcErr)
 			return
 		}
-		metadataConfigGauge.WithLabelValues(metadata.Name, metadataConfig.ClusterName, strconv.Itoa(metadataConfig.TimeRefreshInMs), metadataConfig.GroupReplicationID)
+		metadataConfigGauge.WithLabelValues(m.Name, metadataConfig.ClusterName, strconv.Itoa(metadataConfig.TimeRefreshInMs), metadataConfig.GroupReplicationID)
 
 		// config nodes count
-		metadataConfigNodesGauge.WithLabelValues(metadata.Name, router.Hostname, metadataConfig.ClusterName).Set(float64(len(metadataConfig.Nodes)))
+		metadataConfigNodesGauge.WithLabelValues(m.Name, router.Hostname, metadataConfig.ClusterName).Set(float64(len(metadataConfig.Nodes)))
 
 		// status
 		if args.CollectMetadataStatus {
-			metadataStatus, gmsErr := mysqlRouterClient.GetMetadataStatus(metadata.Name)
+			metadataStatus, gmsErr := mysqlRouterClient.GetMetadataStatus(m.Name)
 			if gmsErr != nil {
 				writeError(gmsErr)
 				return
 			}
 			metadataStatusGauge.Reset()
-			metadataStatusGauge.WithLabelValues(metadata.Name, strconv.Itoa(metadataStatus.RefreshFailed), metadataStatus.TimeLastRefreshSucceeded.String(), metadataStatus.LastRefreshHostname, strconv.Itoa(metadataStatus.LastRefreshPort))
+			metadataStatusGauge.WithLabelValues(m.Name, strconv.Itoa(metadataStatus.RefreshFailed), metadataStatus.TimeLastRefreshSucceeded.String(), metadataStatus.LastRefreshHostname, strconv.Itoa(metadataStatus.LastRefreshPort))
 		}
 	}
+}
+
+func collectRouteMetrics(router *mysqlrouter.Router) {
+	// nil means router is down
+	// so route metrics will be 0
+	if router == nil {
+		for _, route := range lastRoutes {
+			routeActiveConnectionsGauge.WithLabelValues(route.Name, lastRouter.Hostname).Set(0)
+			routeTotalConnectionsGauge.WithLabelValues(route.Name, lastRouter.Hostname).Set(0)
+			routeBlockedHostsGauge.WithLabelValues(route.Name, lastRouter.Hostname).Set(0)
+			routeHealthGauge.WithLabelValues(route.Name, lastRouter.Hostname).Set(0)
+
+			for _, routeConnection := range lastRouteConnections {
+				if args.CollectRouteConnectionsByteFromServer {
+					routeConnectionsByteFromServerGauge.WithLabelValues(route.Name, router.Hostname, routeConnection.SourceAddress, routeConnection.DestinationAddress).Set(0)
+				}
+				if args.CollectRouteConnectionsByteToServer {
+					routeConnectionsByteToServerGauge.WithLabelValues(route.Name, router.Hostname, routeConnection.SourceAddress, routeConnection.DestinationAddress).Set(0)
+				}
+				if args.CollectRouteConnectionsTimeStarted {
+					routeConnectionsTimeStartedGauge.WithLabelValues(route.Name, router.Hostname, routeConnection.SourceAddress, routeConnection.DestinationAddress).Set(0)
+				}
+				if args.CollectRouteConnectionsTimeConnectedToServer {
+					routeConnectionsTimeConnectedToServerGauge.WithLabelValues(route.Name, router.Hostname, routeConnection.SourceAddress, routeConnection.DestinationAddress).Set(0)
+				}
+				if args.CollectRouteConnectionsTimeLastSentToServer {
+					routeConnectionsTimeLastSentToServerGauge.WithLabelValues(route.Name, router.Hostname, routeConnection.SourceAddress, routeConnection.DestinationAddress).Set(0)
+				}
+				if args.CollectRouteConnectionsTimeReceivedFromServer {
+					routeConnectionsTimeLastReceivedFromServerGauge.WithLabelValues(route.Name, router.Hostname, routeConnection.SourceAddress, routeConnection.DestinationAddress).Set(0)
+				}
+			}
+		}
+		return
+	}
 
-	// routes
 	routes, err := mysqlRouterClient.GetAllRoutes()
 	if err != nil {
 		writeError(err)
 		return
 	}
-	
+
 	if args.CollectRouteConnectionsByteFromServer {
 		routeConnectionsByteFromServerGauge.Reset()
 	}
@@ -253,7 +315,9 @@ func collectMetrics() {
 				routeConnectionsTimeLastReceivedFromServerGauge.WithLabelValues(route.Name, router.Hostname, routeConnection.SourceAddress, routeConnection.DestinationAddress).Set(float64(routeConnection.TimeLastReceivedFromServer.Unix() * 1000)) // nolint
 			}
 		}
+		lastRouteConnections = routeConnections
 	}
+	lastRoutes = routes
 }
 
 func writeError(err error) {