lndmon: cache closedChannels response

collectors/channels_collector.go

@@ -4,13 +4,17 @@ import (
 	"context"
 	"fmt"
 	"strconv"
+	"sync"
+	"time"
 
 	"github.com/btcsuite/btcd/btcutil"
 	"github.com/lightninglabs/lndclient"
 	"github.com/lightningnetwork/lnd/routing/route"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+const closedChannelsKey = "closedChannels"
+
 // ChannelsCollector is a collector that keeps track of channel information.
 type ChannelsCollector struct {
 	channelBalanceDesc           *prometheus.Desc
@@ -51,6 +55,14 @@ type ChannelsCollector struct {
 	// errChan is a channel that we send any errors that we encounter into.
 	// This channel should be buffered so that it does not block sends.
 	errChan chan<- error
+
+	// quit is a channel that we use to signal for graceful shutdown
+	quit chan struct{}
+
+	// cache is a map storing results from a ticker to reduce grpc server
+	// load on lnd
+	closedChannelsCache []lndclient.ClosedChannel
+	cacheMutex          sync.RWMutex
 }
 
 // NewChannelsCollector returns a new instance of the ChannelsCollector for the
@@ -61,7 +73,7 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error,
 	// Our set of labels, status should either be active or inactive. The
 	// initiator is "true" if we are the initiator, and "false" otherwise.
 	labels := []string{"chan_id", "status", "initiator", "peer"}
-	return &ChannelsCollector{
+	collector := &ChannelsCollector{
 		channelBalanceDesc: prometheus.NewDesc(
 			"lnd_channels_open_balance_sat",
 			"total balance of channels in satoshis",
@@ -174,10 +186,43 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error,
 			[]string{"amount"}, nil,
 		),
 
-		lnd:         lnd,
-		primaryNode: cfg.PrimaryNode,
-		errChan:     errChan,
+		lnd:                 lnd,
+		primaryNode:         cfg.PrimaryNode,
+		closedChannelsCache: []lndclient.ClosedChannel{},
+		errChan:             errChan,
+		quit:                make(chan struct{}),
+	}
+
+	// Perform an initial refresh for the cache
+	collector.refreshClosedChannelsCache()
+
+	// Start a ticker to update the cache once per 10m
+	go func() {
+		ticker := time.NewTicker(10 * time.Minute)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ticker.C:
+				collector.refreshClosedChannelsCache()
+
+			case <-collector.quit:
+				return
+			}
+		}
+	}()
+
+	return collector
+}
+
+func (c *ChannelsCollector) refreshClosedChannelsCache() {
+	data, err := c.lnd.ClosedChannels(context.Background())
+	if err != nil {
+		c.errChan <- err
+		return
 	}
+	c.cacheMutex.Lock()
+	c.closedChannelsCache = data
+	c.cacheMutex.Unlock()
 }
 
 // Describe sends the super-set of all possible descriptors of metrics
@@ -452,12 +497,9 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) {
 	)
 
 	// Get the list of closed channels.
-	closedChannelsResp, err := c.lnd.ClosedChannels(context.Background())
-	if err != nil {
-		c.errChan <- fmt.Errorf("ChannelsCollector ClosedChannels "+
-			"failed with: %v", err)
-		return
-	}
+	c.cacheMutex.RLock()
+	closedChannelsResp := c.closedChannelsCache
+	c.cacheMutex.RUnlock()
 	closeCounts := make(map[string]int)
 	for _, channel := range closedChannelsResp {
 		typeString, ok := closeTypeLabelMap[channel.CloseType]

collectors/prometheus.go

@@ -5,7 +5,9 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"os/signal"
 	"path/filepath"
+	"syscall"
 
 	"github.com/btcsuite/btcd/btcutil"
 	"github.com/lightninglabs/lndclient"
@@ -94,11 +96,21 @@ func NewPrometheusExporter(cfg *PrometheusConfig, lnd *lndclient.LndServices,
 
 	htlcMonitor := newHtlcMonitor(lnd.Router, errChan)
 
+	// Setup signalling so SIGTERM || SIGINT will stop the cache goroutine
+	// from ChannelsCollector
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	chanCollector := NewChannelsCollector(
+		lnd.Client, errChan, monitoringCfg,
+	)
+	go func() {
+		<-sigs
+		close(chanCollector.quit)
+	}()
+
 	collectors := []prometheus.Collector{
 		NewChainCollector(lnd.Client, errChan),
-		NewChannelsCollector(
-			lnd.Client, errChan, monitoringCfg,
-		),
+		chanCollector,
 		NewWalletCollector(lnd, errChan),
 		NewPeerCollector(lnd.Client, errChan),
 		NewInfoCollector(lnd.Client, errChan),