diff --git a/collectors/channels_collector.go b/collectors/channels_collector.go index 085e308..aaf9c98 100644 --- a/collectors/channels_collector.go +++ b/collectors/channels_collector.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "strconv" + "sync" + "time" "github.com/btcsuite/btcd/btcutil" "github.com/lightninglabs/lndclient" @@ -11,6 +13,8 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +const closedChannelsKey = "closedChannels" + // ChannelsCollector is a collector that keeps track of channel information. type ChannelsCollector struct { channelBalanceDesc *prometheus.Desc @@ -51,6 +55,14 @@ type ChannelsCollector struct { // errChan is a channel that we send any errors that we encounter into. // This channel should be buffered so that it does not block sends. errChan chan<- error + + // quit is a channel that we use to signal for graceful shutdown + quit chan struct{} + + // cache is a map storing results from a ticker to reduce grpc server + // load on lnd + closedChannelsCache []lndclient.ClosedChannel + cacheMutex sync.RWMutex } // NewChannelsCollector returns a new instance of the ChannelsCollector for the @@ -61,7 +73,7 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error, // Our set of labels, status should either be active or inactive. The // initiator is "true" if we are the initiator, and "false" otherwise. labels := []string{"chan_id", "status", "initiator", "peer"} - return &ChannelsCollector{ + collector := &ChannelsCollector{ channelBalanceDesc: prometheus.NewDesc( "lnd_channels_open_balance_sat", "total balance of channels in satoshis", @@ -174,10 +186,43 @@ func NewChannelsCollector(lnd lndclient.LightningClient, errChan chan<- error, []string{"amount"}, nil, ), - lnd: lnd, - primaryNode: cfg.PrimaryNode, - errChan: errChan, + lnd: lnd, + primaryNode: cfg.PrimaryNode, + closedChannelsCache: []lndclient.ClosedChannel{}, + errChan: errChan, + quit: make(chan struct{}), + } + + // Perform an initial refresh for the cache + collector.refreshClosedChannelsCache() + + // Start a ticker to update the cache once per 10m + go func() { + ticker := time.NewTicker(10 * time.Minute) + defer ticker.Stop() + for { + select { + case <-ticker.C: + collector.refreshClosedChannelsCache() + + case <-collector.quit: + return + } + } + }() + + return collector +} + +func (c *ChannelsCollector) refreshClosedChannelsCache() { + data, err := c.lnd.ClosedChannels(context.Background()) + if err != nil { + c.errChan <- err + return } + c.cacheMutex.Lock() + c.closedChannelsCache = data + c.cacheMutex.Unlock() } // Describe sends the super-set of all possible descriptors of metrics @@ -452,12 +497,9 @@ func (c *ChannelsCollector) Collect(ch chan<- prometheus.Metric) { ) // Get the list of closed channels. - closedChannelsResp, err := c.lnd.ClosedChannels(context.Background()) - if err != nil { - c.errChan <- fmt.Errorf("ChannelsCollector ClosedChannels "+ - "failed with: %v", err) - return - } + c.cacheMutex.RLock() + closedChannelsResp := c.closedChannelsCache + c.cacheMutex.RUnlock() closeCounts := make(map[string]int) for _, channel := range closedChannelsResp { typeString, ok := closeTypeLabelMap[channel.CloseType] diff --git a/collectors/prometheus.go b/collectors/prometheus.go index c5368cc..e84bbea 100644 --- a/collectors/prometheus.go +++ b/collectors/prometheus.go @@ -5,7 +5,9 @@ import ( "log" "net/http" "os" + "os/signal" "path/filepath" + "syscall" "github.com/btcsuite/btcd/btcutil" "github.com/lightninglabs/lndclient" @@ -94,11 +96,21 @@ func NewPrometheusExporter(cfg *PrometheusConfig, lnd *lndclient.LndServices, htlcMonitor := newHtlcMonitor(lnd.Router, errChan) + // Setup signalling so SIGTERM || SIGINT will stop the cache goroutine + // from ChannelsCollector + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) + chanCollector := NewChannelsCollector( + lnd.Client, errChan, monitoringCfg, + ) + go func() { + <-sigs + close(chanCollector.quit) + }() + collectors := []prometheus.Collector{ NewChainCollector(lnd.Client, errChan), - NewChannelsCollector( - lnd.Client, errChan, monitoringCfg, - ), + chanCollector, NewWalletCollector(lnd, errChan), NewPeerCollector(lnd.Client, errChan), NewInfoCollector(lnd.Client, errChan),