Skip to content

Commit

Permalink
feat: new export for stuck packets (#24)
Browse files Browse the repository at this point in the history
* feat: combine all IBC messages to one collector

* feat: add new IBC collector - ibc channel

This is used to have metrics about IBC channel, now just have stuck packets

* feat: split one metric to three

this splits one total metric so that it will produce three metrics, src,dst and total

* fix: error if config does not have IBC RPC information

* fix: remove magic number and have stateOpen const

* chore: change naming scheme

* chore: have path info in the error message

* fix: ensure that all RPC endpoints are working

* feat: have src, dst channel IDs

* fix: have status=error on bad RPC nodes

* feat: change metric logic to only have one

* feat: add logger to PrepChain

this removes the need of having two identical checks

* refactor: readd the check and add separate statuscheck
  • Loading branch information
jlehtimaki authored Oct 31, 2023
1 parent 49a7413 commit 2bba835
Show file tree
Hide file tree
Showing 5 changed files with 207 additions and 23 deletions.
18 changes: 13 additions & 5 deletions cmd/relayer_exporter/relayer_exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,12 @@ import (
"net/http"
"os"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"

"github.com/archway-network/relayer_exporter/pkg/collector"
"github.com/archway-network/relayer_exporter/pkg/config"
log "github.com/archway-network/relayer_exporter/pkg/logger"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
Expand Down Expand Up @@ -42,7 +43,14 @@ func main() {
log.Fatal(err.Error())
}

log.Info(fmt.Sprintf("Getting IBC paths from %s/%s/%s on GitHub", cfg.GitHub.Org, cfg.GitHub.Repo, cfg.GitHub.IBCDir))
log.Info(
fmt.Sprintf(
"Getting IBC paths from %s/%s/%s on GitHub",
cfg.GitHub.Org,
cfg.GitHub.Repo,
cfg.GitHub.IBCDir,
),
)

// TODO: Add a feature to refresh paths at configured interval
paths, err := cfg.IBCPaths()
Expand All @@ -52,7 +60,7 @@ func main() {

rpcs := cfg.GetRPCsMap()

clientsCollector := collector.IBCClientsCollector{
ibcCollector := collector.IBCCollector{
RPCs: rpcs,
Paths: paths,
}
Expand All @@ -62,7 +70,7 @@ func main() {
Accounts: cfg.Accounts,
}

prometheus.MustRegister(clientsCollector)
prometheus.MustRegister(ibcCollector)
prometheus.MustRegister(balancesCollector)

http.Handle("/metrics", promhttp.Handler())
Expand Down
2 changes: 1 addition & 1 deletion config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ rpc:
url: https://noble-rpc.polkachu.com:443
- chainName: nois
chainId: nois-1
url: https://rpc.cosmos.directory/nois:443
url: https://nois.rpc.kjnodes.com:443
- chainName: osmosistestnet
chainId: osmo-test-5
url: https://rpc.osmotest5.osmosis.zone:443
Expand Down
7 changes: 4 additions & 3 deletions pkg/chain/chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (

"github.com/cosmos/relayer/v2/relayer"
"github.com/cosmos/relayer/v2/relayer/chains/cosmos"
"go.uber.org/zap"
)

const (
Expand All @@ -19,7 +20,7 @@ type Info struct {
}

func PrepChain(info Info) (*relayer.Chain, error) {
chain := relayer.Chain{}
logger := zap.NewNop()
providerConfig := cosmos.CosmosProviderConfig{
ChainID: info.ChainID,
Timeout: rpcTimeout,
Expand All @@ -37,12 +38,12 @@ func PrepChain(info Info) (*relayer.Chain, error) {
return nil, err
}

chain.ChainProvider = provider
chain := relayer.NewChain(logger, provider, false)

err = chain.SetPath(&relayer.PathEnd{ClientID: info.ClientID})
if err != nil {
return nil, err
}

return &chain, nil
return chain, nil
}
86 changes: 75 additions & 11 deletions pkg/collector/collector.go
Original file line number Diff line number Diff line change
@@ -1,22 +1,26 @@
package collector

import (
"fmt"
"math/big"
"reflect"
"sync"

"github.com/archway-network/relayer_exporter/pkg/config"
"github.com/archway-network/relayer_exporter/pkg/ibc"
log "github.com/archway-network/relayer_exporter/pkg/logger"
"github.com/cosmos/relayer/v2/relayer"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"

"github.com/archway-network/relayer_exporter/pkg/config"
"github.com/archway-network/relayer_exporter/pkg/ibc"
log "github.com/archway-network/relayer_exporter/pkg/logger"
)

const (
successStatus = "success"
errorStatus = "error"
clientExpiryMetricName = "cosmos_ibc_client_expiry"
walletBalanceMetricName = "cosmos_wallet_balance"
successStatus = "success"
errorStatus = "error"
clientExpiryMetricName = "cosmos_ibc_client_expiry"
walletBalanceMetricName = "cosmos_wallet_balance"
channelStuckPacketsMetricName = "cosmos_ibc_stuck_packets"
)

var (
Expand All @@ -25,14 +29,26 @@ var (
"Returns light client expiry in unixtime.",
[]string{"host_chain_id", "client_id", "target_chain_id", "status"}, nil,
)
channelStuckPackets = prometheus.NewDesc(
channelStuckPacketsMetricName,
"Returns stuck packets for a channel.",
[]string{
"src_channel_id",
"dst_channel_id",
"src_chain_id",
"dst_chain_id",
"status",
},
nil,
)
walletBalance = prometheus.NewDesc(
walletBalanceMetricName,
"Returns wallet balance for an address on a chain.",
[]string{"account", "chain_id", "denom", "status"}, nil,
)
)

type IBCClientsCollector struct {
type IBCCollector struct {
RPCs *map[string]config.RPC
Paths []*relayer.IBCdata
}
Expand All @@ -42,12 +58,19 @@ type WalletBalanceCollector struct {
Accounts []config.Account
}

func (cc IBCClientsCollector) Describe(ch chan<- *prometheus.Desc) {
func (cc IBCCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- clientExpiry
ch <- channelStuckPackets
}

func (cc IBCClientsCollector) Collect(ch chan<- prometheus.Metric) {
log.Debug("Start collecting", zap.String("metric", clientExpiryMetricName))
func (cc IBCCollector) Collect(ch chan<- prometheus.Metric) {
log.Debug(
"Start collecting",
zap.String(
"metrics",
fmt.Sprintf("%s, %s", clientExpiryMetricName, channelStuckPacketsMetricName),
),
)

var wg sync.WaitGroup

Expand All @@ -57,6 +80,7 @@ func (cc IBCClientsCollector) Collect(ch chan<- prometheus.Metric) {
go func(path *relayer.IBCdata) {
defer wg.Done()

// Client info
ci, err := ibc.GetClientsInfo(path, cc.RPCs)
status := successStatus

Expand All @@ -79,6 +103,46 @@ func (cc IBCClientsCollector) Collect(ch chan<- prometheus.Metric) {
float64(ci.ChainBClientExpiration.Unix()),
[]string{(*cc.RPCs)[path.Chain2.ChainName].ChainID, path.Chain2.ClientID, (*cc.RPCs)[path.Chain1.ChainName].ChainID, status}...,
)

// Stuck packets
status = successStatus

stuckPackets, err := ibc.GetChannelsInfo(path, cc.RPCs)
if err != nil {
status = errorStatus

log.Error(err.Error())
}

if !reflect.DeepEqual(stuckPackets, ibc.ChannelsInfo{}) {
for _, sp := range stuckPackets.Channels {
ch <- prometheus.MustNewConstMetric(
channelStuckPackets,
prometheus.GaugeValue,
float64(sp.StuckPackets.Source),
[]string{
sp.Source,
sp.Destination,
(*cc.RPCs)[path.Chain1.ChainName].ChainID,
(*cc.RPCs)[path.Chain2.ChainName].ChainID,
status,
}...,
)

ch <- prometheus.MustNewConstMetric(
channelStuckPackets,
prometheus.GaugeValue,
float64(sp.StuckPackets.Destination),
[]string{
sp.Destination,
sp.Source,
(*cc.RPCs)[path.Chain2.ChainName].ChainID,
(*cc.RPCs)[path.Chain1.ChainName].ChainID,
status,
}...,
)
}
}
}(p)
}

Expand Down
117 changes: 114 additions & 3 deletions pkg/ibc/ibc.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,15 @@ import (
"fmt"
"time"

chantypes "github.com/cosmos/ibc-go/v7/modules/core/04-channel/types"
"github.com/cosmos/relayer/v2/relayer"

"github.com/archway-network/relayer_exporter/pkg/chain"
"github.com/archway-network/relayer_exporter/pkg/config"
"github.com/cosmos/relayer/v2/relayer"
)

const stateOpen = 3

type ClientsInfo struct {
ChainA *relayer.Chain
ChainAClientInfo relayer.ClientStateInfo
Expand All @@ -19,6 +23,22 @@ type ClientsInfo struct {
ChainBClientExpiration time.Time
}

type ChannelsInfo struct {
Channels []Channel
}

type Channel struct {
Source string
Destination string
SourcePort string
DestinationPort string
Ordering string
StuckPackets struct {
Source int
Destination int
}
}

func GetClientsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ClientsInfo, error) {
clientsInfo := ClientsInfo{}

Expand Down Expand Up @@ -50,15 +70,106 @@ func GetClientsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ClientsI

ctx := context.Background()

clientsInfo.ChainAClientExpiration, clientsInfo.ChainAClientInfo, err = relayer.QueryClientExpiration(ctx, chainA, chainB)
clientsInfo.ChainAClientExpiration, clientsInfo.ChainAClientInfo, err = relayer.QueryClientExpiration(
ctx,
chainA,
chainB,
)
if err != nil {
return ClientsInfo{}, fmt.Errorf("Error: %w path %v <-> %v", err, cdA, cdB)
}

clientsInfo.ChainBClientExpiration, clientsInfo.ChainBClientInfo, err = relayer.QueryClientExpiration(ctx, chainB, chainA)
clientsInfo.ChainBClientExpiration, clientsInfo.ChainBClientInfo, err = relayer.QueryClientExpiration(
ctx,
chainB,
chainA,
)
if err != nil {
return ClientsInfo{}, fmt.Errorf("Error: %w path %v <-> %v", err, cdB, cdA)
}

return clientsInfo, nil
}

func GetChannelsInfo(ibc *relayer.IBCdata, rpcs *map[string]config.RPC) (ChannelsInfo, error) {
ctx := context.Background()
channelInfo := ChannelsInfo{}

// Init channel data
for _, c := range ibc.Channels {
var channel Channel
channel.Source = c.Chain1.ChannelID
channel.Destination = c.Chain2.ChannelID
channel.SourcePort = c.Chain1.PortID
channel.DestinationPort = c.Chain2.PortID
channel.Ordering = c.Ordering
channelInfo.Channels = append(channelInfo.Channels, channel)
}

if (*rpcs)[ibc.Chain1.ChainName].ChainID == "" || (*rpcs)[ibc.Chain2.ChainName].ChainID == "" {
return channelInfo, fmt.Errorf(
"Error: RPC data is missing, cannot retrieve channel data: %v",
ibc.Channels,
)
}

cdA := chain.Info{
ChainID: (*rpcs)[ibc.Chain1.ChainName].ChainID,
RPCAddr: (*rpcs)[ibc.Chain1.ChainName].URL,
ClientID: ibc.Chain1.ClientID,
}

chainA, err := chain.PrepChain(cdA)
if err != nil {
return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdA)
}

cdB := chain.Info{
ChainID: (*rpcs)[ibc.Chain2.ChainName].ChainID,
RPCAddr: (*rpcs)[ibc.Chain2.ChainName].URL,
ClientID: ibc.Chain2.ClientID,
}

chainB, err := chain.PrepChain(cdB)
if err != nil {
return ChannelsInfo{}, fmt.Errorf("Error: %w for %v", err, cdB)
}

// test that RPC endpoints are working
if _, _, err := relayer.QueryLatestHeights(
ctx, chainA, chainB,
); err != nil {
return channelInfo, fmt.Errorf("Error: %w for %v", err, cdA)
}

for i, c := range channelInfo.Channels {
var order chantypes.Order

switch c.Ordering {
case "none":
order = chantypes.NONE
case "unordered":
order = chantypes.UNORDERED
case "ordered":
order = chantypes.ORDERED
}

ch := chantypes.IdentifiedChannel{
State: stateOpen,
Ordering: order,
Counterparty: chantypes.Counterparty{
PortId: c.DestinationPort,
ChannelId: c.Destination,
},
PortId: c.SourcePort,
ChannelId: c.Source,
}

unrelayedSequences := relayer.UnrelayedSequences(ctx, chainA, chainB, &ch)

channelInfo.Channels[i].StuckPackets.Source += len(unrelayedSequences.Src)
channelInfo.Channels[i].StuckPackets.Destination += len(unrelayedSequences.Dst)
}

return channelInfo, nil
}

0 comments on commit 2bba835

Please sign in to comment.