Skip to content

Commit

Permalink
Merge pull request #160 from bhandras/paginate-listinvoices
Browse files Browse the repository at this point in the history
challenger: paginage ListInvoices to avoid resource exhaustion
  • Loading branch information
guggero authored Jan 13, 2025
2 parents 25ef16b + 8ec6c28 commit 3d3e042
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 40 deletions.
96 changes: 58 additions & 38 deletions challenger/lnd.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
"math"
"strings"
"sync"
"time"
Expand All @@ -13,6 +12,12 @@ import (
"github.com/lightningnetwork/lnd/lntypes"
)

const (
// invoiceQueryPageSize is the maximum number of invoices that will be
// queried in a single request.
invoiceQueryPageSize = 1000
)

// LndChallenger is a challenger that uses an lnd backend to create new L402
// payment challenges.
type LndChallenger struct {
Expand Down Expand Up @@ -74,7 +79,7 @@ func NewLndChallenger(client InvoiceClient,

// Start starts the challenger's main work which is to keep track of all
// invoices and their states. For that the backing lnd node is queried for all
// invoices on startup and the a subscription to all subsequent invoice updates
// invoices on startup and a subscription to all subsequent invoice updates
// is created.
func (l *LndChallenger) Start() error {
// These are the default values for the subscription. In case there are
Expand All @@ -84,49 +89,64 @@ func (l *LndChallenger) Start() error {
addIndex := uint64(0)
settleIndex := uint64(0)

// Get a list of all existing invoices on startup and add them to our
// cache. We need to keep track of all invoices, even quite old ones to
// make sure tokens are valid. But to save space we only keep track of
// an invoice's state.
log.Debugf("Starting LND challenger")
// Paginate through all existing invoices on startup and add them to our
// cache. We need to keep track of all invoices to ensure tokens are
// valid.
ctx := l.clientCtx()
invoiceResp, err := l.client.ListInvoices(
ctx, &lnrpc.ListInvoiceRequest{
NumMaxInvoices: math.MaxUint64,
},
)
if err != nil {
return err
}

// Advance our indices to the latest known one so we'll only receive
// updates for new invoices and/or newly settled invoices.
l.invoicesMtx.Lock()
for _, invoice := range invoiceResp.Invoices {
// Some invoices like AMP invoices may not have a payment hash
// populated.
if invoice.RHash == nil {
continue
indexOffset := uint64(0)
for {
log.Debugf("Querying invoices from index %d", indexOffset)
invoiceResp, err := l.client.ListInvoices(
ctx, &lnrpc.ListInvoiceRequest{
IndexOffset: indexOffset,
NumMaxInvoices: invoiceQueryPageSize,
},
)
if err != nil {
return err
}

if invoice.AddIndex > addIndex {
addIndex = invoice.AddIndex
}
if invoice.SettleIndex > settleIndex {
settleIndex = invoice.SettleIndex
}
hash, err := lntypes.MakeHash(invoice.RHash)
if err != nil {
l.invoicesMtx.Unlock()
return fmt.Errorf("error parsing invoice hash: %v", err)
// If there are no more invoices, stop pagination.
if len(invoiceResp.Invoices) == 0 {
break
}

// Don't track the state of canceled or expired invoices.
if invoiceIrrelevant(invoice) {
continue
// Lock the mutex to safely update the invoice states.
l.invoicesMtx.Lock()
for _, invoice := range invoiceResp.Invoices {
// Skip invoices that do not have a payment hash
// populated.
if invoice.RHash == nil {
continue
}

if invoice.AddIndex > addIndex {
addIndex = invoice.AddIndex
}
if invoice.SettleIndex > settleIndex {
settleIndex = invoice.SettleIndex
}
hash, err := lntypes.MakeHash(invoice.RHash)
if err != nil {
l.invoicesMtx.Unlock()
return fmt.Errorf("error parsing invoice "+
"hash: %v", err)
}

// Skip tracking the state of canceled or expired
// invoices.
if invoiceIrrelevant(invoice) {
continue
}
l.invoiceStates[hash] = invoice.State
}
l.invoiceStates[hash] = invoice.State
l.invoicesMtx.Unlock()

// Update the index offset for the next batch.
indexOffset = invoiceResp.LastIndexOffset
}
l.invoicesMtx.Unlock()
log.Debugf("Finished querying invoices")

// We need to be able to cancel any subscription we make.
ctxc, cancel := context.WithCancel(l.clientCtx())
Expand Down
14 changes: 12 additions & 2 deletions challenger/lnd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,21 @@ type mockInvoiceClient struct {

// ListInvoices returns a paginated list of all invoices known to lnd.
func (m *mockInvoiceClient) ListInvoices(_ context.Context,
_ *lnrpc.ListInvoiceRequest,
r *lnrpc.ListInvoiceRequest,
_ ...grpc.CallOption) (*lnrpc.ListInvoiceResponse, error) {

if r.IndexOffset >= uint64(len(m.invoices)) {
return &lnrpc.ListInvoiceResponse{}, nil
}

endIndex := r.IndexOffset + r.NumMaxInvoices
if endIndex > uint64(len(m.invoices)) {
endIndex = uint64(len(m.invoices))
}

return &lnrpc.ListInvoiceResponse{
Invoices: m.invoices,
Invoices: m.invoices[r.IndexOffset:endIndex],
LastIndexOffset: endIndex,
}, nil
}

Expand Down

0 comments on commit 3d3e042

Please sign in to comment.