From 747d41c44f538d927e7db730a52dfca6c991e8d1 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 7 Sep 2023 12:25:11 +0200 Subject: [PATCH 1/5] fix lnd init bug --- init_lnd.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/init_lnd.go b/init_lnd.go index b3618ba3..44d64405 100644 --- a/init_lnd.go +++ b/init_lnd.go @@ -30,6 +30,9 @@ func InitSingleLNDClient(c *service.Config, ctx context.Context) (result lnd.Lig CertFile: c.LNDCertFile, CertHex: c.LNDCertHex, }, ctx) + if err != nil { + return nil, err + } getInfo, err := client.GetInfo(ctx, &lnrpc.GetInfoRequest{}) if err != nil { return nil, err From dee42bb26dc5098122f58289a618c72e93f32c31 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 7 Sep 2023 13:46:25 +0200 Subject: [PATCH 2/5] add json fields to rabbitmq error logs --- rabbitmq/rabbitmq.go | 67 +++++++++++++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/rabbitmq/rabbitmq.go b/rabbitmq/rabbitmq.go index e59498b9..fea33995 100644 --- a/rabbitmq/rabbitmq.go +++ b/rabbitmq/rabbitmq.go @@ -164,10 +164,11 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv return err } - client.logger.Infof("Payment finalizer: Found %d pending invoices", len(pendingInvoices)) - - client.logger.Info("Starting payment finalizer rabbitmq consumer") - + client.logger.Infoj(log.JSON{ + "subroutine": "payment finalizer", + "num_pending_payments": len(pendingInvoices), + "message": "starting payment finalizer loop", + }) for { select { case <-ctx.Done(): @@ -182,6 +183,9 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv err := json.Unmarshal(delivery.Body, &payment) if err != nil { + captureErr(client.logger, err, log.JSON{ + "message": "error unmarshalling payment json", + }) delivery.Nack(false, false) continue @@ -191,7 +195,10 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv if invoice, ok := pendingInvoices[payment.PaymentHash]; ok { t, err := svc.GetTransactionEntryByInvoiceId(ctx, invoice.ID) if err != nil { - captureErr(client.logger, err) + captureErr(client.logger, err, log.JSON{ + "payment_hash": invoice.RHash, + "message": "error fetching transaction entry by id", + }) delivery.Nack(false, false) continue @@ -203,7 +210,10 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv invoice.Preimage = payment.PaymentPreimage if err = svc.HandleSuccessfulPayment(ctx, &invoice, t); err != nil { - captureErr(client.logger, err) + captureErr(client.logger, err, log.JSON{ + "payment_hash": invoice.RHash, + "message": "error handling succesful payment", + }) delivery.Nack(false, false) continue @@ -214,7 +224,10 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv case lnrpc.Payment_FAILED: if err = svc.HandleFailedPayment(ctx, &invoice, t, fmt.Errorf(payment.FailureReason.String())); err != nil { - captureErr(client.logger, err) + captureErr(client.logger, err, log.JSON{ + "message": "error handling failed payment", + "payment_hash": invoice.RHash, + }) delivery.Nack(false, false) continue @@ -249,14 +262,19 @@ func (client *DefaultClient) SubscribeToLndInvoices(ctx context.Context, handler err := json.Unmarshal(delivery.Body, &invoice) if err != nil { - captureErr(client.logger, err) + captureErr(client.logger, err, log.JSON{ + "message": "error unmarshalling invoice json", + }) // If we can't even Unmarshall the message we are dealing with // badly formatted events. In that case we simply Nack the message // and explicitly do not requeue it. err = delivery.Nack(false, false) if err != nil { - captureErr(client.logger, err) + captureErr(client.logger, err, log.JSON{ + "message": "error nacking invoice", + "payment_hash": invoice.RHash, + }) } continue @@ -264,14 +282,20 @@ func (client *DefaultClient) SubscribeToLndInvoices(ctx context.Context, handler err = handler(ctx, &invoice) if err != nil { - captureErr(client.logger, err) + captureErr(client.logger, err, log.JSON{ + "message": "error handling invoice", + "payment_hash": invoice.RHash, + }) // If for some reason we can't handle the message we also don't requeue // because this can lead to an endless loop that puts pressure on the // database and logs. err := delivery.Nack(false, false) if err != nil { - captureErr(client.logger, err) + captureErr(client.logger, err, log.JSON{ + "message": "error nacking event", + "payment_hash": invoice.RHash, + }) } continue @@ -279,7 +303,10 @@ func (client *DefaultClient) SubscribeToLndInvoices(ctx context.Context, handler err = delivery.Ack(false) if err != nil { - captureErr(client.logger, err) + captureErr(client.logger, err, log.JSON{ + "message": "error acking event", + "payment_hash": invoice.RHash, + }) } } } @@ -320,13 +347,19 @@ func (client *DefaultClient) StartPublishInvoices(ctx context.Context, invoicesS err = client.publishToLndhubExchange(ctx, incomingInvoice, payloadFunc) if err != nil { - captureErr(client.logger, err) + captureErr(client.logger, err, log.JSON{ + "message": "error publishing invoice", + "payment_hash": incomingInvoice.RHash, + }) } case outgoing := <-out: err = client.publishToLndhubExchange(ctx, outgoing, payloadFunc) if err != nil { - captureErr(client.logger, err) + captureErr(client.logger, err, log.JSON{ + "message": "error publishing invoice", + "payment_hash": outgoing.RHash, + }) } } } @@ -352,7 +385,6 @@ func (client *DefaultClient) publishToLndhubExchange(ctx context.Context, invoic }, ) if err != nil { - captureErr(client.logger, err) return err } @@ -361,7 +393,8 @@ func (client *DefaultClient) publishToLndhubExchange(ctx context.Context, invoic return nil } -func captureErr(logger *lecho.Logger, err error) { - logger.Error(err) +func captureErr(logger *lecho.Logger, err error, j log.JSON) { + j["error"] = err + logger.Errorj(j) sentry.CaptureException(err) } From 62025479162054ca4c6512f8457c9a2c9b7efee8 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 7 Sep 2023 14:12:10 +0200 Subject: [PATCH 3/5] add subroutine fields --- rabbitmq/rabbitmq.go | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/rabbitmq/rabbitmq.go b/rabbitmq/rabbitmq.go index fea33995..9d65943d 100644 --- a/rabbitmq/rabbitmq.go +++ b/rabbitmq/rabbitmq.go @@ -184,7 +184,8 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv err := json.Unmarshal(delivery.Body, &payment) if err != nil { captureErr(client.logger, err, log.JSON{ - "message": "error unmarshalling payment json", + "subroutine": "payment finalizer", + "message": "error unmarshalling payment json", }) delivery.Nack(false, false) @@ -196,6 +197,7 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv t, err := svc.GetTransactionEntryByInvoiceId(ctx, invoice.ID) if err != nil { captureErr(client.logger, err, log.JSON{ + "subroutine": "payment finalizer", "payment_hash": invoice.RHash, "message": "error fetching transaction entry by id", }) @@ -211,6 +213,7 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv if err = svc.HandleSuccessfulPayment(ctx, &invoice, t); err != nil { captureErr(client.logger, err, log.JSON{ + "subroutine": "payment finalizer", "payment_hash": invoice.RHash, "message": "error handling succesful payment", }) @@ -225,6 +228,7 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv case lnrpc.Payment_FAILED: if err = svc.HandleFailedPayment(ctx, &invoice, t, fmt.Errorf(payment.FailureReason.String())); err != nil { captureErr(client.logger, err, log.JSON{ + "subroutine": "payment finalizer", "message": "error handling failed payment", "payment_hash": invoice.RHash, }) @@ -248,7 +252,10 @@ func (client *DefaultClient) SubscribeToLndInvoices(ctx context.Context, handler return err } - client.logger.Info("Starting RabbitMQ invoice consumer loop") + client.logger.Infoj(log.JSON{ + "subroutine": "invoice consumer", + "message": "starting loop", + }) for { select { case <-ctx.Done(): @@ -263,7 +270,8 @@ func (client *DefaultClient) SubscribeToLndInvoices(ctx context.Context, handler err := json.Unmarshal(delivery.Body, &invoice) if err != nil { captureErr(client.logger, err, log.JSON{ - "message": "error unmarshalling invoice json", + "subroutine": "invoice consumer", + "message": "error unmarshalling invoice json", }) // If we can't even Unmarshall the message we are dealing with @@ -272,6 +280,7 @@ func (client *DefaultClient) SubscribeToLndInvoices(ctx context.Context, handler err = delivery.Nack(false, false) if err != nil { captureErr(client.logger, err, log.JSON{ + "subroutine": "invoice consumer", "message": "error nacking invoice", "payment_hash": invoice.RHash, }) @@ -283,6 +292,7 @@ func (client *DefaultClient) SubscribeToLndInvoices(ctx context.Context, handler err = handler(ctx, &invoice) if err != nil { captureErr(client.logger, err, log.JSON{ + "subroutine": "invoice consumer", "message": "error handling invoice", "payment_hash": invoice.RHash, }) @@ -293,6 +303,7 @@ func (client *DefaultClient) SubscribeToLndInvoices(ctx context.Context, handler err := delivery.Nack(false, false) if err != nil { captureErr(client.logger, err, log.JSON{ + "subroutine": "invoice consumer", "message": "error nacking event", "payment_hash": invoice.RHash, }) @@ -304,6 +315,7 @@ func (client *DefaultClient) SubscribeToLndInvoices(ctx context.Context, handler err = delivery.Ack(false) if err != nil { captureErr(client.logger, err, log.JSON{ + "subroutine": "invoice consumer", "message": "error acking event", "payment_hash": invoice.RHash, }) @@ -332,7 +344,10 @@ func (client *DefaultClient) StartPublishInvoices(ctx context.Context, invoicesS return err } - client.logger.Info("Starting rabbitmq publisher") + client.logger.Infoj(log.JSON{ + "subroutine": "invoice publisher", + "message": "starting publisher", + }) in, out, err := invoicesSubscribeFunc() if err != nil { @@ -348,6 +363,7 @@ func (client *DefaultClient) StartPublishInvoices(ctx context.Context, invoicesS if err != nil { captureErr(client.logger, err, log.JSON{ + "subroutine": "invoice consumer", "message": "error publishing invoice", "payment_hash": incomingInvoice.RHash, }) @@ -357,6 +373,7 @@ func (client *DefaultClient) StartPublishInvoices(ctx context.Context, invoicesS if err != nil { captureErr(client.logger, err, log.JSON{ + "subroutine": "invoice consumer", "message": "error publishing invoice", "payment_hash": outgoing.RHash, }) @@ -388,7 +405,11 @@ func (client *DefaultClient) publishToLndhubExchange(ctx context.Context, invoic return err } - client.logger.Debugf("Successfully published invoice to rabbitmq with RHash %s", invoice.RHash) + client.logger.Infoj(log.JSON{ + "subroutine": "invoice publisher", + "message": "succesfully published invoice", + "payment_hash": invoice.RHash, + }) return nil } From f973c3edbc70a77cf7f73faa0f060531205f55e2 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Thu, 7 Sep 2023 14:35:26 +0200 Subject: [PATCH 4/5] add info for invoice events --- rabbitmq/rabbitmq.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/rabbitmq/rabbitmq.go b/rabbitmq/rabbitmq.go index 9d65943d..4587ead1 100644 --- a/rabbitmq/rabbitmq.go +++ b/rabbitmq/rabbitmq.go @@ -205,6 +205,11 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv continue } + client.logger.Infoj(log.JSON{ + "subroutine": "payment finalizer", + "payment_hash": invoice.RHash, + "message": "updating payment", + }) switch payment.Status { case lnrpc.Payment_SUCCEEDED: @@ -222,7 +227,11 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv continue } - client.logger.Infof("Payment finalizer: updated successful payment with hash: %s", payment.PaymentHash) + client.logger.Infoj(log.JSON{ + "subroutine": "payment finalizer", + "message": "updated succesful payment", + "payment_hash": payment.PaymentHash, + }) delete(pendingInvoices, payment.PaymentHash) case lnrpc.Payment_FAILED: @@ -236,8 +245,11 @@ func (client *DefaultClient) FinalizeInitializedPayments(ctx context.Context, sv continue } - - client.logger.Infof("Payment finalizer: updated failed payment with hash: %s", payment.PaymentHash) + client.logger.Infoj(log.JSON{ + "subroutine": "payment finalizer", + "message": "updated failed payment", + "payment_hash": payment.PaymentHash, + }) delete(pendingInvoices, payment.PaymentHash) } } @@ -288,6 +300,11 @@ func (client *DefaultClient) SubscribeToLndInvoices(ctx context.Context, handler continue } + log.Infoj(log.JSON{ + "subroutine": "invoice consumer", + "message": "adding invoice", + "payment_hash": invoice.RHash, + }) err = handler(ctx, &invoice) if err != nil { From b929ca30e57b681ee09d8586740106c99bbc22d8 Mon Sep 17 00:00:00 2001 From: kiwiidb Date: Mon, 11 Sep 2023 15:21:37 +0200 Subject: [PATCH 5/5] mainnet: rabbitmq routing key --- rabbitmq/rabbitmq.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rabbitmq/rabbitmq.go b/rabbitmq/rabbitmq.go index 4587ead1..2139d7f7 100644 --- a/rabbitmq/rabbitmq.go +++ b/rabbitmq/rabbitmq.go @@ -423,9 +423,10 @@ func (client *DefaultClient) publishToLndhubExchange(ctx context.Context, invoic } client.logger.Infoj(log.JSON{ - "subroutine": "invoice publisher", - "message": "succesfully published invoice", - "payment_hash": invoice.RHash, + "subroutine": "invoice publisher", + "message": "succesfully published invoice", + "payment_hash": invoice.RHash, + "rabbitmq_routing_key": key, }) return nil