Chore/refactor structure

Dockerfile

@@ -12,12 +12,18 @@ RUN go mod download
 COPY . .
 
 # Build the application
-RUN go build -o main
+RUN go build -o main ./cmd/server
+
+# Build the utility scripts
+RUN go build ./cmd/invoice-republishing
+RUN go build ./cmd/payment-reconciliation
 
 # Start a new, final image to reduce size.
 FROM alpine as final
 
 # Copy the binaries and entrypoint from the builder image.
 COPY --from=builder /build/main /bin/
+COPY --from=builder /build/invoice-republishing /bin/
+COPY --from=builder /build/payment-reconciliation /bin/
 
 ENTRYPOINT [ "/bin/main" ]

cmd/invoice-republishing/main.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/getAlby/lndhub.go/db"
+	"github.com/getAlby/lndhub.go/db/models"
+	"github.com/getAlby/lndhub.go/lib"
+	"github.com/getAlby/lndhub.go/lib/service"
+	"github.com/getAlby/lndhub.go/rabbitmq"
+	"github.com/joho/godotenv"
+	"github.com/kelseyhightower/envconfig"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+
+	c := &service.Config{}
+	// Load configruation from environment variables
+	err := godotenv.Load(".env")
+	if err != nil {
+		fmt.Println("Failed to load .env file")
+	}
+	logger := lib.Logger(c.LogFilePath)
+	startDate, endDate, err := loadStartAndEndIdFromEnv()
+	if err != nil {
+		logger.Fatalf("Could not load start and end id from env %v", err)
+	}
+	err = envconfig.Process("", c)
+	if err != nil {
+		logger.Fatalf("Error loading environment variables: %v", err)
+	}
+	// Open a DB connection based on the configured DATABASE_URI
+	dbConn, err := db.Open(c)
+	if err != nil {
+		logger.Fatalf("Error initializing db connection: %v", err)
+	}
+	amqpClient, err := rabbitmq.DialAMQP(c.RabbitMQUri, rabbitmq.WithAmqpLogger(logger))
+	if err != nil {
+		logger.Fatal(err)
+	}
+
+	defer amqpClient.Close()
+
+	rabbitmqClient, err := rabbitmq.NewClient(amqpClient,
+		rabbitmq.WithLogger(logger),
+		rabbitmq.WithLndInvoiceExchange(c.RabbitMQLndInvoiceExchange),
+		rabbitmq.WithLndHubInvoiceExchange(c.RabbitMQLndhubInvoiceExchange),
+		rabbitmq.WithLndInvoiceConsumerQueueName(c.RabbitMQInvoiceConsumerQueueName),
+		rabbitmq.WithLndPaymentExchange(c.RabbitMQLndPaymentExchange),
+		rabbitmq.WithLndPaymentConsumerQueueName(c.RabbitMQPaymentConsumerQueueName),
+	)
+	if err != nil {
+		logger.Fatal(err)
+	}
+	//small hack to get the script to work decently
+	defaultClient := rabbitmqClient.(*rabbitmq.DefaultClient)
+
+	// close the connection gently at the end of the runtime
+	defer rabbitmqClient.Close()
+
+	result := []models.Invoice{}
+	err = dbConn.NewSelect().Model(&result).Where("settled_at > ?", startDate).Where("settled_at < ?", endDate).Scan(context.Background())
+	if err != nil {
+		logger.Fatal(err)
+	}
+	logrus.Infof("Found %d invoices", len(result))
+	svc := &service.LndhubService{
+		Config:         c,
+		DB:             dbConn,
+		Logger:         logger,
+		RabbitMQClient: rabbitmqClient,
+		InvoicePubSub:  service.NewPubsub(),
+	}
+	ctx := context.Background()
+	dryRun := os.Getenv("DRY_RUN") == "true"
+	errCount := 0
+	for _, inv := range result {
+		logger.Infof("Publishing invoice with hash %s", inv.RHash)
+		if dryRun {
+			continue
+		}
+		err = defaultClient.PublishToLndhubExchange(ctx, inv, svc.EncodeInvoiceWithUserLogin)
+		if err != nil {
+			logrus.WithError(err).Error("errror publishing to lndhub exchange")
+		}
+	}
+	logger.Infof("Published %d invoices, # errors %d", len(result), errCount)
+
+}
+
+func loadStartAndEndIdFromEnv() (start, end time.Time, err error) {
+	start, err = time.Parse(time.RFC3339, os.Getenv("START_DATE"))
+	if err != nil {
+		return
+	}
+	end, err = time.Parse(time.RFC3339, os.Getenv("END_DATE"))
+	return
+}

reconciliation_lost_invoices/main.go → cmd/payment-reconciliation/main.go

@@ -47,12 +47,16 @@ func main() {
 	e := echo.New()
 
 	// Init new LND client
+	lnCfg, err := lnd.LoadConfig()
+	if err != nil {
+		logger.Fatalf("Failed to load lnd config %v", err)
+	}
 	lndClient, err := lnd.NewLNDclient(lnd.LNDoptions{
-		Address:      c.LNDAddress,
-		MacaroonFile: c.LNDMacaroonFile,
-		MacaroonHex:  c.LNDMacaroonHex,
-		CertFile:     c.LNDCertFile,
-		CertHex:      c.LNDCertHex,
+		Address:      lnCfg.LNDAddress,
+		MacaroonFile: lnCfg.LNDMacaroonFile,
+		MacaroonHex:  lnCfg.LNDMacaroonHex,
+		CertFile:     lnCfg.LNDCertFile,
+		CertHex:      lnCfg.LNDCertHex,
 	}, startupCtx)
 	if err != nil {
 		e.Logger.Fatalf("Error initializing the LND connection: %v", err)

main.go → cmd/server/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"embed"
 	"fmt"
 	"log"
 	"net/http"
@@ -11,6 +10,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/getAlby/lndhub.go/lnd"
 	"github.com/getAlby/lndhub.go/rabbitmq"
 	ddEcho "gopkg.in/DataDog/dd-trace-go.v1/contrib/labstack/echo.v4"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
@@ -21,6 +21,7 @@ import (
 	"github.com/getAlby/lndhub.go/lib"
 	"github.com/getAlby/lndhub.go/lib/service"
 	"github.com/getAlby/lndhub.go/lib/tokens"
+	"github.com/getAlby/lndhub.go/lib/transport"
 	"github.com/getsentry/sentry-go"
 	"github.com/joho/godotenv"
 	"github.com/kelseyhightower/envconfig"
@@ -29,12 +30,6 @@ import (
 	"github.com/uptrace/bun/migrate"
 )
 
-//go:embed templates/index.html
-var indexHtml string
-
-//go:embed static/*
-var staticContent embed.FS
-
 // @title        LndHub.go
 // @version      0.9.0
 // @description  Accounting wrapper for the Lightning Network providing separate accounts for end-users.
@@ -99,12 +94,16 @@ func main() {
 		}
 	}
 	// Init new LND client
-	lndClient, err := InitLNClient(c, logger, startupCtx)
+	lnCfg, err := lnd.LoadConfig()
+	if err != nil {
+		logger.Fatalf("Error loading LN config: %v", err)
+	}
+	lndClient, err := lnd.InitLNClient(lnCfg, logger, startupCtx)
 	if err != nil {
-		logger.Fatalf("Error initializing the %s connection: %v", c.LNClientType, err)
+		logger.Fatalf("Error initializing the %s connection: %v", lnCfg.LNClientType, err)
 	}
 
-	logger.Infof("Connected to %s: %s", c.LNClientType, lndClient.GetMainPubkey())
+	logger.Infof("Connected to %s: %s", lnCfg.LNClientType, lndClient.GetMainPubkey())
 
 	// If no RABBITMQ_URI was provided we will not attempt to create a client
 	// No rabbitmq features will be available in this case.
@@ -143,22 +142,22 @@ func main() {
 	}
 
 	//init echo server
-	e := initEcho(c, logger)
+	e := transport.InitEcho(c, logger)
 	//if Datadog is configured, add datadog middleware
 	if c.DatadogAgentUrl != "" {
 		tracer.Start(tracer.WithAgentAddr(c.DatadogAgentUrl))
 		defer tracer.Stop()
 		e.Use(ddEcho.Middleware(ddEcho.WithServiceName("lndhub.go")))
 	}
 
-	logMw := createLoggingMiddleware(logger)
+	logMw := transport.CreateLoggingMiddleware(logger)
 	// strict rate limit for requests for sending payments
-	strictRateLimitMiddleware := createRateLimitMiddleware(c.StrictRateLimit, c.BurstRateLimit)
+	strictRateLimitMiddleware := transport.CreateRateLimitMiddleware(c.StrictRateLimit, c.BurstRateLimit)
 	secured := e.Group("", tokens.Middleware(c.JWTSecret), logMw)
 	securedWithStrictRateLimit := e.Group("", tokens.Middleware(c.JWTSecret), strictRateLimitMiddleware, logMw)
 
-	RegisterLegacyEndpoints(svc, e, secured, securedWithStrictRateLimit, strictRateLimitMiddleware, tokens.AdminTokenMiddleware(c.AdminToken), logMw)
-	RegisterV2Endpoints(svc, e, secured, securedWithStrictRateLimit, strictRateLimitMiddleware, tokens.AdminTokenMiddleware(c.AdminToken), logMw)
+	transport.RegisterLegacyEndpoints(svc, e, secured, securedWithStrictRateLimit, strictRateLimitMiddleware, tokens.AdminTokenMiddleware(c.AdminToken), logMw)
+	transport.RegisterV2Endpoints(svc, e, secured, securedWithStrictRateLimit, strictRateLimitMiddleware, tokens.AdminTokenMiddleware(c.AdminToken), logMw)
 
 	//Swagger API spec
 	docs.SwaggerInfo.Host = c.Host
@@ -169,7 +168,7 @@ func main() {
 	// Subscribe to LND invoice updates in the background
 	backgroundWg.Add(1)
 	go func() {
-		err = StartInvoiceRoutine(svc, backGroundCtx)
+		err = svc.StartInvoiceRoutine(backGroundCtx)
 		if err != nil {
 			sentry.CaptureException(err)
 			//we want to restart in case of an error here
@@ -182,7 +181,7 @@ func main() {
 	// Check the status of all pending outgoing payments
 	backgroundWg.Add(1)
 	go func() {
-		err = StartPendingPaymentRoutine(svc, backGroundCtx)
+		err = svc.StartPendingPaymentRoutine(backGroundCtx)
 		if err != nil {
 			sentry.CaptureException(err)
 			//in case of an error here no restart is necessary
@@ -223,7 +222,7 @@ func main() {
 	//Start Prometheus server if necessary
 	var echoPrometheus *echo.Echo
 	if svc.Config.EnablePrometheus {
-		go startPrometheusEcho(logger, svc, e)
+		go transport.StartPrometheusEcho(logger, svc, e)
 	}
 
 	// Start server

integration_tests/util.go

@@ -54,8 +54,6 @@ func LndHubTestServiceInit(lndClientMock lnd.LightningClientWrapper) (svc *servi
 		JWTSecret:               []byte("SECRET"),
 		JWTAccessTokenExpiry:    3600,
 		JWTRefreshTokenExpiry:   3600,
-		LNDAddress:              mockLNDAddress,
-		LNDMacaroonHex:          mockLNDMacaroonHex,
 	}
 
 	rabbitmqUri, ok := os.LookupEnv("RABBITMQ_URI")

lib/service/background_routines.go

@@ -0,0 +1,32 @@
+package service
+
+import (
+	"context"
+)
+
+func (svc *LndhubService) StartInvoiceRoutine(ctx context.Context) (err error) {
+	if svc.RabbitMQClient != nil {
+		err = svc.RabbitMQClient.SubscribeToLndInvoices(ctx, svc.ProcessInvoiceUpdate)
+		if err != nil && err != context.Canceled {
+			return err
+		}
+
+		return nil
+	} else {
+		err = svc.InvoiceUpdateSubscription(ctx)
+		if err != nil && err != context.Canceled {
+			// in case of an error in this routine, we want to restart LNDhub
+			return err
+		}
+
+		return nil
+	}
+}
+
+func (svc *LndhubService) StartPendingPaymentRoutine(ctx context.Context) (err error) {
+	if svc.RabbitMQClient != nil {
+		return svc.RabbitMQClient.FinalizeInitializedPayments(ctx, svc)
+	} else {
+		return svc.CheckAllPendingOutgoingPayments(ctx)
+	}
+}

lib/service/config.go

@@ -5,12 +5,6 @@ import (
 	"strings"
 )
 
-const (
-	LND_CLIENT_TYPE         = "lnd"
-	LND_CLUSTER_CLIENT_TYPE = "lnd_cluster"
-	ECLAIR_CLIENT_TYPE      = "eclair"
-)
-
 type Config struct {
 	DatabaseUri                      string  `envconfig:"DATABASE_URI" required:"true"`
 	DatabaseMaxConns                 int     `envconfig:"DATABASE_MAX_CONNS" default:"10"`
@@ -25,14 +19,6 @@ type Config struct {
 	AdminToken                       string  `envconfig:"ADMIN_TOKEN"`
 	JWTRefreshTokenExpiry            int     `envconfig:"JWT_REFRESH_EXPIRY" default:"604800"` // in seconds, default 7 days
 	JWTAccessTokenExpiry             int     `envconfig:"JWT_ACCESS_EXPIRY" default:"172800"`  // in seconds, default 2 days
-	LNClientType                     string  `envconfig:"LN_CLIENT_TYPE" default:"lnd"`        //lnd, lnd_cluster, eclair
-	LNDAddress                       string  `envconfig:"LND_ADDRESS" required:"true"`
-	LNDMacaroonFile                  string  `envconfig:"LND_MACAROON_FILE"`
-	LNDCertFile                      string  `envconfig:"LND_CERT_FILE"`
-	LNDMacaroonHex                   string  `envconfig:"LND_MACAROON_HEX"`
-	LNDCertHex                       string  `envconfig:"LND_CERT_HEX"`
-	LNDClusterLivenessPeriod         int     `envconfig:"LND_CLUSTER_LIVENESS_PERIOD" default:"10"`
-	LNDClusterActiveChannelRatio     float64 `envconfig:"LND_CLUSTER_ACTIVE_CHANNEL_RATIO" default:"0.5"`
 	CustomName                       string  `envconfig:"CUSTOM_NAME"`
 	Host                             string  `envconfig:"HOST" default:"localhost:3000"`
 	Port                             int     `envconfig:"PORT" default:"3000"`