From 464c49004c12dc6bba3a2bfbf39dc3be1c615de8 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 17:03:14 +0200 Subject: [PATCH 01/29] Add env-var support for flags --- main.go | 74 ++++++++++++++++++++++++++++++++------------------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/main.go b/main.go index 9fea8b1..cf74fea 100644 --- a/main.go +++ b/main.go @@ -33,9 +33,10 @@ func main() { app.Flags = []cli.Flag{ &cli.StringFlag{ - Name: "datadir", - Value: "", - Usage: "specify the directory that cache data will be stored", + Name: "datadir", + Value: "", + EnvVars: []string{"RAINBOW_DATADIR"}, + Usage: "specify the directory that cache data will be stored", }, &cli.StringFlag{ Name: "seed", @@ -50,45 +51,53 @@ func main() { Usage: "Specify an index to derivate the peerID from the key (needs --seed)", }, &cli.IntFlag{ - Name: "gateway-port", - Value: 8090, - Usage: "specify the listen address for the gateway endpoint", + Name: "gateway-port", + Value: 8090, + EnvVars: []string{"RAINBOW_GATEWAY_PORT"}, + Usage: "specify the listen address for the gateway endpoint", }, &cli.IntFlag{ - Name: "ctl-port", - Value: 8091, - Usage: "specify the api listening address for the internal control api", + Name: "ctl-port", + Value: 8091, + EnvVars: []string{"RAINBOW_CTL_PORT"}, + Usage: "specify the api listening address for the internal control api", }, &cli.IntFlag{ - Name: "connmgr-low", - Value: 100, - Usage: "libp2p connection manager 'low' water mark", + Name: "connmgr-low", + Value: 100, + EnvVars: []string{"RAINBOW_CONNMGR_LOW"}, + Usage: "libp2p connection manager 'low' water mark", }, &cli.IntFlag{ - Name: "connmgr-hi", - Value: 3000, - Usage: "libp2p connection manager 'high' water mark", + Name: "connmgr-high", + Value: 3000, + EnvVars: []string{"RAINBOW_CONNMGR_HIGH"}, + Usage: "libp2p connection manager 'high' water mark", + }, + &cli.DurationFlag{ + Name: "connmgr-grace", + Value: time.Minute, + EnvVars: []string{"RAINBOW_CONNMGR_GRACE_PERIOD"}, + Usage: "libp2p connection manager grace period", }, &cli.IntFlag{ - Name: "inmem-block-cache", - Value: 1 << 30, - Usage: "Size of the in-memory block cache. 0 to disable (disables compression too)", + Name: "inmem-block-cache", + Value: 1 << 30, + EnvVars: []string{"RAINBOW_INMEM_BLOCK_CACHE"}, + Usage: "Size of the in-memory block cache. 0 to disable (disables compression too)", }, &cli.Uint64Flag{ - Name: "max-memory", - Value: 0, - Usage: "Libp2p resource manager max memory. Defaults to system's memory * 0.85", + Name: "max-memory", + Value: 0, + EnvVars: []string{"RAINBOW_MAX_MEMORY"}, + Usage: "Libp2p resource manager max memory. Defaults to system's memory * 0.85", }, &cli.Uint64Flag{ - Name: "max-fd", - Value: 0, - Usage: "Libp2p resource manager file description limit. Defaults to the process' fd-limit/2", - }, - &cli.DurationFlag{ - Name: "connmgr-grace", - Value: time.Minute, - Usage: "libp2p connection manager grace period", + Name: "max-fd", + Value: 0, + EnvVars: []string{"RAINBOW_MAX_FD"}, + Usage: "Libp2p resource manager file description limit. Defaults to the process' fd-limit/2", }, &cli.StringFlag{ Name: "routing", @@ -101,9 +110,10 @@ func main() { Usage: "If using an Amino DHT client should the libp2p host be shared with the data downloading host", }, &cli.StringFlag{ - Name: "denylists", - Value: "https://denyli.st/badbits.deny", - Usage: "Denylist subscriptions (comma-separated)", + Name: "denylists", + Value: "https://denyli.st/badbits.deny", + EnvVars: []string{"RAINBOW_DENYLISTS"}, + Usage: "Denylist subscriptions (comma-separated)", }, } From 8acaf7ab212772a6087ec238aeb8e0ee48d15483 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 17:27:04 +0200 Subject: [PATCH 02/29] Read credentials from file in CREDENTIALS_DIRECTORY This is what systemd uses to safely provide credentials to processes. --- main.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index cf74fea..d754a77 100644 --- a/main.go +++ b/main.go @@ -42,7 +42,7 @@ func main() { Name: "seed", Value: "", EnvVars: []string{"RAINBOW_SEED"}, - Usage: "Specify a seed to derive peerID from (needs --seed-index)", + Usage: "Specify a seed to derive peerID from (needs --seed-index). Best to use CREDENTIALS_DIRECTORY/seed", }, &cli.IntFlag{ Name: "seed-index", @@ -146,9 +146,20 @@ to create libp2p identities for the gateway. cdns := newCachedDNS(dnsCacheRefreshInterval) defer cdns.Close() + var seed string var priv crypto.PrivKey var err error - seed := cctx.String("seed") + + // attempt to read seed from disk + if credDir := os.Getenv("CREDENTIALS_DIRECTORY"); credDir != "" { + seedBytes, err := os.ReadFile(filepath.Join(credDir, "seed")) + if err != nil { + return fmt.Errorf("error reading seed credentials: %w", err) + } + seed = strings.TrimSpace(string(seedBytes)) + } else { + seed = cctx.String("seed") + } index := cctx.Int("seed-index") if len(seed) > 0 && index >= 0 { From 40104f20b33572dac6cd88feb203e21bb2cb9bf1 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 17:52:16 +0200 Subject: [PATCH 03/29] Harden the path specifying how we read the seed --- main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index d754a77..adf1b09 100644 --- a/main.go +++ b/main.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "io/fs" "log" "net/http" _ "net/http/pprof" @@ -153,7 +154,7 @@ to create libp2p identities for the gateway. // attempt to read seed from disk if credDir := os.Getenv("CREDENTIALS_DIRECTORY"); credDir != "" { seedBytes, err := os.ReadFile(filepath.Join(credDir, "seed")) - if err != nil { + if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("error reading seed credentials: %w", err) } seed = strings.TrimSpace(string(seedBytes)) @@ -163,8 +164,10 @@ to create libp2p identities for the gateway. index := cctx.Int("seed-index") if len(seed) > 0 && index >= 0 { + fmt.Println("Deriving identity from seed") priv, err = deriveKey(seed, []byte(fmt.Sprintf("rainbow-%d", index))) } else { + fmt.Println("Setting identity from libp2p.key") keyFile := filepath.Join(ddir, "libp2p.key") priv, err = loadOrInitPeerKey(keyFile) } From d336f352ba64236a7b18aeaf610edb4c0b46a6e2 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 17:58:55 +0200 Subject: [PATCH 04/29] Use CREDENTIALS_DIRECTORY only when available Otherwise fallback to DATADIR --- main.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index adf1b09..c5534c2 100644 --- a/main.go +++ b/main.go @@ -151,15 +151,24 @@ to create libp2p identities for the gateway. var priv crypto.PrivKey var err error + credDir := os.Getenv("CREDENTIALS_DIRECTORY") + secretsDir := ddir + + if len(credDir) > 0 { + secretsDir = credDir + } + // attempt to read seed from disk - if credDir := os.Getenv("CREDENTIALS_DIRECTORY"); credDir != "" { - seedBytes, err := os.ReadFile(filepath.Join(credDir, "seed")) - if err != nil && !errors.Is(err, fs.ErrNotExist) { + seedBytes, err := os.ReadFile(filepath.Join(secretsDir, "seed")) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // set seed from command line or env-var + seed = cctx.String("seed") + } else { return fmt.Errorf("error reading seed credentials: %w", err) } - seed = strings.TrimSpace(string(seedBytes)) } else { - seed = cctx.String("seed") + seed = strings.TrimSpace(string(seedBytes)) } index := cctx.Int("seed-index") @@ -168,7 +177,7 @@ to create libp2p identities for the gateway. priv, err = deriveKey(seed, []byte(fmt.Sprintf("rainbow-%d", index))) } else { fmt.Println("Setting identity from libp2p.key") - keyFile := filepath.Join(ddir, "libp2p.key") + keyFile := filepath.Join(secretsDir, "libp2p.key") priv, err = loadOrInitPeerKey(keyFile) } if err != nil { From ffa5c1f4945038c733323165622076c1715df66b Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 18:04:07 +0200 Subject: [PATCH 05/29] Fix datastore location --- go.mod | 2 -- go.sum | 9 --------- setup.go | 13 +++---------- 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 06afffa..50d3cf7 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981 - github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-ipfs-delay v0.0.1 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-metrics-interface v0.0.1 @@ -150,7 +149,6 @@ require ( github.com/samber/lo v1.36.0 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect diff --git a/go.sum b/go.sum index f764352..6dc203f 100644 --- a/go.sum +++ b/go.sum @@ -308,7 +308,6 @@ github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= -github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= @@ -317,8 +316,6 @@ github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9 github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981 h1:GOKV62VnjerKwO7mwOyeoArzlaVrDLyoC/YPNtxxGwg= github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981/go.mod h1:LUU2FbhNdmhAbJmMeoahVRbe4GsduAODSJHWJJh2Vo4= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= -github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= -github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12XwJyvYvMM= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= @@ -397,7 +394,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -529,10 +525,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -658,7 +652,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= @@ -1105,7 +1098,6 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -1113,7 +1105,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/setup.go b/setup.go index 3e3d53e..2e19923 100644 --- a/setup.go +++ b/setup.go @@ -29,7 +29,6 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" badger4 "github.com/ipfs/go-ds-badger4" - levelds "github.com/ipfs/go-ds-leveldb" delay "github.com/ipfs/go-ipfs-delay" metri "github.com/ipfs/go-metrics-interface" mprome "github.com/ipfs/go-metrics-prometheus" @@ -195,12 +194,6 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { } else { // If there are no delegated routing endpoints run an accelerated Amino DHT client and send IPNI requests to cid.contact - // TODO: This datastore shouldn't end up containing anything anyway so this could potentially just be a null datastore - memDS, err := levelds.NewDatastore("", nil) - if err != nil { - return nil, err - } - var dhtHost host.Host if cfg.DHTSharedHost { dhtHost = h @@ -217,7 +210,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { } standardClient, err := dht.New(ctx, dhtHost, - dht.Datastore(memDS), + dht.Datastore(ds), dht.BootstrapPeers(dht.GetDefaultBootstrapPeerAddrInfos()...), dht.Mode(dht.ModeClient), ) @@ -231,7 +224,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { "pk": record.PublicKeyValidator{}, "ipns": ipns.Validator{KeyBook: h.Peerstore()}, }), - dht.Datastore(memDS), + dht.Datastore(ds), dht.BootstrapPeers(dht.GetDefaultBootstrapPeerAddrInfos()...), dht.BucketSize(20), )) @@ -401,7 +394,7 @@ func setupDatastore(cfg Config) (datastore.Batching, error) { Options: badgerOpts, } - return badger4.NewDatastore("badger4", &opts) + return badger4.NewDatastore(filepath.Join(cfg.DataDir, "badger4"), &opts) } type bundledDHT struct { From 57a4bac8d1197ad696e4c02ef86c8a218de793cb Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 18:05:49 +0200 Subject: [PATCH 06/29] Fix denylists folder location --- setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.go b/setup.go index 2e19923..4569b9f 100644 --- a/setup.go +++ b/setup.go @@ -285,7 +285,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { ) bn.Start(bswap) - err = os.Mkdir("denylists", 0755) + err = os.Mkdir(filepath.Join(cfg.DataDir, "denylists"), 0755) if err != nil && !errors.Is(err, fs.ErrExist) { return nil, err } From 5f51d1038c8ba042301d906dbddc2cdb2e0227f4 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 18:13:43 +0200 Subject: [PATCH 07/29] Add systemd notifications --- go.mod | 2 +- main.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 50d3cf7..c564f8e 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/ipfs/rainbow go 1.20 require ( + github.com/coreos/go-systemd/v22 v22.5.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dustin/go-humanize v1.0.0 github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97 @@ -47,7 +48,6 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/cskr/pubsub v1.0.2 // indirect diff --git a/main.go b/main.go index c5534c2..57f59dc 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( "sync" "time" + sddaemon "github.com/coreos/go-systemd/v22/daemon" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/crypto" peer "github.com/libp2p/go-libp2p/core/peer" @@ -275,8 +276,10 @@ to create libp2p identities for the gateway. } }() + sddaemon.SdNotify(false, sddaemon.SdNotifyReady) signal.Notify(quit, os.Interrupt) <-quit + sddaemon.SdNotify(false, sddaemon.SdNotifyStopping) log.Printf("Closing servers...\n") go gatewaySrv.Close() go apiSrv.Close() From a306ba095dbc6c20869fc0c2dc4edf0d38699fc5 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 18:29:31 +0200 Subject: [PATCH 08/29] Handle more signals to quit --- main.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 57f59dc..8d8f985 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/fs" - "log" "net/http" _ "net/http/pprof" "os" @@ -14,6 +13,7 @@ import ( "runtime" "strings" "sync" + "syscall" "time" sddaemon "github.com/coreos/go-systemd/v22/daemon" @@ -277,10 +277,15 @@ to create libp2p identities for the gateway. }() sddaemon.SdNotify(false, sddaemon.SdNotifyReady) - signal.Notify(quit, os.Interrupt) + signal.Notify( + quit, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGHUP, + ) <-quit sddaemon.SdNotify(false, sddaemon.SdNotifyStopping) - log.Printf("Closing servers...\n") + goLog.Info("Closing servers...") go gatewaySrv.Close() go apiSrv.Close() for _, sub := range gnd.denylistSubs { @@ -291,7 +296,7 @@ to create libp2p identities for the gateway. } if err := app.Run(os.Args); err != nil { - log.Print(err) + goLog.Error(err) os.Exit(1) } } From fa2e4af264dd4ccf8152d6e168072ccbde6c2997 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 19:26:25 +0200 Subject: [PATCH 09/29] Support custom gateway domain --- handlers.go | 8 +++++--- main.go | 14 +++++++++++--- setup.go | 1 + 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/handlers.go b/handlers.go index ef92d5e..79b2172 100644 --- a/handlers.go +++ b/handlers.go @@ -89,7 +89,7 @@ func withRequestLogger(next http.Handler) http.Handler { }) } -func setupGatewayHandler(nd *Node) (http.Handler, error) { +func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) { backend, err := gateway.NewBlocksBackend(nd.bsrv, gateway.WithValueStore(nd.vs), gateway.WithNameSystem(nd.ns)) if err != nil { return nil, err @@ -110,13 +110,15 @@ func setupGatewayHandler(nd *Node) (http.Handler, error) { DeserializedResponses: true, UseSubdomains: true, }, - "dweb.link": { + } + if cfg.GatewayDomain != "" { + publicGateways[cfg.GatewayDomain] = &gateway.PublicGateway{ Paths: []string{"/ipfs", "/ipns", "/version", "/api/v0"}, NoDNSLink: noDNSLink, InlineDNSLink: true, DeserializedResponses: true, UseSubdomains: true, - }, + } } // If we're doing tests, ensure the right public gateways are enabled. diff --git a/main.go b/main.go index 8d8f985..03d9a89 100644 --- a/main.go +++ b/main.go @@ -52,6 +52,12 @@ func main() { EnvVars: []string{"RAINBOW_SEED_INDEX"}, Usage: "Specify an index to derivate the peerID from the key (needs --seed)", }, + &cli.StringFlag{ + Name: "gateway-domain", + Value: "", + EnvVars: []string{"RAINBOW_GATEWAY_DOMAIN"}, + Usage: "Set to enable subdomain gateway on this domain", + }, &cli.IntFlag{ Name: "gateway-port", Value: 8090, @@ -185,8 +191,9 @@ to create libp2p identities for the gateway. return err } - gnd, err := Setup(cctx.Context, Config{ + cfg := Config{ DataDir: ddir, + GatewayDomain: cctx.String("gateway-domain"), ConnMgrLow: cctx.Int("connmgr-low"), ConnMgrHi: cctx.Int("connmgr-hi"), ConnMgrGrace: cctx.Duration("connmgr-grace"), @@ -199,7 +206,8 @@ to create libp2p identities for the gateway. DHTSharedHost: cctx.Bool("dht-fallback-shared-host"), DNSCache: cdns, DenylistSubs: strings.Split(cctx.String("denylists"), ","), - }) + } + gnd, err := Setup(cctx.Context, cfg) if err != nil { return err } @@ -207,7 +215,7 @@ to create libp2p identities for the gateway. gatewayPort := cctx.Int("gateway-port") apiPort := cctx.Int("ctl-port") - handler, err := setupGatewayHandler(gnd) + handler, err := setupGatewayHandler(cfg, gnd) if err != nil { return err } diff --git a/setup.go b/setup.go index 4569b9f..a647aab 100644 --- a/setup.go +++ b/setup.go @@ -94,6 +94,7 @@ type Config struct { MaxMemory uint64 MaxFD int + GatewayDomain string RoutingV1 string KuboRPCURLs []string DHTSharedHost bool From ecce13d519d112ae29e557145a34809d75e052c7 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 20:03:09 +0200 Subject: [PATCH 10/29] Enable info logging. --- go.mod | 4 ++-- go.sum | 4 ++++ main.go | 5 +++++ setup.go | 5 ++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c564f8e..a55d91a 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dustin/go-humanize v1.0.0 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97 - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97 + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017180002-8e447c27631b + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017180002-8e447c27631b github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 6dc203f..10e3ea7 100644 --- a/go.sum +++ b/go.sum @@ -289,8 +289,12 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97 h1:zK5+tjnusBotrFPYPI1rgOe5VIwJHpf3j131M1KAm3I= github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017180002-8e447c27631b h1:bco9XIJi5GIvwMUCcKI4tCKaqt4oDvQ7Sxf/qI5gkjw= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017180002-8e447c27631b/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97 h1:XHANeOLCEz9pTxUK6gtKAxuAjCoMZ8B2QLcKUQNgJUA= github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017180002-8e447c27631b h1:wcIAEoI+mDUNFp7/3vIXy4zVHUM8hdh/8soJ6Vo8pcU= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017180002-8e447c27631b/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 h1:WFpBlVbrvRyMzukb2TS+JPqrc8TkuILdGPoZoyX2na0= diff --git a/main.go b/main.go index 03d9a89..eeda11c 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,11 @@ const EnvKuboRPC = "KUBO_RPC_URL" var goLog = logging.Logger("rainbow") +func init() { + logging.SetLogLevel("rainbow", "INFO") + logging.SetLogLevel("nopfs", "INFO") +} + func main() { app := cli.NewApp() diff --git a/setup.go b/setup.go index a647aab..7409520 100644 --- a/setup.go +++ b/setup.go @@ -293,7 +293,10 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { var denylists []*nopfs.HTTPSubscriber for _, dl := range cfg.DenylistSubs { - s := nopfs.NewHTTPSubscriber(dl, filepath.Join(cfg.DataDir, "denylists", filepath.Base(dl)), time.Minute) + s, err := nopfs.NewHTTPSubscriber(dl, filepath.Join(cfg.DataDir, "denylists", filepath.Base(dl)), time.Minute) + if err != nil { + return nil, err + } denylists = append(denylists, s) } From 91c6ce272dc2e5ff1ea9c2789b90feb9b5e89ac1 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 20:12:01 +0200 Subject: [PATCH 11/29] udpate nopfs --- go.mod | 4 ++-- go.sum | 12 ++++-------- main.go | 1 - 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index a55d91a..d159a60 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dustin/go-humanize v1.0.0 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017180002-8e447c27631b - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017180002-8e447c27631b + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017211857-6dcea67b3d0a + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017211857-6dcea67b3d0a github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 10e3ea7..17e815d 100644 --- a/go.sum +++ b/go.sum @@ -287,14 +287,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97 h1:zK5+tjnusBotrFPYPI1rgOe5VIwJHpf3j131M1KAm3I= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017180002-8e447c27631b h1:bco9XIJi5GIvwMUCcKI4tCKaqt4oDvQ7Sxf/qI5gkjw= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017180002-8e447c27631b/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97 h1:XHANeOLCEz9pTxUK6gtKAxuAjCoMZ8B2QLcKUQNgJUA= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017180002-8e447c27631b h1:wcIAEoI+mDUNFp7/3vIXy4zVHUM8hdh/8soJ6Vo8pcU= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017180002-8e447c27631b/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017211857-6dcea67b3d0a h1:3bKOhJIyQdQC+On4/zIwPyQzRHt+xg51RadsnyYQdD8= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017211857-6dcea67b3d0a/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017211857-6dcea67b3d0a h1:/KZenwTfLR/NjuTjClYXnJNEXRQARnQxQVdt7qlGJrU= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017211857-6dcea67b3d0a/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 h1:WFpBlVbrvRyMzukb2TS+JPqrc8TkuILdGPoZoyX2na0= diff --git a/main.go b/main.go index eeda11c..89e4ab4 100644 --- a/main.go +++ b/main.go @@ -31,7 +31,6 @@ var goLog = logging.Logger("rainbow") func init() { logging.SetLogLevel("rainbow", "INFO") - logging.SetLogLevel("nopfs", "INFO") } func main() { From 6655c437d9841127baaad51ba126a4a082cdd416 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 17 Oct 2023 23:53:31 +0200 Subject: [PATCH 12/29] Forgot to add the wrapped resolver when initializing BlocksBackend --- handlers.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/handlers.go b/handlers.go index 79b2172..7c4dcc4 100644 --- a/handlers.go +++ b/handlers.go @@ -90,7 +90,12 @@ func withRequestLogger(next http.Handler) http.Handler { } func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) { - backend, err := gateway.NewBlocksBackend(nd.bsrv, gateway.WithValueStore(nd.vs), gateway.WithNameSystem(nd.ns)) + backend, err := gateway.NewBlocksBackend( + nd.bsrv, + gateway.WithValueStore(nd.vs), + gateway.WithNameSystem(nd.ns), + gateway.WithResolver(nd.resolver), + ) if err != nil { return nil, err } From 42d02d50e9356ecafdb06fece90b4177fc7e2278 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 18 Oct 2023 16:30:57 +0200 Subject: [PATCH 13/29] Update README. Bring it up to date. --- README.md | 88 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index f0d726a..51d77c3 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,26 @@ # rainbow -> The rainbow bridge between your CIDs and your html, javascript, wasm, jpegs, mp4s, tars, CARs, blocks, and more! +> An specialized IPFS HTTP gateway. ## About -Rainbow is an experimental specifically dedicated implementation of the [IPFS HTTP Gateway API](https://specs.ipfs.tech/http-gateways), +Rainbow is an implementation of the [IPFS HTTP Gateway API](https://specs.ipfs.tech/http-gateways), based on [boxo](https://github.com/ipfs/boxo) which is the tooling the powers [kubo](https://github.com/ipfs/kubo). +Rainbow uses the same Go code as the HTTP gateway in Kubo, but is fully specialized to just be a gateway: + + * Rainbow acts as DHT and Bitswap client only. Rainbow is not a server for the network. + * Rainbow does not pin, or permanently store any content. It is just meant + to act as gateway to content present in the network. GC strategy + * Rainbow settings are optimized for production deployments and streamlined + for specific choices (badger datastore, writethrough uncached blockstore + etc.) + * Denylist and denylist subscription support is included. + * And more to come... + + ## Building + ``` go build ``` @@ -18,31 +31,54 @@ go build rainbow ``` +Use `rainbow --help` for documentation. + ## Configuration -``` -NAME: - rainbow - a standalone ipfs gateway - -USAGE: - rainbow [global options] command [command options] [arguments...] - -VERSION: - v0.0.1-dev - -COMMANDS: - help, h Shows a list of commands or help for one command - -GLOBAL OPTIONS: - --datadir value specify the directory that cache data will be stored - --listen value specify the listen address for the gateway endpoint (default: ":8080") - --api-listen value specify the api listening address for the internal control api (default: "127.0.0.1:8081") - --connmgr-low value libp2p connection manager 'low' water mark (default: 100) - --connmgr-hi value libp2p connection manager 'high' water mark (default: 3000) - --connmgr-grace value libp2p connection manager grace period (default: 1m0s) - --routing value RoutingV1 Endpoint (default: "http://127.0.0.1:8090") - --help, -h show help (default: false) - --version, -v print the version (default: false) -``` + +Rainbow can be configured via command-line arguments or environment variables. + +See `rainbow --help` for information on the available options. + +Rainbow uses a `--datadir` (or `RAINBOW_DATADIR` environment variable) as +location for persisted data. It defaults to the folder in which `rainbow` is +run. + +### Peer Identity + +**Using a key file**: By default generates a `libp2p.key` in its data folder if none exist yet. This +file stores the libp2p peer identity. + +**Using a seed + index**: Alternatively, random can be initialized with a +32-byte, b58 encoded seed and a derivation index. This allows to use the same +seed for multiple instances of rainbow, and only change the derivation index. + +The seed and index can be provided as command line arguments or environment +vars (`--seed` , `--seed-index`). The seed can also be provided as a `seed` +file in the datadir folder. A new random seed can be generated with: + + rainbow gen-seed > seed + +To facilitate the use of rainbow with systemd +[`LoadCredential=`](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#LoadCredential=ID:PATH) +directive, we look for both `libp2p.key` and `seed` in +`$CREDENTIALS_DIRECTORY` first. + +### Denylists + +Rainbow can subscribe to append-only denylists using the `--denylists` flag. The value is a comma-separated list of URLs to subscribe to, for example: `https://denyli.st/badbits.deny`. This will download and update the denylist automatically when it is updated with new entries. + +Denylists can be manually placed in the `$RAINBOW_DATADIR/denylists` folder too. + +See [NoPFS](https://github.com/ipfs-shipyard/nopfs) for an explanation of the denylist format. Note that denylists should only be appended to while Rainbow is running. Editing differently, or adding new denylist files, should be done with Rainbow stopped. + +## Deployment + +An ansible role to deploy Rainbow is available within the ipfs.ipfs collection in Ansible Galaxy (https://github.com/ipfs-shipyard/ansible). It includes a systemd service unit file. + +Automated Docker container releases are available from the [Github container registry](https://github.com/ipfs/rainbow/pkgs/container/rainbow): + + docker pull ghcr.io/ipfs/rainbow:main-latest + ## License From 1964b018853e68486c7a41f8db141dff3a6a705b Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 18 Oct 2023 16:37:55 +0200 Subject: [PATCH 14/29] Do not subscribe by default to badbits --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 89e4ab4..881ad8a 100644 --- a/main.go +++ b/main.go @@ -123,7 +123,7 @@ func main() { }, &cli.StringFlag{ Name: "denylists", - Value: "https://denyli.st/badbits.deny", + Value: "", EnvVars: []string{"RAINBOW_DENYLISTS"}, Usage: "Denylist subscriptions (comma-separated)", }, From 19602c9f98484badbcecbb4cb61fd23953caf66e Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 18 Oct 2023 19:33:26 +0200 Subject: [PATCH 15/29] Do not end up with empty-name denylist when none specified --- main.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 881ad8a..c5c8c15 100644 --- a/main.go +++ b/main.go @@ -195,6 +195,11 @@ to create libp2p identities for the gateway. return err } + var denylists []string + if list := cctx.String("denylists"); len(list) > 0 { + denylists = strings.Split(list, ",") + } + cfg := Config{ DataDir: ddir, GatewayDomain: cctx.String("gateway-domain"), @@ -209,7 +214,7 @@ to create libp2p identities for the gateway. KuboRPCURLs: getEnvs(EnvKuboRPC, DefaultKuboRPC), DHTSharedHost: cctx.Bool("dht-fallback-shared-host"), DNSCache: cdns, - DenylistSubs: strings.Split(cctx.String("denylists"), ","), + DenylistSubs: denylists, } gnd, err := Setup(cctx.Context, cfg) if err != nil { From 3fd83ef64deae42d3ec2df679443737bf8f13ba0 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 19 Oct 2023 01:51:46 +0200 Subject: [PATCH 16/29] Just control all logging via env-var --- main.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/main.go b/main.go index c5c8c15..8731e7c 100644 --- a/main.go +++ b/main.go @@ -29,10 +29,6 @@ const EnvKuboRPC = "KUBO_RPC_URL" var goLog = logging.Logger("rainbow") -func init() { - logging.SetLogLevel("rainbow", "INFO") -} - func main() { app := cli.NewApp() From 4e121dd586671924b2861d1c59af6e46efc6c451 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 19 Oct 2023 01:52:47 +0200 Subject: [PATCH 17/29] Update nopfs with fix for dnslink-encoded ipns --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d159a60..baed9eb 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dustin/go-humanize v1.0.0 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017211857-6dcea67b3d0a - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017211857-6dcea67b3d0a + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231018184959-5433362b11ac + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231018184959-5433362b11ac github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 17e815d..7458dac 100644 --- a/go.sum +++ b/go.sum @@ -287,10 +287,10 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017211857-6dcea67b3d0a h1:3bKOhJIyQdQC+On4/zIwPyQzRHt+xg51RadsnyYQdD8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231017211857-6dcea67b3d0a/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017211857-6dcea67b3d0a h1:/KZenwTfLR/NjuTjClYXnJNEXRQARnQxQVdt7qlGJrU= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231017211857-6dcea67b3d0a/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231018184959-5433362b11ac h1:K9wupxUa42nCjjfQLBEUjn2t5QdcdZT1g8tuxnKzuAE= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231018184959-5433362b11ac/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231018184959-5433362b11ac h1:SCDQyXmYSDRYusUiZ19DcP8/fOKj69P9W/xShKI2o8g= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231018184959-5433362b11ac/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 h1:WFpBlVbrvRyMzukb2TS+JPqrc8TkuILdGPoZoyX2na0= From a88f3d57e5914a2f4e4a94b507f1b9966cad48b4 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 19 Oct 2023 14:14:02 +0200 Subject: [PATCH 18/29] Fix connmgr-high not getting set --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 8731e7c..f96429f 100644 --- a/main.go +++ b/main.go @@ -200,7 +200,7 @@ to create libp2p identities for the gateway. DataDir: ddir, GatewayDomain: cctx.String("gateway-domain"), ConnMgrLow: cctx.Int("connmgr-low"), - ConnMgrHi: cctx.Int("connmgr-hi"), + ConnMgrHi: cctx.Int("connmgr-high"), ConnMgrGrace: cctx.Duration("connmgr-grace"), MaxMemory: cctx.Uint64("max-memory"), MaxFD: cctx.Int("max-fd"), From 4ffc67d226239a4cefe51ac0246e6c3e223bd469 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 19 Oct 2023 14:20:19 +0200 Subject: [PATCH 19/29] Print configuration on boot --- main.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/main.go b/main.go index f96429f..785873d 100644 --- a/main.go +++ b/main.go @@ -212,6 +212,15 @@ to create libp2p identities for the gateway. DNSCache: cdns, DenylistSubs: denylists, } + + fmt.Printf(` + +Rainbow config: + +%+v + +`, cfg) + gnd, err := Setup(cctx.Context, cfg) if err != nil { return err From f1a629b741752a28b190f392d7c5d15e0a60540a Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 19 Oct 2023 14:25:30 +0200 Subject: [PATCH 20/29] Avoid passing key and dnscache as config objects Better to print actual command-line configuration. --- main.go | 4 +--- setup.go | 9 +++------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/main.go b/main.go index 785873d..3d77619 100644 --- a/main.go +++ b/main.go @@ -205,11 +205,9 @@ to create libp2p identities for the gateway. MaxMemory: cctx.Uint64("max-memory"), MaxFD: cctx.Int("max-fd"), InMemBlockCache: cctx.Int64("inmem-block-cache"), - Libp2pKey: priv, RoutingV1: cctx.String("routing"), KuboRPCURLs: getEnvs(EnvKuboRPC, DefaultKuboRPC), DHTSharedHost: cctx.Bool("dht-fallback-shared-host"), - DNSCache: cdns, DenylistSubs: denylists, } @@ -221,7 +219,7 @@ Rainbow config: `, cfg) - gnd, err := Setup(cctx.Context, cfg) + gnd, err := Setup(cctx.Context, cfg, priv, cdns) if err != nil { return err } diff --git a/setup.go b/setup.go index 7409520..f10bb39 100644 --- a/setup.go +++ b/setup.go @@ -84,8 +84,6 @@ type Config struct { ListenAddrs []string AnnounceAddrs []string - Libp2pKey crypto.PrivKey - ConnMgrLow int ConnMgrHi int ConnMgrGrace time.Duration @@ -98,12 +96,11 @@ type Config struct { RoutingV1 string KuboRPCURLs []string DHTSharedHost bool - DNSCache *cachedDNS DenylistSubs []string } -func Setup(ctx context.Context, cfg Config) (*Node, error) { +func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cachedDNS) (*Node, error) { ds, err := setupDatastore(cfg) if err != nil { return nil, err @@ -126,7 +123,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { libp2p.ListenAddrStrings(cfg.ListenAddrs...), libp2p.NATPortMap(), libp2p.ConnectionManager(cmgr), - libp2p.Identity(cfg.Libp2pKey), + libp2p.Identity(key), libp2p.BandwidthReporter(bwc), libp2p.DefaultTransports, libp2p.DefaultMuxers, @@ -175,7 +172,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { MaxConnsPerHost: 100, MaxIdleConnsPerHost: 100, IdleConnTimeout: 90 * time.Second, - DialContext: cfg.DNSCache.dialWithCachedDNS, + DialContext: dnsCache.dialWithCachedDNS, ForceAttemptHTTP2: true, }, }, From 4207c9a5b904703bc97a3ff6980a53d617bb8827 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 19 Oct 2023 14:27:48 +0200 Subject: [PATCH 21/29] Log configuration with debug logging only --- main.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/main.go b/main.go index 3d77619..d1d18df 100644 --- a/main.go +++ b/main.go @@ -211,13 +211,7 @@ to create libp2p identities for the gateway. DenylistSubs: denylists, } - fmt.Printf(` - -Rainbow config: - -%+v - -`, cfg) + goLog.Debugf("Rainbow config: %+v", cfg) gnd, err := Setup(cctx.Context, cfg, priv, cdns) if err != nil { From 37c379d0199d90f1602251307322973a98ccff3f Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 19 Oct 2023 17:25:26 +0200 Subject: [PATCH 22/29] change the bitswap metrics scope to "ipfs" --- setup.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.go b/setup.go index f10bb39..23f908a 100644 --- a/setup.go +++ b/setup.go @@ -154,8 +154,6 @@ func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cached ) blkst = blockstore.NewIdStore(blkst) - bsctx := metri.CtxScope(ctx, "rainbow") - var pr routing.PeerRouting var vs routing.ValueStore var cr routing.ContentRouting @@ -270,6 +268,7 @@ func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cached return nil, err } + bsctx := metri.CtxScope(ctx, "ipfs") bn := bsnet.NewFromIpfsHost(h, cr) bswap := bsclient.New(bsctx, bn, blkst, // default is 1 minute to search for a random live-want (1 From 36552d04dd01ca938aac3a122401aa1739978f5c Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 19 Oct 2023 18:12:27 +0200 Subject: [PATCH 23/29] The metric namespace for bitswap is actually ipfs_bitswap :/ --- setup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.go b/setup.go index 23f908a..aa5824e 100644 --- a/setup.go +++ b/setup.go @@ -268,7 +268,7 @@ func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cached return nil, err } - bsctx := metri.CtxScope(ctx, "ipfs") + bsctx := metri.CtxScope(ctx, "ipfs_bitswap") bn := bsnet.NewFromIpfsHost(h, cr) bswap := bsclient.New(bsctx, bn, blkst, // default is 1 minute to search for a random live-want (1 From 2f8cf9a0ea2b9e189a9fcee2bf0d53d5f73233ad Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 20 Oct 2023 15:22:03 +0200 Subject: [PATCH 24/29] Support a list of subdomain gateway domains and legacy gateway domains --- handlers.go | 14 ++++++++++++-- main.go | 53 +++++++++++++++++++++++++++++++++-------------------- setup.go | 9 +++++---- 3 files changed, 50 insertions(+), 26 deletions(-) diff --git a/handlers.go b/handlers.go index 7c4dcc4..d422636 100644 --- a/handlers.go +++ b/handlers.go @@ -116,8 +116,18 @@ func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) { UseSubdomains: true, }, } - if cfg.GatewayDomain != "" { - publicGateways[cfg.GatewayDomain] = &gateway.PublicGateway{ + for _, domain := range cfg.GatewayDomains { + publicGateways[domain] = &gateway.PublicGateway{ + Paths: []string{"/ipfs", "/ipns", "/version", "/api/v0"}, + NoDNSLink: noDNSLink, + InlineDNSLink: true, + DeserializedResponses: true, + UseSubdomains: false, + } + } + + for _, domain := range cfg.SubdomainGatewayDomains { + publicGateways[domain] = &gateway.PublicGateway{ Paths: []string{"/ipfs", "/ipns", "/version", "/api/v0"}, NoDNSLink: noDNSLink, InlineDNSLink: true, diff --git a/main.go b/main.go index d1d18df..7f845e3 100644 --- a/main.go +++ b/main.go @@ -53,10 +53,16 @@ func main() { Usage: "Specify an index to derivate the peerID from the key (needs --seed)", }, &cli.StringFlag{ - Name: "gateway-domain", + Name: "gateway-domains", Value: "", - EnvVars: []string{"RAINBOW_GATEWAY_DOMAIN"}, - Usage: "Set to enable subdomain gateway on this domain", + EnvVars: []string{"RAINBOW_GATEWAY_DOMAINS"}, + Usage: "Set to enable legacy gateway on these domains. Comma-separated list.", + }, + &cli.StringFlag{ + Name: "subdomain-gateway-domains", + Value: "", + EnvVars: []string{"RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS"}, + Usage: "Set to enable legacy gateway on these domains. Comma-separated list.", }, &cli.IntFlag{ Name: "gateway-port", @@ -191,24 +197,20 @@ to create libp2p identities for the gateway. return err } - var denylists []string - if list := cctx.String("denylists"); len(list) > 0 { - denylists = strings.Split(list, ",") - } - cfg := Config{ - DataDir: ddir, - GatewayDomain: cctx.String("gateway-domain"), - ConnMgrLow: cctx.Int("connmgr-low"), - ConnMgrHi: cctx.Int("connmgr-high"), - ConnMgrGrace: cctx.Duration("connmgr-grace"), - MaxMemory: cctx.Uint64("max-memory"), - MaxFD: cctx.Int("max-fd"), - InMemBlockCache: cctx.Int64("inmem-block-cache"), - RoutingV1: cctx.String("routing"), - KuboRPCURLs: getEnvs(EnvKuboRPC, DefaultKuboRPC), - DHTSharedHost: cctx.Bool("dht-fallback-shared-host"), - DenylistSubs: denylists, + DataDir: ddir, + GatewayDomains: getCommaSeparatedList(cctx.String("gateway-domains")), + SubdomainGatewayDomains: getCommaSeparatedList(cctx.String("subdomain-gateway-domains")), + ConnMgrLow: cctx.Int("connmgr-low"), + ConnMgrHi: cctx.Int("connmgr-high"), + ConnMgrGrace: cctx.Duration("connmgr-grace"), + MaxMemory: cctx.Uint64("max-memory"), + MaxFD: cctx.Int("max-fd"), + InMemBlockCache: cctx.Int64("inmem-block-cache"), + RoutingV1: cctx.String("routing"), + KuboRPCURLs: getEnvs(EnvKuboRPC, DefaultKuboRPC), + DHTSharedHost: cctx.Bool("dht-fallback-shared-host"), + DenylistSubs: getCommaSeparatedList(cctx.String("denylists")), } goLog.Debugf("Rainbow config: %+v", cfg) @@ -344,3 +346,14 @@ func getEnvs(key, defaultValue string) []string { value = strings.TrimSpace(value) return strings.Split(value, ",") } + +func getCommaSeparatedList(val string) []string { + if val == "" { + return nil + } + items := strings.Split(val, ",") + for i, item := range items { + items[i] = strings.TrimSpace(item) + } + return items +} diff --git a/setup.go b/setup.go index aa5824e..0b7a21d 100644 --- a/setup.go +++ b/setup.go @@ -92,10 +92,11 @@ type Config struct { MaxMemory uint64 MaxFD int - GatewayDomain string - RoutingV1 string - KuboRPCURLs []string - DHTSharedHost bool + GatewayDomains []string + SubdomainGatewayDomains []string + RoutingV1 string + KuboRPCURLs []string + DHTSharedHost bool DenylistSubs []string } From 246f0f8ba1902e02485e328ee28a884bc1f4ca1c Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Sat, 21 Oct 2023 01:08:22 +0200 Subject: [PATCH 25/29] update nopfs --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index baed9eb..89e2d89 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dustin/go-humanize v1.0.0 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231018184959-5433362b11ac - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231018184959-5433362b11ac - github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020230641-82a98a2ab538 + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020230641-82a98a2ab538 + github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981 diff --git a/go.sum b/go.sum index 7458dac..e368abf 100644 --- a/go.sum +++ b/go.sum @@ -289,12 +289,18 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs-shipyard/nopfs v0.0.12-0.20231018184959-5433362b11ac h1:K9wupxUa42nCjjfQLBEUjn2t5QdcdZT1g8tuxnKzuAE= github.com/ipfs-shipyard/nopfs v0.0.12-0.20231018184959-5433362b11ac/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020230641-82a98a2ab538 h1:qBg62kjpyF0TtPmsSD+/RGJFjauLNu5cPt774vKoLsM= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020230641-82a98a2ab538/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231018184959-5433362b11ac h1:SCDQyXmYSDRYusUiZ19DcP8/fOKj69P9W/xShKI2o8g= github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231018184959-5433362b11ac/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020230641-82a98a2ab538 h1:kQ8BZnHzFKGAAHe9GpJWDsHZinRbMTsNTzJu0RCFjJo= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020230641-82a98a2ab538/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 h1:WFpBlVbrvRyMzukb2TS+JPqrc8TkuILdGPoZoyX2na0= github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= +github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= From 704db53f6a23d1e56492a8c864035516a6fb4635 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 23 Oct 2023 15:53:57 +0200 Subject: [PATCH 26/29] Do not hardcode ports in Docker --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8e26cbb..7bfa085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,4 +50,3 @@ WORKDIR $RAINBOW_GATEWAY_PATH ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/entrypoint.sh"] -CMD ["--gateway-port", "8080", "--ctl-port", "8081"] From 28cdcd388714df44c31caa97820a3c413edfe035 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 24 Oct 2023 10:34:24 +0200 Subject: [PATCH 27/29] Replace port flags with listen-address flags --- main.go | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/main.go b/main.go index 7f845e3..8f0adb2 100644 --- a/main.go +++ b/main.go @@ -64,17 +64,17 @@ func main() { EnvVars: []string{"RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS"}, Usage: "Set to enable legacy gateway on these domains. Comma-separated list.", }, - &cli.IntFlag{ - Name: "gateway-port", - Value: 8090, - EnvVars: []string{"RAINBOW_GATEWAY_PORT"}, - Usage: "specify the listen address for the gateway endpoint", + &cli.StringFlag{ + Name: "gateway-listen-address", + Value: "127.0.0.1:8090", + EnvVars: []string{"RAINBOW_GATEWAY_LISTEN_ADDRESS"}, + Usage: "listen address for the gateway endpoint", }, - &cli.IntFlag{ - Name: "ctl-port", - Value: 8091, - EnvVars: []string{"RAINBOW_CTL_PORT"}, - Usage: "specify the api listening address for the internal control api", + &cli.StringFlag{ + Name: "ctl-listen-address", + Value: "127.0.0.1:8091", + EnvVars: []string{"RAINBOW_CTL_LISTEN_ADDRESS"}, + Usage: "listen address for the internal control api and metrics", }, &cli.IntFlag{ @@ -220,8 +220,8 @@ to create libp2p identities for the gateway. return err } - gatewayPort := cctx.Int("gateway-port") - apiPort := cctx.Int("ctl-port") + gatewayListen := cctx.String("gateway-listen-address") + ctlListen := cctx.String("ctl-listen-address") handler, err := setupGatewayHandler(cfg, gnd) if err != nil { @@ -229,7 +229,7 @@ to create libp2p identities for the gateway. } gatewaySrv := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", gatewayPort), + Addr: gatewayListen, Handler: handler, } @@ -255,7 +255,7 @@ to create libp2p identities for the gateway. apiMux.HandleFunc("/mgr/gc", GCHandler(gnd)) apiSrv := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", apiPort), + Addr: ctlListen, Handler: apiMux, } @@ -263,14 +263,10 @@ to create libp2p identities for the gateway. var wg sync.WaitGroup wg.Add(2) + fmt.Printf("Gateway listening at %s\n", gatewayListen) fmt.Printf("Legacy RPC at /api/v0 (%s): %s\n", EnvKuboRPC, strings.Join(gnd.kuboRPCs, " ")) - fmt.Printf("Path gateway: http://127.0.0.1:%d\n", gatewayPort) - fmt.Printf(" Smoke test (JPG): http://127.0.0.1:%d/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi\n", gatewayPort) - fmt.Printf("Subdomain gateway: http://localhost:%d\n", gatewayPort) - fmt.Printf(" Smoke test (Subdomain+DNSLink+UnixFS+HAMT): http://localhost:%d/ipns/en.wikipedia-on-ipfs.org/wiki/\n\n\n", gatewayPort) - - fmt.Printf("CTL port: http://127.0.0.1:%d\n", apiPort) - fmt.Printf("Metrics: http://127.0.0.1:%d/debug/metrics/prometheus\n\n", apiPort) + fmt.Printf("CTL endpoint listening at http://%s\n", ctlListen) + fmt.Printf("Metrics: http://%s/debug/metrics/prometheus\n\n", ctlListen) go func() { defer wg.Done() From 4c3f4a92dcb53df98902d7f07aa163aa82f28b39 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 24 Oct 2023 11:37:15 +0200 Subject: [PATCH 28/29] update nopfs --- go.mod | 4 ++-- go.sum | 14 ++++---------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 89e2d89..71d7c5c 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/coreos/go-systemd/v22 v22.5.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dustin/go-humanize v1.0.0 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020230641-82a98a2ab538 - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020230641-82a98a2ab538 + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231023110018-765003708b27 + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231023110018-765003708b27 github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index e368abf..fe56f48 100644 --- a/go.sum +++ b/go.sum @@ -287,18 +287,12 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231018184959-5433362b11ac h1:K9wupxUa42nCjjfQLBEUjn2t5QdcdZT1g8tuxnKzuAE= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231018184959-5433362b11ac/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020230641-82a98a2ab538 h1:qBg62kjpyF0TtPmsSD+/RGJFjauLNu5cPt774vKoLsM= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020230641-82a98a2ab538/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231018184959-5433362b11ac h1:SCDQyXmYSDRYusUiZ19DcP8/fOKj69P9W/xShKI2o8g= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231018184959-5433362b11ac/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020230641-82a98a2ab538 h1:kQ8BZnHzFKGAAHe9GpJWDsHZinRbMTsNTzJu0RCFjJo= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020230641-82a98a2ab538/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231023110018-765003708b27 h1:5BOAWLsBGjBnVaMP/tLvGDzVfT6V8AMiIFWCUJB0woQ= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231023110018-765003708b27/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231023110018-765003708b27 h1:Z1z4xfqZZLbPuIWyj5aIbXpXp9GvGl0Cqrq8UtYq1OU= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231023110018-765003708b27/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 h1:WFpBlVbrvRyMzukb2TS+JPqrc8TkuILdGPoZoyX2na0= -github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= From 3e1d4cfd31012287376169a7a65a5b8ac53bd19d Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 24 Oct 2023 12:20:18 +0200 Subject: [PATCH 29/29] Documentation --- README.md | 2 +- main.go | 91 +++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 70 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 51d77c3..7b31c40 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ## About Rainbow is an implementation of the [IPFS HTTP Gateway API](https://specs.ipfs.tech/http-gateways), -based on [boxo](https://github.com/ipfs/boxo) which is the tooling the powers [kubo](https://github.com/ipfs/kubo). +based on [boxo](https://github.com/ipfs/boxo) which is the tooling the powers [Kubo](https://github.com/ipfs/kubo). Rainbow uses the same Go code as the HTTP gateway in Kubo, but is fully specialized to just be a gateway: diff --git a/main.go b/main.go index 8f0adb2..b832ce0 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,46 @@ var goLog = logging.Logger("rainbow") func main() { app := cli.NewApp() + app.Name = "rainbow" + app.Usage = "The IPFS HTTP gateway daemon" + app.Version = version + app.Description = ` +Rainbow runs an IPFS HTTP gateway. + +An IPFS HTTP gateway is able to fetch content from the IPFS network and serve +it via HTTP, so that it becomes seamless to browse the web, when the web is +stored and provided by peers in the IPFS network. + +HTTP gateways are also able to facilitate download of any IPFS content (not +only websites, but any supported content-addressed Merkle-DAG), in formats +that are suitable for verification client-side (i.e. CAR files). + +Rainbow is optimized to perform the tasks of a gateway and only that, making +opinionated choices on the configration and setup of internal +components. Rainbow aims to serve production environments, where gateways are +deployed as a public service meant to be accessible by anyone. Rainbow acts as +a client to the IPFS network and does not serve or provide content to +it. Rainbow cannot be used to store or pin IPFS content, other than that +temporailly served over HTTP. Rainbow is just a gateway. + +Persistent configuration and data is stored in $RAINBOW_DATADIR (by default, +the folder in which rainbow is run). + +EXAMPLES + +Launch a gateway with randomly generated libp2p.key (will be written to +$RAINBOW_DATADIR/libp2p.key and used in subsequent runs): + + $ rainbow + +Generate an identity seed and launch a gateway: + + $ rainbow gen-seed > $RAINBOW_DATADIR/seed + $ rainbow --seed-index 0 + +(other rainbow gateways can use the same seed with different indexes to + derivate their identities) +` app.Flags = []cli.Flag{ @@ -38,96 +78,97 @@ func main() { Name: "datadir", Value: "", EnvVars: []string{"RAINBOW_DATADIR"}, - Usage: "specify the directory that cache data will be stored", + Usage: "Directory for persistent data (keys, blocks, denylists)", }, &cli.StringFlag{ Name: "seed", Value: "", EnvVars: []string{"RAINBOW_SEED"}, - Usage: "Specify a seed to derive peerID from (needs --seed-index). Best to use CREDENTIALS_DIRECTORY/seed", + Usage: "Seed to derive peerID from. Generate with gen-seed. Needs --seed-index. Best to use $CREDENTIALS_DIRECTORY/seed or $RAINBOW_DATADIR/seed.", }, &cli.IntFlag{ Name: "seed-index", Value: -1, EnvVars: []string{"RAINBOW_SEED_INDEX"}, - Usage: "Specify an index to derivate the peerID from the key (needs --seed)", + Usage: "Index to derivate the peerID (needs --seed)", }, &cli.StringFlag{ Name: "gateway-domains", Value: "", EnvVars: []string{"RAINBOW_GATEWAY_DOMAINS"}, - Usage: "Set to enable legacy gateway on these domains. Comma-separated list.", + Usage: "Legacy path-gateway domains. Comma-separated list.", }, &cli.StringFlag{ Name: "subdomain-gateway-domains", Value: "", EnvVars: []string{"RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS"}, - Usage: "Set to enable legacy gateway on these domains. Comma-separated list.", + Usage: "Subdomain gateway domains. Comma-separated list.", }, &cli.StringFlag{ Name: "gateway-listen-address", Value: "127.0.0.1:8090", EnvVars: []string{"RAINBOW_GATEWAY_LISTEN_ADDRESS"}, - Usage: "listen address for the gateway endpoint", + Usage: "Listen address for the gateway endpoint", }, &cli.StringFlag{ Name: "ctl-listen-address", Value: "127.0.0.1:8091", EnvVars: []string{"RAINBOW_CTL_LISTEN_ADDRESS"}, - Usage: "listen address for the internal control api and metrics", + Usage: "Listen address for the management api and metrics", }, &cli.IntFlag{ Name: "connmgr-low", Value: 100, EnvVars: []string{"RAINBOW_CONNMGR_LOW"}, - Usage: "libp2p connection manager 'low' water mark", + Usage: "Minimum number of connections to keep", }, &cli.IntFlag{ Name: "connmgr-high", Value: 3000, EnvVars: []string{"RAINBOW_CONNMGR_HIGH"}, - Usage: "libp2p connection manager 'high' water mark", + Usage: "Maximum number of connections to keep", }, &cli.DurationFlag{ Name: "connmgr-grace", Value: time.Minute, EnvVars: []string{"RAINBOW_CONNMGR_GRACE_PERIOD"}, - Usage: "libp2p connection manager grace period", + Usage: "Minimum connection TTL", }, &cli.IntFlag{ Name: "inmem-block-cache", Value: 1 << 30, EnvVars: []string{"RAINBOW_INMEM_BLOCK_CACHE"}, - Usage: "Size of the in-memory block cache. 0 to disable (disables compression too)", + Usage: "Size of the in-memory block cache. 0 to disable (disables compression on disk too)", }, &cli.Uint64Flag{ Name: "max-memory", Value: 0, EnvVars: []string{"RAINBOW_MAX_MEMORY"}, - Usage: "Libp2p resource manager max memory. Defaults to system's memory * 0.85", + Usage: "Max memory to use. Defaults to 85% of the system's available RAM", }, &cli.Uint64Flag{ Name: "max-fd", Value: 0, EnvVars: []string{"RAINBOW_MAX_FD"}, - Usage: "Libp2p resource manager file description limit. Defaults to the process' fd-limit/2", + Usage: "Maximum number of file descriptors. Defaults to 50% of the process' limit", }, &cli.StringFlag{ Name: "routing", Value: "", - Usage: "RoutingV1 Endpoint (if none is supplied use the Amino DHT and cid.contact)", + Usage: "RoutingV1 Endpoint (otherwise Amino DHT and cid.contact is used)", }, &cli.BoolFlag{ - Name: "dht-fallback-shared-host", - Value: false, - Usage: "If using an Amino DHT client should the libp2p host be shared with the data downloading host", + Name: "dht-share-host", + Value: false, + EnvVars: []string{"RAINBOW_DHT_SHARED_HOST"}, + Usage: "If false, DHT operations are run using an ephemeral peer, separate from the main one", }, &cli.StringFlag{ Name: "denylists", Value: "", EnvVars: []string{"RAINBOW_DENYLISTS"}, - Usage: "Denylist subscriptions (comma-separated)", + Usage: "Denylist HTTP subscriptions (comma-separated). Must be append-only denylists", }, } @@ -139,6 +180,15 @@ func main() { Running this command will generate a random seed and print it. The value can be used with the RAINBOW_SEED env-var to use key-derivation from a single seed to create libp2p identities for the gateway. + +The seed can be provided to rainbow by: + + - Storing it in $RAINBOW_DATADIR/seed + - Storing it in $CREDENTIALS_DIRECTORY/seed + - Passing the --seed flag + +In all cases the --seed-index flag will be necessary. Multiple gateways can +share the same seed as long as the indexes are different. `, Flags: []cli.Flag{}, Action: func(c *cli.Context) error { @@ -152,9 +202,6 @@ to create libp2p identities for the gateway. }, } - app.Name = "rainbow" - app.Usage = "a standalone ipfs gateway" - app.Version = version app.Action = func(cctx *cli.Context) error { ddir := cctx.String("datadir") cdns := newCachedDNS(dnsCacheRefreshInterval) @@ -209,7 +256,7 @@ to create libp2p identities for the gateway. InMemBlockCache: cctx.Int64("inmem-block-cache"), RoutingV1: cctx.String("routing"), KuboRPCURLs: getEnvs(EnvKuboRPC, DefaultKuboRPC), - DHTSharedHost: cctx.Bool("dht-fallback-shared-host"), + DHTSharedHost: cctx.Bool("dht-shared-host"), DenylistSubs: getCommaSeparatedList(cctx.String("denylists")), }