diff --git a/Dockerfile b/Dockerfile index 8e26cbb..7bfa085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,4 +50,3 @@ WORKDIR $RAINBOW_GATEWAY_PATH ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/entrypoint.sh"] -CMD ["--gateway-port", "8080", "--ctl-port", "8081"] diff --git a/README.md b/README.md index f0d726a..7b31c40 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,26 @@ # rainbow -> The rainbow bridge between your CIDs and your html, javascript, wasm, jpegs, mp4s, tars, CARs, blocks, and more! +> An specialized IPFS HTTP gateway. ## About -Rainbow is an experimental specifically dedicated implementation of the [IPFS HTTP Gateway API](https://specs.ipfs.tech/http-gateways), -based on [boxo](https://github.com/ipfs/boxo) which is the tooling the powers [kubo](https://github.com/ipfs/kubo). +Rainbow is an implementation of the [IPFS HTTP Gateway API](https://specs.ipfs.tech/http-gateways), +based on [boxo](https://github.com/ipfs/boxo) which is the tooling the powers [Kubo](https://github.com/ipfs/kubo). + +Rainbow uses the same Go code as the HTTP gateway in Kubo, but is fully specialized to just be a gateway: + + * Rainbow acts as DHT and Bitswap client only. Rainbow is not a server for the network. + * Rainbow does not pin, or permanently store any content. It is just meant + to act as gateway to content present in the network. GC strategy + * Rainbow settings are optimized for production deployments and streamlined + for specific choices (badger datastore, writethrough uncached blockstore + etc.) + * Denylist and denylist subscription support is included. + * And more to come... + ## Building + ``` go build ``` @@ -18,31 +31,54 @@ go build rainbow ``` +Use `rainbow --help` for documentation. + ## Configuration -``` -NAME: - rainbow - a standalone ipfs gateway - -USAGE: - rainbow [global options] command [command options] [arguments...] - -VERSION: - v0.0.1-dev - -COMMANDS: - help, h Shows a list of commands or help for one command - -GLOBAL OPTIONS: - --datadir value specify the directory that cache data will be stored - --listen value specify the listen address for the gateway endpoint (default: ":8080") - --api-listen value specify the api listening address for the internal control api (default: "127.0.0.1:8081") - --connmgr-low value libp2p connection manager 'low' water mark (default: 100) - --connmgr-hi value libp2p connection manager 'high' water mark (default: 3000) - --connmgr-grace value libp2p connection manager grace period (default: 1m0s) - --routing value RoutingV1 Endpoint (default: "http://127.0.0.1:8090") - --help, -h show help (default: false) - --version, -v print the version (default: false) -``` + +Rainbow can be configured via command-line arguments or environment variables. + +See `rainbow --help` for information on the available options. + +Rainbow uses a `--datadir` (or `RAINBOW_DATADIR` environment variable) as +location for persisted data. It defaults to the folder in which `rainbow` is +run. + +### Peer Identity + +**Using a key file**: By default generates a `libp2p.key` in its data folder if none exist yet. This +file stores the libp2p peer identity. + +**Using a seed + index**: Alternatively, random can be initialized with a +32-byte, b58 encoded seed and a derivation index. This allows to use the same +seed for multiple instances of rainbow, and only change the derivation index. + +The seed and index can be provided as command line arguments or environment +vars (`--seed` , `--seed-index`). The seed can also be provided as a `seed` +file in the datadir folder. A new random seed can be generated with: + + rainbow gen-seed > seed + +To facilitate the use of rainbow with systemd +[`LoadCredential=`](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#LoadCredential=ID:PATH) +directive, we look for both `libp2p.key` and `seed` in +`$CREDENTIALS_DIRECTORY` first. + +### Denylists + +Rainbow can subscribe to append-only denylists using the `--denylists` flag. The value is a comma-separated list of URLs to subscribe to, for example: `https://denyli.st/badbits.deny`. This will download and update the denylist automatically when it is updated with new entries. + +Denylists can be manually placed in the `$RAINBOW_DATADIR/denylists` folder too. + +See [NoPFS](https://github.com/ipfs-shipyard/nopfs) for an explanation of the denylist format. Note that denylists should only be appended to while Rainbow is running. Editing differently, or adding new denylist files, should be done with Rainbow stopped. + +## Deployment + +An ansible role to deploy Rainbow is available within the ipfs.ipfs collection in Ansible Galaxy (https://github.com/ipfs-shipyard/ansible). It includes a systemd service unit file. + +Automated Docker container releases are available from the [Github container registry](https://github.com/ipfs/rainbow/pkgs/container/rainbow): + + docker pull ghcr.io/ipfs/rainbow:main-latest + ## License diff --git a/go.mod b/go.mod index 06afffa..71d7c5c 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,15 @@ module github.com/ipfs/rainbow go 1.20 require ( + github.com/coreos/go-systemd/v22 v22.5.0 github.com/dgraph-io/badger/v4 v4.2.0 github.com/dustin/go-humanize v1.0.0 - github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97 - github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97 - github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 + github.com/ipfs-shipyard/nopfs v0.0.12-0.20231023110018-765003708b27 + github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231023110018-765003708b27 + github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981 - github.com/ipfs/go-ds-leveldb v0.5.0 github.com/ipfs/go-ipfs-delay v0.0.1 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-metrics-interface v0.0.1 @@ -48,7 +48,6 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect - github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/cskr/pubsub v1.0.2 // indirect @@ -150,7 +149,6 @@ require ( github.com/samber/lo v1.36.0 // indirect github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/syndtr/goleveldb v1.0.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect diff --git a/go.sum b/go.sum index f764352..fe56f48 100644 --- a/go.sum +++ b/go.sum @@ -287,14 +287,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97 h1:zK5+tjnusBotrFPYPI1rgOe5VIwJHpf3j131M1KAm3I= -github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97 h1:XHANeOLCEz9pTxUK6gtKAxuAjCoMZ8B2QLcKUQNgJUA= -github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231023110018-765003708b27 h1:5BOAWLsBGjBnVaMP/tLvGDzVfT6V8AMiIFWCUJB0woQ= +github.com/ipfs-shipyard/nopfs v0.0.12-0.20231023110018-765003708b27/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231023110018-765003708b27 h1:Z1z4xfqZZLbPuIWyj5aIbXpXp9GvGl0Cqrq8UtYq1OU= +github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231023110018-765003708b27/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 h1:WFpBlVbrvRyMzukb2TS+JPqrc8TkuILdGPoZoyX2na0= -github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ= +github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -308,7 +308,6 @@ github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= -github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8= github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk= @@ -317,8 +316,6 @@ github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9 github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981 h1:GOKV62VnjerKwO7mwOyeoArzlaVrDLyoC/YPNtxxGwg= github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981/go.mod h1:LUU2FbhNdmhAbJmMeoahVRbe4GsduAODSJHWJJh2Vo4= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= -github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= -github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12XwJyvYvMM= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= @@ -397,7 +394,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -529,10 +525,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -658,7 +652,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= @@ -1105,7 +1098,6 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -1113,7 +1105,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/handlers.go b/handlers.go index ef92d5e..d422636 100644 --- a/handlers.go +++ b/handlers.go @@ -89,8 +89,13 @@ func withRequestLogger(next http.Handler) http.Handler { }) } -func setupGatewayHandler(nd *Node) (http.Handler, error) { - backend, err := gateway.NewBlocksBackend(nd.bsrv, gateway.WithValueStore(nd.vs), gateway.WithNameSystem(nd.ns)) +func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) { + backend, err := gateway.NewBlocksBackend( + nd.bsrv, + gateway.WithValueStore(nd.vs), + gateway.WithNameSystem(nd.ns), + gateway.WithResolver(nd.resolver), + ) if err != nil { return nil, err } @@ -110,13 +115,25 @@ func setupGatewayHandler(nd *Node) (http.Handler, error) { DeserializedResponses: true, UseSubdomains: true, }, - "dweb.link": { + } + for _, domain := range cfg.GatewayDomains { + publicGateways[domain] = &gateway.PublicGateway{ + Paths: []string{"/ipfs", "/ipns", "/version", "/api/v0"}, + NoDNSLink: noDNSLink, + InlineDNSLink: true, + DeserializedResponses: true, + UseSubdomains: false, + } + } + + for _, domain := range cfg.SubdomainGatewayDomains { + publicGateways[domain] = &gateway.PublicGateway{ Paths: []string{"/ipfs", "/ipns", "/version", "/api/v0"}, NoDNSLink: noDNSLink, InlineDNSLink: true, DeserializedResponses: true, UseSubdomains: true, - }, + } } // If we're doing tests, ensure the right public gateways are enabled. diff --git a/main.go b/main.go index 9fea8b1..b832ce0 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" "io" - "log" + "io/fs" "net/http" _ "net/http/pprof" "os" @@ -13,8 +13,10 @@ import ( "runtime" "strings" "sync" + "syscall" "time" + sddaemon "github.com/coreos/go-systemd/v22/daemon" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/crypto" peer "github.com/libp2p/go-libp2p/core/peer" @@ -29,81 +31,144 @@ var goLog = logging.Logger("rainbow") func main() { app := cli.NewApp() + app.Name = "rainbow" + app.Usage = "The IPFS HTTP gateway daemon" + app.Version = version + app.Description = ` +Rainbow runs an IPFS HTTP gateway. + +An IPFS HTTP gateway is able to fetch content from the IPFS network and serve +it via HTTP, so that it becomes seamless to browse the web, when the web is +stored and provided by peers in the IPFS network. + +HTTP gateways are also able to facilitate download of any IPFS content (not +only websites, but any supported content-addressed Merkle-DAG), in formats +that are suitable for verification client-side (i.e. CAR files). + +Rainbow is optimized to perform the tasks of a gateway and only that, making +opinionated choices on the configration and setup of internal +components. Rainbow aims to serve production environments, where gateways are +deployed as a public service meant to be accessible by anyone. Rainbow acts as +a client to the IPFS network and does not serve or provide content to +it. Rainbow cannot be used to store or pin IPFS content, other than that +temporailly served over HTTP. Rainbow is just a gateway. + +Persistent configuration and data is stored in $RAINBOW_DATADIR (by default, +the folder in which rainbow is run). + +EXAMPLES + +Launch a gateway with randomly generated libp2p.key (will be written to +$RAINBOW_DATADIR/libp2p.key and used in subsequent runs): + + $ rainbow + +Generate an identity seed and launch a gateway: + + $ rainbow gen-seed > $RAINBOW_DATADIR/seed + $ rainbow --seed-index 0 + +(other rainbow gateways can use the same seed with different indexes to + derivate their identities) +` app.Flags = []cli.Flag{ &cli.StringFlag{ - Name: "datadir", - Value: "", - Usage: "specify the directory that cache data will be stored", + Name: "datadir", + Value: "", + EnvVars: []string{"RAINBOW_DATADIR"}, + Usage: "Directory for persistent data (keys, blocks, denylists)", }, &cli.StringFlag{ Name: "seed", Value: "", EnvVars: []string{"RAINBOW_SEED"}, - Usage: "Specify a seed to derive peerID from (needs --seed-index)", + Usage: "Seed to derive peerID from. Generate with gen-seed. Needs --seed-index. Best to use $CREDENTIALS_DIRECTORY/seed or $RAINBOW_DATADIR/seed.", }, &cli.IntFlag{ Name: "seed-index", Value: -1, EnvVars: []string{"RAINBOW_SEED_INDEX"}, - Usage: "Specify an index to derivate the peerID from the key (needs --seed)", + Usage: "Index to derivate the peerID (needs --seed)", }, - &cli.IntFlag{ - Name: "gateway-port", - Value: 8090, - Usage: "specify the listen address for the gateway endpoint", + &cli.StringFlag{ + Name: "gateway-domains", + Value: "", + EnvVars: []string{"RAINBOW_GATEWAY_DOMAINS"}, + Usage: "Legacy path-gateway domains. Comma-separated list.", }, - &cli.IntFlag{ - Name: "ctl-port", - Value: 8091, - Usage: "specify the api listening address for the internal control api", + &cli.StringFlag{ + Name: "subdomain-gateway-domains", + Value: "", + EnvVars: []string{"RAINBOW_SUBDOMAIN_GATEWAY_DOMAINS"}, + Usage: "Subdomain gateway domains. Comma-separated list.", + }, + &cli.StringFlag{ + Name: "gateway-listen-address", + Value: "127.0.0.1:8090", + EnvVars: []string{"RAINBOW_GATEWAY_LISTEN_ADDRESS"}, + Usage: "Listen address for the gateway endpoint", + }, + &cli.StringFlag{ + Name: "ctl-listen-address", + Value: "127.0.0.1:8091", + EnvVars: []string{"RAINBOW_CTL_LISTEN_ADDRESS"}, + Usage: "Listen address for the management api and metrics", }, &cli.IntFlag{ - Name: "connmgr-low", - Value: 100, - Usage: "libp2p connection manager 'low' water mark", + Name: "connmgr-low", + Value: 100, + EnvVars: []string{"RAINBOW_CONNMGR_LOW"}, + Usage: "Minimum number of connections to keep", }, &cli.IntFlag{ - Name: "connmgr-hi", - Value: 3000, - Usage: "libp2p connection manager 'high' water mark", + Name: "connmgr-high", + Value: 3000, + EnvVars: []string{"RAINBOW_CONNMGR_HIGH"}, + Usage: "Maximum number of connections to keep", + }, + &cli.DurationFlag{ + Name: "connmgr-grace", + Value: time.Minute, + EnvVars: []string{"RAINBOW_CONNMGR_GRACE_PERIOD"}, + Usage: "Minimum connection TTL", }, &cli.IntFlag{ - Name: "inmem-block-cache", - Value: 1 << 30, - Usage: "Size of the in-memory block cache. 0 to disable (disables compression too)", + Name: "inmem-block-cache", + Value: 1 << 30, + EnvVars: []string{"RAINBOW_INMEM_BLOCK_CACHE"}, + Usage: "Size of the in-memory block cache. 0 to disable (disables compression on disk too)", }, &cli.Uint64Flag{ - Name: "max-memory", - Value: 0, - Usage: "Libp2p resource manager max memory. Defaults to system's memory * 0.85", + Name: "max-memory", + Value: 0, + EnvVars: []string{"RAINBOW_MAX_MEMORY"}, + Usage: "Max memory to use. Defaults to 85% of the system's available RAM", }, &cli.Uint64Flag{ - Name: "max-fd", - Value: 0, - Usage: "Libp2p resource manager file description limit. Defaults to the process' fd-limit/2", - }, - &cli.DurationFlag{ - Name: "connmgr-grace", - Value: time.Minute, - Usage: "libp2p connection manager grace period", + Name: "max-fd", + Value: 0, + EnvVars: []string{"RAINBOW_MAX_FD"}, + Usage: "Maximum number of file descriptors. Defaults to 50% of the process' limit", }, &cli.StringFlag{ Name: "routing", Value: "", - Usage: "RoutingV1 Endpoint (if none is supplied use the Amino DHT and cid.contact)", + Usage: "RoutingV1 Endpoint (otherwise Amino DHT and cid.contact is used)", }, &cli.BoolFlag{ - Name: "dht-fallback-shared-host", - Value: false, - Usage: "If using an Amino DHT client should the libp2p host be shared with the data downloading host", + Name: "dht-share-host", + Value: false, + EnvVars: []string{"RAINBOW_DHT_SHARED_HOST"}, + Usage: "If false, DHT operations are run using an ephemeral peer, separate from the main one", }, &cli.StringFlag{ - Name: "denylists", - Value: "https://denyli.st/badbits.deny", - Usage: "Denylist subscriptions (comma-separated)", + Name: "denylists", + Value: "", + EnvVars: []string{"RAINBOW_DENYLISTS"}, + Usage: "Denylist HTTP subscriptions (comma-separated). Must be append-only denylists", }, } @@ -115,6 +180,15 @@ func main() { Running this command will generate a random seed and print it. The value can be used with the RAINBOW_SEED env-var to use key-derivation from a single seed to create libp2p identities for the gateway. + +The seed can be provided to rainbow by: + + - Storing it in $RAINBOW_DATADIR/seed + - Storing it in $CREDENTIALS_DIRECTORY/seed + - Passing the --seed flag + +In all cases the --seed-index flag will be necessary. Multiple gateways can +share the same seed as long as the indexes are different. `, Flags: []cli.Flag{}, Action: func(c *cli.Context) error { @@ -128,58 +202,81 @@ to create libp2p identities for the gateway. }, } - app.Name = "rainbow" - app.Usage = "a standalone ipfs gateway" - app.Version = version app.Action = func(cctx *cli.Context) error { ddir := cctx.String("datadir") cdns := newCachedDNS(dnsCacheRefreshInterval) defer cdns.Close() + var seed string var priv crypto.PrivKey var err error - seed := cctx.String("seed") + + credDir := os.Getenv("CREDENTIALS_DIRECTORY") + secretsDir := ddir + + if len(credDir) > 0 { + secretsDir = credDir + } + + // attempt to read seed from disk + seedBytes, err := os.ReadFile(filepath.Join(secretsDir, "seed")) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + // set seed from command line or env-var + seed = cctx.String("seed") + } else { + return fmt.Errorf("error reading seed credentials: %w", err) + } + } else { + seed = strings.TrimSpace(string(seedBytes)) + } index := cctx.Int("seed-index") if len(seed) > 0 && index >= 0 { + fmt.Println("Deriving identity from seed") priv, err = deriveKey(seed, []byte(fmt.Sprintf("rainbow-%d", index))) } else { - keyFile := filepath.Join(ddir, "libp2p.key") + fmt.Println("Setting identity from libp2p.key") + keyFile := filepath.Join(secretsDir, "libp2p.key") priv, err = loadOrInitPeerKey(keyFile) } if err != nil { return err } - gnd, err := Setup(cctx.Context, Config{ - DataDir: ddir, - ConnMgrLow: cctx.Int("connmgr-low"), - ConnMgrHi: cctx.Int("connmgr-hi"), - ConnMgrGrace: cctx.Duration("connmgr-grace"), - MaxMemory: cctx.Uint64("max-memory"), - MaxFD: cctx.Int("max-fd"), - InMemBlockCache: cctx.Int64("inmem-block-cache"), - Libp2pKey: priv, - RoutingV1: cctx.String("routing"), - KuboRPCURLs: getEnvs(EnvKuboRPC, DefaultKuboRPC), - DHTSharedHost: cctx.Bool("dht-fallback-shared-host"), - DNSCache: cdns, - DenylistSubs: strings.Split(cctx.String("denylists"), ","), - }) + cfg := Config{ + DataDir: ddir, + GatewayDomains: getCommaSeparatedList(cctx.String("gateway-domains")), + SubdomainGatewayDomains: getCommaSeparatedList(cctx.String("subdomain-gateway-domains")), + ConnMgrLow: cctx.Int("connmgr-low"), + ConnMgrHi: cctx.Int("connmgr-high"), + ConnMgrGrace: cctx.Duration("connmgr-grace"), + MaxMemory: cctx.Uint64("max-memory"), + MaxFD: cctx.Int("max-fd"), + InMemBlockCache: cctx.Int64("inmem-block-cache"), + RoutingV1: cctx.String("routing"), + KuboRPCURLs: getEnvs(EnvKuboRPC, DefaultKuboRPC), + DHTSharedHost: cctx.Bool("dht-shared-host"), + DenylistSubs: getCommaSeparatedList(cctx.String("denylists")), + } + + goLog.Debugf("Rainbow config: %+v", cfg) + + gnd, err := Setup(cctx.Context, cfg, priv, cdns) if err != nil { return err } - gatewayPort := cctx.Int("gateway-port") - apiPort := cctx.Int("ctl-port") + gatewayListen := cctx.String("gateway-listen-address") + ctlListen := cctx.String("ctl-listen-address") - handler, err := setupGatewayHandler(gnd) + handler, err := setupGatewayHandler(cfg, gnd) if err != nil { return err } gatewaySrv := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", gatewayPort), + Addr: gatewayListen, Handler: handler, } @@ -205,7 +302,7 @@ to create libp2p identities for the gateway. apiMux.HandleFunc("/mgr/gc", GCHandler(gnd)) apiSrv := &http.Server{ - Addr: fmt.Sprintf("127.0.0.1:%d", apiPort), + Addr: ctlListen, Handler: apiMux, } @@ -213,14 +310,10 @@ to create libp2p identities for the gateway. var wg sync.WaitGroup wg.Add(2) + fmt.Printf("Gateway listening at %s\n", gatewayListen) fmt.Printf("Legacy RPC at /api/v0 (%s): %s\n", EnvKuboRPC, strings.Join(gnd.kuboRPCs, " ")) - fmt.Printf("Path gateway: http://127.0.0.1:%d\n", gatewayPort) - fmt.Printf(" Smoke test (JPG): http://127.0.0.1:%d/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi\n", gatewayPort) - fmt.Printf("Subdomain gateway: http://localhost:%d\n", gatewayPort) - fmt.Printf(" Smoke test (Subdomain+DNSLink+UnixFS+HAMT): http://localhost:%d/ipns/en.wikipedia-on-ipfs.org/wiki/\n\n\n", gatewayPort) - - fmt.Printf("CTL port: http://127.0.0.1:%d\n", apiPort) - fmt.Printf("Metrics: http://127.0.0.1:%d/debug/metrics/prometheus\n\n", apiPort) + fmt.Printf("CTL endpoint listening at http://%s\n", ctlListen) + fmt.Printf("Metrics: http://%s/debug/metrics/prometheus\n\n", ctlListen) go func() { defer wg.Done() @@ -242,9 +335,16 @@ to create libp2p identities for the gateway. } }() - signal.Notify(quit, os.Interrupt) + sddaemon.SdNotify(false, sddaemon.SdNotifyReady) + signal.Notify( + quit, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGHUP, + ) <-quit - log.Printf("Closing servers...\n") + sddaemon.SdNotify(false, sddaemon.SdNotifyStopping) + goLog.Info("Closing servers...") go gatewaySrv.Close() go apiSrv.Close() for _, sub := range gnd.denylistSubs { @@ -255,7 +355,7 @@ to create libp2p identities for the gateway. } if err := app.Run(os.Args); err != nil { - log.Print(err) + goLog.Error(err) os.Exit(1) } } @@ -289,3 +389,14 @@ func getEnvs(key, defaultValue string) []string { value = strings.TrimSpace(value) return strings.Split(value, ",") } + +func getCommaSeparatedList(val string) []string { + if val == "" { + return nil + } + items := strings.Split(val, ",") + for i, item := range items { + items[i] = strings.TrimSpace(item) + } + return items +} diff --git a/setup.go b/setup.go index 3e3d53e..0b7a21d 100644 --- a/setup.go +++ b/setup.go @@ -29,7 +29,6 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" badger4 "github.com/ipfs/go-ds-badger4" - levelds "github.com/ipfs/go-ds-leveldb" delay "github.com/ipfs/go-ipfs-delay" metri "github.com/ipfs/go-metrics-interface" mprome "github.com/ipfs/go-metrics-prometheus" @@ -85,8 +84,6 @@ type Config struct { ListenAddrs []string AnnounceAddrs []string - Libp2pKey crypto.PrivKey - ConnMgrLow int ConnMgrHi int ConnMgrGrace time.Duration @@ -95,15 +92,16 @@ type Config struct { MaxMemory uint64 MaxFD int - RoutingV1 string - KuboRPCURLs []string - DHTSharedHost bool - DNSCache *cachedDNS + GatewayDomains []string + SubdomainGatewayDomains []string + RoutingV1 string + KuboRPCURLs []string + DHTSharedHost bool DenylistSubs []string } -func Setup(ctx context.Context, cfg Config) (*Node, error) { +func Setup(ctx context.Context, cfg Config, key crypto.PrivKey, dnsCache *cachedDNS) (*Node, error) { ds, err := setupDatastore(cfg) if err != nil { return nil, err @@ -126,7 +124,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { libp2p.ListenAddrStrings(cfg.ListenAddrs...), libp2p.NATPortMap(), libp2p.ConnectionManager(cmgr), - libp2p.Identity(cfg.Libp2pKey), + libp2p.Identity(key), libp2p.BandwidthReporter(bwc), libp2p.DefaultTransports, libp2p.DefaultMuxers, @@ -157,8 +155,6 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { ) blkst = blockstore.NewIdStore(blkst) - bsctx := metri.CtxScope(ctx, "rainbow") - var pr routing.PeerRouting var vs routing.ValueStore var cr routing.ContentRouting @@ -175,7 +171,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { MaxConnsPerHost: 100, MaxIdleConnsPerHost: 100, IdleConnTimeout: 90 * time.Second, - DialContext: cfg.DNSCache.dialWithCachedDNS, + DialContext: dnsCache.dialWithCachedDNS, ForceAttemptHTTP2: true, }, }, @@ -195,12 +191,6 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { } else { // If there are no delegated routing endpoints run an accelerated Amino DHT client and send IPNI requests to cid.contact - // TODO: This datastore shouldn't end up containing anything anyway so this could potentially just be a null datastore - memDS, err := levelds.NewDatastore("", nil) - if err != nil { - return nil, err - } - var dhtHost host.Host if cfg.DHTSharedHost { dhtHost = h @@ -217,7 +207,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { } standardClient, err := dht.New(ctx, dhtHost, - dht.Datastore(memDS), + dht.Datastore(ds), dht.BootstrapPeers(dht.GetDefaultBootstrapPeerAddrInfos()...), dht.Mode(dht.ModeClient), ) @@ -231,7 +221,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { "pk": record.PublicKeyValidator{}, "ipns": ipns.Validator{KeyBook: h.Peerstore()}, }), - dht.Datastore(memDS), + dht.Datastore(ds), dht.BootstrapPeers(dht.GetDefaultBootstrapPeerAddrInfos()...), dht.BucketSize(20), )) @@ -279,6 +269,7 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { return nil, err } + bsctx := metri.CtxScope(ctx, "ipfs_bitswap") bn := bsnet.NewFromIpfsHost(h, cr) bswap := bsclient.New(bsctx, bn, blkst, // default is 1 minute to search for a random live-want (1 @@ -292,14 +283,17 @@ func Setup(ctx context.Context, cfg Config) (*Node, error) { ) bn.Start(bswap) - err = os.Mkdir("denylists", 0755) + err = os.Mkdir(filepath.Join(cfg.DataDir, "denylists"), 0755) if err != nil && !errors.Is(err, fs.ErrExist) { return nil, err } var denylists []*nopfs.HTTPSubscriber for _, dl := range cfg.DenylistSubs { - s := nopfs.NewHTTPSubscriber(dl, filepath.Join(cfg.DataDir, "denylists", filepath.Base(dl)), time.Minute) + s, err := nopfs.NewHTTPSubscriber(dl, filepath.Join(cfg.DataDir, "denylists", filepath.Base(dl)), time.Minute) + if err != nil { + return nil, err + } denylists = append(denylists, s) } @@ -401,7 +395,7 @@ func setupDatastore(cfg Config) (datastore.Batching, error) { Options: badgerOpts, } - return badger4.NewDatastore("badger4", &opts) + return badger4.NewDatastore(filepath.Join(cfg.DataDir, "badger4"), &opts) } type bundledDHT struct {