Merge pull request #29 from ipfs/develop

Fixes fixes and more fixes to get this to working state

Dockerfile

@@ -50,4 +50,3 @@ WORKDIR $RAINBOW_GATEWAY_PATH
 
 ENTRYPOINT ["/sbin/tini", "--", "/usr/local/bin/entrypoint.sh"]
 
-CMD ["--gateway-port", "8080", "--ctl-port", "8081"]

README.md

@@ -1,13 +1,26 @@
 # rainbow 
 
-> The rainbow bridge between your CIDs and your html, javascript, wasm, jpegs, mp4s, tars, CARs, blocks, and more!
+> An specialized IPFS HTTP gateway.
 
 ## About
 
-Rainbow is an experimental specifically dedicated implementation of the [IPFS HTTP Gateway API](https://specs.ipfs.tech/http-gateways), 
-based on [boxo](https://github.com/ipfs/boxo) which is the tooling the powers [kubo](https://github.com/ipfs/kubo).
+Rainbow is an implementation of the [IPFS HTTP Gateway API](https://specs.ipfs.tech/http-gateways),
+based on [boxo](https://github.com/ipfs/boxo) which is the tooling the powers [Kubo](https://github.com/ipfs/kubo).
+
+Rainbow uses the same Go code as the HTTP gateway in Kubo, but is fully specialized to just be a gateway:
+
+  * Rainbow acts as DHT and Bitswap client only. Rainbow is not a server for the network.
+  * Rainbow does not pin, or permanently store any content. It is just meant
+    to act as gateway to content present in the network. GC strategy 
+  * Rainbow settings are optimized for production deployments and streamlined
+    for specific choices (badger datastore, writethrough uncached blockstore
+    etc.)
+  * Denylist and denylist subscription support is included.
+  * And more to come...
+
 
 ## Building
+
 ```
 go build
 ```
@@ -18,31 +31,54 @@ go build
 rainbow
 ```
 
+Use `rainbow --help` for documentation.
+
 ## Configuration
-```
-NAME:
-   rainbow - a standalone ipfs gateway
-
-USAGE:
-   rainbow [global options] command [command options] [arguments...]
-
-VERSION:
-   v0.0.1-dev
-
-COMMANDS:
-   help, h  Shows a list of commands or help for one command
-
-GLOBAL OPTIONS:
-   --datadir value        specify the directory that cache data will be stored
-   --listen value         specify the listen address for the gateway endpoint (default: ":8080")
-   --api-listen value     specify the api listening address for the internal control api (default: "127.0.0.1:8081")
-   --connmgr-low value    libp2p connection manager 'low' water mark (default: 100)
-   --connmgr-hi value     libp2p connection manager 'high' water mark (default: 3000)
-   --connmgr-grace value  libp2p connection manager grace period (default: 1m0s)
-   --routing value        RoutingV1 Endpoint (default: "http://127.0.0.1:8090")
-   --help, -h             show help (default: false)
-   --version, -v          print the version (default: false)
-```
+
+Rainbow can be configured via command-line arguments or environment variables.
+
+See `rainbow --help` for information on the available options.
+
+Rainbow uses a `--datadir` (or `RAINBOW_DATADIR` environment variable) as
+location for persisted data. It defaults to the folder in which `rainbow` is
+run.
+
+### Peer Identity
+
+**Using a key file**: By default generates a `libp2p.key` in its data folder if none exist yet. This
+file stores the libp2p peer identity.
+
+**Using a seed + index**: Alternatively, random can be initialized with a
+32-byte, b58 encoded seed and a derivation index. This allows to use the same
+seed for multiple instances of rainbow, and only change the derivation index.
+
+The seed and index can be provided as command line arguments or environment
+vars (`--seed` , `--seed-index`). The seed can also be provided as a `seed`
+file in the datadir folder. A new random seed can be generated with:
+
+    rainbow gen-seed > seed
+
+To facilitate the use of rainbow with systemd
+[`LoadCredential=`](https://www.freedesktop.org/software/systemd/man/systemd.exec.html#LoadCredential=ID:PATH)
+directive, we look for both `libp2p.key` and `seed` in
+`$CREDENTIALS_DIRECTORY` first.
+
+### Denylists
+
+Rainbow can subscribe to append-only denylists using the `--denylists` flag. The value is a comma-separated list of URLs to subscribe to, for example: `https://denyli.st/badbits.deny`. This will download and update the denylist automatically when it is updated with new entries.
+
+Denylists can be manually placed in the `$RAINBOW_DATADIR/denylists` folder too.
+
+See [NoPFS](https://github.com/ipfs-shipyard/nopfs) for an explanation of the denylist format. Note that denylists should only be appended to while Rainbow is running. Editing differently, or adding new denylist files, should be done with Rainbow stopped.
+
+## Deployment
+
+An ansible role to deploy Rainbow is available within the ipfs.ipfs collection in Ansible Galaxy (https://github.com/ipfs-shipyard/ansible). It includes a systemd service unit file.
+
+Automated Docker container releases are available from the [Github container registry](https://github.com/ipfs/rainbow/pkgs/container/rainbow):
+
+    docker pull ghcr.io/ipfs/rainbow:main-latest
+
 
 ## License
 

go.mod

@@ -3,15 +3,15 @@ module github.com/ipfs/rainbow
 go 1.20
 
 require (
+	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/dgraph-io/badger/v4 v4.2.0
 	github.com/dustin/go-humanize v1.0.0
-	github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97
-	github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97
-	github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299
+	github.com/ipfs-shipyard/nopfs v0.0.12-0.20231023110018-765003708b27
+	github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231023110018-765003708b27
+	github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-datastore v0.6.0
 	github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981
-	github.com/ipfs/go-ds-leveldb v0.5.0
 	github.com/ipfs/go-ipfs-delay v0.0.1
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipfs/go-metrics-interface v0.0.1
@@ -48,7 +48,6 @@ require (
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/containerd/cgroups v1.1.0 // indirect
-	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
 	github.com/cskr/pubsub v1.0.2 // indirect
@@ -150,7 +149,6 @@ require (
 	github.com/samber/lo v1.36.0 // indirect
 	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
 	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect

go.sum

@@ -287,14 +287,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
 github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97 h1:zK5+tjnusBotrFPYPI1rgOe5VIwJHpf3j131M1KAm3I=
-github.com/ipfs-shipyard/nopfs v0.0.12-0.20231016145930-853ce79d9b97/go.mod h1:kuVnRVtaUO33/H05F/TH0pia1Pr/VW3DrFRGZGP4bpg=
-github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97 h1:XHANeOLCEz9pTxUK6gtKAxuAjCoMZ8B2QLcKUQNgJUA=
-github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231016145930-853ce79d9b97/go.mod h1:+hCrN49NLsrgas30m4xwqD6V8xPRl/I1ZMqy0iNEzUE=
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231023110018-765003708b27 h1:5BOAWLsBGjBnVaMP/tLvGDzVfT6V8AMiIFWCUJB0woQ=
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231023110018-765003708b27/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231023110018-765003708b27 h1:Z1z4xfqZZLbPuIWyj5aIbXpXp9GvGl0Cqrq8UtYq1OU=
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231023110018-765003708b27/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299 h1:WFpBlVbrvRyMzukb2TS+JPqrc8TkuILdGPoZoyX2na0=
-github.com/ipfs/boxo v0.13.2-0.20231013160006-ef6adea52299/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
+github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ=
+github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -308,7 +308,6 @@ github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=
 github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk=
 github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE=
 github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw=
-github.com/ipfs/go-datastore v0.5.0/go.mod h1:9zhEApYMTl17C8YDp7JmU7sQZi2/wqiYh73hakZ90Bk=
 github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk=
 github.com/ipfs/go-datastore v0.6.0/go.mod h1:rt5M3nNbSO/8q1t4LNkLyUwRs8HupMeN/8O4Vn9YAT8=
 github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ6cOk=
@@ -317,8 +316,6 @@ github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9
 github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981 h1:GOKV62VnjerKwO7mwOyeoArzlaVrDLyoC/YPNtxxGwg=
 github.com/ipfs/go-ds-badger4 v0.0.0-20231006150127-9137bcc6b981/go.mod h1:LUU2FbhNdmhAbJmMeoahVRbe4GsduAODSJHWJJh2Vo4=
 github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8=
-github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo=
-github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
 github.com/ipfs/go-ipfs-blockstore v1.3.0 h1:m2EXaWgwTzAfsmt5UdJ7Is6l4gJcaM/A12XwJyvYvMM=
 github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ=
 github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8=
@@ -397,7 +394,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0=
 github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -529,10 +525,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n
 github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
 github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
 github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
-github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU=
 github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -658,7 +652,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
@@ -1105,15 +1098,13 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs
 google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8=
 gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

handlers.go

@@ -89,8 +89,13 @@ func withRequestLogger(next http.Handler) http.Handler {
 	})
 }
 
-func setupGatewayHandler(nd *Node) (http.Handler, error) {
-	backend, err := gateway.NewBlocksBackend(nd.bsrv, gateway.WithValueStore(nd.vs), gateway.WithNameSystem(nd.ns))
+func setupGatewayHandler(cfg Config, nd *Node) (http.Handler, error) {
+	backend, err := gateway.NewBlocksBackend(
+		nd.bsrv,
+		gateway.WithValueStore(nd.vs),
+		gateway.WithNameSystem(nd.ns),
+		gateway.WithResolver(nd.resolver),
+	)
 	if err != nil {
 		return nil, err
 	}
@@ -110,13 +115,25 @@ func setupGatewayHandler(nd *Node) (http.Handler, error) {
 			DeserializedResponses: true,
 			UseSubdomains:         true,
 		},
-		"dweb.link": {
+	}
+	for _, domain := range cfg.GatewayDomains {
+		publicGateways[domain] = &gateway.PublicGateway{
+			Paths:                 []string{"/ipfs", "/ipns", "/version", "/api/v0"},
+			NoDNSLink:             noDNSLink,
+			InlineDNSLink:         true,
+			DeserializedResponses: true,
+			UseSubdomains:         false,
+		}
+	}
+
+	for _, domain := range cfg.SubdomainGatewayDomains {
+		publicGateways[domain] = &gateway.PublicGateway{
 			Paths:                 []string{"/ipfs", "/ipns", "/version", "/api/v0"},
 			NoDNSLink:             noDNSLink,
 			InlineDNSLink:         true,
 			DeserializedResponses: true,
 			UseSubdomains:         true,
-		},
+		}
 	}
 
 	// If we're doing tests, ensure the right public gateways are enabled.