From 276ccbab854302e799c61b216c0adb75ed171a38 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Thu, 2 Feb 2023 14:08:53 +0100 Subject: [PATCH] feat: migrate subdomain and dnslink code --- examples/gateway/car/README.md | 2 +- examples/gateway/car/main.go | 25 +- examples/gateway/common/blocks.go | 14 + examples/gateway/proxy/README.md | 2 +- examples/gateway/proxy/main.go | 25 +- gateway/gateway.go | 6 + gateway/gateway_test.go | 103 ++++++ gateway/hostname.go | 587 ++++++++++++++++++++++++++++++ gateway/hostname_test.go | 299 +++++++++++++++ go.mod | 9 +- go.sum | 13 + 11 files changed, 1070 insertions(+), 15 deletions(-) create mode 100644 gateway/gateway_test.go create mode 100644 gateway/hostname.go create mode 100644 gateway/hostname_test.go diff --git a/examples/gateway/car/README.md b/examples/gateway/car/README.md index c9f972403..4d41341fc 100644 --- a/examples/gateway/car/README.md +++ b/examples/gateway/car/README.md @@ -26,7 +26,7 @@ Then, you can start the gateway with: ./gateway -c data.car -p 8040 ``` -Now you can access the gateway in [127.0.0.1:8040](http://127.0.0.1:8040). It will +Now you can access the gateway in [localhost:8040](http://localhost:8040). It will behave like a regular IPFS Gateway, except for the fact that all contents are provided from the CAR file. Therefore, things such as IPNS resolution and fetching contents from nodes in the IPFS network won't work. diff --git a/examples/gateway/car/main.go b/examples/gateway/car/main.go index a0c03fa0b..2c1918d72 100644 --- a/examples/gateway/car/main.go +++ b/examples/gateway/car/main.go @@ -12,6 +12,7 @@ import ( "github.com/ipfs/go-cid" offline "github.com/ipfs/go-ipfs-exchange-offline" "github.com/ipfs/go-libipfs/examples/gateway/common" + "github.com/ipfs/go-libipfs/gateway" carblockstore "github.com/ipld/go-car/v2/blockstore" ) @@ -26,20 +27,32 @@ func main() { } defer f.Close() - gateway, err := common.NewBlocksGateway(blockService, nil) + gwAPI, err := common.NewBlocksGateway(blockService, nil) if err != nil { log.Fatal(err) } + handler := common.NewBlocksHandler(gwAPI, *portPtr) - handler := common.NewBlocksHandler(gateway, *portPtr) + // Initialize the public gateways that we will want to have available through + // Host header rewritting. This step is optional and only required if you're + // running multiple public gateways and want different settings and support + // for DNSLink and Subdomain Gateways. + publicGateways := map[string]*gateway.Specification{ + "localhost": { + Paths: []string{"/ipfs", "/ipns"}, + NoDNSLink: true, + UseSubdomains: true, + }, + } + noDNSLink := true + handler = gateway.WithHostname(handler, gwAPI, publicGateways, noDNSLink) - address := "127.0.0.1:" + strconv.Itoa(*portPtr) - log.Printf("Listening on http://%s", address) + log.Printf("Listening on http://localhost:%d", *portPtr) for _, cid := range roots { - log.Printf("Hosting CAR root at http://%s/ipfs/%s", address, cid.String()) + log.Printf("Hosting CAR root at http://localhost:%d/ipfs/%s", *portPtr, cid.String()) } - if err := http.ListenAndServe(address, handler); err != nil { + if err := http.ListenAndServe(":"+strconv.Itoa(*portPtr), handler); err != nil { log.Fatal(err) } } diff --git a/examples/gateway/common/blocks.go b/examples/gateway/common/blocks.go index 40288136c..a34acae01 100644 --- a/examples/gateway/common/blocks.go +++ b/examples/gateway/common/blocks.go @@ -2,6 +2,7 @@ package common import ( "context" + "errors" "fmt" "net/http" gopath "path" @@ -24,6 +25,7 @@ import ( uio "github.com/ipfs/go-unixfs/io" "github.com/ipfs/go-unixfsnode" iface "github.com/ipfs/interface-go-ipfs-core" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" ifacepath "github.com/ipfs/interface-go-ipfs-core/path" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime" @@ -143,6 +145,18 @@ func (api *BlocksGateway) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, return nil, routing.ErrNotSupported } +func (api *BlocksGateway) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { + if api.namesys != nil { + p, err := api.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + if err == namesys.ErrResolveRecursion { + err = nil + } + return ifacepath.New(p.String()), err + } + + return nil, errors.New("not implemented") +} + func (api *BlocksGateway) IsCached(ctx context.Context, p ifacepath.Path) bool { rp, err := api.ResolvePath(ctx, p) if err != nil { diff --git a/examples/gateway/proxy/README.md b/examples/gateway/proxy/README.md index 0c54eb60e..604806360 100644 --- a/examples/gateway/proxy/README.md +++ b/examples/gateway/proxy/README.md @@ -31,6 +31,6 @@ types. Once you have it, run the proxy gateway with its address as the host para ./verifying-proxy -h https://ipfs.io -p 8040 ``` -Now you can access the gateway in [127.0.0.1:8040](http://127.0.0.1:8040). It will +Now you can access the gateway in [localhost:8040](http://localhost:8040). It will behave like a regular IPFS Gateway, except for the fact that it runs no libp2p, and has no local blockstore. All contents are provided by a remote gateway and fetched as RAW Blocks and Records, and verified locally. diff --git a/examples/gateway/proxy/main.go b/examples/gateway/proxy/main.go index f4d8c0b7c..5e99e61ba 100644 --- a/examples/gateway/proxy/main.go +++ b/examples/gateway/proxy/main.go @@ -9,6 +9,7 @@ import ( "github.com/ipfs/go-blockservice" offline "github.com/ipfs/go-ipfs-exchange-offline" "github.com/ipfs/go-libipfs/examples/gateway/common" + "github.com/ipfs/go-libipfs/gateway" ) func main() { @@ -24,16 +25,28 @@ func main() { routing := newProxyRouting(*gatewayUrlPtr, nil) // Creates the gateway with the block service and the routing. - gateway, err := common.NewBlocksGateway(blockService, routing) + gwAPI, err := common.NewBlocksGateway(blockService, routing) if err != nil { log.Fatal(err) } + handler := common.NewBlocksHandler(gwAPI, *portPtr) + + // Initialize the public gateways that we will want to have available through + // Host header rewritting. This step is optional and only required if you're + // running multiple public gateways and want different settings and support + // for DNSLink and Subdomain Gateways. + publicGateways := map[string]*gateway.Specification{ + "localhost": { + Paths: []string{"/ipfs", "/ipns"}, + NoDNSLink: true, + UseSubdomains: true, + }, + } + noDNSLink := true + handler = gateway.WithHostname(handler, gwAPI, publicGateways, noDNSLink) - handler := common.NewBlocksHandler(gateway, *portPtr) - address := "127.0.0.1:" + strconv.Itoa(*portPtr) - log.Printf("Listening on http://%s", address) - - if err := http.ListenAndServe(address, handler); err != nil { + log.Printf("Listening on http://localhost:%d", *portPtr) + if err := http.ListenAndServe(":"+strconv.Itoa(*portPtr), handler); err != nil { log.Fatal(err) } } diff --git a/gateway/gateway.go b/gateway/gateway.go index f50d80332..718bd3021 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -32,6 +32,12 @@ type API interface { // from the routing system. GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) + // GetDNSLinkRecord returns the DNSLink TXT record for the provided FQDN. + // Unlike ResolvePath, it does not perform recursive resolution. It only + // checks for the existence of a DNSLink TXT record with path starting with + // /ipfs/ or /ipns/ and returns the path as-is. + GetDNSLinkRecord(context.Context, string) (path.Path, error) + // IsCached returns whether or not the path exists locally. IsCached(context.Context, path.Path) bool diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go new file mode 100644 index 000000000..f878cb631 --- /dev/null +++ b/gateway/gateway_test.go @@ -0,0 +1,103 @@ +package gateway + +import ( + "context" + "errors" + "strings" + + cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/files" + "github.com/ipfs/go-namesys" + path "github.com/ipfs/go-path" + iface "github.com/ipfs/interface-go-ipfs-core" + nsopts "github.com/ipfs/interface-go-ipfs-core/options/namesys" + ipath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/libp2p/go-libp2p/core/crypto" +) + +type mockNamesys map[string]path.Path + +func (m mockNamesys) Resolve(ctx context.Context, name string, opts ...nsopts.ResolveOpt) (value path.Path, err error) { + cfg := nsopts.DefaultResolveOpts() + for _, o := range opts { + o(&cfg) + } + depth := cfg.Depth + if depth == nsopts.UnlimitedDepth { + // max uint + depth = ^uint(0) + } + for strings.HasPrefix(name, "/ipns/") { + if depth == 0 { + return value, namesys.ErrResolveRecursion + } + depth-- + + var ok bool + value, ok = m[name] + if !ok { + return "", namesys.ErrResolveFailed + } + name = value.String() + } + return value, nil +} + +func (m mockNamesys) ResolveAsync(ctx context.Context, name string, opts ...nsopts.ResolveOpt) <-chan namesys.Result { + out := make(chan namesys.Result, 1) + v, err := m.Resolve(ctx, name, opts...) + out <- namesys.Result{Path: v, Err: err} + close(out) + return out +} + +func (m mockNamesys) Publish(ctx context.Context, name crypto.PrivKey, value path.Path, opts ...nsopts.PublishOption) error { + return errors.New("not implemented for mockNamesys") +} + +func (m mockNamesys) GetResolver(subs string) (namesys.Resolver, bool) { + return nil, false +} + +type mockApi struct { + ns mockNamesys +} + +func newMockApi() *mockApi { + return &mockApi{ + ns: mockNamesys{}, + } +} + +func (m *mockApi) GetUnixFsNode(context.Context, ipath.Resolved) (files.Node, error) { + return nil, errors.New("not implemented") +} + +func (m *mockApi) LsUnixFsDir(context.Context, ipath.Resolved) (<-chan iface.DirEntry, error) { + return nil, errors.New("not implemented") +} + +func (m *mockApi) GetBlock(context.Context, cid.Cid) (blocks.Block, error) { + return nil, errors.New("not implemented") +} + +func (m *mockApi) GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) { + return nil, errors.New("not implemented") +} + +func (m *mockApi) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { + p, err := m.ns.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + if err == namesys.ErrResolveRecursion { + err = nil + } + return ipath.New(p.String()), err +} + +func (m *mockApi) IsCached(context.Context, ipath.Path) bool { + return false +} + +func (m *mockApi) ResolvePath(context.Context, ipath.Path) (ipath.Resolved, error) { + return nil, errors.New("not implemented") +} diff --git a/gateway/hostname.go b/gateway/hostname.go new file mode 100644 index 000000000..61c0dc16c --- /dev/null +++ b/gateway/hostname.go @@ -0,0 +1,587 @@ +package gateway + +import ( + "context" + "fmt" + "net" + "net/http" + "net/url" + "regexp" + "strings" + + cid "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/peer" + dns "github.com/miekg/dns" + + mbase "github.com/multiformats/go-multibase" +) + +// Specification is the specification of an IPFS Public Gateway. +type Specification struct { + // Paths is explicit list of path prefixes that should be handled by + // this gateway. Example: `["/ipfs", "/ipns"]` + Paths []string + + // UseSubdomains indicates whether or not this gateway uses subdomains + // for IPFS resources instead of paths. That is: http://CID.ipfs.GATEWAY/... + // + // If this flag is set, any /ipns/$id and/or /ipfs/$id paths in Paths + // will be permanently redirected to http://$id.[ipns|ipfs].$gateway/. + // + // We do not support using both paths and subdomains for a single domain + // for security reasons (Origin isolation). + UseSubdomains bool + + // NoDNSLink configures this gateway to _not_ resolve DNSLink for the FQDN + // provided in `Host` HTTP header. + NoDNSLink bool + + // InlineDNSLink configures this gateway to always inline DNSLink names + // (FQDN) into a single DNS label in order to interop with wildcard TLS certs + // and Origin per CID isolation provided by rules like https://publicsuffix.org + InlineDNSLink bool +} + +// WithHostname is a middleware that can wrap an http.Handler in order to parse the +// Host header and translating it to the content path. This is useful for Subdomain +// and DNSLink gateways. +// +// publicGateways configures the behavior of known public gateways. Each key is a +// fully qualified domain name (FQDN). +// +// noDNSLink configures the gateway to _not_ perform DNS TXT record lookups in +// response to requests with values in `Host` HTTP header. This flag can be overridden +// per FQDN in publicGateways. +func WithHostname(next http.Handler, api API, publicGateways map[string]*Specification, noDNSLink bool) http.HandlerFunc { + gateways := prepareHostnameGateways(publicGateways) + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Unfortunately, many (well, ipfs.io) gateways use + // DNSLink so if we blindly rewrite with DNSLink, we'll + // break /ipfs links. + // + // We fix this by maintaining a list of known gateways + // and the paths that they serve "gateway" content on. + // That way, we can use DNSLink for everything else. + + // Support X-Forwarded-Host if added by a reverse proxy + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Host + host := r.Host + if xHost := r.Header.Get("X-Forwarded-Host"); xHost != "" { + host = xHost + } + + // HTTP Host & Path check: is this one of our "known gateways"? + if gw, ok := gateways.isKnownHostname(host); ok { + // This is a known gateway but request is not using + // the subdomain feature. + + // Does this gateway _handle_ this path? + if hasPrefix(r.URL.Path, gw.Paths...) { + // It does. + + // Should this gateway use subdomains instead of paths? + if gw.UseSubdomains { + // Yes, redirect if applicable + // Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link + useInlinedDNSLink := gw.InlineDNSLink + newURL, err := toSubdomainURL(host, r.URL.Path, r, useInlinedDNSLink, api) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Set "Location" header with redirect destination. + // It is ignored by curl in default mode, but will + // be respected by user agents that follow + // redirects by default, namely web browsers + w.Header().Set("Location", newURL) + + // Note: we continue regular gateway processing: + // HTTP Status Code http.StatusMovedPermanently + // will be set later, in statusResponseWriter + } + } + + // Not a subdomain resource, continue with path processing + // Example: 127.0.0.1:8080/ipfs/{CID}, ipfs.io/ipfs/{CID} etc + next.ServeHTTP(w, r) + return + } + // Not a whitelisted path + + // Try DNSLink, if it was not explicitly disabled for the hostname + if !gw.NoDNSLink && hasDNSLinkRecord(r.Context(), api, host) { + // rewrite path and handle as DNSLink + r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path + next.ServeHTTP(w, withHostnameContext(r, host)) + return + } + + // If not, resource does not exist on the hostname, return 404 + http.NotFound(w, r) + return + } + + // HTTP Host check: is this one of our subdomain-based "known gateways"? + // IPFS details extracted from the host: {rootID}.{ns}.{gwHostname} + // /ipfs/ example: {cid}.ipfs.localhost:8080, {cid}.ipfs.dweb.link + // /ipns/ example: {libp2p-key}.ipns.localhost:8080, {inlined-dnslink-fqdn}.ipns.dweb.link + if gw, gwHostname, ns, rootID, ok := gateways.knownSubdomainDetails(host); ok { + // Looks like we're using a known gateway in subdomain mode. + + // Assemble original path prefix. + pathPrefix := "/" + ns + "/" + rootID + + // Retrieve whether or not we should inline DNSLink. + useInlinedDNSLink := gw.InlineDNSLink + + // Does this gateway _handle_ subdomains AND this path? + if !(gw.UseSubdomains && hasPrefix(pathPrefix, gw.Paths...)) { + // If not, resource does not exist, return 404 + http.NotFound(w, r) + return + } + + // Check if rootID is a valid CID + if rootCID, err := cid.Decode(rootID); err == nil { + // Do we need to redirect root CID to a canonical DNS representation? + dnsCID, err := toDNSLabel(rootID, rootCID) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if !strings.HasPrefix(r.Host, dnsCID) { + dnsPrefix := "/" + ns + "/" + dnsCID + newURL, err := toSubdomainURL(gwHostname, dnsPrefix+r.URL.Path, r, useInlinedDNSLink, api) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to deterministic CID to ensure CID + // always gets the same Origin on the web + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + + // Do we need to fix multicodec in PeerID represented as CIDv1? + if isPeerIDNamespace(ns) { + if rootCID.Type() != cid.Libp2pKey { + newURL, err := toSubdomainURL(gwHostname, pathPrefix+r.URL.Path, r, useInlinedDNSLink, api) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to CID fixed inside of toSubdomainURL() + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + } + } else { // rootID is not a CID.. + // Check if rootID is a single DNS label with an inlined + // DNSLink FQDN a single DNS label. We support this so + // loading DNSLink names over TLS "just works" on public + // HTTP gateways. + // + // Rationale for doing this can be found under "Option C" + // at: https://github.com/ipfs/in-web-browsers/issues/169 + // + // TLDR is: + // https://dweb.link/ipns/my.v-long.example.com + // can be loaded from a subdomain gateway with a wildcard + // TLS cert if represented as a single DNS label: + // https://my-v--long-example-com.ipns.dweb.link + if ns == "ipns" && !strings.Contains(rootID, ".") { + // if there is no TXT recordfor rootID + if !hasDNSLinkRecord(r.Context(), api, rootID) { + // my-v--long-example-com → my.v-long.example.com + dnslinkFQDN := toDNSLinkFQDN(rootID) + if hasDNSLinkRecord(r.Context(), api, dnslinkFQDN) { + // update path prefix to use real FQDN with DNSLink + pathPrefix = "/ipns/" + dnslinkFQDN + } + } + } + } + + // Rewrite the path to not use subdomains + r.URL.Path = pathPrefix + r.URL.Path + + // Serve path request + next.ServeHTTP(w, withHostnameContext(r, gwHostname)) + return + } + + // We don't have a known gateway. Fallback on DNSLink lookup + + // Wildcard HTTP Host check: + // 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)? + // 2. does Host header include a fully qualified domain name (FQDN)? + // 3. does DNSLink record exist in DNS? + if !noDNSLink && hasDNSLinkRecord(r.Context(), api, host) { + // rewrite path and handle as DNSLink + r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path + ctx := context.WithValue(r.Context(), DNSLinkHostnameKey, host) + next.ServeHTTP(w, withHostnameContext(r.WithContext(ctx), host)) + return + } + + // else, treat it as an old school gateway, I guess. + next.ServeHTTP(w, r) + + }) +} + +// Extends request context to include hostname of a canonical gateway root +// (subdomain root or dnslink fqdn) +func withHostnameContext(r *http.Request, hostname string) *http.Request { + // This is required for links on directory listing pages to work correctly + // on subdomain and dnslink gateways. While DNSlink could read value from + // Host header, subdomain gateways have more comples rules (knownSubdomainDetails) + // More: https://github.com/ipfs/dir-index-html/issues/42 + // nolint: staticcheck // non-backward compatible change + ctx := context.WithValue(r.Context(), GatewayHostnameKey, hostname) + return r.WithContext(ctx) +} + +// isDomainNameAndNotPeerID returns bool if string looks like a valid DNS name AND is not a PeerID +func isDomainNameAndNotPeerID(hostname string) bool { + if len(hostname) == 0 { + return false + } + if _, err := peer.Decode(hostname); err == nil { + return false + } + _, ok := dns.IsDomainName(hostname) + return ok +} + +// hasDNSLinkRecord returns if a DNS TXT record exists for the provided host. +func hasDNSLinkRecord(ctx context.Context, api API, host string) bool { + dnslinkName := stripPort(host) + + if !isDomainNameAndNotPeerID(dnslinkName) { + return false + } + + _, err := api.GetDNSLinkRecord(ctx, dnslinkName) + return err == nil +} + +func isSubdomainNamespace(ns string) bool { + switch ns { + case "ipfs", "ipns", "p2p", "ipld": + // Note: 'p2p' and 'ipld' is only kept here for compatibility with Kubo. + return true + default: + return false + } +} + +func isPeerIDNamespace(ns string) bool { + switch ns { + case "ipns", "p2p": + // Note: 'p2p' and 'ipld' is only kept here for compatibility with Kubo. + return true + default: + return false + } +} + +// Label's max length in DNS (https://tools.ietf.org/html/rfc1034#page-7) +const dnsLabelMaxLength int = 63 + +// Converts a CID to DNS-safe representation that fits in 63 characters +func toDNSLabel(rootID string, rootCID cid.Cid) (dnsCID string, err error) { + // Return as-is if things fit + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Convert to Base36 and see if that helped + rootID, err = cid.NewCidV1(rootCID.Type(), rootCID.Hash()).StringOfBase(mbase.Base36) + if err != nil { + return "", err + } + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Can't win with DNS at this point, return error + return "", fmt.Errorf("CID incompatible with DNS label length limit of 63: %s", rootID) +} + +// Returns true if HTTP request involves TLS certificate. +// See https://github.com/ipfs/in-web-browsers/issues/169 to understand how it +// impacts DNSLink websites on public gateways. +func isHTTPSRequest(r *http.Request) bool { + // X-Forwarded-Proto if added by a reverse proxy + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-Proto + xproto := r.Header.Get("X-Forwarded-Proto") + // Is request a native TLS (not used atm, but future-proofing) + // or a proxied HTTPS (eg. go-ipfs behind nginx at a public gw)? + return r.URL.Scheme == "https" || xproto == "https" +} + +// Converts a FQDN to DNS-safe representation that fits in 63 characters: +// my.v-long.example.com → my-v--long-example-com +func toDNSLinkDNSLabel(fqdn string) (dnsLabel string, err error) { + dnsLabel = strings.ReplaceAll(fqdn, "-", "--") + dnsLabel = strings.ReplaceAll(dnsLabel, ".", "-") + if len(dnsLabel) > dnsLabelMaxLength { + return "", fmt.Errorf("DNSLink representation incompatible with DNS label length limit of 63: %s", dnsLabel) + } + return dnsLabel, nil +} + +// Converts a DNS-safe representation of DNSLink FQDN to real FQDN: +// my-v--long-example-com → my.v-long.example.com +func toDNSLinkFQDN(dnsLabel string) (fqdn string) { + fqdn = strings.ReplaceAll(dnsLabel, "--", "@") // @ placeholder is unused in DNS labels + fqdn = strings.ReplaceAll(fqdn, "-", ".") + fqdn = strings.ReplaceAll(fqdn, "@", "-") + return fqdn +} + +// Converts a hostname/path to a subdomain-based URL, if applicable. +func toSubdomainURL(hostname, path string, r *http.Request, inlineDNSLink bool, api API) (redirURL string, err error) { + var scheme, ns, rootID, rest string + + query := r.URL.RawQuery + parts := strings.SplitN(path, "/", 4) + isHTTPS := isHTTPSRequest(r) + safeRedirectURL := func(in string) (out string, err error) { + safeURI, err := url.ParseRequestURI(in) + if err != nil { + return "", err + } + return safeURI.String(), nil + } + + if isHTTPS { + scheme = "https:" + } else { + scheme = "http:" + } + + switch len(parts) { + case 4: + rest = parts[3] + fallthrough + case 3: + ns = parts[1] + rootID = parts[2] + default: + return "", nil + } + + if !isSubdomainNamespace(ns) { + return "", nil + } + + // add prefix if query is present + if query != "" { + query = "?" + query + } + + // Normalize problematic PeerIDs (eg. ed25519+identity) to CID representation + if isPeerIDNamespace(ns) && !isDomainNameAndNotPeerID(rootID) { + peerID, err := peer.Decode(rootID) + // Note: PeerID CIDv1 with protobuf multicodec will fail, but we fix it + // in the next block + if err == nil { + rootID = peer.ToCid(peerID).String() + } + } + + // If rootID is a CID, ensure it uses DNS-friendly text representation + if rootCID, err := cid.Decode(rootID); err == nil { + multicodec := rootCID.Type() + var base mbase.Encoding = mbase.Base32 + + // Normalizations specific to /ipns/{libp2p-key} + if isPeerIDNamespace(ns) { + // Using Base36 for /ipns/ for consistency + // Context: https://github.com/ipfs/kubo/pull/7441#discussion_r452372828 + base = mbase.Base36 + + // PeerIDs represented as CIDv1 are expected to have libp2p-key + // multicodec (https://github.com/libp2p/specs/pull/209). + // We ease the transition by fixing multicodec on the fly: + // https://github.com/ipfs/kubo/issues/5287#issuecomment-492163929 + if multicodec != cid.Libp2pKey { + multicodec = cid.Libp2pKey + } + } + + // Ensure CID text representation used in subdomain is compatible + // with the way DNS and URIs are implemented in user agents. + // + // 1. Switch to CIDv1 and enable case-insensitive Base encoding + // to avoid issues when user agent force-lowercases the hostname + // before making the request + // (https://github.com/ipfs/in-web-browsers/issues/89) + rootCID = cid.NewCidV1(multicodec, rootCID.Hash()) + rootID, err = rootCID.StringOfBase(base) + if err != nil { + return "", err + } + // 2. Make sure CID fits in a DNS label, adjust encoding if needed + // (https://github.com/ipfs/kubo/issues/7318) + rootID, err = toDNSLabel(rootID, rootCID) + if err != nil { + return "", err + } + } else { // rootID is not a CID + + // Check if rootID is a FQDN with DNSLink and convert it to TLS-safe + // representation that fits in a single DNS label. We support this so + // loading DNSLink names over TLS "just works" on public HTTP gateways + // that pass 'https' in X-Forwarded-Proto to go-ipfs. + // + // Rationale can be found under "Option C" + // at: https://github.com/ipfs/in-web-browsers/issues/169 + // + // TLDR is: + // /ipns/my.v-long.example.com + // can be loaded from a subdomain gateway with a wildcard TLS cert if + // represented as a single DNS label: + // https://my-v--long-example-com.ipns.dweb.link + if (inlineDNSLink || isHTTPS) && ns == "ipns" && strings.Contains(rootID, ".") { + if hasDNSLinkRecord(r.Context(), api, rootID) { + // my.v-long.example.com → my-v--long-example-com + dnsLabel, err := toDNSLinkDNSLabel(rootID) + if err != nil { + return "", err + } + // update path prefix to use real FQDN with DNSLink + rootID = dnsLabel + } + } + } + + return safeRedirectURL(fmt.Sprintf( + "%s//%s.%s.%s/%s%s", + scheme, + rootID, + ns, + hostname, + rest, + query, + )) +} + +func hasPrefix(path string, prefixes ...string) bool { + for _, prefix := range prefixes { + // Assume people are creative with trailing slashes in Gateway config + p := strings.TrimSuffix(prefix, "/") + // Support for both /version and /ipfs/$cid + if p == path || strings.HasPrefix(path, p+"/") { + return true + } + } + return false +} + +func stripPort(hostname string) string { + host, _, err := net.SplitHostPort(hostname) + if err == nil { + return host + } + return hostname +} + +type hostnameGateways struct { + exact map[string]*Specification + wildcard map[*regexp.Regexp]*Specification +} + +// prepareHostnameGateways converts the user given gateways into an internal format +// split between exact and wildcard-based gateway hostnames. +func prepareHostnameGateways(gateways map[string]*Specification) *hostnameGateways { + h := &hostnameGateways{ + exact: map[string]*Specification{}, + wildcard: map[*regexp.Regexp]*Specification{}, + } + + for hostname, gw := range gateways { + if strings.Contains(hostname, "*") { + // from *.domain.tld, construct a regexp that match any direct subdomain + // of .domain.tld. + // + // Regexp will be in the form of ^[^.]+\.domain.tld(?::\d+)?$ + escaped := strings.ReplaceAll(hostname, ".", `\.`) + regexed := strings.ReplaceAll(escaped, "*", "[^.]+") + + re, err := regexp.Compile(fmt.Sprintf(`^%s(?::\d+)?$`, regexed)) + if err != nil { + log.Warn("invalid wildcard gateway hostname \"%s\"", hostname) + } + + h.wildcard[re] = gw + } else { + h.exact[hostname] = gw + } + } + + return h +} + +// isKnownHostname checks the given hostname gateways and returns a matching +// specification with graceful fallback to version without port. +func (gws *hostnameGateways) isKnownHostname(hostname string) (gw *Specification, ok bool) { + // Try hostname (host+optional port - value from Host header as-is) + if gw, ok := gws.exact[hostname]; ok { + return gw, ok + } + // Also test without port + if gw, ok = gws.exact[stripPort(hostname)]; ok { + return gw, ok + } + + // Wildcard support. Test both with and without port. + for re, spec := range gws.wildcard { + if re.MatchString(hostname) { + return spec, true + } + } + + return nil, false +} + +// knownSubdomainDetails parses the Host header and looks for a known gateway matching +// the subdomain host. If found, returns a Specification and the subdomain components +// extracted from Host header: {rootID}.{ns}.{gwHostname}. +// Note: hostname is host + optional port +func (gws *hostnameGateways) knownSubdomainDetails(hostname string) (gw *Specification, gwHostname, ns, rootID string, ok bool) { + labels := strings.Split(hostname, ".") + // Look for FQDN of a known gateway hostname. + // Example: given "dist.ipfs.tech.ipns.dweb.link": + // 1. Lookup "link" TLD in knownGateways: negative + // 2. Lookup "dweb.link" in knownGateways: positive + // + // Stops when we have 2 or fewer labels left as we need at least a + // rootId and a namespace. + for i := len(labels) - 1; i >= 2; i-- { + fqdn := strings.Join(labels[i:], ".") + gw, ok := gws.isKnownHostname(fqdn) + if !ok { + continue + } + + ns := labels[i-1] + if !isSubdomainNamespace(ns) { + continue + } + + // Merge remaining labels (could be a FQDN with DNSLink) + rootID := strings.Join(labels[:i-1], ".") + return gw, fqdn, ns, rootID, true + } + // no match + return nil, "", "", "", false +} diff --git a/gateway/hostname_test.go b/gateway/hostname_test.go new file mode 100644 index 000000000..f4ce73241 --- /dev/null +++ b/gateway/hostname_test.go @@ -0,0 +1,299 @@ +package gateway + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + + cid "github.com/ipfs/go-cid" + path "github.com/ipfs/go-path" +) + +func TestToSubdomainURL(t *testing.T) { + gwAPI := newMockApi() + testCID, err := cid.Decode("bafkqaglimvwgy3zakrsxg5cun5jxkyten5wwc2lokvjeycq") + if err != nil { + t.Fatal(err) + } + + gwAPI.ns["/ipns/dnslink.long-name.example.com"] = path.FromString(testCID.String()) + gwAPI.ns["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = path.FromString(testCID.String()) + httpRequest := httptest.NewRequest("GET", "http://127.0.0.1:8080", nil) + httpsRequest := httptest.NewRequest("GET", "https://https-request-stub.example.com", nil) + httpsProxiedRequest := httptest.NewRequest("GET", "http://proxied-https-request-stub.example.com", nil) + httpsProxiedRequest.Header.Set("X-Forwarded-Proto", "https") + + for _, test := range []struct { + // in: + request *http.Request + gwHostname string + inlineDNSLink bool + path string + // out: + url string + err error + }{ + + // DNSLink + {httpRequest, "localhost", false, "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", nil}, + // Hostname with port + {httpRequest, "localhost:8080", false, "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", nil}, + // CIDv0 → CIDv1base32 + {httpRequest, "localhost", false, "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", nil}, + // CIDv1 with long sha512 + {httpRequest, "localhost", false, "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, + // PeerID as CIDv1 needs to have libp2p-key multicodec + {httpRequest, "localhost", false, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://k2k4r8n0flx3ra0y5dr8fmyvwbzy3eiztmtq6th694k5a3rznayp3e4o.ipns.localhost/", nil}, + {httpRequest, "localhost", false, "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://k2k4r8l9ja7hkzynavdqup76ou46tnvuaqegbd04a4o1mpbsey0meucb.ipns.localhost/", nil}, + // PeerID: ed25519+identity multihash → CIDv1Base36 + {httpRequest, "localhost", false, "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil}, + {httpRequest, "sub.localhost", false, "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.sub.localhost/", nil}, + // HTTPS requires DNSLink name to fit in a single DNS label – see "Option C" from https://github.com/ipfs/in-web-browsers/issues/169 + {httpRequest, "dweb.link", false, "/ipns/dnslink.long-name.example.com", "http://dnslink.long-name.example.com.ipns.dweb.link/", nil}, + {httpsRequest, "dweb.link", false, "/ipns/dnslink.long-name.example.com", "https://dnslink-long--name-example-com.ipns.dweb.link/", nil}, + {httpsProxiedRequest, "dweb.link", false, "/ipns/dnslink.long-name.example.com", "https://dnslink-long--name-example-com.ipns.dweb.link/", nil}, + // HTTP requests can also be converted to fit into a single DNS label - https://github.com/ipfs/kubo/issues/9243 + {httpRequest, "localhost", true, "/ipns/dnslink.long-name.example.com", "http://dnslink-long--name-example-com.ipns.localhost/", nil}, + {httpRequest, "dweb.link", true, "/ipns/dnslink.long-name.example.com", "http://dnslink-long--name-example-com.ipns.dweb.link/", nil}, + } { + + url, err := toSubdomainURL(test.gwHostname, test.path, test.request, test.inlineDNSLink, gwAPI) + if url != test.url || !equalError(err, test.err) { + t.Errorf("(%s, %v, %s) returned (%s, %v), expected (%s, %v)", test.gwHostname, test.inlineDNSLink, test.path, url, err, test.url, test.err) + } + } +} + +func TestToDNSLinkDNSLabel(t *testing.T) { + for _, test := range []struct { + in string + out string + err error + }{ + {"dnslink.long-name.example.com", "dnslink-long--name-example-com", nil}, + {"dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com", "", errors.New("DNSLink representation incompatible with DNS label length limit of 63: dnslink-too--long-f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o-example-com")}, + } { + out, err := toDNSLinkDNSLabel(test.in) + if out != test.out || !equalError(err, test.err) { + t.Errorf("(%s) returned (%s, %v), expected (%s, %v)", test.in, out, err, test.out, test.err) + } + } +} + +func TestToDNSLinkFQDN(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"singlelabel", "singlelabel"}, + {"docs-ipfs-tech", "docs.ipfs.tech"}, + {"dnslink-long--name-example-com", "dnslink.long-name.example.com"}, + } { + out := toDNSLinkFQDN(test.in) + if out != test.out { + t.Errorf("(%s) returned (%s), expected (%s)", test.in, out, test.out) + } + } +} + +func TestIsHTTPSRequest(t *testing.T) { + httpRequest := httptest.NewRequest("GET", "http://127.0.0.1:8080", nil) + httpsRequest := httptest.NewRequest("GET", "https://https-request-stub.example.com", nil) + httpsProxiedRequest := httptest.NewRequest("GET", "http://proxied-https-request-stub.example.com", nil) + httpsProxiedRequest.Header.Set("X-Forwarded-Proto", "https") + httpProxiedRequest := httptest.NewRequest("GET", "http://proxied-http-request-stub.example.com", nil) + httpProxiedRequest.Header.Set("X-Forwarded-Proto", "http") + oddballRequest := httptest.NewRequest("GET", "foo://127.0.0.1:8080", nil) + for _, test := range []struct { + in *http.Request + out bool + }{ + {httpRequest, false}, + {httpsRequest, true}, + {httpsProxiedRequest, true}, + {httpProxiedRequest, false}, + {oddballRequest, false}, + } { + out := isHTTPSRequest(test.in) + if out != test.out { + t.Errorf("(%+v): returned %t, expected %t", test.in, out, test.out) + } + } +} + +func TestHasPrefix(t *testing.T) { + for _, test := range []struct { + prefixes []string + path string + out bool + }{ + {[]string{"/ipfs"}, "/ipfs/cid", true}, + {[]string{"/ipfs/"}, "/ipfs/cid", true}, + {[]string{"/version/"}, "/version", true}, + {[]string{"/version"}, "/version", true}, + } { + out := hasPrefix(test.path, test.prefixes...) + if out != test.out { + t.Errorf("(%+v, %s) returned '%t', expected '%t'", test.prefixes, test.path, out, test.out) + } + } +} + +func TestIsDomainNameAndNotPeerID(t *testing.T) { + for _, test := range []struct { + hostname string + out bool + }{ + {"", false}, + {"example.com", true}, + {"non-icann.something", true}, + {"..", false}, + {"12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", false}, // valid peerid + {"k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna", false}, // valid peerid + } { + out := isDomainNameAndNotPeerID(test.hostname) + if out != test.out { + t.Errorf("(%s) returned '%t', expected '%t'", test.hostname, out, test.out) + } + } +} + +func TestPortStripping(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"localhost:8080", "localhost"}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost"}, + {"example.com:443", "example.com"}, + {"example.com", "example.com"}, + {"foo-dweb.ipfs.pvt.k12.ma.us:8080", "foo-dweb.ipfs.pvt.k12.ma.us"}, + {"localhost", "localhost"}, + {"[::1]:8080", "::1"}, + } { + out := stripPort(test.in) + if out != test.out { + t.Errorf("(%s): returned '%s', expected '%s'", test.in, out, test.out) + } + } +} + +func TestToDNSLabel(t *testing.T) { + for _, test := range []struct { + in string + out string + err error + }{ + // <= 63 + {"QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", nil}, + {"bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", nil}, + // > 63 + // PeerID: ed25519+identity multihash → CIDv1Base36 + {"bafzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk", "k51qzi5uqu5dj16qyiq0tajolkojyl9qdkr254920wxv7ghtuwcz593tp69z9m", nil}, + // CIDv1 with long sha512 → error + {"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, + } { + inCID, _ := cid.Decode(test.in) + out, err := toDNSLabel(test.in, inCID) + if out != test.out || !equalError(err, test.err) { + t.Errorf("(%s): returned (%s, %v) expected (%s, %v)", test.in, out, err, test.out, test.err) + } + } + +} + +func TestKnownSubdomainDetails(t *testing.T) { + gwLocalhost := &Specification{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwDweb := &Specification{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwLong := &Specification{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwWildcard1 := &Specification{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + gwWildcard2 := &Specification{Paths: []string{"/ipfs", "/ipns", "/api"}, UseSubdomains: true} + + gateways := prepareHostnameGateways(map[string]*Specification{ + "localhost": gwLocalhost, + "dweb.link": gwDweb, + "devgateway.dweb.link": gwDweb, + "dweb.ipfs.pvt.k12.ma.us": gwLong, // note the sneaky ".ipfs." ;-) + "*.wildcard1.tld": gwWildcard1, + "*.*.wildcard2.tld": gwWildcard2, + }) + + for _, test := range []struct { + // in: + hostHeader string + // out: + gw *Specification + hostname string + ns string + rootID string + ok bool + }{ + // no subdomain + {"127.0.0.1:8080", nil, "", "", "", false}, + {"[::1]:8080", nil, "", "", "", false}, + {"hey.look.example.com", nil, "", "", "", false}, + {"dweb.link", nil, "", "", "", false}, + // malformed Host header + {".....dweb.link", nil, "", "", "", false}, + {"link", nil, "", "", "", false}, + {"8080:dweb.link", nil, "", "", "", false}, + {" ", nil, "", "", "", false}, + {"", nil, "", "", "", false}, + // unknown gateway host + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.unknown.example.com", nil, "", "", "", false}, + // cid in subdomain, known gateway + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.localhost:8080", gwLocalhost, "localhost:8080", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.devgateway.dweb.link", gwDweb, "devgateway.dweb.link", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + // capture everything before .ipfs. + {"foo.bar.boo-buzz.ipfs.dweb.link", gwDweb, "dweb.link", "ipfs", "foo.bar.boo-buzz", true}, + // ipns + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + // edge case check: public gateway under long TLD (see: https://publicsuffix.org) + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + // dnslink in subdomain + {"en.wikipedia-on-ipfs.org.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"en.wikipedia-on-ipfs.org.ipns.localhost", gwLocalhost, "localhost", "ipns", "en.wikipedia-on-ipfs.org", true}, + {"dist.ipfs.tech.ipns.localhost:8080", gwLocalhost, "localhost:8080", "ipns", "dist.ipfs.tech", true}, + {"en.wikipedia-on-ipfs.org.ipns.dweb.link", gwDweb, "dweb.link", "ipns", "en.wikipedia-on-ipfs.org", true}, + // edge case check: public gateway under long TLD (see: https://publicsuffix.org) + {"foo.dweb.ipfs.pvt.k12.ma.us", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju.ipns.dweb.ipfs.pvt.k12.ma.us", gwLong, "dweb.ipfs.pvt.k12.ma.us", "ipns", "bafzbeihe35nmjqar22thmxsnlsgxppd66pseq6tscs4mo25y55juhh6bju", true}, + // other namespaces + {"api.localhost", nil, "", "", "", false}, + {"peerid.p2p.localhost", gwLocalhost, "localhost", "p2p", "peerid", true}, + // wildcards + {"wildcard1.tld", nil, "", "", "", false}, + {".wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub.wildcard1.tld", gwWildcard1, "sub.wildcard1.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard1.tld", nil, "", "", "", false}, + {"bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am.ipfs.sub1.sub2.wildcard2.tld", gwWildcard2, "sub1.sub2.wildcard2.tld", "ipfs", "bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am", true}, + } { + gw, hostname, ns, rootID, ok := gateways.knownSubdomainDetails(test.hostHeader) + if ok != test.ok { + t.Errorf("knownSubdomainDetails(%s): ok is %t, expected %t", test.hostHeader, ok, test.ok) + } + if rootID != test.rootID { + t.Errorf("knownSubdomainDetails(%s): rootID is '%s', expected '%s'", test.hostHeader, rootID, test.rootID) + } + if ns != test.ns { + t.Errorf("knownSubdomainDetails(%s): ns is '%s', expected '%s'", test.hostHeader, ns, test.ns) + } + if hostname != test.hostname { + t.Errorf("knownSubdomainDetails(%s): hostname is '%s', expected '%s'", test.hostHeader, hostname, test.hostname) + } + if gw != test.gw { + t.Errorf("knownSubdomainDetails(%s): gw is %+v, expected %+v", test.hostHeader, gw, test.gw) + } + } + +} + +func equalError(a, b error) bool { + return (a == nil && b == nil) || (a != nil && b != nil && a.Error() == b.Error()) +} diff --git a/go.mod b/go.mod index cd213febb..9dc88453c 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-metrics-interface v0.0.1 + github.com/ipfs/go-namesys v0.7.0 github.com/ipfs/go-path v0.3.0 github.com/ipfs/go-peertaskqueue v0.8.0 github.com/ipfs/interface-go-ipfs-core v0.10.0 @@ -38,6 +39,7 @@ require ( github.com/libp2p/go-libp2p-record v0.2.0 github.com/libp2p/go-libp2p-testing v0.12.0 github.com/libp2p/go-msgio v0.2.0 + github.com/miekg/dns v1.1.50 github.com/multiformats/go-multiaddr v0.8.0 github.com/multiformats/go-multibase v0.1.1 github.com/multiformats/go-multicodec v0.6.0 @@ -63,6 +65,8 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gopacket v1.1.19 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/ipfs/bbloom v0.0.4 // indirect @@ -81,13 +85,14 @@ require ( github.com/koron/go-ssdp v0.0.3 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.19.0 // indirect + github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-nat v0.1.0 // indirect github.com/libp2p/go-netroute v0.2.0 // indirect github.com/libp2p/go-openssl v0.1.0 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/miekg/dns v1.1.50 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect @@ -106,7 +111,9 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect + github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0 // indirect + github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect go.uber.org/atomic v1.10.0 // indirect golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b // indirect diff --git a/go.sum b/go.sum index 98732459b..ee49196ca 100644 --- a/go.sum +++ b/go.sum @@ -309,10 +309,14 @@ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmv github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -460,6 +464,8 @@ github.com/ipfs/go-merkledag v0.9.0 h1:DFC8qZ96Dz1hMT7dtIpcY524eFFDiEWAF8hNJHWW2 github.com/ipfs/go-merkledag v0.9.0/go.mod h1:bPHqkHt5OZ0p1n3iqPeDiw2jIBkjAytRjS3WSBwjq90= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/ipfs/go-namesys v0.7.0 h1:xqosk71GIVRkFDtF2UNRcXn4LdNeo7tzuy8feHD6NbU= +github.com/ipfs/go-namesys v0.7.0/go.mod h1:KYSZBVZG3VJC34EfqqJPG7T48aWgxseoMPAPA5gLyyQ= github.com/ipfs/go-path v0.3.0 h1:tkjga3MtpXyM5v+3EbRvOHEoo+frwi4oumw5K+KYWyA= github.com/ipfs/go-path v0.3.0/go.mod h1:NOScsVgxfC/eIw4nz6OiGwK42PjaSJ4Y/ZFPn1Xe07I= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= @@ -609,6 +615,10 @@ github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFT github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= +github.com/libp2p/go-libp2p-kad-dht v0.19.0 h1:2HuiInHZTm9ZvQajaqdaPLHr0PCKKigWiflakimttE0= +github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= +github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= +github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= @@ -1068,10 +1078,13 @@ github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvS github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= +github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0 h1:obKzQ1ey5AJg5NKjgtTo/CKwLImVP4ETLRcsmzFJ4Qw= github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= +github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE=