Update types with a clearer API, and fix compile issues

api.go → api/api.go

@@ -1,9 +1,11 @@
-package gateway
+package api
 
 import (
 	"context"
 
+	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-fetcher"
+	files "github.com/ipfs/go-ipfs-files"
 	"github.com/ipld/go-ipld-prime"
 )
 
@@ -17,7 +19,23 @@ type API interface {
 	// FetcherForSession describes dags that that the session is requesting to load. These dags should
 	// be fetchd into the local linksystem if not already present.
 	FetcherForSession(*ipld.LinkSystem) fetcher.Fetcher
+	// GetUnixFSNode requests a unixfs file or directory. This could be requested through a fetcher,
+	// but our fetcher paths do not currently support parallel fetchign or pre-loading of files.
+	GetUnixFSNode(*ipld.LinkSystem, cid.Cid) (files.Node, error)
+	// GetUnixFSDir requests the entries of a unixfs directories. This eventaully should be an
+	// API on the go-ipfs-files interface, but is included here explictly until all implementations
+	// support it directly on Directory objects.
+	GetUnixFSDir(*ipld.LinkSystem, files.Directory) ([]DirectoryItem, error)
 	// Resolver requests resolutions of dns names, and acts as an interface over go-namesys.
 	// If resolution is not supported, the name argument should be returned directly.
 	Resolve(ctx context.Context, name string) (string, error)
 }
+
+// DirectoryItem defines an entry in a UnixFS directory.
+type DirectoryItem interface {
+	GetSize() string
+	GetName() string
+	GetPath() string
+	GetHash() string
+	GetShortHash() string
+}

commands.go

@@ -5,8 +5,10 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"path/filepath"
+	"os"
 	"strings"
+
+	"github.com/ipfs-shipyard/gateway-prime/api"
 )
 
 // APIPath is the path at which the API is mounted.
@@ -16,15 +18,41 @@ var (
 	errAPIVersionMismatch = errors.New("api version mismatch")
 )
 
+var defaultLocalhostOrigins = []string{
+	"http://127.0.0.1:<port>",
+	"https://127.0.0.1:<port>",
+	"http://[::1]:<port>",
+	"https://[::1]:<port>",
+	"http://localhost:<port>",
+	"https://localhost:<port>",
+}
+
+var companionBrowserExtensionOrigins = []string{
+	"chrome-extension://nibjojkomfdiaoajekhjakgkdhaomnch", // ipfs-companion
+	"chrome-extension://hjoieblefckbooibpepigmacodalfndh", // ipfs-companion-beta
+}
+
+func commandsOption(handler http.Handler) ServeOption {
+	return func(_ api.API, gc *GatewayConfig, l net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
+		mux.Handle(APIPath+"/", handler)
+		return mux, nil
+	}
+}
+
+// CommandsOption constructs a ServerOption for hooking an additional endpoint into the
+// HTTP server.
+func CommandsOption(handler http.Handler) ServeOption {
+	return commandsOption(handler)
+}
+
 // CheckVersionOption returns a ServeOption that checks whether the client ipfs version matches. Does nothing when the user agent string does not contain `/go-ipfs/`
 func CheckVersionOption(daemonVersion string) ServeOption {
-	return ServeOption(func(_ API, _ *GatewayConfig, l net.Listener, parent *http.ServeMux) (*http.ServeMux, error) {
+	return ServeOption(func(_ api.API, _ *GatewayConfig, l net.Listener, parent *http.ServeMux) (*http.ServeMux, error) {
 		mux := http.NewServeMux()
 		parent.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 			if strings.HasPrefix(r.URL.Path, APIPath) {
 				cmdqry := r.URL.Path[len(APIPath):]
-				pth := filepath.SplitList(cmdqry)
-
+				pth := strings.Split(cmdqry, string(os.PathSeparator))
 				// backwards compatibility to previous version check
 				if len(pth) >= 2 && pth[1] != "version" {
 					clientVersion := r.UserAgent()

config.go

@@ -67,4 +67,15 @@ type GatewayConfig struct {
 	// PublicGateways configures behavior of known public gateways.
 	// Each key is a fully qualified domain name (FQDN).
 	PublicGateways map[string]*GatewaySpec
+
+	// FastDirIndexThreshold defines a threshold of directory size under which
+	// the direct children of the directory will be loaded in order to provide
+	// more details (like the number of items in child directories).
+	FastDirIndexThreshold int
+}
+
+func setConfigDefaults(gc *GatewayConfig) {
+	if gc.FastDirIndexThreshold == 0 {
+		gc.FastDirIndexThreshold = 100
+	}
 }

gateway.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"sort"
 
+	"github.com/ipfs-shipyard/gateway-prime/api"
 	id "github.com/libp2p/go-libp2p/p2p/protocol/identify"
 	ma "github.com/multiformats/go-multiaddr"
 	manet "github.com/multiformats/go-multiaddr/net"
@@ -15,7 +16,7 @@ import (
 // It returns the mux to expose to future options, which may be a new mux if it
 // is interested in mediating requests to future options, or the same mux
 // initially passed in if not.
-type ServeOption func(API, *GatewayConfig, net.Listener, *http.ServeMux) (*http.ServeMux, error)
+type ServeOption func(api.API, *GatewayConfig, net.Listener, *http.ServeMux) (*http.ServeMux, error)
 
 // A helper function to clean up a set of headers:
 // 1. Canonicalizes.
@@ -38,7 +39,7 @@ func cleanHeaderSet(headers []string) []string {
 }
 
 func GatewayOption(paths ...string) ServeOption {
-	return func(a API, cfg *GatewayConfig, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
+	return func(a api.API, cfg *GatewayConfig, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
 		headers := make(map[string][]string, len(cfg.HTTPHeaders))
 		for h, v := range cfg.HTTPHeaders {
 			headers[http.CanonicalHeaderKey(h)] = v
@@ -84,7 +85,7 @@ func GatewayOption(paths ...string) ServeOption {
 }
 
 func VersionOption(UserAgentVersion, CurrentCommit string) ServeOption {
-	return func(a API, cfg *GatewayConfig, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
+	return func(a api.API, cfg *GatewayConfig, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
 		mux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
 			fmt.Fprintf(w, "Commit: %s\n", CurrentCommit)
 			fmt.Fprintf(w, "Client Version: %s\n", UserAgentVersion)
@@ -96,7 +97,7 @@ func VersionOption(UserAgentVersion, CurrentCommit string) ServeOption {
 
 // makeHandler turns a list of ServeOptions into a http.Handler that implements
 // all of the given options, in order.
-func makeHandler(a API, gc *GatewayConfig, l net.Listener, options ...ServeOption) (http.Handler, error) {
+func makeHandler(a api.API, gc *GatewayConfig, l net.Listener, options ...ServeOption) (http.Handler, error) {
 	topMux := http.NewServeMux()
 	mux := topMux
 	for _, option := range options {
@@ -125,7 +126,7 @@ func makeHandler(a API, gc *GatewayConfig, l net.Listener, options ...ServeOptio
 // TODO intelligently parse address strings in other formats so long as they
 // unambiguously map to a valid multiaddr. e.g. for convenience, ":8080" should
 // map to "/ip4/0.0.0.0/tcp/8080".
-func ListenAndServe(a API, gc *GatewayConfig, listeningMultiAddr string, options ...ServeOption) (*http.Server, error) {
+func ListenAndServe(a api.API, gc *GatewayConfig, listeningMultiAddr string, options ...ServeOption) (*http.Server, error) {
 	addr, err := ma.NewMultiaddr(listeningMultiAddr)
 	if err != nil {
 		return nil, err
@@ -145,9 +146,8 @@ func ListenAndServe(a API, gc *GatewayConfig, listeningMultiAddr string, options
 
 // Serve accepts incoming HTTP connections on the listener and pass them
 // to ServeOption handlers.
-func Serve(a API, gc *GatewayConfig, lis net.Listener, options ...ServeOption) *http.Server {
-	// make sure we close this no matter what.
-	//defer lis.Close()
+func Serve(a api.API, gc *GatewayConfig, lis net.Listener, options ...ServeOption) *http.Server {
+	setConfigDefaults(gc)
 
 	handler, err := makeHandler(a, gc, lis, options...)
 	if err != nil {

gateway_handler.go

@@ -16,15 +16,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ipfs-shipyard/gateway-prime/api"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-fetcher"
+	files "github.com/ipfs/go-ipfs-files"
 	logging "github.com/ipfs/go-log"
 	resolver "github.com/ipfs/go-path/resolver"
-	"github.com/ipld/go-ipld-prime"
-	"github.com/ipld/go-ipld-prime/datamodel"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	basicnode "github.com/ipld/go-ipld-prime/node/basic"
-	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
@@ -68,7 +66,7 @@ type redirectTemplateData struct {
 // (it serves requests like GET /ipfs/QmVRzPKPzNtSrEzBFm2UZfxmPAgnaLke4DMcerbsGGSaFe/link)
 type gatewayHandler struct {
 	config *GatewayConfig
-	api    API
+	api    api.API
 
 	// generic metrics
 	firstContentBlockGetMetric *prometheus.HistogramVec
@@ -212,7 +210,7 @@ func newGatewayHistogramMetric(name string, help string) *prometheus.HistogramVe
 	return histogramMetric
 }
 
-func newGatewayHandler(c *GatewayConfig, api API) *gatewayHandler {
+func newGatewayHandler(c *GatewayConfig, api api.API) *gatewayHandler {
 	i := &gatewayHandler{
 		config: c,
 		api:    api,
@@ -361,6 +359,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 			return
 		}
 
+		fmt.Printf("err on resolve: %s\n", err)
 		webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusNotFound)
 		return
 	}
@@ -427,44 +426,27 @@ func (i *gatewayHandler) servePretty404IfPresent(w http.ResponseWriter, r *http.
 	}
 
 	ls := i.api.NewSession(r.Context())
-	f := i.api.FetcherForSession(ls)
-	sel := selectorparse.CommonSelector_ExploreAllRecursively
-	if err := f.NodeMatching(r.Context(), basicnode.NewLink(cidlink.Link{Cid: resolved404Path.Cid()}), sel, func(result fetcher.FetchResult) error { return nil }); err != nil {
-		return false
-	}
-	file, err := ls.Load(ipld.LinkContext{Ctx: r.Context()}, cidlink.Link{Cid: resolved404Path.Cid()}, basicnode.Prototype.Any)
+	dr, err := i.api.GetUnixFSNode(ls, resolved404Path.Cid())
 	if err != nil {
 		return false
 	}
+	defer dr.Close()
 
-	byteReader, ok := file.(datamodel.LargeBytesNode)
-	if ok {
-		rs, err := byteReader.AsLargeBytes()
-		if err == nil {
-			size, err := rs.Seek(0, io.SeekEnd)
-			if err == nil {
-				_, _ = rs.Seek(0, io.SeekStart)
-				log.Debugw("using pretty 404 file", "path", contentPath)
-				w.Header().Set("Content-Type", ctype)
-				w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
-				w.WriteHeader(http.StatusNotFound)
-				_, err = io.CopyN(w, rs, size)
-				return err == nil
-			}
-		}
+	f, ok := dr.(files.File)
+	if !ok {
+		return false
 	}
-	// fallback to non-large-bytes
-	bytes, err := file.AsBytes()
+
+	size, err := f.Size()
 	if err != nil {
 		return false
 	}
-	size := len(bytes)
 
 	log.Debugw("using pretty 404 file", "path", contentPath)
 	w.Header().Set("Content-Type", ctype)
-	w.Header().Set("Content-Length", strconv.FormatInt(int64(size), 10))
+	w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
 	w.WriteHeader(http.StatusNotFound)
-	_, err = w.Write(bytes)
+	_, err = io.CopyN(w, f, size)
 	return err == nil
 }
 

gateway_handler_unixfs.go

@@ -2,18 +2,12 @@ package gateway
 
 import (
 	"context"
+	"fmt"
 	"html"
 	"net/http"
 	"time"
 
-	"github.com/ipfs/go-fetcher"
-	"github.com/ipfs/go-unixfsnode"
-	"github.com/ipld/go-ipld-prime"
-	"github.com/ipld/go-ipld-prime/linking"
-	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
-	"github.com/ipld/go-ipld-prime/node/basicnode"
-	"github.com/ipld/go-ipld-prime/traversal/selector"
-	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
+	files "github.com/ipfs/go-ipfs-files"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
@@ -25,41 +19,27 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter,
 	defer span.End()
 	// Handling UnixFS
 	ls := i.api.NewSession(ctx)
-	fetchSession := i.api.FetcherForSession(ls)
-	ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
-	selSpec := ssb.ExploreInterpretAs("unixfs", ssb.ExploreRecursive(selector.RecursionLimitDepth(1), ssb.ExploreAll(ssb.ExploreRecursiveEdge())))
-	sel := selSpec.Node()
-	err := fetchSession.NodeMatching(ctx, basicnode.NewLink(cidlink.Link{Cid: resolvedPath.Cid()}), sel, func(_ fetcher.FetchResult) error { return nil })
-	if err != nil {
-		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
-		return
-	}
-	f := i.api.FetcherForSession(ls)
-	proto, _ := f.PrototypeFromLink(cidlink.Link{Cid: resolvedPath.Cid()})
-	node, err := ls.Load(ipld.LinkContext{Ctx: ctx}, cidlink.Link{Cid: resolvedPath.Cid()}, proto)
-	if err != nil {
-		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
-		return
-	}
-	if node == nil {
-		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
-		return
-	}
-	unode, err := unixfsnode.Reify(linking.LinkContext{Ctx: ctx}, node, ls)
+	dr, err := i.api.GetUnixFSNode(ls, resolvedPath.Cid())
 	if err != nil {
 		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
 		return
 	}
+	defer dr.Close()
 
 	// Handling Unixfs file
-	if unode.Kind() == ipld.Kind_Bytes {
+	if f, ok := dr.(files.File); ok {
 		logger.Debugw("serving unixfs file", "path", contentPath)
-		i.serveFile(ctx, w, r, resolvedPath, contentPath, unode, begin)
+		i.serveFile(ctx, w, r, resolvedPath, contentPath, f, begin)
 		return
 	}
 
 	// Handling Unixfs directory
-	logger.Debugf("resolved node is of type: %v", unode)
+	dir, ok := dr.(files.Directory)
+	if !ok {
+		internalWebError(w, fmt.Errorf("unsupported UnixFS type"))
+		return
+	}
+
 	logger.Debugw("serving unixfs directory", "path", contentPath)
-	i.serveDirectory(ctx, w, r, resolvedPath, contentPath, unode, begin, logger)
+	i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger)
 }