Skip to content

Commit

Permalink
feat(gateway): make backend api more explicit. switch to only process…
Browse files Browse the repository at this point in the history
…ing single range requests
  • Loading branch information
aschmahmann committed Jun 27, 2023
1 parent 8424cf4 commit db5809e
Show file tree
Hide file tree
Showing 13 changed files with 756 additions and 165 deletions.
63 changes: 57 additions & 6 deletions gateway/blocks_backend.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,11 +153,30 @@ func (bb *BlocksBackend) Get(ctx context.Context, path ImmutablePath, ranges ...
return md, nil, err
}

// Only a single range is supported
var ra *ByteRange
if len(ranges) > 0 {
ra = &ranges[0]
}

rootCodec := nd.Cid().Prefix().GetCodec()
// This covers both Raw blocks and terminal IPLD codecs like dag-cbor and dag-json
// Note: while only cbor, json, dag-cbor, and dag-json are currently supported by gateways this could change
if rootCodec != uint64(mc.DagPb) {
return md, NewGetResponseFromFile(files.NewBytesFile(nd.RawData())), nil
f := files.NewBytesFile(nd.RawData())

fileSize, err := f.Size()
if err != nil {
return ContentPathMetadata{}, nil, err
}

if ra != nil && ra.From != 0 {
if _, err := f.Seek(int64(ra.From), io.SeekStart); err != nil {
return ContentPathMetadata{}, nil, err
}
}

return md, NewGetResponseFromReader(f, fileSize), nil
}

// This code path covers full graph, single file/directory, and range requests
Expand All @@ -179,10 +198,25 @@ func (bb *BlocksBackend) Get(ctx context.Context, path ImmutablePath, ranges ...
if sz < 0 {
return ContentPathMetadata{}, nil, fmt.Errorf("directory cumulative DAG size cannot be negative")
}
return md, NewGetResponseFromDirectoryListing(uint64(sz), dir.EnumLinksAsync(ctx)), nil
return md, NewGetResponseFromDirectoryListing(uint64(sz), dir.EnumLinksAsync(ctx), nil), nil
}
if file, ok := f.(files.File); ok {
return md, NewGetResponseFromFile(file), nil
fileSize, err := f.Size()
if err != nil {
return ContentPathMetadata{}, nil, err
}

if ra != nil && ra.From != 0 {
if _, err := file.Seek(int64(ra.From), io.SeekStart); err != nil {
return ContentPathMetadata{}, nil, err
}
}

if s, ok := f.(*files.Symlink); ok {
return md, NewGetResponseFromSymlink(s, fileSize), nil
}

return md, NewGetResponseFromReader(file, fileSize), nil
}

return ContentPathMetadata{}, nil, fmt.Errorf("data was not a valid file or directory: %w", ErrInternalServerError) // TODO: should there be a gateway invalid content type to abstract over the various IPLD error types?
Expand Down Expand Up @@ -211,15 +245,15 @@ func (bb *BlocksBackend) GetBlock(ctx context.Context, path ImmutablePath) (Cont
return md, files.NewBytesFile(nd.RawData()), nil
}

func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) {
func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *HeadResponse, error) {
md, nd, err := bb.getNode(ctx, path)
if err != nil {
return md, nil, err
}

rootCodec := nd.Cid().Prefix().GetCodec()
if rootCodec != uint64(mc.DagPb) {
return md, files.NewBytesFile(nd.RawData()), nil
return md, NewHeadResponseForFile(files.NewBytesFile(nd.RawData()), int64(len(nd.RawData()))), nil
}

// TODO: We're not handling non-UnixFS dag-pb. There's a bit of a discrepancy
Expand All @@ -229,7 +263,24 @@ func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentP
return ContentPathMetadata{}, nil, err
}

return md, fileNode, nil
sz, err := fileNode.Size()
if err != nil {
return ContentPathMetadata{}, nil, err
}

if _, ok := fileNode.(files.Directory); ok {
return md, NewHeadResponseForDirectory(sz), nil
}

if _, ok := fileNode.(*files.Symlink); ok {
return md, NewHeadResponseForSymlink(sz), nil
}

if f, ok := fileNode.(files.File); ok {
return md, NewHeadResponseForFile(f, sz), nil
}

return ContentPathMetadata{}, nil, fmt.Errorf("unsupported UnixFS file type")
}

func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) {
Expand Down
79 changes: 68 additions & 11 deletions gateway/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,21 +209,74 @@ type ByteRange struct {
}

type GetResponse struct {
bytes files.File
bytes io.ReadCloser
bytesSize int64
symlink *files.Symlink
directoryMetadata *directoryMetadata
}

func (r *GetResponse) Close() error {
if r.bytes != nil {
return r.bytes.Close()
}
if r.symlink != nil {
return r.symlink.Close()
}
if r.directoryMetadata != nil {
if r.directoryMetadata.closeFn == nil {
return nil
}
return r.directoryMetadata.closeFn()
}
// Should be unreachable
return nil
}

var _ io.Closer = (*GetResponse)(nil)

type directoryMetadata struct {
dagSize uint64
entries <-chan unixfs.LinkResult
closeFn func() error
}

func NewGetResponseFromReader(file io.ReadCloser, fullFileSize int64) *GetResponse {
return &GetResponse{bytes: file, bytesSize: fullFileSize}
}

func NewGetResponseFromSymlink(symlink *files.Symlink, size int64) *GetResponse {
return &GetResponse{symlink: symlink, bytesSize: size}
}

func NewGetResponseFromFile(file files.File) *GetResponse {
return &GetResponse{bytes: file}
func NewGetResponseFromDirectoryListing(dagSize uint64, entries <-chan unixfs.LinkResult, closeFn func() error) *GetResponse {
return &GetResponse{directoryMetadata: &directoryMetadata{dagSize: dagSize, entries: entries, closeFn: closeFn}}
}

func NewGetResponseFromDirectoryListing(dagSize uint64, entries <-chan unixfs.LinkResult) *GetResponse {
return &GetResponse{directoryMetadata: &directoryMetadata{dagSize, entries}}
type HeadResponse struct {
size int64
startingBytes io.ReadCloser
isFile bool
isSymLink bool
isDir bool
}

func (r *HeadResponse) Close() error {
if r.startingBytes != nil {
return r.startingBytes.Close()
}
return nil
}

func NewHeadResponseForFile(startingBytes io.ReadCloser, size int64) *HeadResponse {
return &HeadResponse{startingBytes: startingBytes, isFile: true, size: size}
}

func NewHeadResponseForSymlink(symlinkSize int64) *HeadResponse {
return &HeadResponse{isSymLink: true, size: symlinkSize}
}

func NewHeadResponseForDirectory(dagSize int64) *HeadResponse {
return &HeadResponse{isDir: true, size: dagSize}
}

// IPFSBackend is the required set of functionality used to implement the IPFS
Expand Down Expand Up @@ -265,12 +318,16 @@ type IPFSBackend interface {
// GetBlock returns a single block of data
GetBlock(context.Context, ImmutablePath) (ContentPathMetadata, files.File, error)

// Head returns a file or directory depending on what the path is that has been requested.
// For UnixFS files should return a file which has the correct file size and either returns the ContentType in ContentPathMetadata or
// enough data (e.g. 3kiB) such that the content type can be determined by sniffing.
// For all other data types returning just size information is sufficient
// TODO: give function more explicit return types
Head(context.Context, ImmutablePath) (ContentPathMetadata, files.Node, error)
// Head returns a [HeadResponse] depending on what the path is that has been requested.
// For UnixFS files (and raw blocks) should return the size of the file and either set the ContentType in
// ContentPathMetadata or send back a reader from the beginning of the file with enough data (e.g. 3kiB) such that
// the content type can be determined by sniffing.
//
// For UnixFS directories and symlinks only setting the size and type are necessary.
//
// For all other data types (e.g. (DAG-)CBOR/JSON blocks) returning the size information as a file while setting
// the content-type is sufficient.
Head(context.Context, ImmutablePath) (ContentPathMetadata, *HeadResponse, error)

// ResolvePath resolves the path using UnixFS resolver. If the path does not
// exist due to a missing link, it should return an error of type:
Expand Down
4 changes: 2 additions & 2 deletions gateway/gateway_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -665,7 +665,7 @@ func (mb *errorMockBackend) GetBlock(ctx context.Context, path ImmutablePath) (C
return ContentPathMetadata{}, nil, mb.err
}

func (mb *errorMockBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) {
func (mb *errorMockBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *HeadResponse, error) {
return ContentPathMetadata{}, nil, mb.err
}

Expand Down Expand Up @@ -749,7 +749,7 @@ func (mb *panicMockBackend) GetBlock(ctx context.Context, immutablePath Immutabl
panic("i am panicking")
}

func (mb *panicMockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) {
func (mb *panicMockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, *HeadResponse, error) {
panic("i am panicking")
}

Expand Down
46 changes: 9 additions & 37 deletions gateway/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,17 @@ func NewHandler(c Config, backend IPFSBackend) http.Handler {

// serveContent replies to the request using the content in the provided ReadSeeker
// and returns the status code written and any error encountered during a write.
// It wraps http.serveContent which takes care of If-None-Match+Etag,
// It wraps httpServeContent (a close clone of http.ServeContent) which takes care of If-None-Match+Etag,
// Content-Length and range requests.
func serveContent(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, content io.ReadSeeker) (int, bool, error) {
//
// Notes:
// 1. For HEAD requests the io.Reader may be nil/undefined
// 2. When the io.Reader is needed it must start at the beginning of the first Range Request component if it exists
// 3. Only a single HTTP Range Request is supported, if more than one are requested only the first will be honored
// 4. The Content-Type header must already be set
func serveContent(w http.ResponseWriter, req *http.Request, modtime time.Time, size int64, content io.Reader) (int, bool, error) {
ew := &errRecordingResponseWriter{ResponseWriter: w}
http.ServeContent(ew, req, name, modtime, content)
httpServeContent(ew, req, modtime, size, content)

// When we calculate some metrics we want a flag that lets us to ignore
// errors and 304 Not Modified, and only care when requested data
Expand Down Expand Up @@ -554,40 +560,6 @@ func etagMatch(ifNoneMatchHeader string, etagsToCheck ...string) bool {
return false
}

// scanETag determines if a syntactically valid ETag is present at s. If so,
// the ETag and remaining text after consuming ETag is returned. Otherwise,
// it returns "", "".
// (This is the same logic as one executed inside of http.ServeContent)
func scanETag(s string) (etag string, remain string) {
s = textproto.TrimString(s)
start := 0
if strings.HasPrefix(s, "W/") {
start = 2
}
if len(s[start:]) < 2 || s[start] != '"' {
return "", ""
}
// ETag is either W/"text" or "text".
// See RFC 7232 2.3.
for i := start + 1; i < len(s); i++ {
c := s[i]
switch {
// Character values allowed in ETags.
case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80:
case c == '"':
return s[:i+1], s[i+1:]
default:
return "", ""
}
}
return "", ""
}

// etagWeakMatch reports whether a and b match using weak ETag comparison.
func etagWeakMatch(a, b string) bool {
return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/")
}

// getEtag generates an ETag value based on an HTTP Request, a CID and a response
// format. This function DOES NOT generate ETags for CARs or IPNS Records.
func getEtag(r *http.Request, cid cid.Cid, responseFormat string) string {
Expand Down
12 changes: 11 additions & 1 deletion gateway/handler_block.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,19 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h
w.Header().Set("Content-Type", rawResponseFormat)
w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)

sz, err := data.Size()
if err != nil {
i.handleRequestErrors(w, r, rq.contentPath, err)
return false
}

if !i.seekToStartOfFirstRange(w, r, data) {
return false
}

// ServeContent will take care of
// If-None-Match+Etag, Content-Length and range requests
_, dataSent, _ := serveContent(w, r, name, modtime, data)
_, dataSent, _ := serveContent(w, r, modtime, sz, data)

if dataSent {
// Update metrics
Expand Down
Loading

0 comments on commit db5809e

Please sign in to comment.