chore: update to tagged, add more tests and fix Content-Type header

go.mod

@@ -11,8 +11,9 @@ require (
 	github.com/ipfs/go-log/v2 v2.5.1
 	github.com/ipfs/go-unixfsnode v1.8.0
 	github.com/ipld/go-car/v2 v2.13.1
+	github.com/ipld/go-codec-dagpb v1.6.0
 	github.com/ipld/go-ipld-prime v0.21.0
-	github.com/ipld/go-trustless-utils v0.1.1-0.20230907055204-83d1e838c1c1
+	github.com/ipld/go-trustless-utils v0.2.0
 	github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474
 	github.com/ipni/go-libipni v0.3.4
 	github.com/ipni/index-provider v0.13.5
@@ -74,7 +75,6 @@ require (
 	github.com/ipfs/go-ipld-cbor v0.1.0 // indirect
 	github.com/ipfs/go-log v1.0.5 // indirect
 	github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
-	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
 	github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect

go.sum

@@ -280,8 +280,8 @@ github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0/go.mod h1:od
 github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
 github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
-github.com/ipld/go-trustless-utils v0.1.1-0.20230907055204-83d1e838c1c1 h1:E1YXkAnvFn0lXkiocREtp0pUBydIwMW94myLBLCvhsY=
-github.com/ipld/go-trustless-utils v0.1.1-0.20230907055204-83d1e838c1c1/go.mod h1:nbUA6YuKZqNHYd3mYKALxOQa5aC8fnr2w+OxirSSOqc=
+github.com/ipld/go-trustless-utils v0.2.0 h1:XPP2gJJ2snhWiwP8m5Gw66pmTSMHAERxo/DMV1NVOkU=
+github.com/ipld/go-trustless-utils v0.2.0/go.mod h1:nbUA6YuKZqNHYd3mYKALxOQa5aC8fnr2w+OxirSSOqc=
 github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474 h1:CiDtcUFyzRwhKyiS4Gn+fzdraMoNVtPHd+wVIekExPc=
 github.com/ipld/ipld/specs v0.0.0-20230907004443-0e4ff95ff474/go.mod h1:WcT0DfRe+e2QFY0kcbsOnuT6jL5Q0JNZ83I5DHIdStg=
 github.com/ipni/go-libipni v0.3.4 h1:ZYgCE2TOZt/QJJcBZb+R63FaBLlA2suZGP2IH1fKv4A=

httpipfs.go

@@ -116,6 +116,9 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) {
 		return
 	}
 
+	// get the preferred `Accept` header if one exists; we should be able to
+	// handle whatever comes back from here, primarily we're looking for
+	// the `dups` parameter
 	accept, err := trustlesshttp.CheckFormat(req)
 	if err != nil {
 		logError(http.StatusBadRequest, err)
@@ -166,7 +169,7 @@ func (hi *HttpIpfs) ServeHTTP(res http.ResponseWriter, req *http.Request) {
 		// called once we start writing blocks into the CAR (on the first Put())
 		res.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", fileName))
 		res.Header().Set("Cache-Control", trustlesshttp.ResponseCacheControlHeader)
-		res.Header().Set("Content-Type", accept.String())
+		res.Header().Set("Content-Type", accept.WithMimeType(trustlesshttp.MimeTypeCar).WithQuality(1).String())
 		res.Header().Set("Etag", request.Etag())
 		res.Header().Set("X-Content-Type-Options", "nosniff")
 		res.Header().Set("X-Ipfs-Path", "/"+datamodel.ParsePath(req.URL.Path).String())

httpipfs_test.go

@@ -2,19 +2,27 @@ package frisbii_test
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-unixfsnode"
 	"github.com/ipld/frisbii"
 	"github.com/ipld/go-car/v2"
+	dagpb "github.com/ipld/go-codec-dagpb"
+	"github.com/ipld/go-ipld-prime/datamodel"
+	"github.com/ipld/go-ipld-prime/fluent/qp"
+	"github.com/ipld/go-ipld-prime/linking"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	"github.com/ipld/go-ipld-prime/node/basicnode"
 	"github.com/ipld/go-ipld-prime/storage/memstore"
 	trustlesshttp "github.com/ipld/go-trustless-utils/http"
 	trustlesspathing "github.com/ipld/ipld/specs/pkg-go/trustless-pathing"
+	"github.com/multiformats/go-multihash"
 	"github.com/stretchr/testify/require"
 )
 
@@ -95,7 +103,7 @@ func TestHttpIpfsHandler(t *testing.T) {
 	}
 }
 
-func TestIntegration_Unixfs20mVariety(t *testing.T) {
+func TestHttpIpfsIntegration_Unixfs20mVariety(t *testing.T) {
 	req := require.New(t)
 
 	testCases, _, err := trustlesspathing.Unixfs20mVarietyCases()
@@ -144,3 +152,163 @@ func TestIntegration_Unixfs20mVariety(t *testing.T) {
 		})
 	}
 }
+
+func TestHttpIpfsDuplicates(t *testing.T) {
+	store := &CorrectedMemStore{Store: &memstore.Store{}}
+	lsys := cidlink.DefaultLinkSystem()
+	lsys.SetReadStorage(store)
+	lsys.SetWriteStorage(store)
+	dupyLinks, dupyLinksDeduped := mkDupy(lsys)
+
+	handler := frisbii.NewHttpIpfs(context.Background(), lsys)
+	testServer := httptest.NewServer(handler)
+	defer testServer.Close()
+
+	for _, tc := range []struct {
+		name                string
+		accepts             []string
+		expectedContentType string
+		expectedCids        []cid.Cid
+	}{
+		{
+			name:                "default",
+			accepts:             []string{trustlesshttp.DefaultContentType().String()},
+			expectedContentType: trustlesshttp.DefaultContentType().String(),
+			expectedCids:        dupyLinks,
+		},
+		{
+			// note that we're pretty permissive, as long as you send an Accept
+			// that vaguely signals you're willing to accept what we give
+			name:                "*/*",
+			accepts:             []string{"*/*"},
+			expectedContentType: trustlesshttp.DefaultContentType().String(),
+			expectedCids:        dupyLinks,
+		},
+		{
+			name:                "dups",
+			accepts:             []string{trustlesshttp.DefaultContentType().WithDuplicates(true).String()},
+			expectedContentType: trustlesshttp.DefaultContentType().WithDuplicates(true).String(),
+			expectedCids:        dupyLinks,
+		},
+		{
+			name:                "no dups",
+			accepts:             []string{trustlesshttp.DefaultContentType().WithDuplicates(false).String()},
+			expectedContentType: trustlesshttp.DefaultContentType().WithDuplicates(false).String(),
+			expectedCids:        dupyLinksDeduped,
+		},
+		{
+			name: "ranked w/ dups",
+			accepts: []string{
+				"text/html",
+				trustlesshttp.DefaultContentType().WithDuplicates(false).WithQuality(0.7).String(),
+				trustlesshttp.DefaultContentType().WithDuplicates(true).WithQuality(0.8).String(),
+				"*/*;q=0.2",
+			},
+			expectedContentType: trustlesshttp.DefaultContentType().WithDuplicates(true).String(),
+			expectedCids:        dupyLinks,
+		},
+		{
+			name: "ranked w/ no dups",
+			accepts: []string{
+				"text/html",
+				trustlesshttp.DefaultContentType().WithDuplicates(false).WithQuality(0.8).String(),
+				trustlesshttp.DefaultContentType().WithDuplicates(true).WithQuality(0.7).String(),
+				"*/*;q=0.2",
+			},
+			expectedContentType: trustlesshttp.DefaultContentType().WithDuplicates(false).String(),
+			expectedCids:        dupyLinksDeduped,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			req := require.New(t)
+
+			request, err := http.NewRequest(http.MethodGet, testServer.URL+"/ipfs/"+dupyLinks[0].String(), nil)
+			req.NoError(err)
+			var accept strings.Builder
+			for _, a := range tc.accepts {
+				if accept.Len() > 0 {
+					accept.WriteString(", ")
+				}
+				accept.WriteString(a)
+			}
+			request.Header.Set("Accept", accept.String())
+			res, err := http.DefaultClient.Do(request)
+			req.NoError(err)
+			req.Equal(http.StatusOK, res.StatusCode)
+			req.Equal(tc.expectedContentType, res.Header.Get("Content-Type"))
+
+			carReader, err := car.NewBlockReader(res.Body)
+			req.NoError(err)
+			req.Equal(uint64(1), carReader.Version)
+			req.Equal([]cid.Cid{dupyLinks[0]}, carReader.Roots)
+
+			for ii, expectedCid := range tc.expectedCids {
+				blk, err := carReader.Next()
+				if err != nil {
+					req.Equal(io.EOF, err)
+					req.Len(tc.expectedCids, ii+1)
+					break
+				}
+				req.Equal(expectedCid, blk.Cid())
+			}
+		})
+	}
+}
+
+var pblp = cidlink.LinkPrototype{
+	Prefix: cid.Prefix{
+		Version:  1,
+		Codec:    cid.DagProtobuf,
+		MhType:   multihash.SHA2_256,
+		MhLength: 32,
+	},
+}
+
+var rawlp = cidlink.LinkPrototype{
+	Prefix: cid.Prefix{
+		Version:  1,
+		Codec:    cid.Raw,
+		MhType:   multihash.SHA2_256,
+		MhLength: 32,
+	},
+}
+
+func mkBlockWithBytes(lsys linking.LinkSystem, bytes []byte) cid.Cid {
+	l, err := lsys.Store(linking.LinkContext{}, rawlp, basicnode.NewBytes(bytes))
+	if err != nil {
+		panic(err)
+	}
+	return l.(cidlink.Link).Cid
+}
+
+func mkDupy(lsys linking.LinkSystem) ([]cid.Cid, []cid.Cid) {
+	dupy := mkBlockWithBytes(lsys, []byte("duplicate data"))
+
+	n, err := qp.BuildMap(dagpb.Type.PBNode, 1, func(ma datamodel.MapAssembler) {
+		qp.MapEntry(ma, "Links", qp.List(100, func(la datamodel.ListAssembler) {
+			for i := 0; i < 100; i++ {
+				qp.ListEntry(la, qp.Map(2, func(ma datamodel.MapAssembler) {
+					qp.MapEntry(ma, "Name", qp.String(fmt.Sprintf("%03d", i)))
+					qp.MapEntry(ma, "Hash", qp.Link(cidlink.Link{Cid: dupy}))
+				}))
+			}
+		}))
+	})
+	if err != nil {
+		panic(err)
+	}
+	l, err := lsys.Store(linking.LinkContext{}, pblp, n)
+	if err != nil {
+		panic(err)
+	}
+
+	// dupyLinks contains the duplicates
+	dupyLinks := []cid.Cid{l.(cidlink.Link).Cid}
+	for i := 0; i < 100; i++ {
+		dupyLinks = append(dupyLinks, dupy)
+	}
+	// dupyLinksDeduped contains just the unique links
+	dupyLinksDeduped := []cid.Cid{l.(cidlink.Link).Cid, dupy}
+
+	return dupyLinks, dupyLinksDeduped
+}