Update ipfs-lite, test data races, and other implications (#324)

* update ipfs-lite Signed-off-by: Ignacio Hagopian <[email protected]> * go mod tidy Signed-off-by: Ignacio Hagopian <[email protected]> * fix deleting log keys Signed-off-by: Ignacio Hagopian <[email protected]> * avoid unnecessary custom peerstore Signed-off-by: Ignacio Hagopian <[email protected]> * fix test data races Signed-off-by: Ignacio Hagopian <[email protected]> * switch to child string for indexed values Signed-off-by: Ignacio Hagopian <[email protected]>
textileio · Apr 28, 2020 · 61506ac · 61506ac
1 parent 04789b5
commit 61506ac
Show file tree

Hide file tree

Showing 7 changed files with 269 additions and 124 deletions.
diff --git a/common/common.go b/common/common.go
@@ -13,9 +13,7 @@ import (
 	connmgr "github.com/libp2p/go-libp2p-connmgr"
 	host "github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/peer"
-	"github.com/libp2p/go-libp2p-core/peerstore"
-	dht "github.com/libp2p/go-libp2p-kad-dht"
-	"github.com/libp2p/go-libp2p-peerstore/pstoreds"
+	"github.com/libp2p/go-libp2p-kad-dht/dual"
 	ma "github.com/multiformats/go-multiaddr"
 	"github.com/textileio/go-threads/core/app"
 	"github.com/textileio/go-threads/logstore/lstoreds"
@@ -62,20 +60,14 @@ func DefaultNetwork(repoPath string, opts ...NetOption) (NetBoostrapper, error)
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())
-	pstore, err := pstoreds.NewPeerstore(ctx, litestore, pstoreds.DefaultOpts())
-	if err != nil {
-		litestore.Close()
-		cancel()
-		return nil, err
-	}
 	priv := util.LoadKey(filepath.Join(ipfsLitePath, "key"))
 	h, d, err := ipfslite.SetupLibp2p(
 		ctx,
 		priv,
 		nil,
 		[]ma.Multiaddr{config.HostAddr},
+		litestore,
 		libp2p.ConnectionManager(connmgr.NewConnManager(100, 400, time.Minute)),
-		libp2p.Peerstore(pstore),
 	)
 	if err != nil {
 		cancel()
@@ -128,7 +120,6 @@ func DefaultNetwork(repoPath string, opts ...NetOption) (NetBoostrapper, error)
 		cancel:    cancel,
 		Net:       api,
 		litepeer:  lite,
-		pstore:    pstore,
 		logstore:  logstore,
 		litestore: litestore,
 		host:      h,
@@ -169,11 +160,10 @@ type netBoostrapper struct {
 	cancel context.CancelFunc
 	app.Net
 	litepeer  *ipfslite.Peer
-	pstore    peerstore.Peerstore
 	logstore  datastore.Datastore
 	litestore datastore.Datastore
 	host      host.Host
-	dht       *dht.IpfsDHT
+	dht       *dual.DHT
 }
 
 var _ NetBoostrapper = (*netBoostrapper)(nil)
@@ -197,9 +187,6 @@ func (tsb *netBoostrapper) Close() error {
 	if err := tsb.host.Close(); err != nil {
 		return err
 	}
-	if err := tsb.pstore.Close(); err != nil {
-		return err
-	}
 	if err := tsb.litestore.Close(); err != nil {
 		return err
 	}

diff --git a/db/index.go b/db/index.go
@@ -88,7 +88,7 @@ func indexUpdate(baseKey ds.Key, path string, index Index, tx ds.Txn, key ds.Key
 		return nil
 	}
 
-	indexKey := indexPrefix.Child(baseKey).ChildString(path).Instance(valueKey.String()[1:])
+	indexKey := indexPrefix.Child(baseKey).ChildString(path).ChildString(valueKey.String()[1:])
 
 	data, err := tx.Get(indexKey)
 	if err != nil && err != ds.ErrNotFound {
@@ -124,7 +124,6 @@ func indexUpdate(baseKey ds.Key, path string, index Index, tx ds.Txn, key ds.Key
 	if err != nil {
 		return err
 	}
-
 	return tx.Put(indexKey, iVal)
 }
 
@@ -221,7 +220,7 @@ func newIterator(txn ds.Txn, baseKey ds.Key, q *Query) *iterator {
 			first = false
 			// result.Key contains the indexed value, extract here first
 			key := ds.RawKey(result.Key)
-			base := key.Type()
+			base := indexKey.Name()
 			name := key.Name()
 			val := gjson.Parse(name).Value()
 			if val == nil {

diff --git a/db/query_more_test.go b/db/query_more_test.go
@@ -94,7 +94,7 @@ var (
 
 func TestCollectionQuery(t *testing.T) {
 	t.Parallel()
-	c, clean := createCollectionWithData(t)
+	c, data, clean := createCollectionWithData(t)
 	defer clean()
 	for _, q := range queries {
 		q := q
@@ -123,12 +123,12 @@ func TestCollectionQuery(t *testing.T) {
 					return strings.Compare(res[i].ID.String(), res[j].ID.String()) == -1
 				})
 				sort.Slice(expectedIdx, func(i, j int) bool {
-					return strings.Compare(sampleData[expectedIdx[i]].ID.String(), sampleData[expectedIdx[j]].ID.String()) == -1
+					return strings.Compare(data[expectedIdx[i]].ID.String(), data[expectedIdx[j]].ID.String()) == -1
 				})
 			}
 			for i, idx := range expectedIdx {
-				if !reflect.DeepEqual(sampleData[idx], *res[i]) {
-					t.Fatalf("wrong query item result, expected: %v, got: %v", sampleData[idx], *res[i])
+				if !reflect.DeepEqual(data[idx], *res[i]) {
+					t.Fatalf("wrong query item result, expected: %v, got: %v", data[idx], *res[i])
 				}
 			}
 		})
@@ -138,21 +138,22 @@ func TestCollectionQuery(t *testing.T) {
 func TestInvalidSortField(t *testing.T) {
 	t.Parallel()
 
-	c, clean := createCollectionWithData(t)
+	c, _, clean := createCollectionWithData(t)
 	defer clean()
 	_, err := c.Find(OrderBy("WrongFieldName"))
 	if !errors.Is(err, ErrInvalidSortingField) {
 		t.Fatal("query should fail using an invalid field")
 	}
 }
 
-func createCollectionWithData(t *testing.T) (*Collection, func()) {
+func createCollectionWithData(t *testing.T) (*Collection, []book, func()) {
 	db, clean := createTestDB(t)
 	c, err := db.NewCollection(CollectionConfig{
 		Name:   "Book",
 		Schema: util.SchemaFromInstance(&book{}, false),
 	})
 	checkErr(t, err)
+	sampleDataCopy := make([]book, len(sampleData))
 	for i := range sampleData {
 		sampleDataJSON := util.JSONFromInstance(sampleData[i])
 		id, err := c.Create(sampleDataJSON)
@@ -162,7 +163,7 @@ func createCollectionWithData(t *testing.T) (*Collection, func()) {
 		sampleDataJSON = util.SetJSONID(id, sampleDataJSON)
 		updated := book{}
 		util.InstanceFromJSON(sampleDataJSON, &updated)
-		sampleData[i] = updated
+		sampleDataCopy[i] = updated
 	}
-	return c, clean
+	return c, sampleDataCopy, clean
 }
diff --git a/db/query_test.go b/db/query_test.go
@@ -6,6 +6,7 @@ import (
 	"strings"
 	"testing"
 
+	logging "github.com/ipfs/go-log"
 	"github.com/textileio/go-threads/core/db"
 	"github.com/textileio/go-threads/util"
 )
@@ -262,7 +263,8 @@ var (
 )
 
 func TestQuery(t *testing.T) {
-	c, clean := createCollectionWithJSONData(t)
+	logging.SetAllLoggers(logging.LevelError)
+	c, d, clean := createCollectionWithJSONData(t)
 	defer clean()
 
 	for _, q := range jsonQueries {
@@ -292,19 +294,19 @@ func TestQuery(t *testing.T) {
 					return strings.Compare(foundBooks[i].ID.String(), foundBooks[j].ID.String()) == -1
 				})
 				sort.Slice(expectedIdx, func(i, j int) bool {
-					return strings.Compare(data[expectedIdx[i]].ID.String(), data[expectedIdx[j]].ID.String()) == -1
+					return strings.Compare(d[expectedIdx[i]].ID.String(), d[expectedIdx[j]].ID.String()) == -1
 				})
 			}
 			for i, idx := range expectedIdx {
-				if !reflect.DeepEqual(data[idx], foundBooks[i]) {
-					t.Fatalf("wrong query item result, expected: %v, got: %v", data[idx], foundBooks[i])
+				if !reflect.DeepEqual(d[idx], foundBooks[i]) {
+					t.Fatalf("wrong query item result, expected: %v, got: %v", d[idx], foundBooks[i])
 				}
 			}
 		})
 	}
 }
 
-func createCollectionWithJSONData(t *testing.T) (*Collection, func()) {
+func createCollectionWithJSONData(t *testing.T) (*Collection, []Book, func()) {
 	s, clean := createTestDB(t)
 	c, err := s.NewCollection(CollectionConfig{
 		Name:   "Book",
@@ -318,12 +320,14 @@ func createCollectionWithJSONData(t *testing.T) (*Collection, func()) {
 			}},
 	})
 	checkErr(t, err)
+	dataCopy := make([]Book, len(data))
 	for i := range data {
 		id, err := c.Create(util.JSONFromInstance(data[i]))
 		if err != nil {
 			t.Fatalf("failed to create sample data: %v", err)
 		}
-		data[i].ID = id
+		dataCopy[i] = data[i]
+		dataCopy[i].ID = id
 	}
-	return c, clean
+	return c, dataCopy, clean
 }
diff --git a/go.mod b/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/alecthomas/jsonschema v0.0.0-20191017121752-4bb6e3fae4f2
 	github.com/c-bata/go-prompt v0.2.3
 	github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect
-	github.com/dgraph-io/badger v1.6.0
+	github.com/dgraph-io/badger v1.6.1
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/evanphx/json-patch v4.5.0+incompatible
 	github.com/fatih/color v1.7.0
@@ -18,35 +18,35 @@ require (
 	github.com/gogo/status v1.1.0
 	github.com/golang/protobuf v1.3.3
 	github.com/grpc-ecosystem/go-grpc-middleware v1.2.0
-	github.com/hashicorp/go-multierror v1.0.0
-	github.com/hashicorp/golang-lru v0.5.3
-	github.com/hsanjuan/ipfs-lite v0.1.8
+	github.com/hashicorp/go-multierror v1.1.0
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/hsanjuan/ipfs-lite v1.1.11
 	github.com/improbable-eng/grpc-web v0.12.0
 	github.com/ipfs/go-block-format v0.0.2
-	github.com/ipfs/go-blockservice v0.1.2
+	github.com/ipfs/go-blockservice v0.1.3
 	github.com/ipfs/go-cid v0.0.5
-	github.com/ipfs/go-datastore v0.3.1
-	github.com/ipfs/go-ds-badger v0.2.0
-	github.com/ipfs/go-ipfs-blockstore v0.1.1
+	github.com/ipfs/go-datastore v0.4.4
+	github.com/ipfs/go-ds-badger v0.2.4
+	github.com/ipfs/go-ipfs-blockstore v1.0.0
 	github.com/ipfs/go-ipfs-exchange-offline v0.0.1
 	github.com/ipfs/go-ipfs-files v0.0.4 // indirect
-	github.com/ipfs/go-ipld-cbor v0.0.3
-	github.com/ipfs/go-ipld-format v0.0.2
-	github.com/ipfs/go-log v1.0.0
-	github.com/ipfs/go-merkledag v0.2.3
-	github.com/libp2p/go-libp2p v0.4.2
-	github.com/libp2p/go-libp2p-connmgr v0.1.1
-	github.com/libp2p/go-libp2p-core v0.3.0
+	github.com/ipfs/go-ipld-cbor v0.0.4
+	github.com/ipfs/go-ipld-format v0.2.0
+	github.com/ipfs/go-log v1.0.4
+	github.com/ipfs/go-merkledag v0.3.2
+	github.com/libp2p/go-libp2p v0.8.2
+	github.com/libp2p/go-libp2p-connmgr v0.2.1
+	github.com/libp2p/go-libp2p-core v0.5.2
 	github.com/libp2p/go-libp2p-gostream v0.2.0
-	github.com/libp2p/go-libp2p-kad-dht v0.3.0
-	github.com/libp2p/go-libp2p-peerstore v0.1.4
+	github.com/libp2p/go-libp2p-kad-dht v0.7.10
+	github.com/libp2p/go-libp2p-peerstore v0.2.3
 	github.com/libp2p/go-libp2p-pubsub v0.2.4
-	github.com/libp2p/go-libp2p-swarm v0.2.2
+	github.com/libp2p/go-libp2p-swarm v0.2.3
 	github.com/mattn/go-isatty v0.0.11 // indirect
 	github.com/mattn/go-runewidth v0.0.8 // indirect
 	github.com/mattn/go-tty v0.0.3 // indirect
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/multiformats/go-multiaddr v0.2.0
+	github.com/multiformats/go-multiaddr v0.2.1
 	github.com/multiformats/go-multibase v0.0.1
 	github.com/multiformats/go-multihash v0.0.13
 	github.com/multiformats/go-varint v0.0.5
@@ -63,11 +63,10 @@ require (
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0
-	go.uber.org/zap v1.10.0
-	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413
+	go.uber.org/zap v1.14.1
+	golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d
 	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/sys v0.0.0-20191210023423-ac6580df4449 // indirect
 	google.golang.org/genproto v0.0.0-20191206224255-0243a4be9c8f // indirect
 	google.golang.org/grpc v1.29.0
 )