From 115cda59dba79935e07421e41f86eacf47d6841a Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Wed, 17 Aug 2022 01:02:23 -0700 Subject: [PATCH 1/7] feat(booster-bitswap): booster bitswap MVP untested --- cmd/booster-bitswap/indexbs/indexbs.go | 312 +++++++++++++++++++++++++ cmd/booster-bitswap/main.go | 42 ++++ cmd/booster-bitswap/run.go | 267 +++++++++++++++++++++ cmd/booster-bitswap/server.go | 122 ++++++++++ go.mod | 27 +-- go.sum | 31 +++ 6 files changed, 788 insertions(+), 13 deletions(-) create mode 100644 cmd/booster-bitswap/indexbs/indexbs.go create mode 100644 cmd/booster-bitswap/main.go create mode 100644 cmd/booster-bitswap/run.go create mode 100644 cmd/booster-bitswap/server.go diff --git a/cmd/booster-bitswap/indexbs/indexbs.go b/cmd/booster-bitswap/indexbs/indexbs.go new file mode 100644 index 000000000..4bdc3b662 --- /dev/null +++ b/cmd/booster-bitswap/indexbs/indexbs.go @@ -0,0 +1,312 @@ +package indexbs + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + + "github.com/filecoin-project/dagstore" + "github.com/filecoin-project/go-fil-markets/piecestore" + "github.com/filecoin-project/go-state-types/abi" + blocks "github.com/ipfs/go-block-format" + logging "github.com/ipfs/go-log/v2" + "github.com/ipld/go-car/v2" + "github.com/multiformats/go-multihash" + + "github.com/filecoin-project/dagstore/mount" + "github.com/hashicorp/go-multierror" + lru "github.com/hashicorp/golang-lru" + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" + carbs "github.com/ipld/go-car/v2/blockstore" +) + +var logbs = logging.Logger("dagstore-all-readblockstore") + +var ErrBlockNotFound = errors.New("block not found") +var ErrNotFound = errors.New("not found") + +var _ blockstore.Blockstore = (*IndexBackedBlockstore)(nil) + +// ErrNoPieceSelected means that the piece selection function rejected all of the given pieces. +var ErrNoPieceSelected = errors.New("no piece selected") + +// PieceSelectorF helps select a piece to fetch a cid from if the given cid is present in multiple pieces. +// It should return `ErrNoPieceSelected` if none of the given piece is selected. +type PieceSelectorF func(c cid.Cid, pieceCids []cid.Cid) (cid.Cid, error) + +type accessorWithBlockstore struct { + mount mount.Reader + bs dagstore.ReadBlockstore +} + +type IndexBackedBlockstoreAPI interface { + PiecesContainingMultihash(mh multihash.Multihash) ([]cid.Cid, error) + GetMaxPieceOffset(pieceCid cid.Cid) (uint64, error) + GetPieceInfo(pieceCID cid.Cid) (*piecestore.PieceInfo, error) + IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) + UnsealSectorAt(ctx context.Context, sectorID abi.SectorNumber, pieceOffset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) +} + +// IndexBackedBlockstore is a read only blockstore over all cids across all pieces on a provider. +type IndexBackedBlockstore struct { + api IndexBackedBlockstoreAPI + pieceSelectF PieceSelectorF + + bsStripedLocks [256]sync.Mutex + blockstoreCache *lru.Cache // caches the blockstore for a given piece for piece read affinity i.e. further reads will likely be from the same piece. Maps (piece CID -> blockstore). +} + +func NewIndexBackedBlockstore(api IndexBackedBlockstoreAPI, pieceSelector PieceSelectorF, maxCacheSize int) (blockstore.Blockstore, error) { + // instantiate the blockstore cache + bslru, err := lru.NewWithEvict(maxCacheSize, func(_ interface{}, val interface{}) { + // ensure we close the blockstore for a piece when it's evicted from the cache so dagstore can gc it. + abs := val.(*accessorWithBlockstore) + abs.mount.Close() + }) + if err != nil { + return nil, fmt.Errorf("failed to create lru cache for read only blockstores") + } + + return &IndexBackedBlockstore{ + api: api, + pieceSelectF: pieceSelector, + blockstoreCache: bslru, + }, nil +} + +func (ro *IndexBackedBlockstore) Get(ctx context.Context, c cid.Cid) (b blocks.Block, finalErr error) { + logbs.Debugw("Get called", "cid", c) + defer func() { + if finalErr != nil { + logbs.Debugw("Get: got error", "cid", c, "error", finalErr) + } + }() + + mhash := c.Hash() + + // fetch all the pieceCIDs containing the multihash + pieceCIDs, err := ro.api.PiecesContainingMultihash(mhash) + if err != nil { + return nil, fmt.Errorf("failed to fetch pieces containing the block: %w", err) + } + if len(pieceCIDs) == 0 { + return nil, ErrBlockNotFound + } + + // do we have a cached blockstore for a piece containing the required block ? If yes, serve the block from that blockstore + for _, pieceCID := range pieceCIDs { + lk := &ro.bsStripedLocks[pieceCIDToStriped(pieceCID)] + lk.Lock() + + blk, err := ro.readFromBSCacheUnlocked(ctx, c, pieceCID) + if err == nil && blk != nil { + logbs.Debugw("Get: returning from block store cache", "cid", c) + + lk.Unlock() + return blk, nil + } + + lk.Unlock() + } + + // ---- we don't have a cached blockstore for a piece that can serve the block -> let's build one. + + // select a valid piece that can serve the retrieval + pieceCID, err := ro.pieceSelectF(c, pieceCIDs) + if err != nil && err == ErrNoPieceSelected { + return nil, ErrBlockNotFound + } + if err != nil { + return nil, fmt.Errorf("failed to run piece selection function: %w", err) + } + + lk := &ro.bsStripedLocks[pieceCIDToStriped(pieceCID)] + lk.Lock() + defer lk.Unlock() + + // see if we have blockstore in the cache we can serve the retrieval from as the previous code in this critical section + // could have added a blockstore to the cache for the given piece CID. + blk, err := ro.readFromBSCacheUnlocked(ctx, c, pieceCID) + if err == nil && blk != nil { + return blk, nil + } + + // load blockstore for the selected piece and try to serve the cid from that blockstore. + reader, err := ro.getPieceContent(ctx, pieceCID) + + bs, err := ro.getBlockstore(reader) + + blk, err = bs.Get(ctx, c) + if err != nil { + return nil, fmt.Errorf("failed to get block: %w", err) + } + + // update the block cache and the blockstore cache + ro.blockstoreCache.Add(pieceCID, &accessorWithBlockstore{reader, bs}) + + logbs.Debugw("Get: returning after creating new blockstore", "cid", c) + return blk, nil +} + +func (ro *IndexBackedBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) { + logbs.Debugw("Has called", "cid", c) + + // if there is a piece that can serve the retrieval for the given cid, we have the requested cid + // and has should return true. + mhash := c.Hash() + + pieceCIDs, err := ro.api.PiecesContainingMultihash(mhash) + if err != nil { + logbs.Debugw("Has error", "cid", c, "err", err) + return false, nil + } + if len(pieceCIDs) == 0 { + logbs.Debugw("Has: returning false no error", "cid", c) + return false, nil + } + + _, err = ro.pieceSelectF(c, pieceCIDs) + if err != nil && err == ErrNoPieceSelected { + logbs.Debugw("Has error", "cid", c, "err", err) + return false, nil + } + if err != nil { + logbs.Debugw("Has error", "cid", c, "err", err) + return false, fmt.Errorf("failed to run piece selection function: %w", err) + } + + logbs.Debugw("Has: returning true", "cid", c) + return true, nil +} + +func (ro *IndexBackedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) { + logbs.Debugw("GetSize called", "cid", c) + + blk, err := ro.Get(ctx, c) + if err != nil { + logbs.Debugw("GetSize error", "cid", c, "err", err) + return 0, fmt.Errorf("failed to get block: %w", err) + } + + logbs.Debugw("GetSize success", "cid", c) + return len(blk.RawData()), nil +} + +func (ro *IndexBackedBlockstore) readFromBSCacheUnlocked(ctx context.Context, c cid.Cid, pieceCid cid.Cid) (blocks.Block, error) { + // We've already ensured that the given piece has the cid/multihash we are looking for. + val, ok := ro.blockstoreCache.Get(pieceCid) + if !ok { + return nil, ErrBlockNotFound + } + + rbs := val.(*accessorWithBlockstore).bs + blk, err := rbs.Get(ctx, c) + if err != nil { + // we know that the cid we want to lookup belongs to a piece with given pieceCID and + // so if we fail to get the corresponding block from the blockstore for that piece, something has gone wrong + // and we should remove the blockstore for that pieceCID from our cache. + ro.blockstoreCache.Remove(pieceCid) + return nil, err + } + + return blk, nil +} + +func pieceCIDToStriped(pieceCid cid.Cid) byte { + return pieceCid.String()[len(pieceCid.String())-1] +} + +// --- UNSUPPORTED BLOCKSTORE METHODS ------- +func (ro *IndexBackedBlockstore) DeleteBlock(context.Context, cid.Cid) error { + return errors.New("unsupported operation DeleteBlock") +} +func (ro *IndexBackedBlockstore) HashOnRead(_ bool) {} +func (ro *IndexBackedBlockstore) Put(context.Context, blocks.Block) error { + return errors.New("unsupported operation Put") +} +func (ro *IndexBackedBlockstore) PutMany(context.Context, []blocks.Block) error { + return errors.New("unsupported operation PutMany") +} +func (ro *IndexBackedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return nil, errors.New("unsupported operation AllKeysChan") +} + +func (ro *IndexBackedBlockstore) getPieceContent(ctx context.Context, pieceCid cid.Cid) (mount.Reader, error) { + // Get the deals for the piece + pieceInfo, err := ro.api.GetPieceInfo(pieceCid) + if err != nil { + return nil, fmt.Errorf("getting sector info for piece %s: %w", pieceCid, err) + } + + // Get the first unsealed deal + di, err := ro.unsealedDeal(ctx, *pieceInfo) + if err != nil { + return nil, fmt.Errorf("getting unsealed CAR file: %w", err) + } + + // Get the raw piece data from the sector + pieceReader, err := ro.api.UnsealSectorAt(ctx, di.SectorID, di.Offset.Unpadded(), di.Length.Unpadded()) + if err != nil { + return nil, fmt.Errorf("getting raw data from sector %d: %w", di.SectorID, err) + } + + return pieceReader, nil +} + +func (ro *IndexBackedBlockstore) getBlockstore(pieceReader mount.Reader) (dagstore.ReadBlockstore, error) { + idx, err := car.ReadOrGenerateIndex(pieceReader, car.ZeroLengthSectionAsEOF(true), car.StoreIdentityCIDs(true)) + if err != nil { + return nil, err + } + return carbs.NewReadOnly(pieceReader, idx, car.ZeroLengthSectionAsEOF(true)) +} + +func (ro *IndexBackedBlockstore) unsealedDeal(ctx context.Context, pieceInfo piecestore.PieceInfo) (*piecestore.DealInfo, error) { + // There should always been deals in the PieceInfo, but check just in case + if len(pieceInfo.Deals) == 0 { + return nil, fmt.Errorf("there are no deals containing piece %s: %w", pieceInfo.PieceCID, ErrNotFound) + } + + // The same piece can be in many deals. Find the first unsealed deal. + sealedCount := 0 + var allErr error + for _, di := range pieceInfo.Deals { + isUnsealed, err := ro.api.IsUnsealed(ctx, di.SectorID, di.Offset.Unpadded(), di.Length.Unpadded()) + if err != nil { + allErr = multierror.Append(allErr, err) + continue + } + if isUnsealed { + return &di, nil + } + sealedCount++ + } + + // Try to return an error message with as much useful information as possible + dealSectors := make([]string, 0, len(pieceInfo.Deals)) + for _, di := range pieceInfo.Deals { + dealSectors = append(dealSectors, fmt.Sprintf("Deal %d: Sector %d", di.DealID, di.SectorID)) + } + + if allErr == nil { + dealSectorsErr := fmt.Errorf("%s: %w", strings.Join(dealSectors, ", "), ErrNotFound) + return nil, fmt.Errorf("checked unsealed status of %d deals containing piece %s: none are unsealed: %w", + len(pieceInfo.Deals), pieceInfo.PieceCID, dealSectorsErr) + } + + if len(pieceInfo.Deals) == 1 { + return nil, fmt.Errorf("checking unsealed status of deal %d (sector %d) containing piece %s: %w", + pieceInfo.Deals[0].DealID, pieceInfo.Deals[0].SectorID, pieceInfo.PieceCID, allErr) + } + + if sealedCount == 0 { + return nil, fmt.Errorf("checking unsealed status of %d deals containing piece %s: %s: %w", + len(pieceInfo.Deals), pieceInfo.PieceCID, dealSectors, allErr) + } + + return nil, fmt.Errorf("checking unsealed status of %d deals containing piece %s - %d are sealed, %d had errors: %s: %w", + len(pieceInfo.Deals), pieceInfo.PieceCID, sealedCount, len(pieceInfo.Deals)-sealedCount, dealSectors, allErr) +} diff --git a/cmd/booster-bitswap/main.go b/cmd/booster-bitswap/main.go new file mode 100644 index 000000000..f270ebc75 --- /dev/null +++ b/cmd/booster-bitswap/main.go @@ -0,0 +1,42 @@ +package main + +import ( + "os" + + "github.com/filecoin-project/boost/build" + cliutil "github.com/filecoin-project/boost/cli/util" + logging "github.com/ipfs/go-log/v2" + "github.com/urfave/cli/v2" +) + +var log = logging.Logger("booster") + +func main() { + app := &cli.App{ + Name: "booster-bitswap", + Usage: "Bitswap endpoint for retrieval from Filecoin", + EnableBashCompletion: true, + Version: build.UserVersion(), + Flags: []cli.Flag{ + cliutil.FlagVeryVerbose, + }, + Commands: []*cli.Command{ + runCmd, + }, + } + app.Setup() + + if err := app.Run(os.Args); err != nil { + os.Stderr.WriteString("Error: " + err.Error() + "\n") + } +} + +func before(cctx *cli.Context) error { + _ = logging.SetLogLevel("booster", "INFO") + + if cliutil.IsVeryVerbose { + _ = logging.SetLogLevel("booster", "DEBUG") + } + + return nil +} diff --git a/cmd/booster-bitswap/run.go b/cmd/booster-bitswap/run.go new file mode 100644 index 000000000..5238f92dc --- /dev/null +++ b/cmd/booster-bitswap/run.go @@ -0,0 +1,267 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + _ "net/http/pprof" + "strings" + + "github.com/filecoin-project/boost/api" + bclient "github.com/filecoin-project/boost/api/client" + cliutil "github.com/filecoin-project/boost/cli/util" + "github.com/filecoin-project/dagstore/mount" + "github.com/filecoin-project/go-fil-markets/piecestore" + "github.com/filecoin-project/go-jsonrpc" + "github.com/filecoin-project/go-state-types/abi" + lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/api/client" + "github.com/filecoin-project/lotus/api/v0api" + "github.com/filecoin-project/lotus/api/v1api" + lcli "github.com/filecoin-project/lotus/cli" + "github.com/filecoin-project/lotus/markets/dagstore" + "github.com/filecoin-project/lotus/markets/sectoraccessor" + lotus_modules "github.com/filecoin-project/lotus/node/modules" + "github.com/filecoin-project/lotus/node/modules/dtypes" + "github.com/filecoin-project/lotus/node/repo" + "github.com/filecoin-project/lotus/storage/paths" + "github.com/filecoin-project/lotus/storage/sealer" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "github.com/urfave/cli/v2" +) + +var runCmd = &cli.Command{ + Name: "run", + Usage: "Start a booster-bitswap process", + Before: before, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "pprof", + Usage: "run pprof web server on localhost:6070", + }, + &cli.UintFlag{ + Name: "port", + Usage: "the port to listen for bitswap requests on", + Value: 8888, + }, + &cli.StringFlag{ + Name: "api-boost", + Usage: "the endpoint for the boost API", + Required: true, + }, + &cli.StringFlag{ + Name: "api-fullnode", + Usage: "the endpoint for the full node API", + Required: true, + }, + &cli.StringFlag{ + Name: "api-sealer", + Usage: "the endpoint for the sealer API", + Required: true, + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Bool("pprof") { + go func() { + err := http.ListenAndServe("localhost:6070", nil) + if err != nil { + log.Error(err) + } + }() + } + + // Connect to the Boost API + ctx := lcli.ReqContext(cctx) + boostAPIInfo := cctx.String("api-boost") + bapi, bcloser, err := getBoostAPI(ctx, boostAPIInfo) + if err != nil { + return fmt.Errorf("getting boost API: %w", err) + } + defer bcloser() + + // Connect to the full node API + fnAPIInfo := cctx.String("api-fullnode") + fullnodeAPI, ncloser, err := getFullNodeAPI(ctx, fnAPIInfo) + if err != nil { + return fmt.Errorf("getting full node API: %w", err) + } + defer ncloser() + + // Connect to the sealing API + sealingAPIInfo := cctx.String("api-sealer") + sauth, err := storageAuthWithURL(sealingAPIInfo) + if err != nil { + return fmt.Errorf("parsing sealing API endpoint: %w", err) + } + sealingService, sealerCloser, err := getMinerAPI(ctx, sealingAPIInfo) + if err != nil { + return fmt.Errorf("getting miner API: %w", err) + } + defer sealerCloser() + + maddr, err := sealingService.ActorAddress(ctx) + if err != nil { + return fmt.Errorf("getting miner actor address: %w", err) + } + log.Infof("Miner address: %s", maddr) + + // Use an in-memory repo because we don't need any functions + // of a real repo, we just need to supply something that satisfies + // the LocalStorage interface to the store + memRepo := repo.NewMemory(nil) + lr, err := memRepo.Lock(repo.StorageMiner) + if err != nil { + return fmt.Errorf("locking mem repo: %w", err) + } + defer lr.Close() + + // Create the store interface + var urls []string + lstor, err := paths.NewLocal(ctx, lr, sealingService, urls) + if err != nil { + return fmt.Errorf("creating new local store: %w", err) + } + storage := lotus_modules.RemoteStorage(lstor, sealingService, sauth, sealer.Config{ + // TODO: Not sure if I need this, or any of the other fields in this struct + ParallelFetchLimit: 1, + }) + // Create the piece provider and sector accessors + pp := sealer.NewPieceProvider(storage, sealingService, sealingService) + sa := sectoraccessor.NewSectorAccessor(dtypes.MinerAddress(maddr), sealingService, pp, fullnodeAPI) + // Create the server API + sapi := serverAPI{ctx: ctx, bapi: bapi, sa: sa} + server := NewBitswapServer(cctx.String("base-path"), cctx.Int("port"), sapi) + + // Start the server + log.Infof("Starting booster-http node on port %d with base path '%s'", + cctx.Int("port"), cctx.String("base-path")) + err = server.Start(ctx) + if err != nil { + return err + } + // Monitor for shutdown. + <-ctx.Done() + + log.Info("Shutting down...") + + err = server.Stop() + if err != nil { + return err + } + log.Info("Graceful shutdown successful") + + // Sync all loggers. + _ = log.Sync() //nolint:errcheck + + return nil + }, +} + +func storageAuthWithURL(apiInfo string) (sealer.StorageAuth, error) { + s := strings.Split(apiInfo, ":") + if len(s) != 2 { + return nil, errors.New("unexpected format of `apiInfo`") + } + headers := http.Header{} + headers.Add("Authorization", "Bearer "+s[0]) + return sealer.StorageAuth(headers), nil +} + +type serverAPI struct { + ctx context.Context + bapi api.Boost + sa dagstore.SectorAccessor +} + +var _ BitswapServerAPI = (*serverAPI)(nil) + +func (s serverAPI) PiecesContainingMultihash(mh multihash.Multihash) ([]cid.Cid, error) { + return s.bapi.BoostDagstorePiecesContainingMultihash(s.ctx, mh) +} + +func (s serverAPI) GetMaxPieceOffset(pieceCid cid.Cid) (uint64, error) { + return s.bapi.PiecesGetMaxOffset(s.ctx, pieceCid) +} + +func (s serverAPI) GetPieceInfo(pieceCID cid.Cid) (*piecestore.PieceInfo, error) { + return s.bapi.PiecesGetPieceInfo(s.ctx, pieceCID) +} + +func (s serverAPI) IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) { + return s.sa.IsUnsealed(ctx, sectorID, offset, length) +} + +func (s serverAPI) UnsealSectorAt(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) { + return s.sa.UnsealSectorAt(ctx, sectorID, offset, length) +} + +func getBoostAPI(ctx context.Context, ai string) (api.Boost, jsonrpc.ClientCloser, error) { + ai = strings.TrimPrefix(strings.TrimSpace(ai), "BOOST_API_INFO=") + info := cliutil.ParseApiInfo(ai) + addr, err := info.DialArgs("v0") + if err != nil { + return nil, nil, fmt.Errorf("could not get DialArgs: %w", err) + } + + log.Infof("Using boost API at %s", addr) + api, closer, err := bclient.NewBoostRPCV0(ctx, addr, info.AuthHeader()) + if err != nil { + return nil, nil, fmt.Errorf("creating full node service API: %w", err) + } + + return api, closer, nil +} + +func getFullNodeAPI(ctx context.Context, ai string) (v1api.FullNode, jsonrpc.ClientCloser, error) { + ai = strings.TrimPrefix(strings.TrimSpace(ai), "FULLNODE_API_INFO=") + info := cliutil.ParseApiInfo(ai) + addr, err := info.DialArgs("v1") + if err != nil { + return nil, nil, fmt.Errorf("could not get DialArgs: %w", err) + } + + log.Infof("Using full node API at %s", addr) + api, closer, err := client.NewFullNodeRPCV1(ctx, addr, info.AuthHeader()) + if err != nil { + return nil, nil, fmt.Errorf("creating full node service API: %w", err) + } + + v, err := api.Version(ctx) + if err != nil { + return nil, nil, fmt.Errorf("checking full node service API version: %w", err) + } + + if !v.APIVersion.EqMajorMinor(lapi.FullAPIVersion1) { + return nil, nil, fmt.Errorf("full node service API version didn't match (expected %s, remote %s)", lapi.FullAPIVersion1, v.APIVersion) + } + + return api, closer, nil +} + +func getMinerAPI(ctx context.Context, ai string) (v0api.StorageMiner, jsonrpc.ClientCloser, error) { + ai = strings.TrimPrefix(strings.TrimSpace(ai), "MINER_API_INFO=") + info := cliutil.ParseApiInfo(ai) + addr, err := info.DialArgs("v0") + if err != nil { + return nil, nil, fmt.Errorf("could not get DialArgs: %w", err) + } + + log.Infof("Using sealing API at %s", addr) + api, closer, err := client.NewStorageMinerRPCV0(ctx, addr, info.AuthHeader()) + if err != nil { + return nil, nil, fmt.Errorf("creating miner service API: %w", err) + } + + v, err := api.Version(ctx) + if err != nil { + return nil, nil, fmt.Errorf("checking miner service API version: %w", err) + } + + if !v.APIVersion.EqMajorMinor(lapi.MinerAPIVersion0) { + return nil, nil, fmt.Errorf("miner service API version didn't match (expected %s, remote %s)", lapi.MinerAPIVersion0, v.APIVersion) + } + + return api, closer, nil +} diff --git a/cmd/booster-bitswap/server.go b/cmd/booster-bitswap/server.go new file mode 100644 index 000000000..274143747 --- /dev/null +++ b/cmd/booster-bitswap/server.go @@ -0,0 +1,122 @@ +package main + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + + indexbs "github.com/filecoin-project/boost/cmd/booster-bitswap/indexbs" + "github.com/filecoin-project/dagstore/mount" + "github.com/filecoin-project/go-fil-markets/piecestore" + "github.com/filecoin-project/go-state-types/abi" + bsnetwork "github.com/ipfs/go-bitswap/network" + "github.com/ipfs/go-bitswap/server" + "github.com/ipfs/go-cid" + nilrouting "github.com/ipfs/go-ipfs-routing/none" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p-core/network" + crypto "github.com/libp2p/go-libp2p-crypto" + "github.com/libp2p/go-libp2p/p2p/muxer/mplex" + "github.com/libp2p/go-libp2p/p2p/muxer/yamux" + quic "github.com/libp2p/go-libp2p/p2p/transport/quic" + "github.com/libp2p/go-tcp-transport" + "github.com/multiformats/go-multihash" +) + +var ErrNotFound = errors.New("not found") + +type BitswapServer struct { + port int + api BitswapServerAPI + + ctx context.Context + cancel context.CancelFunc + server *server.Server +} + +type BitswapServerAPI interface { + PiecesContainingMultihash(mh multihash.Multihash) ([]cid.Cid, error) + GetMaxPieceOffset(pieceCid cid.Cid) (uint64, error) + GetPieceInfo(pieceCID cid.Cid) (*piecestore.PieceInfo, error) + IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) + UnsealSectorAt(ctx context.Context, sectorID abi.SectorNumber, pieceOffset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) +} + +func NewBitswapServer(path string, port int, api BitswapServerAPI) *BitswapServer { + return &BitswapServer{port: port, api: api} +} + +func (s *BitswapServer) Start(ctx context.Context) error { + s.ctx, s.cancel = context.WithCancel(ctx) + sf := indexbs.PieceSelectorF(func(c cid.Cid, pieceCIDs []cid.Cid) (cid.Cid, error) { + for _, pieceCID := range pieceCIDs { + pi, err := s.api.GetPieceInfo(pieceCID) + if err != nil { + return cid.Undef, fmt.Errorf("failed to get piece info: %w", err) + } + isUnsealed := s.pieceInUnsealedSector(s.ctx, *pi) + if isUnsealed { + return pieceCID, nil + } + } + + return cid.Undef, indexbs.ErrNoPieceSelected + }) + + rbs, err := indexbs.NewIndexBackedBlockstore(s.api, sf, 100) + if err != nil { + return fmt.Errorf("failed to create index backed blockstore: %w", err) + } + + // setup libp2p host + privKey, _, err := crypto.GenerateECDSAKeyPair(rand.Reader) + + host, err := libp2p.New( + libp2p.ListenAddrStrings( + fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", s.port), + fmt.Sprintf("/ip4/0.0.0.0/udp/%d/quic", s.port), + ), + libp2p.Transport(tcp.NewTCPTransport), + libp2p.Transport(quic.NewTransport), + libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport), + libp2p.Muxer("/yamux/1.0.0", yamux.DefaultTransport), + libp2p.Identity(privKey), + libp2p.ResourceManager(network.NullResourceManager), + ) + if err != nil { + return err + } + + // start a bitswap session on the provider + nilRouter, err := nilrouting.ConstructNilRouting(ctx, nil, nil, nil) + if err != nil { + return err + } + bsopts := []server.Option{server.MaxOutstandingBytesPerPeer(1 << 20)} + s.server = server.New(ctx, bsnetwork.NewFromIpfsHost(r.h, nilRouter), rbs, bsopts...) + + fmt.Printf("bitswap server running on SP, addrs: %s, peerID: %s", r.h.Addrs(), r.h.ID()) + log.Infow("bitswap server running on SP", "multiaddrs", r.h.Addrs(), "peerId", r.h.ID()) + return nil +} + +func (s *BitswapServer) Stop() error { + s.cancel() + return s.server.Close() +} + +func (s *BitswapServer) pieceInUnsealedSector(ctx context.Context, pieceInfo piecestore.PieceInfo) bool { + for _, di := range pieceInfo.Deals { + isUnsealed, err := s.api.IsUnsealed(ctx, di.SectorID, di.Offset.Unpadded(), di.Length.Unpadded()) + if err != nil { + log.Errorf("failed to find out if sector %d is unsealed, err=%s", di.SectorID, err) + continue + } + if isUnsealed { + return true + } + } + + return false +} diff --git a/go.mod b/go.mod index 33f06aaf2..1927c0977 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/filecoin-project/go-data-transfer v1.15.2 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 - github.com/filecoin-project/go-fil-markets v1.23.2 + github.com/filecoin-project/go-fil-markets v1.23.3-0.20220817063153-cdd032886583 github.com/filecoin-project/go-jsonrpc v0.1.5 github.com/filecoin-project/go-legs v0.4.9 github.com/filecoin-project/go-padreader v0.0.1 @@ -45,8 +45,9 @@ require ( github.com/graph-gophers/graphql-go v1.2.0 github.com/graph-gophers/graphql-transport-ws v0.0.2 github.com/hashicorp/go-multierror v1.1.1 + github.com/ipfs/go-bitswap v0.9.0 // indirect github.com/ipfs/go-block-format v0.0.3 - github.com/ipfs/go-blockservice v0.3.0 + github.com/ipfs/go-blockservice v0.4.0 github.com/ipfs/go-cid v0.2.0 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.5.1 @@ -55,8 +56,8 @@ require ( github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-ds-help v1.1.0 - github.com/ipfs/go-ipfs-exchange-interface v0.1.0 - github.com/ipfs/go-ipfs-exchange-offline v0.2.0 + github.com/ipfs/go-ipfs-exchange-interface v0.2.0 + github.com/ipfs/go-ipfs-exchange-offline v0.3.0 github.com/ipfs/go-ipfs-files v0.1.1 github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-log/v2 v2.5.1 @@ -71,27 +72,27 @@ require ( github.com/jpillora/backoff v1.0.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/libp2p/go-eventbus v0.2.1 - github.com/libp2p/go-libp2p v0.20.3 - github.com/libp2p/go-libp2p-core v0.16.1 + github.com/libp2p/go-libp2p v0.21.0 + github.com/libp2p/go-libp2p-core v0.19.1 github.com/libp2p/go-libp2p-gostream v0.4.1-0.20220720161416-e1952aede109 github.com/libp2p/go-libp2p-http v0.2.1 github.com/libp2p/go-libp2p-kad-dht v0.15.0 - github.com/libp2p/go-libp2p-peerstore v0.7.0 + github.com/libp2p/go-libp2p-peerstore v0.7.1 github.com/libp2p/go-libp2p-pubsub v0.7.1 github.com/libp2p/go-libp2p-record v0.1.3 - github.com/libp2p/go-libp2p-resource-manager v0.3.0 + github.com/libp2p/go-libp2p-resource-manager v0.5.1 github.com/mattn/go-sqlite3 v1.14.10 github.com/mitchellh/go-homedir v1.1.0 - github.com/multiformats/go-multiaddr v0.5.0 - github.com/multiformats/go-multibase v0.0.3 + github.com/multiformats/go-multiaddr v0.6.0 + github.com/multiformats/go-multibase v0.1.1 github.com/multiformats/go-multihash v0.2.0 github.com/multiformats/go-varint v0.0.6 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/pressly/goose/v3 v3.5.3 github.com/prometheus/client_golang v1.12.1 github.com/raulk/clock v1.1.0 - github.com/raulk/go-watchdog v1.2.0 - github.com/stretchr/testify v1.7.1 + github.com/raulk/go-watchdog v1.3.0 + github.com/stretchr/testify v1.8.0 github.com/urfave/cli/v2 v2.8.1 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc github.com/whyrusleeping/cbor-gen v0.0.0-20220323183124-98fa8256a799 @@ -101,6 +102,6 @@ require ( go.uber.org/multierr v1.8.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/tools v0.1.11 - golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f + golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f gopkg.in/cheggaaa/pb.v1 v1.0.28 ) diff --git a/go.sum b/go.sum index 25e05c6b6..f291d2af2 100644 --- a/go.sum +++ b/go.sum @@ -388,6 +388,8 @@ github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0 github.com/filecoin-project/go-fil-markets v1.23.1/go.mod h1:V+1vFO34RZmpdECdikKGiyWhSNJK81Q89Kn0egA9iAk= github.com/filecoin-project/go-fil-markets v1.23.2 h1:9+5CCliLVoTbq3qffT2xZMuGjyl2HyR0RJ7x29ywRi8= github.com/filecoin-project/go-fil-markets v1.23.2/go.mod h1:V+1vFO34RZmpdECdikKGiyWhSNJK81Q89Kn0egA9iAk= +github.com/filecoin-project/go-fil-markets v1.23.3-0.20220817063153-cdd032886583 h1:DaHJ8oLE2TSsea/fnXGOTnPd3qbRhCCbuyaiaFdAKf8= +github.com/filecoin-project/go-fil-markets v1.23.3-0.20220817063153-cdd032886583/go.mod h1:ZOPAjEUia7H60F7p0kEupi0FR7Hy4Zfz90BpR1TMBwI= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= @@ -517,8 +519,13 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.1 h1:DX7uPQ4WgAWfoh+NGGlbJQswnYIVvz0SRlLS3rPZQDA= github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.0 h1:j4LrlVXgrbIWO83mmQUnK0Hi+YnbD+vzrE1z/EphbFE= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.5 h1:t4MGB5xEDZvXI+0rMjjsfBsD7yAgp/s9ZDkL1JndXwY= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= @@ -775,6 +782,9 @@ github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUi github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= github.com/ipfs/go-bitswap v0.6.0 h1:f2rc6GZtoSFhEIzQmddgGiel9xntj02Dg0ZNf2hSC+w= github.com/ipfs/go-bitswap v0.6.0/go.mod h1:Hj3ZXdOC5wBJvENtdqsixmzzRukqd8EHLxZLZc3mzRA= +github.com/ipfs/go-bitswap v0.8.0/go.mod h1:/h8sBij8UVEaNWl8ABzpLRA5Y1cttdNUnpeGo2AA/LQ= +github.com/ipfs/go-bitswap v0.9.0 h1:/dZi/XhUN/aIk78pI4kaZrilUglJ+7/SCmOHWIpiy8E= +github.com/ipfs/go-bitswap v0.9.0/go.mod h1:zkfBcGWp4dQTQd0D0akpudhpOVUAJT9GbH9tDmR8/s4= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3 h1:r8t66QstRp/pd/or4dpnbVfXT5Gt7lOqRvC+/dDTpMc= @@ -786,6 +796,8 @@ github.com/ipfs/go-blockservice v0.1.7/go.mod h1:GmS+BAt4hrwBKkzE11AFDQUrnvqjwFa github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= github.com/ipfs/go-blockservice v0.3.0 h1:cDgcZ+0P0Ih3sl8+qjFr2sVaMdysg/YZpLj5WJ8kiiw= github.com/ipfs/go-blockservice v0.3.0/go.mod h1:P5ppi8IHDC7O+pA0AlGTF09jruB2h+oP3wVVaZl8sfk= +github.com/ipfs/go-blockservice v0.4.0 h1:7MUijAW5SqdsqEW/EhnNFRJXVF8mGU5aGhZ3CQaCWbY= +github.com/ipfs/go-blockservice v0.4.0/go.mod h1:kRjO3wlGW9mS1aKuiCeGhx9K1DagQ10ACpVO59qgAx4= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -881,10 +893,14 @@ github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNo github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-interface v0.1.0 h1:TiMekCrOGQuWYtZO3mf4YJXDIdNgnKWZ9IE3fGlnWfo= github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI= +github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= +github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0= github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= github.com/ipfs/go-ipfs-exchange-offline v0.2.0 h1:2PF4o4A7W656rC0RxuhUace997FTcDTcIQ6NoEtyjAI= github.com/ipfs/go-ipfs-exchange-offline v0.2.0/go.mod h1:HjwBeW0dvZvfOMwDP0TSKXIHf2s+ksdP4E3MLDRtLKY= +github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= +github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= @@ -1178,6 +1194,7 @@ github.com/libp2p/go-libp2p v0.20.0/go.mod h1:g0C5Fu+aXXbCXkusCzLycuBowEih3ElmDq github.com/libp2p/go-libp2p v0.20.1/go.mod h1:XgJHsOhEBVBXp/2Sj9bm/yEyD94uunAaP6oaegdcKks= github.com/libp2p/go-libp2p v0.20.3 h1:tjjDNfp7FqdI/7v1rXtB/BtELaPlAThL2uzlj18kcrw= github.com/libp2p/go-libp2p v0.20.3/go.mod h1:I+vndVanE/p/SjFbnA+BEmmfAUEpWxrdXZeyQ1Dus5c= +github.com/libp2p/go-libp2p v0.21.0/go.mod h1:zvcA6/C4mr5/XQarRICh+L1SN9dAHHlSWDq4x5VYxg4= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= @@ -1252,6 +1269,7 @@ github.com/libp2p/go-libp2p-core v0.14.0/go.mod h1:tLasfcVdTXnixsLB0QYaT1syJOhsb github.com/libp2p/go-libp2p-core v0.15.1/go.mod h1:agSaboYM4hzB1cWekgVReqV5M4g5M+2eNNejV+1EEhs= github.com/libp2p/go-libp2p-core v0.16.1 h1:bWoiEBqVkpJ13hbv/f69tHODp86t6mvc4fBN4DkK73M= github.com/libp2p/go-libp2p-core v0.16.1/go.mod h1:O3i/7y+LqUb0N+qhzXjBjjpchgptWAVMG1Voegk7b4c= +github.com/libp2p/go-libp2p-core v0.19.1/go.mod h1:2uLhmmqDiFY+dw+70KkBLeKvvsJHGWUINRDdeV1ip7k= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= @@ -1331,6 +1349,7 @@ github.com/libp2p/go-libp2p-peerstore v0.4.0/go.mod h1:rDJUFyzEWPpXpEwywkcTYYzDH github.com/libp2p/go-libp2p-peerstore v0.6.0/go.mod h1:DGEmKdXrcYpK9Jha3sS7MhqYdInxJy84bIPtSu65bKc= github.com/libp2p/go-libp2p-peerstore v0.7.0 h1:2iIUwok3vtmnWJTZeTeLgnBO6GbkXcwSRwgZHEKrQZs= github.com/libp2p/go-libp2p-peerstore v0.7.0/go.mod h1:cdUWTHro83vpg6unCpGUr8qJoX3e93Vy8o97u5ppIM0= +github.com/libp2p/go-libp2p-peerstore v0.7.1/go.mod h1:cdUWTHro83vpg6unCpGUr8qJoX3e93Vy8o97u5ppIM0= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= github.com/libp2p/go-libp2p-protocol v0.1.0/go.mod h1:KQPHpAabB57XQxGrXCNvbL6UEXfQqUgC/1adR2Xtflk= @@ -1359,6 +1378,8 @@ github.com/libp2p/go-libp2p-resource-manager v0.1.5/go.mod h1:wJPNjeE4XQlxeidwqV github.com/libp2p/go-libp2p-resource-manager v0.2.1/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= github.com/libp2p/go-libp2p-resource-manager v0.3.0 h1:2+cYxUNi33tcydsVLt6K5Fv2E3OTiVeafltecAj15E0= github.com/libp2p/go-libp2p-resource-manager v0.3.0/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= +github.com/libp2p/go-libp2p-resource-manager v0.5.1 h1:jm0mdqn7yfh7wbUzlj948BYZX0KZ3RW7OqerkGQ5rYY= +github.com/libp2p/go-libp2p-resource-manager v0.5.1/go.mod h1:CggtV6EZb+Y0dGh41q5ezO4udcVKyhcEFpydHD8EMe0= github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE= github.com/libp2p/go-libp2p-routing-helpers v0.2.3 h1:xY61alxJ6PurSi+MXbywZpelvuU4U4p/gPTxjqCqTzY= @@ -1700,6 +1721,7 @@ github.com/multiformats/go-multiaddr v0.4.0/go.mod h1:YcpyLH8ZPudLxQlemYBPhSm0/o github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= github.com/multiformats/go-multiaddr v0.5.0 h1:i/JuOoVg4szYQ4YEzDGtb2h0o8M7CG/Yq6cGlcjWZpM= github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar+bR+gh4zgbfr3SNug= +github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= @@ -1722,6 +1744,7 @@ github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= @@ -1931,6 +1954,8 @@ github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= github.com/raulk/go-watchdog v1.2.0 h1:konN75pw2BMmZ+AfuAm5rtFsWcJpKF3m02rKituuXNo= github.com/raulk/go-watchdog v1.2.0/go.mod h1:lzSbAl5sh4rtI8tYHU01BWIDzgzqaQLj6RcA1i4mlqI= +github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= +github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= @@ -2040,6 +2065,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -2193,6 +2219,8 @@ go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzox go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= go.opentelemetry.io/otel v1.3.0 h1:APxLf0eiBwLl+SOXiJJCVYzA1OOJNyAoV8C5RNRyy7Y= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= +go.opentelemetry.io/otel v1.7.0 h1:Z2lA3Tdch0iDcrhJXDIlC94XE+bxok1F9B+4Lz/lGsM= +go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel/bridge/opencensus v0.25.0/go.mod h1:dkZDdaNwLlIutxK2Kc2m3jwW2M1ISaNf8/rOYVwuVHs= go.opentelemetry.io/otel/exporters/jaeger v1.2.0/go.mod h1:KJLFbEMKTNPIfOxcg/WikIozEoKcPgJRz3Ce1vLlM8E= go.opentelemetry.io/otel/internal/metric v0.25.0/go.mod h1:Nhuw26QSX7d6n4duoqAFi5KOQR4AuzyMcl5eXOgwxtc= @@ -2208,6 +2236,8 @@ go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16g go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= go.opentelemetry.io/otel/trace v1.3.0 h1:doy8Hzb1RJ+I3yFhtDmwNc7tIyw1tNMOIsyPzp1NOGY= go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= +go.opentelemetry.io/otel/trace v1.7.0 h1:O37Iogk1lEkMRXewVtZ1BBTVn5JEp8GrJvP92bJqC6o= +go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -2665,6 +2695,7 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= From 05f50e697d9a9de74b0235233f6ad531dace29aa Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 18 Aug 2022 01:58:52 -0700 Subject: [PATCH 2/7] refactor(booster-bitswap): use API for fetching blocks --- api/api.go | 2 +- api/proxy_gen.go | 13 + cmd/booster-bitswap/indexbs/indexbs.go | 312 ------------------ .../remoteblockstore/remoteblockstore.go | 84 +++++ cmd/booster-bitswap/run.go | 149 +-------- cmd/booster-bitswap/server.go | 64 +--- go.mod | 2 +- go.sum | 41 +++ node/builder.go | 1 + node/impl/boost.go | 15 +- node/modules/dtypes/storage.go | 5 + node/modules/storageminer.go | 28 ++ 12 files changed, 197 insertions(+), 519 deletions(-) delete mode 100644 cmd/booster-bitswap/indexbs/indexbs.go create mode 100644 cmd/booster-bitswap/remoteblockstore/remoteblockstore.go diff --git a/api/api.go b/api/api.go index e17440fe1..cc550f9d2 100644 --- a/api/api.go +++ b/api/api.go @@ -47,7 +47,7 @@ type Boost interface { BoostDagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) //perm:read BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:read - + BoostGetBlock(ctx context.Context, c cid.Cid) ([]byte, error) //perm:read // RuntimeSubsystems returns the subsystems that are enabled // in this instance. RuntimeSubsystems(ctx context.Context) (lapi.MinerSubsystems, error) //perm:read diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 1b481dd78..8b56db2dc 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -60,6 +60,8 @@ type BoostStruct struct { BoostDummyDeal func(p0 context.Context, p1 smtypes.DealParams) (*ProviderDealRejectionInfo, error) `perm:"admin"` + BoostGetBlock func(p0 context.Context, p1 cid.Cid) ([]byte, error) `perm:"read"` + BoostIndexerAnnounceAllDeals func(p0 context.Context) error `perm:"admin"` BoostOfflineDealWithData func(p0 context.Context, p1 uuid.UUID, p2 string) (*ProviderDealRejectionInfo, error) `perm:"admin"` @@ -383,6 +385,17 @@ func (s *BoostStub) BoostDummyDeal(p0 context.Context, p1 smtypes.DealParams) (* return nil, ErrNotSupported } +func (s *BoostStruct) BoostGetBlock(p0 context.Context, p1 cid.Cid) ([]byte, error) { + if s.Internal.BoostGetBlock == nil { + return *new([]byte), ErrNotSupported + } + return s.Internal.BoostGetBlock(p0, p1) +} + +func (s *BoostStub) BoostGetBlock(p0 context.Context, p1 cid.Cid) ([]byte, error) { + return *new([]byte), ErrNotSupported +} + func (s *BoostStruct) BoostIndexerAnnounceAllDeals(p0 context.Context) error { if s.Internal.BoostIndexerAnnounceAllDeals == nil { return ErrNotSupported diff --git a/cmd/booster-bitswap/indexbs/indexbs.go b/cmd/booster-bitswap/indexbs/indexbs.go deleted file mode 100644 index 4bdc3b662..000000000 --- a/cmd/booster-bitswap/indexbs/indexbs.go +++ /dev/null @@ -1,312 +0,0 @@ -package indexbs - -import ( - "context" - "errors" - "fmt" - "strings" - "sync" - - "github.com/filecoin-project/dagstore" - "github.com/filecoin-project/go-fil-markets/piecestore" - "github.com/filecoin-project/go-state-types/abi" - blocks "github.com/ipfs/go-block-format" - logging "github.com/ipfs/go-log/v2" - "github.com/ipld/go-car/v2" - "github.com/multiformats/go-multihash" - - "github.com/filecoin-project/dagstore/mount" - "github.com/hashicorp/go-multierror" - lru "github.com/hashicorp/golang-lru" - "github.com/ipfs/go-cid" - blockstore "github.com/ipfs/go-ipfs-blockstore" - carbs "github.com/ipld/go-car/v2/blockstore" -) - -var logbs = logging.Logger("dagstore-all-readblockstore") - -var ErrBlockNotFound = errors.New("block not found") -var ErrNotFound = errors.New("not found") - -var _ blockstore.Blockstore = (*IndexBackedBlockstore)(nil) - -// ErrNoPieceSelected means that the piece selection function rejected all of the given pieces. -var ErrNoPieceSelected = errors.New("no piece selected") - -// PieceSelectorF helps select a piece to fetch a cid from if the given cid is present in multiple pieces. -// It should return `ErrNoPieceSelected` if none of the given piece is selected. -type PieceSelectorF func(c cid.Cid, pieceCids []cid.Cid) (cid.Cid, error) - -type accessorWithBlockstore struct { - mount mount.Reader - bs dagstore.ReadBlockstore -} - -type IndexBackedBlockstoreAPI interface { - PiecesContainingMultihash(mh multihash.Multihash) ([]cid.Cid, error) - GetMaxPieceOffset(pieceCid cid.Cid) (uint64, error) - GetPieceInfo(pieceCID cid.Cid) (*piecestore.PieceInfo, error) - IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) - UnsealSectorAt(ctx context.Context, sectorID abi.SectorNumber, pieceOffset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) -} - -// IndexBackedBlockstore is a read only blockstore over all cids across all pieces on a provider. -type IndexBackedBlockstore struct { - api IndexBackedBlockstoreAPI - pieceSelectF PieceSelectorF - - bsStripedLocks [256]sync.Mutex - blockstoreCache *lru.Cache // caches the blockstore for a given piece for piece read affinity i.e. further reads will likely be from the same piece. Maps (piece CID -> blockstore). -} - -func NewIndexBackedBlockstore(api IndexBackedBlockstoreAPI, pieceSelector PieceSelectorF, maxCacheSize int) (blockstore.Blockstore, error) { - // instantiate the blockstore cache - bslru, err := lru.NewWithEvict(maxCacheSize, func(_ interface{}, val interface{}) { - // ensure we close the blockstore for a piece when it's evicted from the cache so dagstore can gc it. - abs := val.(*accessorWithBlockstore) - abs.mount.Close() - }) - if err != nil { - return nil, fmt.Errorf("failed to create lru cache for read only blockstores") - } - - return &IndexBackedBlockstore{ - api: api, - pieceSelectF: pieceSelector, - blockstoreCache: bslru, - }, nil -} - -func (ro *IndexBackedBlockstore) Get(ctx context.Context, c cid.Cid) (b blocks.Block, finalErr error) { - logbs.Debugw("Get called", "cid", c) - defer func() { - if finalErr != nil { - logbs.Debugw("Get: got error", "cid", c, "error", finalErr) - } - }() - - mhash := c.Hash() - - // fetch all the pieceCIDs containing the multihash - pieceCIDs, err := ro.api.PiecesContainingMultihash(mhash) - if err != nil { - return nil, fmt.Errorf("failed to fetch pieces containing the block: %w", err) - } - if len(pieceCIDs) == 0 { - return nil, ErrBlockNotFound - } - - // do we have a cached blockstore for a piece containing the required block ? If yes, serve the block from that blockstore - for _, pieceCID := range pieceCIDs { - lk := &ro.bsStripedLocks[pieceCIDToStriped(pieceCID)] - lk.Lock() - - blk, err := ro.readFromBSCacheUnlocked(ctx, c, pieceCID) - if err == nil && blk != nil { - logbs.Debugw("Get: returning from block store cache", "cid", c) - - lk.Unlock() - return blk, nil - } - - lk.Unlock() - } - - // ---- we don't have a cached blockstore for a piece that can serve the block -> let's build one. - - // select a valid piece that can serve the retrieval - pieceCID, err := ro.pieceSelectF(c, pieceCIDs) - if err != nil && err == ErrNoPieceSelected { - return nil, ErrBlockNotFound - } - if err != nil { - return nil, fmt.Errorf("failed to run piece selection function: %w", err) - } - - lk := &ro.bsStripedLocks[pieceCIDToStriped(pieceCID)] - lk.Lock() - defer lk.Unlock() - - // see if we have blockstore in the cache we can serve the retrieval from as the previous code in this critical section - // could have added a blockstore to the cache for the given piece CID. - blk, err := ro.readFromBSCacheUnlocked(ctx, c, pieceCID) - if err == nil && blk != nil { - return blk, nil - } - - // load blockstore for the selected piece and try to serve the cid from that blockstore. - reader, err := ro.getPieceContent(ctx, pieceCID) - - bs, err := ro.getBlockstore(reader) - - blk, err = bs.Get(ctx, c) - if err != nil { - return nil, fmt.Errorf("failed to get block: %w", err) - } - - // update the block cache and the blockstore cache - ro.blockstoreCache.Add(pieceCID, &accessorWithBlockstore{reader, bs}) - - logbs.Debugw("Get: returning after creating new blockstore", "cid", c) - return blk, nil -} - -func (ro *IndexBackedBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) { - logbs.Debugw("Has called", "cid", c) - - // if there is a piece that can serve the retrieval for the given cid, we have the requested cid - // and has should return true. - mhash := c.Hash() - - pieceCIDs, err := ro.api.PiecesContainingMultihash(mhash) - if err != nil { - logbs.Debugw("Has error", "cid", c, "err", err) - return false, nil - } - if len(pieceCIDs) == 0 { - logbs.Debugw("Has: returning false no error", "cid", c) - return false, nil - } - - _, err = ro.pieceSelectF(c, pieceCIDs) - if err != nil && err == ErrNoPieceSelected { - logbs.Debugw("Has error", "cid", c, "err", err) - return false, nil - } - if err != nil { - logbs.Debugw("Has error", "cid", c, "err", err) - return false, fmt.Errorf("failed to run piece selection function: %w", err) - } - - logbs.Debugw("Has: returning true", "cid", c) - return true, nil -} - -func (ro *IndexBackedBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) { - logbs.Debugw("GetSize called", "cid", c) - - blk, err := ro.Get(ctx, c) - if err != nil { - logbs.Debugw("GetSize error", "cid", c, "err", err) - return 0, fmt.Errorf("failed to get block: %w", err) - } - - logbs.Debugw("GetSize success", "cid", c) - return len(blk.RawData()), nil -} - -func (ro *IndexBackedBlockstore) readFromBSCacheUnlocked(ctx context.Context, c cid.Cid, pieceCid cid.Cid) (blocks.Block, error) { - // We've already ensured that the given piece has the cid/multihash we are looking for. - val, ok := ro.blockstoreCache.Get(pieceCid) - if !ok { - return nil, ErrBlockNotFound - } - - rbs := val.(*accessorWithBlockstore).bs - blk, err := rbs.Get(ctx, c) - if err != nil { - // we know that the cid we want to lookup belongs to a piece with given pieceCID and - // so if we fail to get the corresponding block from the blockstore for that piece, something has gone wrong - // and we should remove the blockstore for that pieceCID from our cache. - ro.blockstoreCache.Remove(pieceCid) - return nil, err - } - - return blk, nil -} - -func pieceCIDToStriped(pieceCid cid.Cid) byte { - return pieceCid.String()[len(pieceCid.String())-1] -} - -// --- UNSUPPORTED BLOCKSTORE METHODS ------- -func (ro *IndexBackedBlockstore) DeleteBlock(context.Context, cid.Cid) error { - return errors.New("unsupported operation DeleteBlock") -} -func (ro *IndexBackedBlockstore) HashOnRead(_ bool) {} -func (ro *IndexBackedBlockstore) Put(context.Context, blocks.Block) error { - return errors.New("unsupported operation Put") -} -func (ro *IndexBackedBlockstore) PutMany(context.Context, []blocks.Block) error { - return errors.New("unsupported operation PutMany") -} -func (ro *IndexBackedBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { - return nil, errors.New("unsupported operation AllKeysChan") -} - -func (ro *IndexBackedBlockstore) getPieceContent(ctx context.Context, pieceCid cid.Cid) (mount.Reader, error) { - // Get the deals for the piece - pieceInfo, err := ro.api.GetPieceInfo(pieceCid) - if err != nil { - return nil, fmt.Errorf("getting sector info for piece %s: %w", pieceCid, err) - } - - // Get the first unsealed deal - di, err := ro.unsealedDeal(ctx, *pieceInfo) - if err != nil { - return nil, fmt.Errorf("getting unsealed CAR file: %w", err) - } - - // Get the raw piece data from the sector - pieceReader, err := ro.api.UnsealSectorAt(ctx, di.SectorID, di.Offset.Unpadded(), di.Length.Unpadded()) - if err != nil { - return nil, fmt.Errorf("getting raw data from sector %d: %w", di.SectorID, err) - } - - return pieceReader, nil -} - -func (ro *IndexBackedBlockstore) getBlockstore(pieceReader mount.Reader) (dagstore.ReadBlockstore, error) { - idx, err := car.ReadOrGenerateIndex(pieceReader, car.ZeroLengthSectionAsEOF(true), car.StoreIdentityCIDs(true)) - if err != nil { - return nil, err - } - return carbs.NewReadOnly(pieceReader, idx, car.ZeroLengthSectionAsEOF(true)) -} - -func (ro *IndexBackedBlockstore) unsealedDeal(ctx context.Context, pieceInfo piecestore.PieceInfo) (*piecestore.DealInfo, error) { - // There should always been deals in the PieceInfo, but check just in case - if len(pieceInfo.Deals) == 0 { - return nil, fmt.Errorf("there are no deals containing piece %s: %w", pieceInfo.PieceCID, ErrNotFound) - } - - // The same piece can be in many deals. Find the first unsealed deal. - sealedCount := 0 - var allErr error - for _, di := range pieceInfo.Deals { - isUnsealed, err := ro.api.IsUnsealed(ctx, di.SectorID, di.Offset.Unpadded(), di.Length.Unpadded()) - if err != nil { - allErr = multierror.Append(allErr, err) - continue - } - if isUnsealed { - return &di, nil - } - sealedCount++ - } - - // Try to return an error message with as much useful information as possible - dealSectors := make([]string, 0, len(pieceInfo.Deals)) - for _, di := range pieceInfo.Deals { - dealSectors = append(dealSectors, fmt.Sprintf("Deal %d: Sector %d", di.DealID, di.SectorID)) - } - - if allErr == nil { - dealSectorsErr := fmt.Errorf("%s: %w", strings.Join(dealSectors, ", "), ErrNotFound) - return nil, fmt.Errorf("checked unsealed status of %d deals containing piece %s: none are unsealed: %w", - len(pieceInfo.Deals), pieceInfo.PieceCID, dealSectorsErr) - } - - if len(pieceInfo.Deals) == 1 { - return nil, fmt.Errorf("checking unsealed status of deal %d (sector %d) containing piece %s: %w", - pieceInfo.Deals[0].DealID, pieceInfo.Deals[0].SectorID, pieceInfo.PieceCID, allErr) - } - - if sealedCount == 0 { - return nil, fmt.Errorf("checking unsealed status of %d deals containing piece %s: %s: %w", - len(pieceInfo.Deals), pieceInfo.PieceCID, dealSectors, allErr) - } - - return nil, fmt.Errorf("checking unsealed status of %d deals containing piece %s - %d are sealed, %d had errors: %s: %w", - len(pieceInfo.Deals), pieceInfo.PieceCID, sealedCount, len(pieceInfo.Deals)-sealedCount, dealSectors, allErr) -} diff --git a/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go b/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go new file mode 100644 index 000000000..377c08251 --- /dev/null +++ b/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go @@ -0,0 +1,84 @@ +package remoteblockstore + +import ( + "context" + "errors" + + blocks "github.com/ipfs/go-block-format" + format "github.com/ipfs/go-ipld-format" + logging "github.com/ipfs/go-log/v2" + + "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" +) + +var logbs = logging.Logger("remote-blockstore") + +var ErrBlockNotFound = errors.New("block not found") +var ErrNotFound = errors.New("not found") + +var _ blockstore.Blockstore = (*RemoteBlockstore)(nil) + +// ErrNoPieceSelected means that the piece selection function rejected all of the given pieces. +var ErrNoPieceSelected = errors.New("no piece selected") + +// PieceSelectorF helps select a piece to fetch a cid from if the given cid is present in multiple pieces. +// It should return `ErrNoPieceSelected` if none of the given piece is selected. +type PieceSelectorF func(c cid.Cid, pieceCids []cid.Cid) (cid.Cid, error) + +type RemoteBlockstoreAPI interface { + BoostGetBlock(ctx context.Context, c cid.Cid) ([]byte, error) +} + +// RemoteBlockstore is a read only blockstore over all cids across all pieces on a provider. +type RemoteBlockstore struct { + api RemoteBlockstoreAPI +} + +func NewRemoteBlockstore(api RemoteBlockstoreAPI) blockstore.Blockstore { + return &RemoteBlockstore{ + api: api, + } +} + +func (ro *RemoteBlockstore) Get(ctx context.Context, c cid.Cid) (b blocks.Block, err error) { + data, err := ro.api.BoostGetBlock(ctx, c) + if err != nil { + return nil, err + } + return blocks.NewBlock(data), nil +} + +func (ro *RemoteBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) { + _, err := ro.api.BoostGetBlock(ctx, c) + if err != nil { + if format.IsNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (ro *RemoteBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) { + data, err := ro.api.BoostGetBlock(ctx, c) + if err != nil { + return 0, err + } + return len(data), nil +} + +// --- UNSUPPORTED BLOCKSTORE METHODS ------- +func (ro *RemoteBlockstore) DeleteBlock(context.Context, cid.Cid) error { + return errors.New("unsupported operation DeleteBlock") +} +func (ro *RemoteBlockstore) HashOnRead(_ bool) {} +func (ro *RemoteBlockstore) Put(context.Context, blocks.Block) error { + return errors.New("unsupported operation Put") +} +func (ro *RemoteBlockstore) PutMany(context.Context, []blocks.Block) error { + return errors.New("unsupported operation PutMany") +} +func (ro *RemoteBlockstore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { + return nil, errors.New("unsupported operation AllKeysChan") +} diff --git a/cmd/booster-bitswap/run.go b/cmd/booster-bitswap/run.go index 5238f92dc..a7706364d 100644 --- a/cmd/booster-bitswap/run.go +++ b/cmd/booster-bitswap/run.go @@ -11,24 +11,10 @@ import ( "github.com/filecoin-project/boost/api" bclient "github.com/filecoin-project/boost/api/client" cliutil "github.com/filecoin-project/boost/cli/util" - "github.com/filecoin-project/dagstore/mount" - "github.com/filecoin-project/go-fil-markets/piecestore" + "github.com/filecoin-project/boost/cmd/booster-bitswap/remoteblockstore" "github.com/filecoin-project/go-jsonrpc" - "github.com/filecoin-project/go-state-types/abi" - lapi "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/api/client" - "github.com/filecoin-project/lotus/api/v0api" - "github.com/filecoin-project/lotus/api/v1api" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/markets/dagstore" - "github.com/filecoin-project/lotus/markets/sectoraccessor" - lotus_modules "github.com/filecoin-project/lotus/node/modules" - "github.com/filecoin-project/lotus/node/modules/dtypes" - "github.com/filecoin-project/lotus/node/repo" - "github.com/filecoin-project/lotus/storage/paths" "github.com/filecoin-project/lotus/storage/sealer" - "github.com/ipfs/go-cid" - "github.com/multiformats/go-multihash" "github.com/urfave/cli/v2" ) @@ -81,58 +67,9 @@ var runCmd = &cli.Command{ } defer bcloser() - // Connect to the full node API - fnAPIInfo := cctx.String("api-fullnode") - fullnodeAPI, ncloser, err := getFullNodeAPI(ctx, fnAPIInfo) - if err != nil { - return fmt.Errorf("getting full node API: %w", err) - } - defer ncloser() - - // Connect to the sealing API - sealingAPIInfo := cctx.String("api-sealer") - sauth, err := storageAuthWithURL(sealingAPIInfo) - if err != nil { - return fmt.Errorf("parsing sealing API endpoint: %w", err) - } - sealingService, sealerCloser, err := getMinerAPI(ctx, sealingAPIInfo) - if err != nil { - return fmt.Errorf("getting miner API: %w", err) - } - defer sealerCloser() - - maddr, err := sealingService.ActorAddress(ctx) - if err != nil { - return fmt.Errorf("getting miner actor address: %w", err) - } - log.Infof("Miner address: %s", maddr) - - // Use an in-memory repo because we don't need any functions - // of a real repo, we just need to supply something that satisfies - // the LocalStorage interface to the store - memRepo := repo.NewMemory(nil) - lr, err := memRepo.Lock(repo.StorageMiner) - if err != nil { - return fmt.Errorf("locking mem repo: %w", err) - } - defer lr.Close() - - // Create the store interface - var urls []string - lstor, err := paths.NewLocal(ctx, lr, sealingService, urls) - if err != nil { - return fmt.Errorf("creating new local store: %w", err) - } - storage := lotus_modules.RemoteStorage(lstor, sealingService, sauth, sealer.Config{ - // TODO: Not sure if I need this, or any of the other fields in this struct - ParallelFetchLimit: 1, - }) - // Create the piece provider and sector accessors - pp := sealer.NewPieceProvider(storage, sealingService, sealingService) - sa := sectoraccessor.NewSectorAccessor(dtypes.MinerAddress(maddr), sealingService, pp, fullnodeAPI) + remoteStore := remoteblockstore.NewRemoteBlockstore(bapi) // Create the server API - sapi := serverAPI{ctx: ctx, bapi: bapi, sa: sa} - server := NewBitswapServer(cctx.String("base-path"), cctx.Int("port"), sapi) + server := NewBitswapServer(cctx.String("base-path"), cctx.Int("port"), remoteStore) // Start the server log.Infof("Starting booster-http node on port %d with base path '%s'", @@ -169,34 +106,6 @@ func storageAuthWithURL(apiInfo string) (sealer.StorageAuth, error) { return sealer.StorageAuth(headers), nil } -type serverAPI struct { - ctx context.Context - bapi api.Boost - sa dagstore.SectorAccessor -} - -var _ BitswapServerAPI = (*serverAPI)(nil) - -func (s serverAPI) PiecesContainingMultihash(mh multihash.Multihash) ([]cid.Cid, error) { - return s.bapi.BoostDagstorePiecesContainingMultihash(s.ctx, mh) -} - -func (s serverAPI) GetMaxPieceOffset(pieceCid cid.Cid) (uint64, error) { - return s.bapi.PiecesGetMaxOffset(s.ctx, pieceCid) -} - -func (s serverAPI) GetPieceInfo(pieceCID cid.Cid) (*piecestore.PieceInfo, error) { - return s.bapi.PiecesGetPieceInfo(s.ctx, pieceCID) -} - -func (s serverAPI) IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) { - return s.sa.IsUnsealed(ctx, sectorID, offset, length) -} - -func (s serverAPI) UnsealSectorAt(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) { - return s.sa.UnsealSectorAt(ctx, sectorID, offset, length) -} - func getBoostAPI(ctx context.Context, ai string) (api.Boost, jsonrpc.ClientCloser, error) { ai = strings.TrimPrefix(strings.TrimSpace(ai), "BOOST_API_INFO=") info := cliutil.ParseApiInfo(ai) @@ -213,55 +122,3 @@ func getBoostAPI(ctx context.Context, ai string) (api.Boost, jsonrpc.ClientClose return api, closer, nil } - -func getFullNodeAPI(ctx context.Context, ai string) (v1api.FullNode, jsonrpc.ClientCloser, error) { - ai = strings.TrimPrefix(strings.TrimSpace(ai), "FULLNODE_API_INFO=") - info := cliutil.ParseApiInfo(ai) - addr, err := info.DialArgs("v1") - if err != nil { - return nil, nil, fmt.Errorf("could not get DialArgs: %w", err) - } - - log.Infof("Using full node API at %s", addr) - api, closer, err := client.NewFullNodeRPCV1(ctx, addr, info.AuthHeader()) - if err != nil { - return nil, nil, fmt.Errorf("creating full node service API: %w", err) - } - - v, err := api.Version(ctx) - if err != nil { - return nil, nil, fmt.Errorf("checking full node service API version: %w", err) - } - - if !v.APIVersion.EqMajorMinor(lapi.FullAPIVersion1) { - return nil, nil, fmt.Errorf("full node service API version didn't match (expected %s, remote %s)", lapi.FullAPIVersion1, v.APIVersion) - } - - return api, closer, nil -} - -func getMinerAPI(ctx context.Context, ai string) (v0api.StorageMiner, jsonrpc.ClientCloser, error) { - ai = strings.TrimPrefix(strings.TrimSpace(ai), "MINER_API_INFO=") - info := cliutil.ParseApiInfo(ai) - addr, err := info.DialArgs("v0") - if err != nil { - return nil, nil, fmt.Errorf("could not get DialArgs: %w", err) - } - - log.Infof("Using sealing API at %s", addr) - api, closer, err := client.NewStorageMinerRPCV0(ctx, addr, info.AuthHeader()) - if err != nil { - return nil, nil, fmt.Errorf("creating miner service API: %w", err) - } - - v, err := api.Version(ctx) - if err != nil { - return nil, nil, fmt.Errorf("checking miner service API version: %w", err) - } - - if !v.APIVersion.EqMajorMinor(lapi.MinerAPIVersion0) { - return nil, nil, fmt.Errorf("miner service API version didn't match (expected %s, remote %s)", lapi.MinerAPIVersion0, v.APIVersion) - } - - return api, closer, nil -} diff --git a/cmd/booster-bitswap/server.go b/cmd/booster-bitswap/server.go index 274143747..46adfa2ba 100644 --- a/cmd/booster-bitswap/server.go +++ b/cmd/booster-bitswap/server.go @@ -6,13 +6,9 @@ import ( "errors" "fmt" - indexbs "github.com/filecoin-project/boost/cmd/booster-bitswap/indexbs" - "github.com/filecoin-project/dagstore/mount" - "github.com/filecoin-project/go-fil-markets/piecestore" - "github.com/filecoin-project/go-state-types/abi" bsnetwork "github.com/ipfs/go-bitswap/network" "github.com/ipfs/go-bitswap/server" - "github.com/ipfs/go-cid" + blockstore "github.com/ipfs/go-ipfs-blockstore" nilrouting "github.com/ipfs/go-ipfs-routing/none" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p-core/network" @@ -21,54 +17,25 @@ import ( "github.com/libp2p/go-libp2p/p2p/muxer/yamux" quic "github.com/libp2p/go-libp2p/p2p/transport/quic" "github.com/libp2p/go-tcp-transport" - "github.com/multiformats/go-multihash" ) var ErrNotFound = errors.New("not found") type BitswapServer struct { - port int - api BitswapServerAPI + port int + remoteStore blockstore.Blockstore ctx context.Context cancel context.CancelFunc server *server.Server } -type BitswapServerAPI interface { - PiecesContainingMultihash(mh multihash.Multihash) ([]cid.Cid, error) - GetMaxPieceOffset(pieceCid cid.Cid) (uint64, error) - GetPieceInfo(pieceCID cid.Cid) (*piecestore.PieceInfo, error) - IsUnsealed(ctx context.Context, sectorID abi.SectorNumber, offset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (bool, error) - UnsealSectorAt(ctx context.Context, sectorID abi.SectorNumber, pieceOffset abi.UnpaddedPieceSize, length abi.UnpaddedPieceSize) (mount.Reader, error) -} - -func NewBitswapServer(path string, port int, api BitswapServerAPI) *BitswapServer { - return &BitswapServer{port: port, api: api} +func NewBitswapServer(path string, port int, remoteStore blockstore.Blockstore) *BitswapServer { + return &BitswapServer{port: port, remoteStore: remoteStore} } func (s *BitswapServer) Start(ctx context.Context) error { s.ctx, s.cancel = context.WithCancel(ctx) - sf := indexbs.PieceSelectorF(func(c cid.Cid, pieceCIDs []cid.Cid) (cid.Cid, error) { - for _, pieceCID := range pieceCIDs { - pi, err := s.api.GetPieceInfo(pieceCID) - if err != nil { - return cid.Undef, fmt.Errorf("failed to get piece info: %w", err) - } - isUnsealed := s.pieceInUnsealedSector(s.ctx, *pi) - if isUnsealed { - return pieceCID, nil - } - } - - return cid.Undef, indexbs.ErrNoPieceSelected - }) - - rbs, err := indexbs.NewIndexBackedBlockstore(s.api, sf, 100) - if err != nil { - return fmt.Errorf("failed to create index backed blockstore: %w", err) - } - // setup libp2p host privKey, _, err := crypto.GenerateECDSAKeyPair(rand.Reader) @@ -94,10 +61,10 @@ func (s *BitswapServer) Start(ctx context.Context) error { return err } bsopts := []server.Option{server.MaxOutstandingBytesPerPeer(1 << 20)} - s.server = server.New(ctx, bsnetwork.NewFromIpfsHost(r.h, nilRouter), rbs, bsopts...) + s.server = server.New(ctx, bsnetwork.NewFromIpfsHost(host, nilRouter), s.remoteStore, bsopts...) - fmt.Printf("bitswap server running on SP, addrs: %s, peerID: %s", r.h.Addrs(), r.h.ID()) - log.Infow("bitswap server running on SP", "multiaddrs", r.h.Addrs(), "peerId", r.h.ID()) + fmt.Printf("bitswap server running on SP, addrs: %s, peerID: %s", host.Addrs(), host.ID()) + log.Infow("bitswap server running on SP", "multiaddrs", host.Addrs(), "peerId", host.ID()) return nil } @@ -105,18 +72,3 @@ func (s *BitswapServer) Stop() error { s.cancel() return s.server.Close() } - -func (s *BitswapServer) pieceInUnsealedSector(ctx context.Context, pieceInfo piecestore.PieceInfo) bool { - for _, di := range pieceInfo.Deals { - isUnsealed, err := s.api.IsUnsealed(ctx, di.SectorID, di.Offset.Unpadded(), di.Length.Unpadded()) - if err != nil { - log.Errorf("failed to find out if sector %d is unsealed, err=%s", di.SectorID, err) - continue - } - if isUnsealed { - return true - } - } - - return false -} diff --git a/go.mod b/go.mod index 1927c0977..5952a1db4 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/dustin/go-humanize v1.0.0 github.com/etclabscore/go-openrpc-reflect v0.0.36 github.com/fatih/color v1.13.0 - github.com/filecoin-project/dagstore v0.5.3 + github.com/filecoin-project/dagstore v0.5.4-0.20220818083758-aba4e75dd27a github.com/filecoin-project/go-address v1.0.0 github.com/filecoin-project/go-bitfield v0.2.4 github.com/filecoin-project/go-cbor-util v0.0.1 diff --git a/go.sum b/go.sum index f291d2af2..2fa6799eb 100644 --- a/go.sum +++ b/go.sum @@ -158,6 +158,7 @@ github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= github.com/btcsuite/btcd/btcec/v2 v2.1.3 h1:xM/n3yIhHAhHy04z4i43C8p4ehixJZMsnrVJkgl+MTE= github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= +github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= @@ -221,6 +222,7 @@ github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkb github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBdAP4= github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= +github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= @@ -347,8 +349,16 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/filecoin-project/dagstore v0.5.2/go.mod h1:mdqKzYrRBHf1pRMthYfMv3n37oOw0Tkx7+TxPt240M0= +github.com/filecoin-project/dagstore v0.5.3-0.20220330091151-51396854de25 h1:xaJLkFWVVhk598H7kXk010eEq9jYZrA4OXd/hP5Gu0g= +github.com/filecoin-project/dagstore v0.5.3-0.20220330091151-51396854de25/go.mod h1:s4Jn8iZmbjDOAkukS/fWBISrgUwtv1w5QDGsIXG9Fbs= github.com/filecoin-project/dagstore v0.5.3 h1:++s4pEW/NvHph0N8sCdz7NokU0Y3r2yVB5SFaDTLLWM= github.com/filecoin-project/dagstore v0.5.3/go.mod h1:mdqKzYrRBHf1pRMthYfMv3n37oOw0Tkx7+TxPt240M0= +github.com/filecoin-project/dagstore v0.5.4-0.20220818081538-a8c50b81928f h1:YGV85RPFIpgiL0bT7YiGmySQgLH+suB30RVdPa8V2fE= +github.com/filecoin-project/dagstore v0.5.4-0.20220818081538-a8c50b81928f/go.mod h1:s4Jn8iZmbjDOAkukS/fWBISrgUwtv1w5QDGsIXG9Fbs= +github.com/filecoin-project/dagstore v0.5.4-0.20220818083108-8027d20e9980 h1:E9tIO43b/7N4044ZnMQM6nOsY6fLa/C2GQkbF5xzqHo= +github.com/filecoin-project/dagstore v0.5.4-0.20220818083108-8027d20e9980/go.mod h1:LoStZ5eZM8En+jRTYBU1Q6QimdfjT+FtgG90Hnat+cE= +github.com/filecoin-project/dagstore v0.5.4-0.20220818083758-aba4e75dd27a h1:uf2+323ApLfOsmcRL0gHDFpQgptK9M0MY7kofMrN2v8= +github.com/filecoin-project/dagstore v0.5.4-0.20220818083758-aba4e75dd27a/go.mod h1:LoStZ5eZM8En+jRTYBU1Q6QimdfjT+FtgG90Hnat+cE= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-address v0.0.5/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= github.com/filecoin-project/go-address v0.0.6/go.mod h1:7B0/5DA13n6nHkB8bbGx1gWzG/dbTsZ0fgOJVGsM3TE= @@ -485,6 +495,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= github.com/gammazero/keymutex v0.0.2/go.mod h1:qtzWCCLMisQUmVa4dvqHVgwfh4BP2YB7JxNDGXnsKrs= github.com/gammazero/radixtree v0.2.5/go.mod h1:VPqqCDZ3YZZxAzUUsIF/ytFBigVWV7JIV1Stld8hri0= @@ -617,6 +628,7 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -711,6 +723,7 @@ github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyN github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= @@ -1128,6 +1141,8 @@ github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.0.14 h1:QRqdp6bb9M9S5yyKeYteXKuoKE4p0tGlra81fKOpWH8= +github.com/klauspost/cpuid/v2 v2.0.14/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/koalacxr/quantile v0.0.1/go.mod h1:bGN/mCZLZ4lrSDHRQ6Lglj9chowGux8sGUIND+DQeD0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1135,6 +1150,7 @@ github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfo github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.2 h1:fL3wAoyT6hXHQlORyXUW4Q23kkQpJRgEAYcZB5BR71o= github.com/koron/go-ssdp v0.0.2/go.mod h1:XoLfkAiA2KeZsYh4DbHxD7h3nR2AZNqVQOa+LJuqPYs= +github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -1155,6 +1171,7 @@ github.com/libp2p/go-addr-util v0.1.0/go.mod h1:6I3ZYuFr2O/9D+SoyM0zEw0EF3YkldtT github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-conn-security v0.0.1/go.mod h1:bGmu51N0KU9IEjX7kl2PQjgZa40JQWnayTvNMgD/vyk= @@ -1269,6 +1286,7 @@ github.com/libp2p/go-libp2p-core v0.14.0/go.mod h1:tLasfcVdTXnixsLB0QYaT1syJOhsb github.com/libp2p/go-libp2p-core v0.15.1/go.mod h1:agSaboYM4hzB1cWekgVReqV5M4g5M+2eNNejV+1EEhs= github.com/libp2p/go-libp2p-core v0.16.1 h1:bWoiEBqVkpJ13hbv/f69tHODp86t6mvc4fBN4DkK73M= github.com/libp2p/go-libp2p-core v0.16.1/go.mod h1:O3i/7y+LqUb0N+qhzXjBjjpchgptWAVMG1Voegk7b4c= +github.com/libp2p/go-libp2p-core v0.19.0/go.mod h1:AkA+FUKQfYt1FLNef5fOPlo/naAWjKy/RCjkcPjqzYg= github.com/libp2p/go-libp2p-core v0.19.1/go.mod h1:2uLhmmqDiFY+dw+70KkBLeKvvsJHGWUINRDdeV1ip7k= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= @@ -1423,6 +1441,7 @@ github.com/libp2p/go-libp2p-testing v0.8.0/go.mod h1:gRdsNxQSxAZowTgcLY7CC33xPml github.com/libp2p/go-libp2p-testing v0.9.0/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= github.com/libp2p/go-libp2p-testing v0.9.2 h1:dCpODRtRaDZKF8HXT9qqqgON+OMEB423Knrgeod8j84= github.com/libp2p/go-libp2p-testing v0.9.2/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= +github.com/libp2p/go-libp2p-testing v0.11.0/go.mod h1:qG4sF27dfKFoK9KlVzK2y52LQKhp0VEmLjV5aDqr1Hg= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-tls v0.3.0/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= github.com/libp2p/go-libp2p-tls v0.3.1/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= @@ -1575,6 +1594,7 @@ github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T github.com/lucas-clemente/quic-go v0.27.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= github.com/lucas-clemente/quic-go v0.27.1 h1:sOw+4kFSVrdWOYmUjufQ9GBVPqZ+tu+jMtXxXNmRJyk= github.com/lucas-clemente/quic-go v0.27.1/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= +github.com/lucas-clemente/quic-go v0.28.0/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/iostat v1.1.0/go.mod h1:rEPNA0xXgjHQjuI5Cy05sLlS2oRcSlWHRLrvh/AQ+Pg= @@ -1603,9 +1623,12 @@ github.com/marten-seemann/qtls-go1-17 v0.1.0-rc.1/go.mod h1:fz4HIxByo+LlWcreM4CZ github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8= github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc= github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= +github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI= github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y= github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= github.com/matryer/is v1.4.0 h1:sosSmIWwkYITGrxZ25ULNDeKiMNzFSr4V/eqBQP0PeE= @@ -1656,6 +1679,7 @@ github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJys github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -1744,6 +1768,7 @@ github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= @@ -1931,6 +1956,7 @@ github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+ github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.33.0 h1:rHgav/0a6+uYgGdNt3jwz8FNSesO/Hsang3O0T9A5SE= github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289/go.mod h1:FGbBv5OPKjch+jNUJmEQpMZytIdyW0NdBtWFcfSKusc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -2056,6 +2082,7 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -2069,6 +2096,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= github.com/tidwall/gjson v1.6.0 h1:9VEQWz6LLMUsUl6PueE49ir4Ka6CzLymOAZDxpFsTDc= @@ -2351,6 +2379,8 @@ golang.org/x/exp v0.0.0-20210615023648-acb5c1269671/go.mod h1:DVyR6MI7P4kEQgvZJS golang.org/x/exp v0.0.0-20210714144626-1041f73d31d8/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/exp v0.0.0-20210715201039-d37aa40e8013 h1:Jp57DBw4K7mimZNA3F9f7CndVcUt4kJjmyJf2rzJHoI= golang.org/x/exp v0.0.0-20210715201039-d37aa40e8013/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= +golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5 h1:rxKZ2gOnYxjfmakvUUqh9Gyb6KXfrj7JWTxORTYqb0E= +golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2379,6 +2409,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -2428,6 +2459,7 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -2453,6 +2485,9 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2 h1:6mzvA99KwZxbOrxww4EvWVQUnN1+xEu9tafK5ZxkYeA= golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2553,6 +2588,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200812155832-6a926be9bd1d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2595,6 +2631,8 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -2685,6 +2723,7 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY= @@ -2695,6 +2734,7 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= @@ -2846,6 +2886,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= diff --git a/node/builder.go b/node/builder.go index abe8243d1..c76a44852 100644 --- a/node/builder.go +++ b/node/builder.go @@ -508,6 +508,7 @@ func ConfigBoost(cfg *config.Boost) Option { Override(new(mktsdagstore.MinerAPI), lotus_modules.NewMinerAPI(cfg.DAGStore)), Override(DAGStoreKey, lotus_modules.DAGStore(cfg.DAGStore)), Override(new(dagstore.Interface), From(new(*dagstore.DAGStore))), + Override(new(dtypes.IndexBackedBlockstore), modules.NewIndexBackedBlockstore), // Lotus Markets (retrieval) Override(new(mktsdagstore.SectorAccessor), sectoraccessor.NewSectorAccessor), diff --git a/node/impl/boost.go b/node/impl/boost.go index a5b9fbf4c..1fcd4150f 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/boost/api" "github.com/filecoin-project/boost/gql" "github.com/filecoin-project/boost/indexprovider" + "github.com/filecoin-project/boost/node/modules/dtypes" "github.com/filecoin-project/boost/sealingpipeline" "github.com/filecoin-project/boost/storagemarket" "github.com/filecoin-project/boost/storagemarket/types" @@ -49,9 +50,9 @@ type BoostAPI struct { Host host.Host - DAGStore *dagstore.DAGStore - DagStoreWrapper *mktsdagstore.Wrapper - + DAGStore *dagstore.DAGStore + DagStoreWrapper *mktsdagstore.Wrapper + IndexBackedBlockstore dtypes.IndexBackedBlockstore // Boost StorageProvider *storagemarket.Provider IndexProvider *indexprovider.Wrapper @@ -473,3 +474,11 @@ func (sm *BoostAPI) BoostDagstoreDestroyShard(ctx context.Context, key string) e } return nil } + +func (sm *BoostAPI) BoostGetBlock(ctx context.Context, c cid.Cid) ([]byte, error) { + blk, err := sm.IndexBackedBlockstore.Get(ctx, c) + if err != nil { + return nil, err + } + return blk.RawData(), nil +} diff --git a/node/modules/dtypes/storage.go b/node/modules/dtypes/storage.go index e4bf96d86..fc4f72fcc 100644 --- a/node/modules/dtypes/storage.go +++ b/node/modules/dtypes/storage.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/go-fil-markets/piecestore" "github.com/filecoin-project/go-fil-markets/storagemarket/impl/requestvalidation" + ipfsblockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/node/repo/imports" @@ -58,6 +59,10 @@ type ( // holds a 'false' for that block), the process should signal so by calling // blockstore.AllCaches.Dirty(cid). ExposedBlockstore blockstore.Blockstore + + // IndexBackedBlockstore is an abstraction on top of the DAGStore that provides + // access to any CID in a free or unsealed sector + IndexBackedBlockstore ipfsblockstore.Blockstore ) type ChainBitswap exchange.Interface diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 08a9ce240..2408624db 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -23,6 +23,8 @@ import ( "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/boost/transport/httptransport" "github.com/filecoin-project/dagstore" + "github.com/filecoin-project/dagstore/indexbs" + "github.com/filecoin-project/dagstore/shard" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/shared" @@ -434,3 +436,29 @@ func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo return server } } + +func NewIndexBackedBlockstore(dagst dagstore.Interface, rp retrievalmarket.RetrievalProvider, h host.Host) (dtypes.IndexBackedBlockstore, error) { + sf := indexbs.ShardSelectorF(func(c cid.Cid, shards []shard.Key) (shard.Key, error) { + for _, sk := range shards { + pieceCid, err := cid.Parse(sk.String()) + if err != nil { + return shard.Key{}, fmt.Errorf("failed to parse cid") + } + b, err := rp.IsFreeAndUnsealed(context.TODO(), c, pieceCid) + if err != nil { + return shard.Key{}, fmt.Errorf("failed to verify is piece is free and unsealed") + } + if b { + return sk, nil + } + } + + return shard.Key{}, indexbs.ErrNoShardSelected + }) + + rbs, err := indexbs.NewIndexBackedBlockstore(dagst, sf, 100) + if err != nil { + return nil, fmt.Errorf("failed to create index backed blockstore: %w", err) + } + return dtypes.IndexBackedBlockstore(rbs), nil +} From b943a2b785b2451fea80e2d2eec6df1084a115e0 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 18 Aug 2022 21:57:54 -0700 Subject: [PATCH 3/7] fix(deps): update deps to compile --- cmd/booster-bitswap/server.go | 2 +- go.mod | 10 +++-- go.sum | 84 +++++++++++++---------------------- node/modules/rcmgr.go | 71 +++++++++++++++++++---------- 4 files changed, 86 insertions(+), 81 deletions(-) diff --git a/cmd/booster-bitswap/server.go b/cmd/booster-bitswap/server.go index 46adfa2ba..722c1a227 100644 --- a/cmd/booster-bitswap/server.go +++ b/cmd/booster-bitswap/server.go @@ -16,7 +16,7 @@ import ( "github.com/libp2p/go-libp2p/p2p/muxer/mplex" "github.com/libp2p/go-libp2p/p2p/muxer/yamux" quic "github.com/libp2p/go-libp2p/p2p/transport/quic" - "github.com/libp2p/go-tcp-transport" + "github.com/libp2p/go-libp2p/p2p/transport/tcp" ) var ErrNotFound = errors.New("not found") diff --git a/go.mod b/go.mod index 5952a1db4..116ceef67 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/filecoin-project/go-state-types v0.1.10 github.com/filecoin-project/go-statestore v0.2.0 github.com/filecoin-project/index-provider v0.8.1 - github.com/filecoin-project/lotus v1.17.0 + github.com/filecoin-project/lotus v1.17.1-0.20220819044554-215598c1e9cb github.com/filecoin-project/specs-actors v0.9.15 github.com/filecoin-project/specs-actors/v8 v8.0.1 github.com/filecoin-project/specs-storage v0.4.1 @@ -45,7 +45,7 @@ require ( github.com/graph-gophers/graphql-go v1.2.0 github.com/graph-gophers/graphql-transport-ws v0.0.2 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/go-bitswap v0.9.0 // indirect + github.com/ipfs/go-bitswap v0.9.0 github.com/ipfs/go-block-format v0.0.3 github.com/ipfs/go-blockservice v0.4.0 github.com/ipfs/go-cid v0.2.0 @@ -59,6 +59,7 @@ require ( github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipfs-exchange-offline v0.3.0 github.com/ipfs/go-ipfs-files v0.1.1 + github.com/ipfs/go-ipfs-routing v0.2.1 github.com/ipfs/go-ipld-format v0.4.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.6.0 @@ -74,13 +75,14 @@ require ( github.com/libp2p/go-eventbus v0.2.1 github.com/libp2p/go-libp2p v0.21.0 github.com/libp2p/go-libp2p-core v0.19.1 + github.com/libp2p/go-libp2p-crypto v0.1.0 github.com/libp2p/go-libp2p-gostream v0.4.1-0.20220720161416-e1952aede109 github.com/libp2p/go-libp2p-http v0.2.1 - github.com/libp2p/go-libp2p-kad-dht v0.15.0 + github.com/libp2p/go-libp2p-kad-dht v0.17.0 github.com/libp2p/go-libp2p-peerstore v0.7.1 github.com/libp2p/go-libp2p-pubsub v0.7.1 github.com/libp2p/go-libp2p-record v0.1.3 - github.com/libp2p/go-libp2p-resource-manager v0.5.1 + github.com/libp2p/go-libp2p-resource-manager v0.5.3 github.com/mattn/go-sqlite3 v1.14.10 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multiaddr v0.6.0 diff --git a/go.sum b/go.sum index 2fa6799eb..8b03c7090 100644 --- a/go.sum +++ b/go.sum @@ -156,8 +156,8 @@ github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MR github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= -github.com/btcsuite/btcd/btcec/v2 v2.1.3 h1:xM/n3yIhHAhHy04z4i43C8p4ehixJZMsnrVJkgl+MTE= github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= +github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= @@ -220,8 +220,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.3 h1:ADZftAkglvCiD44c77s5YmMqaP2pzVCFZvBmAlBdAP4= github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= +github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -349,14 +349,6 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/filecoin-project/dagstore v0.5.2/go.mod h1:mdqKzYrRBHf1pRMthYfMv3n37oOw0Tkx7+TxPt240M0= -github.com/filecoin-project/dagstore v0.5.3-0.20220330091151-51396854de25 h1:xaJLkFWVVhk598H7kXk010eEq9jYZrA4OXd/hP5Gu0g= -github.com/filecoin-project/dagstore v0.5.3-0.20220330091151-51396854de25/go.mod h1:s4Jn8iZmbjDOAkukS/fWBISrgUwtv1w5QDGsIXG9Fbs= -github.com/filecoin-project/dagstore v0.5.3 h1:++s4pEW/NvHph0N8sCdz7NokU0Y3r2yVB5SFaDTLLWM= -github.com/filecoin-project/dagstore v0.5.3/go.mod h1:mdqKzYrRBHf1pRMthYfMv3n37oOw0Tkx7+TxPt240M0= -github.com/filecoin-project/dagstore v0.5.4-0.20220818081538-a8c50b81928f h1:YGV85RPFIpgiL0bT7YiGmySQgLH+suB30RVdPa8V2fE= -github.com/filecoin-project/dagstore v0.5.4-0.20220818081538-a8c50b81928f/go.mod h1:s4Jn8iZmbjDOAkukS/fWBISrgUwtv1w5QDGsIXG9Fbs= -github.com/filecoin-project/dagstore v0.5.4-0.20220818083108-8027d20e9980 h1:E9tIO43b/7N4044ZnMQM6nOsY6fLa/C2GQkbF5xzqHo= -github.com/filecoin-project/dagstore v0.5.4-0.20220818083108-8027d20e9980/go.mod h1:LoStZ5eZM8En+jRTYBU1Q6QimdfjT+FtgG90Hnat+cE= github.com/filecoin-project/dagstore v0.5.4-0.20220818083758-aba4e75dd27a h1:uf2+323ApLfOsmcRL0gHDFpQgptK9M0MY7kofMrN2v8= github.com/filecoin-project/dagstore v0.5.4-0.20220818083758-aba4e75dd27a/go.mod h1:LoStZ5eZM8En+jRTYBU1Q6QimdfjT+FtgG90Hnat+cE= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= @@ -396,8 +388,6 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+ github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.23.1/go.mod h1:V+1vFO34RZmpdECdikKGiyWhSNJK81Q89Kn0egA9iAk= -github.com/filecoin-project/go-fil-markets v1.23.2 h1:9+5CCliLVoTbq3qffT2xZMuGjyl2HyR0RJ7x29ywRi8= -github.com/filecoin-project/go-fil-markets v1.23.2/go.mod h1:V+1vFO34RZmpdECdikKGiyWhSNJK81Q89Kn0egA9iAk= github.com/filecoin-project/go-fil-markets v1.23.3-0.20220817063153-cdd032886583 h1:DaHJ8oLE2TSsea/fnXGOTnPd3qbRhCCbuyaiaFdAKf8= github.com/filecoin-project/go-fil-markets v1.23.3-0.20220817063153-cdd032886583/go.mod h1:ZOPAjEUia7H60F7p0kEupi0FR7Hy4Zfz90BpR1TMBwI= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= @@ -441,8 +431,8 @@ github.com/filecoin-project/go-storedcounter v0.1.0 h1:Mui6wSUBC+cQGHbDUBcO7rfh5 github.com/filecoin-project/go-storedcounter v0.1.0/go.mod h1:4ceukaXi4vFURIoxYMfKzaRF5Xv/Pinh2oTnoxpv+z8= github.com/filecoin-project/index-provider v0.8.1 h1:ggoBWvMSWR91HZQCWfv8SZjoTGNyJBwNMLuN9bJZrbU= github.com/filecoin-project/index-provider v0.8.1/go.mod h1:c/Ym5HtWPp9NQgNc9dgSBMpSNsZ/DE9FEi9qVubl5RM= -github.com/filecoin-project/lotus v1.17.0 h1:FiOxLzX1vfDrI46rO9PpqjF8pBAaxhEQGdHw3u5UOaw= -github.com/filecoin-project/lotus v1.17.0/go.mod h1:hZ5L7E4uKWwp8E/8Bw3pBaai4wV7Utq/ZSbl9hIoFnU= +github.com/filecoin-project/lotus v1.17.1-0.20220819044554-215598c1e9cb h1:kpy8YuyDgoUf+nKCkF49UIRG7ywigkZaZpzlfoEbP3U= +github.com/filecoin-project/lotus v1.17.1-0.20220819044554-215598c1e9cb/go.mod h1:FK/JSFBxylpRuzIKgQCthHQ06fiseSzJlpdhQVpQzeU= github.com/filecoin-project/pubsub v1.0.0 h1:ZTmT27U07e54qV1mMiQo4HDr0buo8I1LDHBYLXlsNXM= github.com/filecoin-project/pubsub v1.0.0/go.mod h1:GkpB33CcUtUNrLPhJgfdy4FDx4OMNR9k+46DHx/Lqrg= github.com/filecoin-project/specs-actors v0.9.13/go.mod h1:TS1AW/7LbG+615j4NsjMK1qlpAwaFsG9w0V2tg2gSao= @@ -493,8 +483,8 @@ github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3 github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= github.com/gammazero/keymutex v0.0.2/go.mod h1:qtzWCCLMisQUmVa4dvqHVgwfh4BP2YB7JxNDGXnsKrs= @@ -528,12 +518,10 @@ github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNV github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.1 h1:DX7uPQ4WgAWfoh+NGGlbJQswnYIVvz0SRlLS3rPZQDA= github.com/go-logr/logr v1.2.1/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.0 h1:j4LrlVXgrbIWO83mmQUnK0Hi+YnbD+vzrE1z/EphbFE= github.com/go-logr/stdr v1.2.0/go.mod h1:YkVgnZu1ZjjL7xTxrfm/LLZBfkhTqSR1ydtm6jTKKwI= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= @@ -626,8 +614,8 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -721,8 +709,8 @@ github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoP github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= @@ -793,7 +781,6 @@ github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiL github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUiJ2teMvI= github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= -github.com/ipfs/go-bitswap v0.6.0 h1:f2rc6GZtoSFhEIzQmddgGiel9xntj02Dg0ZNf2hSC+w= github.com/ipfs/go-bitswap v0.6.0/go.mod h1:Hj3ZXdOC5wBJvENtdqsixmzzRukqd8EHLxZLZc3mzRA= github.com/ipfs/go-bitswap v0.8.0/go.mod h1:/h8sBij8UVEaNWl8ABzpLRA5Y1cttdNUnpeGo2AA/LQ= github.com/ipfs/go-bitswap v0.9.0 h1:/dZi/XhUN/aIk78pI4kaZrilUglJ+7/SCmOHWIpiy8E= @@ -807,7 +794,6 @@ github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7s github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-blockservice v0.1.7/go.mod h1:GmS+BAt4hrwBKkzE11AFDQUrnvqjwFatGS2MY7wOjEM= github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= -github.com/ipfs/go-blockservice v0.3.0 h1:cDgcZ+0P0Ih3sl8+qjFr2sVaMdysg/YZpLj5WJ8kiiw= github.com/ipfs/go-blockservice v0.3.0/go.mod h1:P5ppi8IHDC7O+pA0AlGTF09jruB2h+oP3wVVaZl8sfk= github.com/ipfs/go-blockservice v0.4.0 h1:7MUijAW5SqdsqEW/EhnNFRJXVF8mGU5aGhZ3CQaCWbY= github.com/ipfs/go-blockservice v0.4.0/go.mod h1:kRjO3wlGW9mS1aKuiCeGhx9K1DagQ10ACpVO59qgAx4= @@ -904,13 +890,11 @@ github.com/ipfs/go-ipfs-ds-help v1.0.0/go.mod h1:ujAbkeIgkKAWtxxNkoZHWLCyk5JpPoK github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= -github.com/ipfs/go-ipfs-exchange-interface v0.1.0 h1:TiMekCrOGQuWYtZO3mf4YJXDIdNgnKWZ9IE3fGlnWfo= github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI= github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0= github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= -github.com/ipfs/go-ipfs-exchange-offline v0.2.0 h1:2PF4o4A7W656rC0RxuhUace997FTcDTcIQ6NoEtyjAI= github.com/ipfs/go-ipfs-exchange-offline v0.2.0/go.mod h1:HjwBeW0dvZvfOMwDP0TSKXIHf2s+ksdP4E3MLDRtLKY= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= @@ -1139,7 +1123,6 @@ github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/klauspost/cpuid/v2 v2.0.14 h1:QRqdp6bb9M9S5yyKeYteXKuoKE4p0tGlra81fKOpWH8= github.com/klauspost/cpuid/v2 v2.0.14/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= @@ -1148,8 +1131,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= -github.com/koron/go-ssdp v0.0.2 h1:fL3wAoyT6hXHQlORyXUW4Q23kkQpJRgEAYcZB5BR71o= github.com/koron/go-ssdp v0.0.2/go.mod h1:XoLfkAiA2KeZsYh4DbHxD7h3nR2AZNqVQOa+LJuqPYs= +github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -1169,8 +1152,8 @@ github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpz github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= github.com/libp2p/go-addr-util v0.1.0/go.mod h1:6I3ZYuFr2O/9D+SoyM0zEw0EF3YkldtTX406BpdQMqw= github.com/libp2p/go-buffer-pool v0.0.1/go.mod h1:xtyIz9PMobb13WaxR6Zo1Pd1zXJKYg0a8KiIvDp3TzQ= -github.com/libp2p/go-buffer-pool v0.0.2 h1:QNK2iAFa8gjAe1SPz6mHSMuCcjs+X1wlHzeOSqcmlfs= github.com/libp2p/go-buffer-pool v0.0.2/go.mod h1:MvaB6xw5vOrDl8rYZGLFdKAuk/hRoRZd1Vi32+RXyFM= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38yPW7c= github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= @@ -1209,8 +1192,7 @@ github.com/libp2p/go-libp2p v0.18.0/go.mod h1:+veaZ9z1SZQhmc5PW78jvnnxZ89Mgvmh4c github.com/libp2p/go-libp2p v0.19.4/go.mod h1:MIt8y481VDhUe4ErWi1a4bvt/CjjFfOq6kZTothWIXY= github.com/libp2p/go-libp2p v0.20.0/go.mod h1:g0C5Fu+aXXbCXkusCzLycuBowEih3ElmDqtbo61Em7k= github.com/libp2p/go-libp2p v0.20.1/go.mod h1:XgJHsOhEBVBXp/2Sj9bm/yEyD94uunAaP6oaegdcKks= -github.com/libp2p/go-libp2p v0.20.3 h1:tjjDNfp7FqdI/7v1rXtB/BtELaPlAThL2uzlj18kcrw= -github.com/libp2p/go-libp2p v0.20.3/go.mod h1:I+vndVanE/p/SjFbnA+BEmmfAUEpWxrdXZeyQ1Dus5c= +github.com/libp2p/go-libp2p v0.21.0 h1:s9yYScuIFY33FOOzwTXbc8QqbvsRyKIWFf0FCSJKrfM= github.com/libp2p/go-libp2p v0.21.0/go.mod h1:zvcA6/C4mr5/XQarRICh+L1SN9dAHHlSWDq4x5VYxg4= github.com/libp2p/go-libp2p-asn-util v0.0.0-20200825225859-85005c6cf052/go.mod h1:nRMRTab+kZuk0LnKZpxhOVH/ndsdr2Nr//Zltc/vwgo= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= @@ -1241,7 +1223,6 @@ github.com/libp2p/go-libp2p-circuit v0.1.3/go.mod h1:Xqh2TjSy8DD5iV2cCOMzdynd6h8 github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo= github.com/libp2p/go-libp2p-circuit v0.4.0/go.mod h1:t/ktoFIUzM6uLQ+o1G6NuBl2ANhBKN9Bc8jRIk31MoA= -github.com/libp2p/go-libp2p-circuit v0.6.0 h1:rw/HlhmUB3OktS/Ygz6+2XABOmHKzZpPUuMNUMosj8w= github.com/libp2p/go-libp2p-circuit v0.6.0/go.mod h1:kB8hY+zCpMeScyvFrKrGicRdid6vNXbunKE4rXATZ0M= github.com/libp2p/go-libp2p-connmgr v0.1.1/go.mod h1:wZxh8veAmU5qdrfJ0ZBLcU8oJe9L82ciVP/fl1VHjXk= github.com/libp2p/go-libp2p-connmgr v0.2.4/go.mod h1:YV0b/RIm8NGPnnNWM7hG9Q38OeQiQfKhHCCs1++ufn0= @@ -1284,12 +1265,13 @@ github.com/libp2p/go-libp2p-core v0.12.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQR github.com/libp2p/go-libp2p-core v0.13.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= github.com/libp2p/go-libp2p-core v0.14.0/go.mod h1:tLasfcVdTXnixsLB0QYaT1syJOhsbrhG7q6pGrHtBg8= github.com/libp2p/go-libp2p-core v0.15.1/go.mod h1:agSaboYM4hzB1cWekgVReqV5M4g5M+2eNNejV+1EEhs= -github.com/libp2p/go-libp2p-core v0.16.1 h1:bWoiEBqVkpJ13hbv/f69tHODp86t6mvc4fBN4DkK73M= github.com/libp2p/go-libp2p-core v0.16.1/go.mod h1:O3i/7y+LqUb0N+qhzXjBjjpchgptWAVMG1Voegk7b4c= github.com/libp2p/go-libp2p-core v0.19.0/go.mod h1:AkA+FUKQfYt1FLNef5fOPlo/naAWjKy/RCjkcPjqzYg= +github.com/libp2p/go-libp2p-core v0.19.1 h1:zaZQQCeCrFMtxFa1wHy6AhsVynyNmZAvwgWqSSPT3WE= github.com/libp2p/go-libp2p-core v0.19.1/go.mod h1:2uLhmmqDiFY+dw+70KkBLeKvvsJHGWUINRDdeV1ip7k= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= +github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-daemon v0.2.2/go.mod h1:kyrpsLB2JeNYR2rvXSVWyY0iZuRIMhqzWR3im9BV6NQ= github.com/libp2p/go-libp2p-discovery v0.0.5/go.mod h1:YtF20GUxjgoKZ4zmXj8j3Nb2TUSBHFlOCetzYdbZL5I= @@ -1314,8 +1296,10 @@ github.com/libp2p/go-libp2p-interface-connmgr v0.0.4/go.mod h1:GarlRLH0LdeWcLnYM github.com/libp2p/go-libp2p-interface-connmgr v0.0.5/go.mod h1:GarlRLH0LdeWcLnYM/SaBykKFl9U5JFnbBGruAk/D5k= github.com/libp2p/go-libp2p-interface-pnet v0.0.1/go.mod h1:el9jHpQAXK5dnTpKA4yfCNBZXvrzdOU75zz+C6ryp3k= github.com/libp2p/go-libp2p-kad-dht v0.2.1/go.mod h1:k7ONOlup7HKzQ68dE6lSnp07cdxdkmnRa+6B4Fh9/w0= -github.com/libp2p/go-libp2p-kad-dht v0.15.0 h1:Ke+Oj78gX5UDXnA6HBdrgvi+fStJxgYTDa51U0TsCLo= github.com/libp2p/go-libp2p-kad-dht v0.15.0/go.mod h1:rZtPxYu1TnHHz6n1RggdGrxUX/tA1C2/Wiw3ZMUDrU0= +github.com/libp2p/go-libp2p-kad-dht v0.16.1-0.20220811151450-6a2603e9927e/go.mod h1:zeE26Xo+PY7sS2AgkBQQcBnJEazMT26KGZLUFttl+rk= +github.com/libp2p/go-libp2p-kad-dht v0.17.0 h1:HWEjqjNVDuf8yuccuswGy1vYGzB0v4Z+yQ4DMDMSIqk= +github.com/libp2p/go-libp2p-kad-dht v0.17.0/go.mod h1:zeE26Xo+PY7sS2AgkBQQcBnJEazMT26KGZLUFttl+rk= github.com/libp2p/go-libp2p-kbucket v0.2.1/go.mod h1:/Rtu8tqbJ4WQ2KTCOMJhggMukOLNLNPY1EtEWWLxUvc= github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= github.com/libp2p/go-libp2p-kbucket v0.4.7 h1:spZAcgxifvFZHBD8tErvppbnNiKA5uokDu3CV7axu70= @@ -1365,8 +1349,8 @@ github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuD github.com/libp2p/go-libp2p-peerstore v0.2.8/go.mod h1:gGiPlXdz7mIHd2vfAsHzBNAMqSDkt2UBFwgcITgw1lA= github.com/libp2p/go-libp2p-peerstore v0.4.0/go.mod h1:rDJUFyzEWPpXpEwywkcTYYzDHlwza8riYMaUzaN6hX0= github.com/libp2p/go-libp2p-peerstore v0.6.0/go.mod h1:DGEmKdXrcYpK9Jha3sS7MhqYdInxJy84bIPtSu65bKc= -github.com/libp2p/go-libp2p-peerstore v0.7.0 h1:2iIUwok3vtmnWJTZeTeLgnBO6GbkXcwSRwgZHEKrQZs= github.com/libp2p/go-libp2p-peerstore v0.7.0/go.mod h1:cdUWTHro83vpg6unCpGUr8qJoX3e93Vy8o97u5ppIM0= +github.com/libp2p/go-libp2p-peerstore v0.7.1 h1:7FpALlqR+3+oOBXdzm3AVt0vjMYLW1b7jM03E4iEHlw= github.com/libp2p/go-libp2p-peerstore v0.7.1/go.mod h1:cdUWTHro83vpg6unCpGUr8qJoX3e93Vy8o97u5ppIM0= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-protocol v0.0.1/go.mod h1:Af9n4PiruirSDjHycM1QuiMi/1VZNHYcK8cLgFJLZ4s= @@ -1394,10 +1378,10 @@ github.com/libp2p/go-libp2p-record v0.1.3 h1:R27hoScIhQf/A8XJZ8lYpnqh9LatJ5YbHs2 github.com/libp2p/go-libp2p-record v0.1.3/go.mod h1:yNUff/adKIfPnYQXgp6FQmNu3gLJ6EMg7+/vv2+9pY4= github.com/libp2p/go-libp2p-resource-manager v0.1.5/go.mod h1:wJPNjeE4XQlxeidwqVY5G6DLOKqFK33u2n8blpl0I6Y= github.com/libp2p/go-libp2p-resource-manager v0.2.1/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= -github.com/libp2p/go-libp2p-resource-manager v0.3.0 h1:2+cYxUNi33tcydsVLt6K5Fv2E3OTiVeafltecAj15E0= github.com/libp2p/go-libp2p-resource-manager v0.3.0/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= -github.com/libp2p/go-libp2p-resource-manager v0.5.1 h1:jm0mdqn7yfh7wbUzlj948BYZX0KZ3RW7OqerkGQ5rYY= github.com/libp2p/go-libp2p-resource-manager v0.5.1/go.mod h1:CggtV6EZb+Y0dGh41q5ezO4udcVKyhcEFpydHD8EMe0= +github.com/libp2p/go-libp2p-resource-manager v0.5.3 h1:W8rG2abNBO52SRQYj24AvKmutTJZfoc1OrgzGQPwcRU= +github.com/libp2p/go-libp2p-resource-manager v0.5.3/go.mod h1:CggtV6EZb+Y0dGh41q5ezO4udcVKyhcEFpydHD8EMe0= github.com/libp2p/go-libp2p-routing v0.0.1/go.mod h1:N51q3yTr4Zdr7V8Jt2JIktVU+3xBBylx1MZeVA6t1Ys= github.com/libp2p/go-libp2p-routing v0.1.0/go.mod h1:zfLhI1RI8RLEzmEaaPwzonRvXeeSHddONWkcTcB54nE= github.com/libp2p/go-libp2p-routing-helpers v0.2.3 h1:xY61alxJ6PurSi+MXbywZpelvuU4U4p/gPTxjqCqTzY= @@ -1439,8 +1423,8 @@ github.com/libp2p/go-libp2p-testing v0.6.0/go.mod h1:QBk8fqIL1XNcno/l3/hhaIEn4aL github.com/libp2p/go-libp2p-testing v0.7.0/go.mod h1:OLbdn9DbgdMwv00v+tlp1l3oe2Cl+FAjoWIA2pa0X6E= github.com/libp2p/go-libp2p-testing v0.8.0/go.mod h1:gRdsNxQSxAZowTgcLY7CC33xPmleZzoBpqSYbWenqPc= github.com/libp2p/go-libp2p-testing v0.9.0/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= -github.com/libp2p/go-libp2p-testing v0.9.2 h1:dCpODRtRaDZKF8HXT9qqqgON+OMEB423Knrgeod8j84= github.com/libp2p/go-libp2p-testing v0.9.2/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= +github.com/libp2p/go-libp2p-testing v0.11.0 h1:+R7FRl/U3Y00neyBSM2qgDzqz3HkWH24U9nMlascHL4= github.com/libp2p/go-libp2p-testing v0.11.0/go.mod h1:qG4sF27dfKFoK9KlVzK2y52LQKhp0VEmLjV5aDqr1Hg= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-tls v0.3.0/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= @@ -1463,6 +1447,7 @@ github.com/libp2p/go-libp2p-transport-upgrader v0.6.0/go.mod h1:1e07y1ZSZdHo9HPb github.com/libp2p/go-libp2p-transport-upgrader v0.7.0/go.mod h1:GIR2aTRp1J5yjVlkUoFqMkdobfob6RnAwYg/RZPhrzg= github.com/libp2p/go-libp2p-transport-upgrader v0.7.1/go.mod h1:GIR2aTRp1J5yjVlkUoFqMkdobfob6RnAwYg/RZPhrzg= github.com/libp2p/go-libp2p-xor v0.0.0-20210714161855-5c005aca55db/go.mod h1:LSTM5yRnjGZbWNTA/hRwq2gGFrvRIbQJscoIL/u6InY= +github.com/libp2p/go-libp2p-xor v0.1.0/go.mod h1:LSTM5yRnjGZbWNTA/hRwq2gGFrvRIbQJscoIL/u6InY= github.com/libp2p/go-libp2p-yamux v0.1.2/go.mod h1:xUoV/RmYkg6BW/qGxA9XJyg+HzXFYkeXbnhjmnYzKp8= github.com/libp2p/go-libp2p-yamux v0.1.3/go.mod h1:VGSQVrqkh6y4nm0189qqxMtvyBft44MOYYPpYKXiVt4= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= @@ -1496,6 +1481,7 @@ github.com/libp2p/go-mplex v0.2.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= github.com/libp2p/go-mplex v0.4.0/go.mod h1:y26Lx+wNVtMYMaPu300Cbot5LkEZ4tJaNYeHeT9dh6E= github.com/libp2p/go-mplex v0.6.0/go.mod h1:y26Lx+wNVtMYMaPu300Cbot5LkEZ4tJaNYeHeT9dh6E= +github.com/libp2p/go-mplex v0.7.0 h1:BDhFZdlk5tbr0oyFq/xv/NPGfjbnrsDam1EvutpBDbY= github.com/libp2p/go-mplex v0.7.0/go.mod h1:rW8ThnRcYWft/Jb2jeORBmPd6xuG3dGxWN/W168L9EU= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= @@ -1592,8 +1578,8 @@ github.com/lucas-clemente/quic-go v0.23.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2 github.com/lucas-clemente/quic-go v0.24.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg= github.com/lucas-clemente/quic-go v0.27.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= -github.com/lucas-clemente/quic-go v0.27.1 h1:sOw+4kFSVrdWOYmUjufQ9GBVPqZ+tu+jMtXxXNmRJyk= github.com/lucas-clemente/quic-go v0.27.1/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= +github.com/lucas-clemente/quic-go v0.28.0 h1:9eXVRgIkMQQyiyorz/dAaOYIx3TFzXsIFkNFz4cxuJM= github.com/lucas-clemente/quic-go v0.28.0/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= @@ -1621,13 +1607,14 @@ github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtU github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= github.com/marten-seemann/qtls-go1-17 v0.1.0-rc.1/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8= github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8= -github.com/marten-seemann/qtls-go1-17 v0.1.1 h1:DQjHPq+aOzUeh9/lixAGunn6rIOQyWChPSI4+hgW7jc= github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= +github.com/marten-seemann/qtls-go1-17 v0.1.2 h1:JADBlm0LYiVbuSySCHeY863dNkcpMmDR7s0bLKJeYlQ= github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI= -github.com/marten-seemann/qtls-go1-18 v0.1.1 h1:qp7p7XXUFL7fpBvSS1sWD+uSqPvzNQK43DH+/qEkj0Y= github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM= github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 h1:7m/WlWcSROrcK5NxuXaxYD32BZqe/LEEnBrWcH/cOqQ= github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= @@ -1677,8 +1664,8 @@ github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= @@ -1743,8 +1730,8 @@ github.com/multiformats/go-multiaddr v0.3.1/go.mod h1:uPbspcUPd5AfaP6ql3ujFY+QWz github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= github.com/multiformats/go-multiaddr v0.4.0/go.mod h1:YcpyLH8ZPudLxQlemYBPhSm0/oCXAT8Z4mzFpyoPyRc= github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= -github.com/multiformats/go-multiaddr v0.5.0 h1:i/JuOoVg4szYQ4YEzDGtb2h0o8M7CG/Yq6cGlcjWZpM= github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar+bR+gh4zgbfr3SNug= +github.com/multiformats/go-multiaddr v0.6.0 h1:qMnoOPj2s8xxPU5kZ57Cqdr0hHhARz7mFsPMIiYNqzg= github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= @@ -1766,7 +1753,6 @@ github.com/multiformats/go-multiaddr-net v0.1.4/go.mod h1:ilNnaM9HbmVFqsb/qcNysj github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysjCu4PVONlrBZpHIrw/qQuA= github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= -github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= @@ -1954,8 +1940,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.33.0 h1:rHgav/0a6+uYgGdNt3jwz8FNSesO/Hsang3O0T9A5SE= github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289/go.mod h1:FGbBv5OPKjch+jNUJmEQpMZytIdyW0NdBtWFcfSKusc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1978,7 +1964,6 @@ github.com/prometheus/statsd_exporter v0.21.0/go.mod h1:rbT83sZq2V+p73lHhPZfMc3M github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/raulk/clock v1.1.0 h1:dpb29+UKMbLqiU/jqIJptgLR1nn23HLgMY0sTCDza5Y= github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= -github.com/raulk/go-watchdog v1.2.0 h1:konN75pw2BMmZ+AfuAm5rtFsWcJpKF3m02rKituuXNo= github.com/raulk/go-watchdog v1.2.0/go.mod h1:lzSbAl5sh4rtI8tYHU01BWIDzgzqaQLj6RcA1i4mlqI= github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= @@ -2090,12 +2075,12 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g= @@ -2245,7 +2230,6 @@ go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I= -go.opentelemetry.io/otel v1.3.0 h1:APxLf0eiBwLl+SOXiJJCVYzA1OOJNyAoV8C5RNRyy7Y= go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs= go.opentelemetry.io/otel v1.7.0 h1:Z2lA3Tdch0iDcrhJXDIlC94XE+bxok1F9B+4Lz/lGsM= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= @@ -2262,7 +2246,6 @@ go.opentelemetry.io/otel/sdk/export/metric v0.25.0/go.mod h1:Ej7NOa+WpN49EIcr1HM go.opentelemetry.io/otel/sdk/metric v0.25.0/go.mod h1:G4xzj4LvC6xDDSsVXpvRVclQCbofGGg4ZU2VKKtDRfg= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= -go.opentelemetry.io/otel/trace v1.3.0 h1:doy8Hzb1RJ+I3yFhtDmwNc7tIyw1tNMOIsyPzp1NOGY= go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk= go.opentelemetry.io/otel/trace v1.7.0 h1:O37Iogk1lEkMRXewVtZ1BBTVn5JEp8GrJvP92bJqC6o= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= @@ -2377,7 +2360,6 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20210615023648-acb5c1269671/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/exp v0.0.0-20210714144626-1041f73d31d8/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= -golang.org/x/exp v0.0.0-20210715201039-d37aa40e8013 h1:Jp57DBw4K7mimZNA3F9f7CndVcUt4kJjmyJf2rzJHoI= golang.org/x/exp v0.0.0-20210715201039-d37aa40e8013/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5 h1:rxKZ2gOnYxjfmakvUUqh9Gyb6KXfrj7JWTxORTYqb0E= golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= @@ -2483,10 +2465,10 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2 h1:6mzvA99KwZxbOrxww4EvWVQUnN1+xEu9tafK5ZxkYeA= golang.org/x/net v0.0.0-20220418201149-a630d4f3e7a2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220630215102-69896b714898 h1:K7wO6V1IrczY9QOQ2WkVpw4JQSwCd52UsxVEirZUfiw= golang.org/x/net v0.0.0-20220630215102-69896b714898/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2629,7 +2611,6 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150 h1:xHms4gcpe1YE7A3yIllJXP16CMAGuqwO2lX1mTyyRRc= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2732,7 +2713,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= @@ -2884,8 +2864,8 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= diff --git a/node/modules/rcmgr.go b/node/modules/rcmgr.go index dce8f3cb2..d6cf07882 100644 --- a/node/modules/rcmgr.go +++ b/node/modules/rcmgr.go @@ -16,7 +16,10 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/protocol" rcmgr "github.com/libp2p/go-libp2p-resource-manager" + "github.com/libp2p/go-libp2p-resource-manager/obs" + rcmgrObs "github.com/libp2p/go-libp2p-resource-manager/obs" "go.opencensus.io/stats" + "go.opencensus.io/stats/view" "go.opencensus.io/tag" "go.uber.org/fx" ) @@ -42,36 +45,49 @@ func ResourceManager(connMgrHi uint) func(lc fx.Lifecycle, repo repo.LockedRepo) // Adjust default limits // - give it more memory, up to 4G, min of 1G - // - if maxconns are too high, adjust Conn/FD/Stream limits - defaultLimits := rcmgr.DefaultLimits.WithSystemMemory(.125, 1<<30, 4<<30) + // - if maxconns are too high, adjust Conn/FD/Stream defaultLimits + defaultLimits := rcmgr.DefaultLimits + + // TODO: also set appropriate default limits for lotus protocols + libp2p.SetDefaultServiceLimits(&defaultLimits) + + // Minimum 1GB of memory + defaultLimits.SystemBaseLimit.Memory = 1 << 30 + // For every extra 1GB of memory we have available, increase our limit by 1GiB + defaultLimits.SystemLimitIncrease.Memory = 1 << 30 + defaultLimitConfig := defaultLimits.AutoScale() + if defaultLimitConfig.System.Memory > 4<<30 { + // Cap our memory limit + defaultLimitConfig.System.Memory = 4 << 30 + } + maxconns := int(connMgrHi) - if 2*maxconns > defaultLimits.SystemBaseLimit.ConnsInbound { + if 2*maxconns > defaultLimitConfig.System.ConnsInbound { // adjust conns to 2x to allow for two conns per peer (TCP+QUIC) - defaultLimits.SystemBaseLimit.ConnsInbound = logScale(2 * maxconns) - defaultLimits.SystemBaseLimit.ConnsOutbound = logScale(2 * maxconns) - defaultLimits.SystemBaseLimit.Conns = logScale(4 * maxconns) + defaultLimitConfig.System.ConnsInbound = logScale(2 * maxconns) + defaultLimitConfig.System.ConnsOutbound = logScale(2 * maxconns) + defaultLimitConfig.System.Conns = logScale(4 * maxconns) - defaultLimits.SystemBaseLimit.StreamsInbound = logScale(16 * maxconns) - defaultLimits.SystemBaseLimit.StreamsOutbound = logScale(64 * maxconns) - defaultLimits.SystemBaseLimit.Streams = logScale(64 * maxconns) + defaultLimitConfig.System.StreamsInbound = logScale(16 * maxconns) + defaultLimitConfig.System.StreamsOutbound = logScale(64 * maxconns) + defaultLimitConfig.System.Streams = logScale(64 * maxconns) - if 2*maxconns > defaultLimits.SystemBaseLimit.FD { - defaultLimits.SystemBaseLimit.FD = logScale(2 * maxconns) + if 2*maxconns > defaultLimitConfig.System.FD { + defaultLimitConfig.System.FD = logScale(2 * maxconns) } + defaultLimitConfig.ServiceDefault.StreamsInbound = logScale(8 * maxconns) + defaultLimitConfig.ServiceDefault.StreamsOutbound = logScale(32 * maxconns) + defaultLimitConfig.ServiceDefault.Streams = logScale(32 * maxconns) - defaultLimits.ServiceBaseLimit.StreamsInbound = logScale(8 * maxconns) - defaultLimits.ServiceBaseLimit.StreamsOutbound = logScale(32 * maxconns) - defaultLimits.ServiceBaseLimit.Streams = logScale(32 * maxconns) - - defaultLimits.ProtocolBaseLimit.StreamsInbound = logScale(8 * maxconns) - defaultLimits.ProtocolBaseLimit.StreamsOutbound = logScale(32 * maxconns) - defaultLimits.ProtocolBaseLimit.Streams = logScale(32 * maxconns) + defaultLimitConfig.ProtocolDefault.StreamsInbound = logScale(8 * maxconns) + defaultLimitConfig.ProtocolDefault.StreamsOutbound = logScale(32 * maxconns) + defaultLimitConfig.ProtocolDefault.Streams = logScale(32 * maxconns) log.Info("adjusted default resource manager limits") } // initialize - var limiter *rcmgr.BasicLimiter + var limiter rcmgr.Limiter var opts []rcmgr.Option repoPath := repo.Path() @@ -82,22 +98,29 @@ func ResourceManager(connMgrHi uint) func(lc fx.Lifecycle, repo repo.LockedRepo) switch { case err == nil: defer limitsIn.Close() //nolint:errcheck - limiter, err = rcmgr.NewLimiterFromJSON(limitsIn, defaultLimits) + limiter, err = rcmgr.NewLimiterFromJSON(limitsIn, defaultLimitConfig) if err != nil { return nil, fmt.Errorf("error parsing limit file: %w", err) } case errors.Is(err, os.ErrNotExist): - limiter = rcmgr.NewStaticLimiter(defaultLimits) + limiter = rcmgr.NewFixedLimiter(defaultLimitConfig) default: return nil, err } - // TODO: also set appropriate default limits for lotus protocols - libp2p.SetDefaultServiceLimits(limiter) + str, err := rcmgrObs.NewStatsTraceReporter() + if err != nil { + return nil, fmt.Errorf("error creating resource manager stats reporter: %w", err) + } + err = view.Register(obs.DefaultViews...) + if err != nil { + return nil, fmt.Errorf("error registering rcmgr metrics: %w", err) + } - opts = append(opts, rcmgr.WithMetrics(rcmgrMetrics{})) + // Metrics + opts = append(opts, rcmgr.WithMetrics(rcmgrMetrics{}), rcmgr.WithTraceReporter(str)) if os.Getenv("LOTUS_DEBUG_RCMGR") != "" { debugPath := filepath.Join(repoPath, "debug") From 150b0ea18dd5f1a1f740e4549dece23ce2bfddcd Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Fri, 19 Aug 2022 01:30:30 -0700 Subject: [PATCH 4/7] feat(booster-bitswap): makefile & fixes add commands to build booster-bitswap, and very a round tripped successful fetch from booster-bitswap --- .gitignore | 1 + Makefile | 6 ++++++ cmd/booster-bitswap/remoteblockstore/remoteblockstore.go | 6 ++++-- cmd/booster-bitswap/server.go | 8 +++++--- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 2f11a46f3..c6c3063c7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ /boostd /devnet /booster-http +/booster-bitswap /docgen-md /docgen-openrpc extern/filecoin-ffi/rust/target diff --git a/Makefile b/Makefile index 3891100cc..dd5f7343e 100644 --- a/Makefile +++ b/Makefile @@ -98,6 +98,12 @@ booster-http: $(BUILD_DEPS) .PHONY: booster-http BINS+=booster-http +booster-bitswap: $(BUILD_DEPS) + rm -f booster-bitswap + $(GOCC) build $(GOFLAGS) -o booster-bitswap ./cmd/booster-bitswap +.PHONY: booster-bitswap +BINS+=booster-bitswap + devnet: $(BUILD_DEPS) rm -f devnet $(GOCC) build $(GOFLAGS) -o devnet ./cmd/devnet diff --git a/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go b/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go index 377c08251..4a1f5eee3 100644 --- a/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go +++ b/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go @@ -12,7 +12,7 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" ) -var logbs = logging.Logger("remote-blockstore") +var log = logging.Logger("remote-blockstore") var ErrBlockNotFound = errors.New("block not found") var ErrNotFound = errors.New("not found") @@ -42,11 +42,13 @@ func NewRemoteBlockstore(api RemoteBlockstoreAPI) blockstore.Blockstore { } func (ro *RemoteBlockstore) Get(ctx context.Context, c cid.Cid) (b blocks.Block, err error) { + log.Debugw("processing request for block", "cid", c) data, err := ro.api.BoostGetBlock(ctx, c) + log.Debugw("boost api response for get block", "cid", c, "error", err) if err != nil { return nil, err } - return blocks.NewBlock(data), nil + return blocks.NewBlockWithCid(data, c) } func (ro *RemoteBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) { diff --git a/cmd/booster-bitswap/server.go b/cmd/booster-bitswap/server.go index 722c1a227..dd95a5eb9 100644 --- a/cmd/booster-bitswap/server.go +++ b/cmd/booster-bitswap/server.go @@ -11,8 +11,8 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" nilrouting "github.com/ipfs/go-ipfs-routing/none" "github.com/libp2p/go-libp2p" + crypto "github.com/libp2p/go-libp2p-core/crypto" "github.com/libp2p/go-libp2p-core/network" - crypto "github.com/libp2p/go-libp2p-crypto" "github.com/libp2p/go-libp2p/p2p/muxer/mplex" "github.com/libp2p/go-libp2p/p2p/muxer/yamux" quic "github.com/libp2p/go-libp2p/p2p/transport/quic" @@ -61,9 +61,11 @@ func (s *BitswapServer) Start(ctx context.Context) error { return err } bsopts := []server.Option{server.MaxOutstandingBytesPerPeer(1 << 20)} - s.server = server.New(ctx, bsnetwork.NewFromIpfsHost(host, nilRouter), s.remoteStore, bsopts...) + net := bsnetwork.NewFromIpfsHost(host, nilRouter) + s.server = server.New(ctx, net, s.remoteStore, bsopts...) + net.Start(s.server) - fmt.Printf("bitswap server running on SP, addrs: %s, peerID: %s", host.Addrs(), host.ID()) + fmt.Printf("bitswap server running on SP, addrs: %s, peerID: %s\n", host.Addrs(), host.ID()) log.Infow("bitswap server running on SP", "multiaddrs", host.Addrs(), "peerId", host.ID()) return nil } From 6c15199683d3b67304e1e862b4b998c0b20330e7 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 24 Aug 2022 16:17:28 +0200 Subject: [PATCH 5/7] refactor: clean up unused vars etc --- api/api.go | 7 +- api/proxy_gen.go | 52 +++++++++++---- build/openrpc/boost.json.gz | Bin 4922 -> 5070 bytes .../remoteblockstore/remoteblockstore.go | 44 +++++------- cmd/booster-bitswap/run.go | 28 +------- cmd/booster-bitswap/server.go | 8 +-- documentation/en/api-v1-methods.md | 63 +++++++++++++++--- go.mod | 1 - go.sum | 1 - node/config/doc_gen.go | 31 +++++++++ node/impl/boost.go | 10 ++- 11 files changed, 160 insertions(+), 85 deletions(-) diff --git a/api/api.go b/api/api.go index cc550f9d2..b86eb0557 100644 --- a/api/api.go +++ b/api/api.go @@ -47,7 +47,12 @@ type Boost interface { BoostDagstoreGC(ctx context.Context) ([]DagstoreShardResult, error) //perm:admin BoostDagstorePiecesContainingMultihash(ctx context.Context, mh multihash.Multihash) ([]cid.Cid, error) //perm:read BoostDagstoreListShards(ctx context.Context) ([]DagstoreShardInfo, error) //perm:read - BoostGetBlock(ctx context.Context, c cid.Cid) ([]byte, error) //perm:read + + // MethodGroup: Blockstore + BlockstoreGet(ctx context.Context, c cid.Cid) ([]byte, error) //perm:read + BlockstoreHas(ctx context.Context, c cid.Cid) (bool, error) //perm:read + BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) //perm:read + // RuntimeSubsystems returns the subsystems that are enabled // in this instance. RuntimeSubsystems(ctx context.Context) (lapi.MinerSubsystems, error) //perm:read diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 8b56db2dc..03f23469f 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -38,6 +38,12 @@ type BoostStruct struct { Internal struct { ActorSectorSize func(p0 context.Context, p1 address.Address) (abi.SectorSize, error) `perm:"read"` + BlockstoreGet func(p0 context.Context, p1 cid.Cid) ([]byte, error) `perm:"read"` + + BlockstoreGetSize func(p0 context.Context, p1 cid.Cid) (int, error) `perm:"read"` + + BlockstoreHas func(p0 context.Context, p1 cid.Cid) (bool, error) `perm:"read"` + BoostDagstoreDestroyShard func(p0 context.Context, p1 string) error `perm:"admin"` BoostDagstoreGC func(p0 context.Context) ([]DagstoreShardResult, error) `perm:"admin"` @@ -60,8 +66,6 @@ type BoostStruct struct { BoostDummyDeal func(p0 context.Context, p1 smtypes.DealParams) (*ProviderDealRejectionInfo, error) `perm:"admin"` - BoostGetBlock func(p0 context.Context, p1 cid.Cid) ([]byte, error) `perm:"read"` - BoostIndexerAnnounceAllDeals func(p0 context.Context) error `perm:"admin"` BoostOfflineDealWithData func(p0 context.Context, p1 uuid.UUID, p2 string) (*ProviderDealRejectionInfo, error) `perm:"admin"` @@ -264,6 +268,39 @@ func (s *BoostStub) ActorSectorSize(p0 context.Context, p1 address.Address) (abi return *new(abi.SectorSize), ErrNotSupported } +func (s *BoostStruct) BlockstoreGet(p0 context.Context, p1 cid.Cid) ([]byte, error) { + if s.Internal.BlockstoreGet == nil { + return *new([]byte), ErrNotSupported + } + return s.Internal.BlockstoreGet(p0, p1) +} + +func (s *BoostStub) BlockstoreGet(p0 context.Context, p1 cid.Cid) ([]byte, error) { + return *new([]byte), ErrNotSupported +} + +func (s *BoostStruct) BlockstoreGetSize(p0 context.Context, p1 cid.Cid) (int, error) { + if s.Internal.BlockstoreGetSize == nil { + return 0, ErrNotSupported + } + return s.Internal.BlockstoreGetSize(p0, p1) +} + +func (s *BoostStub) BlockstoreGetSize(p0 context.Context, p1 cid.Cid) (int, error) { + return 0, ErrNotSupported +} + +func (s *BoostStruct) BlockstoreHas(p0 context.Context, p1 cid.Cid) (bool, error) { + if s.Internal.BlockstoreHas == nil { + return false, ErrNotSupported + } + return s.Internal.BlockstoreHas(p0, p1) +} + +func (s *BoostStub) BlockstoreHas(p0 context.Context, p1 cid.Cid) (bool, error) { + return false, ErrNotSupported +} + func (s *BoostStruct) BoostDagstoreDestroyShard(p0 context.Context, p1 string) error { if s.Internal.BoostDagstoreDestroyShard == nil { return ErrNotSupported @@ -385,17 +422,6 @@ func (s *BoostStub) BoostDummyDeal(p0 context.Context, p1 smtypes.DealParams) (* return nil, ErrNotSupported } -func (s *BoostStruct) BoostGetBlock(p0 context.Context, p1 cid.Cid) ([]byte, error) { - if s.Internal.BoostGetBlock == nil { - return *new([]byte), ErrNotSupported - } - return s.Internal.BoostGetBlock(p0, p1) -} - -func (s *BoostStub) BoostGetBlock(p0 context.Context, p1 cid.Cid) ([]byte, error) { - return *new([]byte), ErrNotSupported -} - func (s *BoostStruct) BoostIndexerAnnounceAllDeals(p0 context.Context) error { if s.Internal.BoostIndexerAnnounceAllDeals == nil { return ErrNotSupported diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index ce1230bc7d4f93fa065dba073523e4a17cbdd835..2124f961d84d7eb34d8b2ce0046e4d9bdb0c2cc6 100644 GIT binary patch literal 5070 zcmV;<6EW-`iwFP!00000|Li?&bJ{q!f5qebrRh*YfRHvb`ynBH;n}pC8`|AxcbmBt zwg7F6>{@afn&H3Sk^F8;wror`rJi^<-qG)%OOTCuk72j>!ZnWcO-Elry0lHApZ@)zW3Xa9 z-yjPbHp1OAgWB#C@_*5HsCRS|62nCf^#OhP@?~Lv#&ZuF+B2d(Keuf{r_}WfdZ1-G zKXZDTVPgvKX?N`Z-_e{S4Vb123F)5s{~u`2zzlWcejN?EuI(OZ51;X8isl;WPB4Wo z238h7?U!FPj4Z$9XX--gxmbIS;kOLTgO2vvbwAj2>N$?>QfRUs8}?7oN&b>&PLJVz z@ELDD<1b&n=pCJSs}*q9`lP z-o%3oTB;3B@Ql5}riLikqi=$>oor}~mohnyZBE_jo! zVXqG70sFmhT$@e$;f%4>;mopUhburRbPs`p4jtFNU;p0%;_kx!pUeKJr*Af$JP)5+ zwsA|S?ZUT^R^#AQR~i2r$n1`hd7wS}`)0PLh5VWVSir45q)J?x#r3K2XfRFXgw5v` zQ*B~GOwk+(hfbv28*_d-iRKjHQOKi5Q);cdR^%B7^<GmtE$QK-*h-F{O>!`*%gt=?u-vv_Z%Sap`aytyY)SBH zAdv*D^95LNcDaCry7qdy1g=?8D#*adB!s`j2fkzw3I-WOv5=@B9STt0xuggvTgk%p z+iG{f@?dT2R>U12JS)!~!NY{iiu9mm{m`^O=nWNU|Lu4Sihny6&^h3wYLN{Fix|-LX8^PA3FZT5WhzeXkajSAZ`TIft6|T7gT7VG*Lju89Xsqb>69jGt z1Cz|j5{>#NVD^28hNIE3IrqN1gRgTkoUMR4GVccnEzGZaN1u9z0ZANyd(9R~Kh-~b zz23m!%f>%KeRFfu(Gi6K#}|h2oW!Gt5N60=Av^o^I@G}XD+r`lZV7&Fb;!dgxI(J$p`8Y$8eb%C+y130>Ba zyCvm@Ed-XB+A*@K+p9jpJE!ny@M=WfJMsU#5d0JV*^QyP0{0Ui=+UU7uMkdrYs!d~ zuYL~i!lvaD`Jl7*pFCvhr)rbh(Z?=i1306sORsnIg8%*4>z(qyf3YSOf|%0|^UiGo zEcV{iXXgKmCl~B<6NYxjhsaZrPLFznenv;?`E5S85QgcIUP0)xN!E}07lU7I`;WKZ z@4x^0*PnOr-G8a`c63JEPyabL{LM_$`+Ki<3VFYkW+FP-ZhP95yE*S1ZY zTdE13*&a5TM7;uZ87a6xX?66EF2H0C-FW!f7XI+~&@#VV1>z<@PaWMSk71up^b#P< zhR0wW4;jH+c{2-gd1C=LFJ_x-KDJ&}H~-yoAQBV#)fS&3>lFVIO4>b z!`&yy^0|ro2EoEjN8f6+rJZuG%S)N65Mp7cLJWc6too9qGAN9Z2!e?#Z;e9e?NG(*iO3f-&u)dEQ6 znZa8++JbjB>wAj_)#eiaY`Mg3NT3Y~v>|~uB+!Ng+K@mS5@>Z`Af|9 z)=aG&?v@&5`EYkpQx#+8%rKxsV+^k$Ac?HoKwDK*LorYhL4>bRI$`z$oBBsZH4i4( zg!j-r!`Sw)0naRptzexqqY_BMgQLZ{;}S@XWRJ_SeC#TPZFcwNuwQhdKal2Zef~gn z$&I?|WGR4Bn)}_+oPKo9VH@96@bc!oRPo5zNErTYqWCTn#ba7^`cLzKNUOB>LTL|1 zV#&z0)!4>_ty>@gt|4_1yaRPqcR4(9<%hCje&ui*s_>^YfSnY3Xe-NQiFPO{Od?vN z!24~8WRuXEBiJO=9g=GX`IcB8g;zs z1i=zDC0ujB8Y9-7i&hcpPr=1+={if-FC`@oQtSB%-vxp0hG<(1umSoO1KtIVONY=3 zX}%3GKji>(D=RLKUr7=H-UIg*(lNjW|a;at^<_uxMN z4B>QvijQao2?Z-BE=qClAcj;}k^+AHU+RqoV@@am5Wt0wQ$E&Q;JVRmC5EC8V|G~;+?AJ-HQD2gS5lPKxzo@>NNP>{) z3pZ>MNJLd7kp$hWw#Xp88?_-Ac3xMA8>uLWusFy0nuf~Y6DGW_O{^ZSzNb*m77MZLVFNKv? z>PUt(gpxYzsYC50?rIuvGp=e5*+)64-quk67i*|L{nPS4n=3n%r2hCUdWi^4#*!*- zPDy~sa=Z>F0nnTl{fCCKyo!sO^TlGa*xmdC@li`|q}AETNzR`+Gl3^VQ&RkEv6 zEiSp-RSoA9AqmFgQ$Rh$IME1Trs!t-P(-#SpU*Lm{a#^^B#>%VDq2^qfbFW5_>N?p zQW=}QIL@&X>BY zYh9<9sj7u7Srw5r9hVx(YL0P5ZW{{gbb0pEGePu1UgfseX;pydrDFig(AflUCqLJ!;kePg^YtRqC@! zRf)6`W|tZ%0;*bFUu1+-SWs03B8r102`MsJY8Lw6uRy^_a_}tnfwwuDEQ2Bo6J&D; zDqU4Ads$(;p!AJ3BL|B08y~kUp2Bbi5eJn%YOx2fwrpTFyVt>7Atvc8F6$WJYw%6x zp-1*a;Ed%HPpYPwh*DLUeunHVLw-4~FYYd$p3lXngI6NoNifKl7vUvpvCpb_ukhw3 z5m4^2f%E$vx7mNPDX^y8U}h^5xDny4k$w`YESxa*TsKD-mv8}$^_6Q=+pq%#kfMP6?EEmk1*1tN)ARW-c_16oI$x? zZo!GJSWZO&l%FH4p$Rew@4QA%ds1-U zPYSl8=JMo7QqO`b3Gx=w@#I2yVQ8V0$@VTej(DIwd+zWq)OKMv=v1_cmVH4ZrzP_vXLk8d_nLndy{N-e#xq9kk6& zMa@)y4w1&KB7ZJ9Q{2+~kYaOGM}x znT%5^CLQPdZ*f6p9$4d-V+!$YFUH&h)s<5-dzB<~axAHH6ce=nsV@8apBiCa|I-dx zH^Q}L+?H{lI^&Kc)j_U3OwkHXy%|{(zP^<>H`iUvy1bGWKPVfwEaKuE*a9{d!LG#7 zqA`eBI**nQjc9ZUH4;CjOF;P%ABe%sVsA&-Kh=XUp#U4O`x$R3AVlNNEuN&2g4odg k&0?a9IloGvBo`RjFV;Fa;eq@A4*&rF|BvRf8?@>G0G4yoYXATM literal 4922 zcmV-A6UFQwiwFP!00000|Li?~bK1DJU&Z71Pt&1<03mH=`bR?AUhbydywL96yW7mG zumxyiWLJ{Y&&e4PEdJq%OGEdE;cDFq+TPDI& z*F1eN8Db7JPt8l0`P8_+8X9L;BeP}RK|+y>MUQ^F^NLtU=Bi~bAe*~3HBbM1Ffmy0 zp6`(ZEf?YTnZ;Z(hT>oJ6Phj4hSVa+V*#LV-@eVk1&G- z11F20@%F8OkrTB1$_QjW!NyAre`H`Dw2XIzd~(^?_dJ&{X!9N`{!iFR{gM}6hvR+l z6|cVHZ{NO|EtC3-1t3fF)cib$1R8)q1G~&vLS_KGG%RJo#^5jBEIZ<{UC z0|YFBDf0=R{K@i_=TanQIQ%)aINnii88TCt!iayd=5PTxX~281@EpkDnbXcux7Rms zTD-e>9dNFQ>p{X0Z!rZ9g)NgUJ;L)Q$bc1d<$>PvWtGr~!<1lf)CWl=V;J90h6D7)E^9v?WzBh^uAANf3g7fz`pMxG_${(EpzN!7FUrG?Y`p+WuBT} z{Z6N6iDeTXp}D%bX_<(@kmC!Lq?dT~5W$QXEMn(XD;S34`5LSBZxh#bAi$ig^C~zY zDgYs1`MjC;WT3&kBa9dV4*ChV)m<4#^{B55&{{QE9^pob?V}QV^IC%sE)ivd4Codt z5ChRtY9l;#wN+XSTVJ*D7gSelsnI-!g5&6JQg30KM^0GJ=g2J(cteZ1T9E8D*}WOo_+KkhUS3Iw_tZAOqK3iL=D>Ora~%M$Zijj ztt|_bFD?`f;ajQ;grf28Ty;w3PEU`X|9Jb>Oz~lrP)9DJQIUt_VlZ49iSUbm(Gq!w zLf}o8-N{d2LD2g_?fGEY1HA#8^yn>Icz?O~-#vfnF1)`ciz!@Ad;KZu`OAUt!+!U! zpA}lsWC=RP%oy1aGc-k#i5nu@AbiAB1m%E1p)-9Iw3%>Q#tLDCaW!ZgXbr7xj2NHu zsY@2nHYQ607{3LP)s_K_p=(3Sc=2P7ths@(jrgqccL=($_^+=#VBJD{{T9AF2L`~l z0gN@xv`vX6wO{hE+tHAR*U)nBAlcUFPlBn(;!i@X)!fgZ+SvMgXzTZSM_RN$Lma^N z#Gimu7xgDV)=d6sWE=Y5NBZw~i|8MM{A6g5I|Po3x_uwn?Jr+O7YD|Rmvq4=lD4mi zdxvaDct_!uY&L8maOBjEkyGAY4G`Wrg|7#%N93K8;Lj_`KhdAv7^(|!KMH^z4O->` z;l#JPj9B&R=kP9Ux_KfWOy2&Bk8JZ)Z&F+4kU&0wGse4gI!CX>-_M=SsrdT`Z{i?` z1?_O}+@-+b?@fK?{?Bl9!9UkwXtx50Vqf<3sMG7V%-EQ`}VDfON{8R)2V|n zOpo*mLc%B6JnCNbesSI3Km5G^`O6>wy@UVy5A!|@&M5iv?{n+-`+uFkA9w#n-?<+? z-;^xIh||_l@@o;7ydxx!{xxKRV}Z|Hvt@odUouFqp#{+$7aDI?n+`*y zgI}p(N~B;I;R?U6(jzI?{!H}XJ}2OVwgH2$7+S!%R~%uJ3y3n~%JWHy-T@~)m?972L!o!ZsNXSuyEBfw;JunPPy0RrA%1}v9Qx2hEQ;p zeMwRo6vjw$YmpH8I{3xHzBGywL0%R{QkUppE7vP!-J{q0=YQ zXIhMtP>p35D%5DQeU`O))xjZY6qJ^o3aX8ag^H>`U-GP#hGIu4&wM41Gd@u#`nb2j zi9TO)B@XRKGs;5udVaM8lFHSNwr{p}v?cGX*Y}nWYUl3w#;uY)+{#F~#kFky)EIng zLIO=lpa}^yA%P|&(1ZkXhH%_NT3M`G$Da!7!nv1CIR}3#j@(=dem2Wek62^ zgpnbZ`M`MbGU+B=Lp~1VV!iX^(F<$z`4eN<{jrkVe3bDJzX(%X(?e7^BaJ`!c>dsS z9JihOWo5ma#*Jv)h{lb0K5j$}CjXaUf&V^}e}mcHn5m`1-Abb@AMP${s$83gLPMtOQ&gO|mLb@sMfSPlTDK0O`NF)L zsWb;eRW?$iK%5+4?FHeKc<-n{WK$Wfsf<<|L!T&4@d=_HLsoa7a1%(?&I*^F1-pJ) zu(#d*(;dLh5Bxoj3Tn0&FZ-_*JKqXY*wRER23Rj^Y_`+3W{eE-4O{FDTf9;1y5|U? zp5W&Oh;=cBb_{e2&XA+uu*SZy#&M@;+cifCH7X39NNaVr*r~dZq292>-mt_`(WY*K z4r+_gZiHDAYm^DpR%Z|85E>TQ8y4vnZ6zmYp{7{y28cDWhki)7I&&z{Z&+h*SfgLG zX+4Ilj;ON?jEV{3An|5S&bQ>0T#AtM~%bW01raa7N2{Y)T-*W+scxH;Y29_9KD^26vY(n?oQ^) zH!&9-+Yc90@7#W+dR`AZs@#0$}~ z9n_x;&EzwBTeNYW1YS$FQ3l(cY>aOhucW@>ruJG1Ec7-gc!4Y!7JP_YlH{|G2+AkKebT)smGFr*m@ha;y&rn!7#AM0)&#X+wer0MMzgC>VSWC_Ieo=Kb zkp!X87bI#DN<>{Hkpx|@w#Xp88?_-Bc3xM=8|f&Bs5r;@l10kk3#Ov3O{^ZSz;XdG z6EkoJ94^jt4@cM{5YV8SPC3b)1FuyIMfG~UIx5LgmzA8U392aF_CqwhQJg;sSY))| zycAVlsUjKD5H{3VPbIZ)a97iS>v2_c$UaJ@dQ(IFZ>*vI^t0vvVXo|;qd58#u;?YC zEE!8GxjCf(qB;Za01vg3Er)Y}F)U14P`sk>RJiIrrH+ZK$F`$$;wr2qas7g(T7mvV zv6`qmsuDzJZql!6(||L`lD zLvjU0pNjSLqbU$P&CRzl1GB*Via!1MV15b~C-eFXiCBy4E(Yb7xr?YSIRC}7H#fR; zX8{(M?kxWv4G^oF)y*oiBU8c*P0&KQ>Tszwm(}!$DM0q?df6rQ*Ago)D`UJ~!xF=c zJFP3(m8lk&T<)@lbBd58;|VCB8e)QIBrsESvwbKcTaz!~n8$vvG)NLkwK5g0%T~Z9 zx+T8XGES?E&0giK42~2>EMpBj!j{X`6RUDUMhQk`hNJ_@BFMscY0cNE$oiWgNh zL-M2Bks+c$TVjW(TWXmhP1)%km7R|35)XrG8S0tQGuwXeW19+~!M!}ubp-K~XiKD` zUAA73iv5Edw}OBiCxGx;qqgHRpDs9Vhf~TH-ch$FEKCM$z(8@Uu>BY5LL%;j5DR_{r(!WfO(HMiU2oCw=oHd=@sb?U?YGQp@I#zXO5DzLb_b9H zCJub>lH7nVQnxkBc#NF>M9Uf94tvlYm=p7q5u5~6m%7dXR!~wO=_|V ziYiQyJws6Hs%q8C3gZPE-&iqnpjp51am(Uq3|A6y(CMQWdjLzv1$MoA9o!Y-lFkvb zjsw01KU5z2+MWoVvHZc4vd2thM^(6fhWsskaXD@v?k=95&n2cqR3hI=Fes20(IslJ z&#HN^@aiTJQ10-7i~Ak7*?)2waK^%5<|`Ar5z(!Y!68&tIAQ1$lB0`DI0M%5inz?R z+|Wa>?wzJ6<%ugs$%JS*cj+lF?YU`=G`lWkcf8Vqs-*S!_y@_h^O7O~P$mo!aCWs# zvNFMdGPl5%xzV-E-K#n~a8;K;?qh9pE-;po(ko|UEm>m4ghH(pblP^0aIHzT9E@1J z>ol+V1m%9Y1wBQwePG~TL&^YQ+olOhz&uKxpya*voW7I~sL2#;G6kDV!TMy=v7*{R zz=Pw;nSx_grl62OOxz!Rj%FjwgxJYE=ow&w+QT_OSj5j-hSwL5u7GVrdmU748Q!t% z`9`TVF32@3E2!2(P0A*YF~@GFGAcQ1n)jNSn=(17Wiu4Xpr?psUgV?+Zf9a%Y`@na z>ig0GWY}Pbqv&;O7{u|j9Ymu#E(e3c;uquX(+I~K(+Gt!ktNIND1gm*gcUSF2H~C8 z$Z0YK_dQdv5jB^C6Gc4>p(N-B$cCc}>4l+%QYPDbLptJt@#3W?x-gf(cG#(C6D|9K zTC$ZTSnpiQG&zZXV@{$pi#$(=dDS3oqoWz>KgKK$bzHiZIQJI3C6B>W-?vDtLmlz^{Yr=*yjEO5w3fmj!F@P=)pXCg|7Pvu zBg_VUErqj&Dpxt3N(v{5YNK-gX(}h6zWl}d(c|4X>HX?)hNme|-YAOithp@SN3JGN z*8R(wv+HhJ@@3t+UXwP7dm~}~X%a?Cj)i7zvXxfI;|ZS2Drc59r$y>MEwUGO>PbgY zxG4~=mx!_{rBhrb=+Agc#q$^f=t`zzj3K*+|O sTRcT04Y6Sco5j?WbAFjXNiHz5U#!)8Eduxd9{>RV|K=RgbQ0bG06ge@MgRZ+ diff --git a/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go b/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go index 4a1f5eee3..bd9611b08 100644 --- a/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go +++ b/cmd/booster-bitswap/remoteblockstore/remoteblockstore.go @@ -5,7 +5,6 @@ import ( "errors" blocks "github.com/ipfs/go-block-format" - format "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "github.com/ipfs/go-cid" @@ -14,23 +13,15 @@ import ( var log = logging.Logger("remote-blockstore") -var ErrBlockNotFound = errors.New("block not found") -var ErrNotFound = errors.New("not found") - var _ blockstore.Blockstore = (*RemoteBlockstore)(nil) -// ErrNoPieceSelected means that the piece selection function rejected all of the given pieces. -var ErrNoPieceSelected = errors.New("no piece selected") - -// PieceSelectorF helps select a piece to fetch a cid from if the given cid is present in multiple pieces. -// It should return `ErrNoPieceSelected` if none of the given piece is selected. -type PieceSelectorF func(c cid.Cid, pieceCids []cid.Cid) (cid.Cid, error) - type RemoteBlockstoreAPI interface { - BoostGetBlock(ctx context.Context, c cid.Cid) ([]byte, error) + BlockstoreGet(ctx context.Context, c cid.Cid) ([]byte, error) + BlockstoreHas(ctx context.Context, c cid.Cid) (bool, error) + BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) } -// RemoteBlockstore is a read only blockstore over all cids across all pieces on a provider. +// RemoteBlockstore is a read-only blockstore over all cids across all pieces on a provider. type RemoteBlockstore struct { api RemoteBlockstoreAPI } @@ -42,9 +33,9 @@ func NewRemoteBlockstore(api RemoteBlockstoreAPI) blockstore.Blockstore { } func (ro *RemoteBlockstore) Get(ctx context.Context, c cid.Cid) (b blocks.Block, err error) { - log.Debugw("processing request for block", "cid", c) - data, err := ro.api.BoostGetBlock(ctx, c) - log.Debugw("boost api response for get block", "cid", c, "error", err) + log.Debugw("Get", "cid", c) + data, err := ro.api.BlockstoreGet(ctx, c) + log.Debugw("Get response", "cid", c, "error", err) if err != nil { return nil, err } @@ -52,22 +43,17 @@ func (ro *RemoteBlockstore) Get(ctx context.Context, c cid.Cid) (b blocks.Block, } func (ro *RemoteBlockstore) Has(ctx context.Context, c cid.Cid) (bool, error) { - _, err := ro.api.BoostGetBlock(ctx, c) - if err != nil { - if format.IsNotFound(err) { - return false, nil - } - return false, err - } - return true, nil + log.Debugw("Has", "cid", c) + has, err := ro.api.BlockstoreHas(ctx, c) + log.Debugw("Has response", "cid", c, "has", has, "error", err) + return has, err } func (ro *RemoteBlockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) { - data, err := ro.api.BoostGetBlock(ctx, c) - if err != nil { - return 0, err - } - return len(data), nil + log.Debugw("GetSize", "cid", c) + size, err := ro.api.BlockstoreGetSize(ctx, c) + log.Debugw("GetSize response", "cid", c, "size", size, "error", err) + return size, err } // --- UNSUPPORTED BLOCKSTORE METHODS ------- diff --git a/cmd/booster-bitswap/run.go b/cmd/booster-bitswap/run.go index a7706364d..f5c43cb36 100644 --- a/cmd/booster-bitswap/run.go +++ b/cmd/booster-bitswap/run.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "fmt" "net/http" _ "net/http/pprof" @@ -14,7 +13,6 @@ import ( "github.com/filecoin-project/boost/cmd/booster-bitswap/remoteblockstore" "github.com/filecoin-project/go-jsonrpc" lcli "github.com/filecoin-project/lotus/cli" - "github.com/filecoin-project/lotus/storage/sealer" "github.com/urfave/cli/v2" ) @@ -37,16 +35,6 @@ var runCmd = &cli.Command{ Usage: "the endpoint for the boost API", Required: true, }, - &cli.StringFlag{ - Name: "api-fullnode", - Usage: "the endpoint for the full node API", - Required: true, - }, - &cli.StringFlag{ - Name: "api-sealer", - Usage: "the endpoint for the sealer API", - Required: true, - }, }, Action: func(cctx *cli.Context) error { if cctx.Bool("pprof") { @@ -69,11 +57,11 @@ var runCmd = &cli.Command{ remoteStore := remoteblockstore.NewRemoteBlockstore(bapi) // Create the server API - server := NewBitswapServer(cctx.String("base-path"), cctx.Int("port"), remoteStore) + port := cctx.Int("port") + server := NewBitswapServer(port, remoteStore) // Start the server - log.Infof("Starting booster-http node on port %d with base path '%s'", - cctx.Int("port"), cctx.String("base-path")) + log.Infof("Starting booster-bitswap node on port %d", port) err = server.Start(ctx) if err != nil { return err @@ -96,16 +84,6 @@ var runCmd = &cli.Command{ }, } -func storageAuthWithURL(apiInfo string) (sealer.StorageAuth, error) { - s := strings.Split(apiInfo, ":") - if len(s) != 2 { - return nil, errors.New("unexpected format of `apiInfo`") - } - headers := http.Header{} - headers.Add("Authorization", "Bearer "+s[0]) - return sealer.StorageAuth(headers), nil -} - func getBoostAPI(ctx context.Context, ai string) (api.Boost, jsonrpc.ClientCloser, error) { ai = strings.TrimPrefix(strings.TrimSpace(ai), "BOOST_API_INFO=") info := cliutil.ParseApiInfo(ai) diff --git a/cmd/booster-bitswap/server.go b/cmd/booster-bitswap/server.go index dd95a5eb9..5871a37af 100644 --- a/cmd/booster-bitswap/server.go +++ b/cmd/booster-bitswap/server.go @@ -3,7 +3,6 @@ package main import ( "context" "crypto/rand" - "errors" "fmt" bsnetwork "github.com/ipfs/go-bitswap/network" @@ -19,8 +18,6 @@ import ( "github.com/libp2p/go-libp2p/p2p/transport/tcp" ) -var ErrNotFound = errors.New("not found") - type BitswapServer struct { port int remoteStore blockstore.Blockstore @@ -30,7 +27,7 @@ type BitswapServer struct { server *server.Server } -func NewBitswapServer(path string, port int, remoteStore blockstore.Blockstore) *BitswapServer { +func NewBitswapServer(port int, remoteStore blockstore.Blockstore) *BitswapServer { return &BitswapServer{port: port, remoteStore: remoteStore} } @@ -65,8 +62,7 @@ func (s *BitswapServer) Start(ctx context.Context) error { s.server = server.New(ctx, net, s.remoteStore, bsopts...) net.Start(s.server) - fmt.Printf("bitswap server running on SP, addrs: %s, peerID: %s\n", host.Addrs(), host.ID()) - log.Infow("bitswap server running on SP", "multiaddrs", host.Addrs(), "peerId", host.ID()) + log.Infow("bitswap server running", "multiaddrs", host.Addrs(), "peerId", host.ID()) return nil } diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md index 4ed9e1260..2b8be58f1 100644 --- a/documentation/en/api-v1-methods.md +++ b/documentation/en/api-v1-methods.md @@ -4,6 +4,10 @@ * [Auth](#auth) * [AuthNew](#authnew) * [AuthVerify](#authverify) +* [Blockstore](#blockstore) + * [BlockstoreGet](#blockstoreget) + * [BlockstoreGetSize](#blockstoregetsize) + * [BlockstoreHas](#blockstorehas) * [Boost](#boost) * [BoostDagstoreDestroyShard](#boostdagstoredestroyshard) * [BoostDagstoreGC](#boostdagstoregc) @@ -137,6 +141,57 @@ Response: ] ``` +## Blockstore + + +### BlockstoreGet +There are not yet any comments for this method. + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `"Ynl0ZSBhcnJheQ=="` + +### BlockstoreGetSize + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `123` + +### BlockstoreHas + + +Perms: read + +Inputs: +```json +[ + { + "/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4" + } +] +``` + +Response: `true` + ## Boost @@ -1303,11 +1358,7 @@ Inputs: Response: ```json { - "Dynamic": true, "Memory": 9, - "MemoryFraction": 12.3, - "MinMemory": 9, - "MaxMemory": 9, "Streams": 123, "StreamsInbound": 123, "StreamsOutbound": 123, @@ -1473,11 +1524,7 @@ Inputs: [ "string value", { - "Dynamic": true, "Memory": 9, - "MemoryFraction": 12.3, - "MinMemory": 9, - "MaxMemory": 9, "Streams": 123, "StreamsInbound": 123, "StreamsOutbound": 123, diff --git a/go.mod b/go.mod index 116ceef67..2f0625eb4 100644 --- a/go.mod +++ b/go.mod @@ -75,7 +75,6 @@ require ( github.com/libp2p/go-eventbus v0.2.1 github.com/libp2p/go-libp2p v0.21.0 github.com/libp2p/go-libp2p-core v0.19.1 - github.com/libp2p/go-libp2p-crypto v0.1.0 github.com/libp2p/go-libp2p-gostream v0.4.1-0.20220720161416-e1952aede109 github.com/libp2p/go-libp2p-http v0.2.1 github.com/libp2p/go-libp2p-kad-dht v0.17.0 diff --git a/go.sum b/go.sum index 8b03c7090..285c3c10b 100644 --- a/go.sum +++ b/go.sum @@ -1271,7 +1271,6 @@ github.com/libp2p/go-libp2p-core v0.19.1 h1:zaZQQCeCrFMtxFa1wHy6AhsVynyNmZAvwgWq github.com/libp2p/go-libp2p-core v0.19.1/go.mod h1:2uLhmmqDiFY+dw+70KkBLeKvvsJHGWUINRDdeV1ip7k= github.com/libp2p/go-libp2p-crypto v0.0.1/go.mod h1:yJkNyDmO341d5wwXxDUGO0LykUVT72ImHNUqh5D/dBE= github.com/libp2p/go-libp2p-crypto v0.0.2/go.mod h1:eETI5OUfBnvARGOHrJz2eWNyTUxEGZnBxMcbUjfIj4I= -github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-daemon v0.2.2/go.mod h1:kyrpsLB2JeNYR2rvXSVWyY0iZuRIMhqzWR3im9BV6NQ= github.com/libp2p/go-libp2p-discovery v0.0.5/go.mod h1:YtF20GUxjgoKZ4zmXj8j3Nb2TUSBHFlOCetzYdbZL5I= diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index 8a29b89df..30f54ebf2 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -1188,6 +1188,13 @@ This parameter is ONLY applicable if the retrieval pricing policy strategy has b Comment: ``, }, + { + Name: "LocalWorkerName", + Type: "string", + + Comment: `LocalWorkerName specifies a custom name for the builtin worker. +If set to an empty string (default) os hostname will be used`, + }, { Name: "Assigner", Type: "string", @@ -1443,6 +1450,30 @@ the compaction boundary; default is 0.`, A value of 0 disables, while a value 1 will do full GC in every compaction. Default is 20 (about once a week).`, }, + { + Name: "EnableColdStoreAutoPrune", + Type: "bool", + + Comment: `EnableColdStoreAutoPrune turns on compaction of the cold store i.e. pruning +where hotstore compaction occurs every finality epochs pruning happens every 3 finalities +Default is false`, + }, + { + Name: "ColdStoreFullGCFrequency", + Type: "uint64", + + Comment: `ColdStoreFullGCFrequency specifies how often to performa a full (moving) GC on the coldstore. +Only applies if auto prune is enabled. A value of 0 disables while a value of 1 will do +full GC in every prune. +Default is 7 (about once every a week)`, + }, + { + Name: "ColdStoreRetention", + Type: "int64", + + Comment: `ColdStoreRetention specifies the retention policy for data reachable from the chain, in +finalities beyond the compaction boundary, default is 0, -1 retains everything`, + }, }, "lotus_config.StorageMiner": []DocField{ { diff --git a/node/impl/boost.go b/node/impl/boost.go index 1fcd4150f..487e8caa5 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -475,10 +475,18 @@ func (sm *BoostAPI) BoostDagstoreDestroyShard(ctx context.Context, key string) e return nil } -func (sm *BoostAPI) BoostGetBlock(ctx context.Context, c cid.Cid) ([]byte, error) { +func (sm *BoostAPI) BlockstoreGet(ctx context.Context, c cid.Cid) ([]byte, error) { blk, err := sm.IndexBackedBlockstore.Get(ctx, c) if err != nil { return nil, err } return blk.RawData(), nil } + +func (sm *BoostAPI) BlockstoreHas(ctx context.Context, c cid.Cid) (bool, error) { + return sm.IndexBackedBlockstore.Has(ctx, c) +} + +func (sm *BoostAPI) BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) { + return sm.IndexBackedBlockstore.GetSize(ctx, c) +} From 09e63131addbd9fe20a1ebe45702479c312fef64 Mon Sep 17 00:00:00 2001 From: Dirk McCormick Date: Wed, 24 Aug 2022 16:26:26 +0200 Subject: [PATCH 6/7] fix: booster-bitsawp - check error when creating libp2p key --- cmd/booster-bitswap/server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/booster-bitswap/server.go b/cmd/booster-bitswap/server.go index 5871a37af..9d20d0c7b 100644 --- a/cmd/booster-bitswap/server.go +++ b/cmd/booster-bitswap/server.go @@ -35,6 +35,9 @@ func (s *BitswapServer) Start(ctx context.Context) error { s.ctx, s.cancel = context.WithCancel(ctx) // setup libp2p host privKey, _, err := crypto.GenerateECDSAKeyPair(rand.Reader) + if err != nil { + return err + } host, err := libp2p.New( libp2p.ListenAddrStrings( From 91481119827af487096aa81401e1f7bceee6e875 Mon Sep 17 00:00:00 2001 From: hannahhoward Date: Thu, 1 Sep 2022 15:51:35 -0700 Subject: [PATCH 7/7] refactor(node): avoid FreeAndUnsealed method --- go.mod | 2 +- go.sum | 4 +-- node/modules/storageminer.go | 51 +++++++++++++++++++++++++++++++++--- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 2f0625eb4..53fe09eed 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/filecoin-project/go-data-transfer v1.15.2 github.com/filecoin-project/go-fil-commcid v0.1.0 github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 - github.com/filecoin-project/go-fil-markets v1.23.3-0.20220817063153-cdd032886583 + github.com/filecoin-project/go-fil-markets v1.24.1-0.20220901222600-b1b44b266c47 github.com/filecoin-project/go-jsonrpc v0.1.5 github.com/filecoin-project/go-legs v0.4.9 github.com/filecoin-project/go-padreader v0.0.1 diff --git a/go.sum b/go.sum index 285c3c10b..24e7fffea 100644 --- a/go.sum +++ b/go.sum @@ -388,8 +388,8 @@ github.com/filecoin-project/go-fil-commcid v0.1.0/go.mod h1:Eaox7Hvus1JgPrL5+M3+ github.com/filecoin-project/go-fil-commp-hashhash v0.1.0 h1:imrrpZWEHRnNqqv0tN7LXep5bFEVOVmQWHJvl2mgsGo= github.com/filecoin-project/go-fil-commp-hashhash v0.1.0/go.mod h1:73S8WSEWh9vr0fDJVnKADhfIv/d6dCbAGaAGWbdJEI8= github.com/filecoin-project/go-fil-markets v1.23.1/go.mod h1:V+1vFO34RZmpdECdikKGiyWhSNJK81Q89Kn0egA9iAk= -github.com/filecoin-project/go-fil-markets v1.23.3-0.20220817063153-cdd032886583 h1:DaHJ8oLE2TSsea/fnXGOTnPd3qbRhCCbuyaiaFdAKf8= -github.com/filecoin-project/go-fil-markets v1.23.3-0.20220817063153-cdd032886583/go.mod h1:ZOPAjEUia7H60F7p0kEupi0FR7Hy4Zfz90BpR1TMBwI= +github.com/filecoin-project/go-fil-markets v1.24.1-0.20220901222600-b1b44b266c47 h1:AJyJxg87ErVBB5QRrBgnuAp5zMbHNZUgTd9gRF/QiiY= +github.com/filecoin-project/go-fil-markets v1.24.1-0.20220901222600-b1b44b266c47/go.mod h1:ZOPAjEUia7H60F7p0kEupi0FR7Hy4Zfz90BpR1TMBwI= github.com/filecoin-project/go-hamt-ipld v0.1.5 h1:uoXrKbCQZ49OHpsTCkrThPNelC4W3LPEk0OrS/ytIBM= github.com/filecoin-project/go-hamt-ipld v0.1.5/go.mod h1:6Is+ONR5Cd5R6XZoCse1CWaXZc0Hdb/JeX+EQCQzX24= github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 h1:b3UDemBYN2HNfk3KOXNuxgTTxlWi3xVvbQP0IT38fvM= diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 2408624db..e506e091a 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -29,6 +29,7 @@ import ( "github.com/filecoin-project/go-fil-markets/retrievalmarket" "github.com/filecoin-project/go-fil-markets/shared" lotus_storagemarket "github.com/filecoin-project/go-fil-markets/storagemarket" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/api/v1api" ctypes "github.com/filecoin-project/lotus/chain/types" @@ -437,18 +438,60 @@ func NewGraphqlServer(cfg *config.Boost) func(lc fx.Lifecycle, r repo.LockedRepo } } -func NewIndexBackedBlockstore(dagst dagstore.Interface, rp retrievalmarket.RetrievalProvider, h host.Host) (dtypes.IndexBackedBlockstore, error) { +func NewIndexBackedBlockstore(dagst dagstore.Interface, ps lotus_dtypes.ProviderPieceStore, sa retrievalmarket.SectorAccessor, rp retrievalmarket.RetrievalProvider, h host.Host) (dtypes.IndexBackedBlockstore, error) { sf := indexbs.ShardSelectorF(func(c cid.Cid, shards []shard.Key) (shard.Key, error) { for _, sk := range shards { + // parse piece CID pieceCid, err := cid.Parse(sk.String()) if err != nil { return shard.Key{}, fmt.Errorf("failed to parse cid") } - b, err := rp.IsFreeAndUnsealed(context.TODO(), c, pieceCid) + // read piece info from piece store + pieceInfo, err := ps.GetPieceInfo(pieceCid) if err != nil { - return shard.Key{}, fmt.Errorf("failed to verify is piece is free and unsealed") + return shard.Key{}, fmt.Errorf("failed to get piece info: %w", err) } - if b { + + // check if piece is in unsealed sector + isUnsealed := false + for _, di := range pieceInfo.Deals { + isUnsealed, err = sa.IsUnsealed(context.TODO(), di.SectorID, di.Offset.Unpadded(), di.Length.Unpadded()) + if err != nil { + log.Errorf("failed to find out if sector %d is unsealed, err=%s", di.SectorID, err) + continue + } + if isUnsealed { + break + } + } + + // sealed sector, skip + if !isUnsealed { + continue + } + + // The piece is in an unsealed sector + // Is it marked for free retrieval ? + // we don't pass the payload cid so we can potentially cache on a piece cid basis + input := retrievalmarket.PricingInput{ + // piece from which the payload will be retrieved + PieceCID: pieceInfo.PieceCID, + Unsealed: true, + } + + var dealsIds []abi.DealID + for _, d := range pieceInfo.Deals { + dealsIds = append(dealsIds, d.DealID) + } + + ask, err := rp.GetDynamicAsk(context.TODO(), input, dealsIds) + if err != nil { + return shard.Key{}, fmt.Errorf("failed to get retrieval ask: %w", err) + } + + // if piece is free we found a match + // otherwise, go to the next shard + if ask.PricePerByte.NilOrZero() { return sk, nil } }