Skip to content

Commit

Permalink
Merge pull request #10433 from ipfs/release-v0.29.0
Browse files Browse the repository at this point in the history
Release: v0.29.0 [skip changelog]
  • Loading branch information
lidel authored Jun 10, 2024
2 parents e7f0f34 + b35cd62 commit 3f0947b
Show file tree
Hide file tree
Showing 34 changed files with 1,230 additions and 611 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/gotest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
make -j "$PARALLEL" test/unit/gotest.junit.xml &&
[[ ! $(jq -s -c 'map(select(.Action == "fail")) | .[]' test/unit/gotest.json) ]]
- name: Upload coverage to Codecov
uses: codecov/codecov-action@7afa10ed9b269c561c2336fd862446844e0cbf71 # v4.2.0
uses: codecov/codecov-action@6d798873df2b1b8e5846dba6fb86631229fbcb17 # v4.4.0
if: failure() || success()
with:
name: unittests
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/sharness.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ jobs:
# increasing parallelism beyond 10 doesn't speed up the tests much
PARALLEL: ${{ github.repository == 'ipfs/kubo' && 10 || 3 }}
- name: Upload coverage report
uses: codecov/codecov-action@7afa10ed9b269c561c2336fd862446844e0cbf71 # v4.2.0
uses: codecov/codecov-action@6d798873df2b1b8e5846dba6fb86631229fbcb17 # v4.4.0
if: failure() || success()
with:
name: sharness
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Kubo Changelogs

- [v0.29](docs/changelogs/v0.29.md)
- [v0.28](docs/changelogs/v0.28.md)
- [v0.27](docs/changelogs/v0.27.md)
- [v0.26](docs/changelogs/v0.26.md)
Expand Down
2 changes: 1 addition & 1 deletion client/rpc/pin.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,14 @@ package rpc
import (
"context"
"encoding/json"
"errors"
"io"
"strings"

"github.com/ipfs/boxo/path"
"github.com/ipfs/go-cid"
iface "github.com/ipfs/kubo/core/coreiface"
caopts "github.com/ipfs/kubo/core/coreiface/options"
"github.com/pkg/errors"
)

type PinAPI HttpApi
Expand Down
1 change: 1 addition & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ type Config struct {
Experimental Experiments
Plugins Plugins
Pinning Pinning
Import Import

Internal Internal // experimental/unstable options
}
Expand Down
17 changes: 17 additions & 0 deletions config/import.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package config

const (
DefaultCidVersion = 0
DefaultUnixFSRawLeaves = false
DefaultUnixFSChunker = "size-262144"
DefaultHashFunction = "sha2-256"
)

// Import configures the default options for ingesting data. This affects commands
// that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'.
type Import struct {
CidVersion OptionalInteger
UnixFSRawLeaves Flag
UnixFSChunker OptionalString
HashFunction OptionalString
}
22 changes: 22 additions & 0 deletions config/profile.go
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,28 @@ fetching may be degraded.
return nil
},
},
"legacy-cid-v0": {
Description: `Makes UnixFS import produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.`,

Transform: func(c *Config) error {
c.Import.CidVersion = *NewOptionalInteger(0)
c.Import.UnixFSRawLeaves = False
c.Import.UnixFSChunker = *NewOptionalString("size-262144")
c.Import.HashFunction = *NewOptionalString("sha2-256")
return nil
},
},
"test-cid-v1": {
Description: `Makes UnixFS import produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.`,

Transform: func(c *Config) error {
c.Import.CidVersion = *NewOptionalInteger(1)
c.Import.UnixFSRawLeaves = True
c.Import.UnixFSChunker = *NewOptionalString("size-1048576")
c.Import.HashFunction = *NewOptionalString("sha2-256")
return nil
},
},
}

func getAvailablePort() (port int, err error) {
Expand Down
33 changes: 31 additions & 2 deletions core/commands/add.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
gopath "path"
"strings"

"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core/commands/cmdenv"

"github.com/cheggaaa/pb"
Expand Down Expand Up @@ -155,12 +156,12 @@ See 'dag export' and 'dag import' for more information.
cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"),
cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash"),
cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."),
cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."),
cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)"),
cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
Expand Down Expand Up @@ -191,6 +192,16 @@ See 'dag export' and 'dag import' for more information.
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

progress, _ := req.Options[progressOptionName].(bool)
trickle, _ := req.Options[trickleOptionName].(bool)
wrap, _ := req.Options[wrapOptionName].(bool)
Expand All @@ -207,6 +218,24 @@ See 'dag export' and 'dag import' for more information.
inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)

if chunker == "" {
chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)
}

if hashFunStr == "" {
hashFunStr = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
}

if !cidVerSet && !cfg.Import.CidVersion.IsDefault() {
cidVerSet = true
cidVer = int(cfg.Import.CidVersion.WithDefault(config.DefaultCidVersion))
}

if !rbset && cfg.Import.UnixFSRawLeaves != config.Default {
rbset = true
rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
}

if onlyHash && toFilesSet {
return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName)
}
Expand Down
17 changes: 16 additions & 1 deletion core/commands/block.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/ipfs/boxo/files"

"github.com/ipfs/kubo/config"
cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
"github.com/ipfs/kubo/core/commands/cmdutils"

Expand Down Expand Up @@ -153,7 +154,7 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).
},
Options: []cmds.Option{
cmds.StringOption(blockCidCodecOptionName, "Multicodec to use in returned CID").WithDefault("raw"),
cmds.StringOption(mhtypeOptionName, "Multihash hash function").WithDefault("sha2-256"),
cmds.StringOption(mhtypeOptionName, "Multihash hash function"),
cmds.IntOption(mhlenOptionName, "Multihash hash length").WithDefault(-1),
cmds.BoolOption(pinOptionName, "Pin added blocks recursively").WithDefault(false),
cmdutils.AllowBigBlockOption,
Expand All @@ -165,7 +166,21 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

mhtype, _ := req.Options[mhtypeOptionName].(string)
if mhtype == "" {
mhtype = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
}

mhtval, ok := mh.Names[mhtype]
if !ok {
return fmt.Errorf("unrecognized multihash function: %s", mhtype)
Expand Down
2 changes: 1 addition & 1 deletion core/commands/dag/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ into an object of the specified format.
cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"),
cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"),
cmds.BoolOption("pin", "Pin this object when adding."),
cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"),
cmds.StringOption("hash", "Hash function to use"),
cmdutils.AllowBigBlockOption,
},
Run: dagPut,
Expand Down
15 changes: 15 additions & 0 deletions core/commands/dag/put.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
ipldlegacy "github.com/ipfs/go-ipld-legacy"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core/commands/cmdenv"
"github.com/ipfs/kubo/core/commands/cmdutils"
"github.com/ipld/go-ipld-prime/multicodec"
Expand All @@ -32,11 +33,25 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

inputCodec, _ := req.Options["input-codec"].(string)
storeCodec, _ := req.Options["store-codec"].(string)
hash, _ := req.Options["hash"].(string)
dopin, _ := req.Options["pin"].(bool)

if hash == "" {
hash = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)
}

var icodec mc.Code
if err := icodec.Set(inputCodec); err != nil {
return err
Expand Down
19 changes: 15 additions & 4 deletions core/commands/files.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"strings"

humanize "github.com/dustin/go-humanize"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core"
"github.com/ipfs/kubo/core/commands/cmdenv"

Expand Down Expand Up @@ -802,18 +803,28 @@ See '--to-files' in 'ipfs add --help' for more information.
return err
}

nd, err := cmdenv.GetNode(env)
if err != nil {
return err
}

cfg, err := nd.Repo.Config()
if err != nil {
return err
}

create, _ := req.Options[filesCreateOptionName].(bool)
mkParents, _ := req.Options[filesParentsOptionName].(bool)
trunc, _ := req.Options[filesTruncateOptionName].(bool)
flush, _ := req.Options[filesFlushOptionName].(bool)
rawLeaves, rawLeavesDef := req.Options[filesRawLeavesOptionName].(bool)

prefix, err := getPrefixNew(req)
if err != nil {
return err
if !rawLeavesDef && cfg.Import.UnixFSRawLeaves != config.Default {
rawLeavesDef = true
rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)
}

nd, err := cmdenv.GetNode(env)
prefix, err := getPrefixNew(req)
if err != nil {
return err
}
Expand Down
12 changes: 7 additions & 5 deletions core/commands/pin/pin.go
Original file line number Diff line number Diff line change
Expand Up @@ -359,9 +359,10 @@ Example:
},
Options: []cmds.Option{
cmds.StringOption(pinTypeOptionName, "t", "The type of pinned keys to list. Can be \"direct\", \"indirect\", \"recursive\", or \"all\".").WithDefault("all"),
cmds.BoolOption(pinQuietOptionName, "q", "Write just hashes of objects."),
cmds.BoolOption(pinQuietOptionName, "q", "Output only the CIDs of pins."),
cmds.StringOption(pinNameOptionName, "n", "Limit returned pins to ones with names that contain the value provided (case-sensitive, partial match). Implies --names=true."),
cmds.BoolOption(pinStreamOptionName, "s", "Enable streaming of pins as they are discovered."),
cmds.BoolOption(pinNamesOptionName, "n", "Enable displaying pin names (slower)."),
cmds.BoolOption(pinNamesOptionName, "Include pin names in the output (slower, disabled by default)."),
},
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
api, err := cmdenv.GetApi(env, req)
Expand All @@ -372,6 +373,7 @@ Example:
typeStr, _ := req.Options[pinTypeOptionName].(string)
stream, _ := req.Options[pinStreamOptionName].(bool)
displayNames, _ := req.Options[pinNamesOptionName].(bool)
name, _ := req.Options[pinNameOptionName].(string)

switch typeStr {
case "all", "direct", "indirect", "recursive":
Expand All @@ -397,7 +399,7 @@ Example:
if len(req.Arguments) > 0 {
err = pinLsKeys(req, typeStr, api, emit)
} else {
err = pinLsAll(req, typeStr, displayNames, api, emit)
err = pinLsAll(req, typeStr, displayNames || name != "", name, api, emit)
}
if err != nil {
return err
Expand Down Expand Up @@ -537,7 +539,7 @@ func pinLsKeys(req *cmds.Request, typeStr string, api coreiface.CoreAPI, emit fu
return nil
}

func pinLsAll(req *cmds.Request, typeStr string, detailed bool, api coreiface.CoreAPI, emit func(value PinLsOutputWrapper) error) error {
func pinLsAll(req *cmds.Request, typeStr string, detailed bool, name string, api coreiface.CoreAPI, emit func(value PinLsOutputWrapper) error) error {
enc, err := cmdenv.GetCidEncoder(req)
if err != nil {
return err
Expand All @@ -555,7 +557,7 @@ func pinLsAll(req *cmds.Request, typeStr string, detailed bool, api coreiface.Co
panic("unhandled pin type")
}

pins, err := api.Pin().Ls(req.Context, opt, options.Pin.Ls.Detailed(detailed))
pins, err := api.Pin().Ls(req.Context, opt, options.Pin.Ls.Detailed(detailed), options.Pin.Ls.Name(name))
if err != nil {
return err
}
Expand Down
4 changes: 2 additions & 2 deletions core/commands/pubsub.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,14 @@ package commands

import (
"context"
"errors"
"fmt"
"io"
"net/http"
"sort"

cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
mbase "github.com/multiformats/go-multibase"
"github.com/pkg/errors"

cmds "github.com/ipfs/go-ipfs-cmds"
options "github.com/ipfs/kubo/core/coreiface/options"
Expand Down Expand Up @@ -351,7 +351,7 @@ func urlArgsDecoder(req *cmds.Request, env cmds.Environment) error {
for n, arg := range req.Arguments {
encoding, data, err := mbase.Decode(arg)
if err != nil {
return errors.Wrap(err, "URL arg must be multibase encoded")
return fmt.Errorf("URL arg must be multibase encoded: %w", err)
}

// Enforce URL-safe encoding is used for data passed via URL arguments
Expand Down
11 changes: 6 additions & 5 deletions core/coreapi/pin.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package coreapi
import (
"context"
"fmt"
"strings"

bserv "github.com/ipfs/boxo/blockservice"
offline "github.com/ipfs/boxo/exchange/offline"
Expand Down Expand Up @@ -67,7 +68,7 @@ func (api *PinAPI) Ls(ctx context.Context, opts ...caopts.PinLsOption) (<-chan c
return nil, fmt.Errorf("invalid type '%s', must be one of {direct, indirect, recursive, all}", settings.Type)
}

return api.pinLsAll(ctx, settings.Type, settings.Detailed), nil
return api.pinLsAll(ctx, settings.Type, settings.Detailed, settings.Name), nil
}

func (api *PinAPI) IsPinned(ctx context.Context, p path.Path, opts ...caopts.PinIsPinnedOption) (string, bool, error) {
Expand Down Expand Up @@ -276,17 +277,17 @@ func (p *pinInfo) Err() error {
//
// The caller must keep reading results until the channel is closed to prevent
// leaking the goroutine that is fetching pins.
func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string, detailed bool) <-chan coreiface.Pin {
func (api *PinAPI) pinLsAll(ctx context.Context, typeStr string, detailed bool, name string) <-chan coreiface.Pin {
out := make(chan coreiface.Pin, 1)

emittedSet := cid.NewSet()

AddToResultKeys := func(c cid.Cid, name, typeStr string) error {
if emittedSet.Visit(c) {
AddToResultKeys := func(c cid.Cid, pinName, typeStr string) error {
if emittedSet.Visit(c) && (name == "" || strings.Contains(pinName, name)) {
select {
case out <- &pinInfo{
pinType: typeStr,
name: name,
name: pinName,
path: path.FromCid(c),
}:
case <-ctx.Done():
Expand Down
Loading

0 comments on commit 3f0947b

Please sign in to comment.