diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c0f19f..ab59684 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,20 +26,12 @@ jobs: **/**.go go.mod go.sum - - name: install leopard - run: | - git clone https://github.com/celestiaorg/go-leopard.git - pushd go-leopard - make build-cleo - sudo make install-cleo - popd - if: env.GIT_DIFF - name: build - run: GOOS=linux GOARCH=${{ matrix.goarch }} go build -tags leopard + run: GOOS=linux GOARCH=${{ matrix.goarch }} go build if: env.GIT_DIFF - name: test & coverage report creation run: | - GOARCH=${{ matrix.goarch }} go test -tags leopard -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic + GOARCH=${{ matrix.goarch }} go test ./... -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic if: env.GIT_DIFF - uses: codecov/codecov-action@v1.0.15 with: diff --git a/codec_test.go b/codec_test.go index 9fa7d09..6dca840 100644 --- a/codec_test.go +++ b/codec_test.go @@ -15,6 +15,9 @@ func BenchmarkEncoding(b *testing.B) { // generate some fake data data := generateRandData(128) for codecName, codec := range codecs { + // For some implementations we want to ensure the encoder for this data length + // is already cached and initialized. For this run with same sized arbitrary data. + _, _ = codec.Encode(generateRandData(128)) b.Run( fmt.Sprintf("%s 128 shares", codecName), func(b *testing.B) { @@ -46,16 +49,20 @@ func generateRandData(count int) [][]byte { func BenchmarkDecoding(b *testing.B) { // generate some fake data for codecName, codec := range codecs { + // For some implementations we want to ensure the encoder for this data length + // is already cached and initialized. For this run with same sized arbitrary data. + _, _ = codec.Decode(generateMissingData(128, codec)) + data := generateMissingData(128, codec) b.Run( fmt.Sprintf("%s 128 shares", codecName), func(b *testing.B) { for n := 0; n < b.N; n++ { - encodedData, err := codec.Decode(data) + decodedData, err := codec.Decode(data) if err != nil { b.Error(err) } - encodedDataDump = encodedData + decodedDataDump = decodedData } }, ) diff --git a/codecs.go b/codecs.go index 2968525..bbaf441 100644 --- a/codecs.go +++ b/codecs.go @@ -3,9 +3,8 @@ package rsmt2d import "fmt" const ( - LeopardFF16 = "LeopardFF16" - LeopardFF8 = "LeopardFF8" - RSGF8 = "RSFG8" + Leopard = "Leopard" + RSGF8 = "RSFG8" ) type Codec interface { @@ -28,17 +27,3 @@ func registerCodec(ct string, codec Codec) { } codecs[ct] = codec } - -func NewLeoRSFF16Codec() Codec { - if codec, has := codecs[LeopardFF16]; has { - return codec - } - panic("cannot use codec LeopardFF16 without the 'leopard' build tag") -} - -func NewLeoRSFF8Codec() Codec { - if codec, has := codecs[LeopardFF8]; has { - return codec - } - panic("cannot use codec LeopardFF8 without the 'leopard' build tag") -} diff --git a/go.mod b/go.mod index caef08d..dd4e11a 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,25 @@ module github.com/celestiaorg/rsmt2d go 1.18 require ( - // only needed if built with go build -tags leopard - github.com/celestiaorg/go-leopard v0.1.0 github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 github.com/stretchr/testify v1.7.0 github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 ) +require ( + github.com/klauspost/reedsolomon v1.11.1 + golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 +) + require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/klauspost/cpuid/v2 v2.1.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/minio/sha256-simd v1.0.0 github.com/pmezard/go-difflib v1.0.0 // indirect gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 // indirect golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect - golang.org/x/sync v0.0.0-20220907140024-f12130a52804 - golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect + golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) - -require github.com/klauspost/cpuid/v2 v2.0.4 // indirect diff --git a/go.sum b/go.sum index 14bfa2c..c91d1a4 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,14 @@ -github.com/celestiaorg/go-leopard v0.1.0 h1:28z2EkvKJIez5J9CEaiiUEC+OxalRLtTGJJ1oScfE1g= -github.com/celestiaorg/go-leopard v0.1.0/go.mod h1:NtO/rjlB8dw2aq7jr06vZFKGvryQcTDXaNHelmPNOAM= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.1.1 h1:t0wUqjowdm8ezddV5k0tLWVklVuvLJpoHeb4WBdydm0= +github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/reedsolomon v1.11.1 h1:0gCWQXOB8pVe1Y5SGozDA5t2qoVxX3prsV+qHgI/Fik= +github.com/klauspost/reedsolomon v1.11.1/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747hZ4oYp2Ml60Lb/k= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -19,7 +20,6 @@ github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k= @@ -35,14 +35,14 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45mi golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/sync v0.0.0-20220907140024-f12130a52804 h1:0SH2R3f1b1VmIMG7BXbEZCBUu2dKmHschSmjqGUrW8A= -golang.org/x/sync v0.0.0-20220907140024-f12130a52804/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 h1:cu5kTvlzcw1Q5S9f5ip1/cpiB4nXvw1XYzFPGgzLUOY= +golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 h1:F5Gozwx4I1xtr/sr/8CFbb57iKi3297KFs0QDbGN60A= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e h1:CsOuNlbOuf0mzxJIefr6Q4uAUetRUwZE4qt7VfzP+xo= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -50,7 +50,6 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infectiousRSGF8.go b/infectiousRSGF8.go index 7c209fb..b7332a0 100644 --- a/infectiousRSGF8.go +++ b/infectiousRSGF8.go @@ -9,7 +9,7 @@ import ( var _ Codec = &rsGF8Codec{} func init() { - registerCodec("RSGF8", NewRSGF8Codec()) + registerCodec(RSGF8, NewRSGF8Codec()) } type rsGF8Codec struct { diff --git a/leopard.go b/leopard.go index fd058bb..cffedb8 100644 --- a/leopard.go +++ b/leopard.go @@ -1,54 +1,76 @@ -//go:build leopard - -// Note that if the build tag leopard is used, liblibleopard.a -// has to be present where the linker will find it. -// Otherwise go-leopard won't build. package rsmt2d -import "github.com/celestiaorg/go-leopard" +import ( + "sync" + + "github.com/klauspost/reedsolomon" +) -var _ Codec = leoRSFF8Codec{} -var _ Codec = leoRSFF16Codec{} +var _ Codec = &leoRSCodec{} func init() { - registerCodec(LeopardFF8, newLeoRSFF8Codec()) - registerCodec(LeopardFF16, newLeoRSFF16Codec()) + registerCodec(Leopard, NewLeoRSCodec()) } -type leoRSFF8Codec struct{} - -func (l leoRSFF8Codec) Encode(data [][]byte) ([][]byte, error) { - return leopard.Encode(data) +type leoRSCodec struct { + // Cache the encoders of various sizes to not have to re-instantiate those + // as it is costly. + // + // Note that past sizes are not removed from the cache at all as the various + // data sizes are expected to relatively small and will not cause any memory issue. + // + // TODO: switch to a generic version of sync.Map with type reedsolomon.Encoder + // once it made it into the standard lib + encCache sync.Map } -func (l leoRSFF8Codec) Decode(data [][]byte) ([][]byte, error) { - half := len(data) / 2 - return leopard.Decode(data[:half], data[half:]) -} +func (l *leoRSCodec) Encode(data [][]byte) ([][]byte, error) { + dataLen := len(data) + enc, err := l.loadOrInitEncoder(dataLen) + if err != nil { + return nil, err + } -func (l leoRSFF8Codec) maxChunks() int { - return 128 * 128 -} + shards := make([][]byte, dataLen*2) + copy(shards, data) + for i := dataLen; i < len(shards); i++ { + shards[i] = make([]byte, len(data[0])) + } -func newLeoRSFF8Codec() leoRSFF8Codec { - return leoRSFF8Codec{} + if err := enc.Encode(shards); err != nil { + return nil, err + } + return shards[dataLen:], nil } -type leoRSFF16Codec struct{} - -func (leo leoRSFF16Codec) Encode(data [][]byte) ([][]byte, error) { - return leopard.Encode(data) +func (l *leoRSCodec) Decode(data [][]byte) ([][]byte, error) { + half := len(data) / 2 + enc, err := l.loadOrInitEncoder(half) + if err != nil { + return nil, err + } + err = enc.Reconstruct(data) + return data, err } -func (leo leoRSFF16Codec) Decode(data [][]byte) ([][]byte, error) { - half := len(data) / 2 - return leopard.Decode(data[:half], data[half:]) +func (l *leoRSCodec) loadOrInitEncoder(dataLen int) (reedsolomon.Encoder, error) { + enc, ok := l.encCache.Load(dataLen) + if !ok { + var err error + enc, err = reedsolomon.New(dataLen, dataLen, reedsolomon.WithLeopardGF(true)) + if err != nil { + return nil, err + } + l.encCache.Store(dataLen, enc) + } + return enc.(reedsolomon.Encoder), nil + } -func (leo leoRSFF16Codec) maxChunks() int { +func (l *leoRSCodec) maxChunks() int { return 32768 * 32768 } -func newLeoRSFF16Codec() leoRSFF16Codec { - return leoRSFF16Codec{} +func NewLeoRSCodec() *leoRSCodec { + return &leoRSCodec{} } diff --git a/rsmt2d_test.go b/rsmt2d_test.go index 073c513..4052896 100644 --- a/rsmt2d_test.go +++ b/rsmt2d_test.go @@ -1,5 +1,3 @@ -//go:build leopard - package rsmt2d_test import ( @@ -18,8 +16,7 @@ func TestEdsRepairRoundtripSimple(t *testing.T) { shareSize int codec rsmt2d.Codec }{ - {"leopardFF8", bufferSize, rsmt2d.NewLeoRSFF8Codec()}, - {"leopardFF16", bufferSize, rsmt2d.NewLeoRSFF16Codec()}, + {"leopard", bufferSize, rsmt2d.NewLeoRSCodec()}, {"infectiousGF8", bufferSize, rsmt2d.NewRSGF8Codec()}, } @@ -86,8 +83,7 @@ func TestEdsRepairTwice(t *testing.T) { shareSize int codec rsmt2d.Codec }{ - {"leopardFF8", bufferSize, rsmt2d.NewLeoRSFF8Codec()}, - {"leopardFF16", bufferSize, rsmt2d.NewLeoRSFF16Codec()}, + {"leopard", bufferSize, rsmt2d.NewLeoRSCodec()}, {"infectiousGF8", bufferSize, rsmt2d.NewRSGF8Codec()}, }