-
Notifications
You must be signed in to change notification settings - Fork 81
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Switch to klauspost/reedsolomon implementation and cache encoder (#136)
* Switch Reed-Solomon implementation to github.com/klauspost/reedsolomon * chore: update dependencies * chore: Use sync.Map instead of previously implemented double-cache * chore: Consistent usage of constructors and constants * chore: Add comment to indicate this line is only for ensuring the cache is used. * fix: use pointer for leopard codec > A Map must not be copied after first use. https://pkg.go.dev/sync#Map Thanks @Wondertan * chore: extract loading/caching encoder into method * chore: consistent test names Co-authored-by: Elias Naur <[email protected]> Co-authored-by: Rahul Ghangas <[email protected]>
- Loading branch information
3 people
authored
Oct 24, 2022
1 parent
21e6086
commit bee24dc
Showing
8 changed files
with
86 additions
and
84 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -26,20 +26,12 @@ jobs: | |
| **/**.go | ||
| go.mod | ||
| go.sum | ||
| - name: install leopard | ||
| run: | | ||
| git clone https://github.com/celestiaorg/go-leopard.git | ||
| pushd go-leopard | ||
| make build-cleo | ||
| sudo make install-cleo | ||
| popd | ||
| if: env.GIT_DIFF | ||
| - name: build | ||
| run: GOOS=linux GOARCH=${{ matrix.goarch }} go build -tags leopard | ||
| run: GOOS=linux GOARCH=${{ matrix.goarch }} go build | ||
| if: env.GIT_DIFF | ||
| - name: test & coverage report creation | ||
| run: | | ||
| GOARCH=${{ matrix.goarch }} go test -tags leopard -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic | ||
| GOARCH=${{ matrix.goarch }} go test ./... -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic | ||
| if: env.GIT_DIFF | ||
| - uses: codecov/[email protected] | ||
| with: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,54 +1,76 @@ | ||
| //go:build leopard | ||
|
|
||
| // Note that if the build tag leopard is used, liblibleopard.a | ||
| // has to be present where the linker will find it. | ||
| // Otherwise go-leopard won't build. | ||
| package rsmt2d | ||
|
|
||
| import "github.com/celestiaorg/go-leopard" | ||
| import ( | ||
| "sync" | ||
|
|
||
| "github.com/klauspost/reedsolomon" | ||
| ) | ||
|
|
||
| var _ Codec = leoRSFF8Codec{} | ||
| var _ Codec = leoRSFF16Codec{} | ||
| var _ Codec = &leoRSCodec{} | ||
|
|
||
| func init() { | ||
| registerCodec(LeopardFF8, newLeoRSFF8Codec()) | ||
| registerCodec(LeopardFF16, newLeoRSFF16Codec()) | ||
| registerCodec(Leopard, NewLeoRSCodec()) | ||
| } | ||
|
|
||
| type leoRSFF8Codec struct{} | ||
|
|
||
| func (l leoRSFF8Codec) Encode(data [][]byte) ([][]byte, error) { | ||
| return leopard.Encode(data) | ||
| type leoRSCodec struct { | ||
| // Cache the encoders of various sizes to not have to re-instantiate those | ||
| // as it is costly. | ||
| // | ||
| // Note that past sizes are not removed from the cache at all as the various | ||
| // data sizes are expected to relatively small and will not cause any memory issue. | ||
| // | ||
| // TODO: switch to a generic version of sync.Map with type reedsolomon.Encoder | ||
| // once it made it into the standard lib | ||
| encCache sync.Map | ||
| } | ||
|
|
||
| func (l leoRSFF8Codec) Decode(data [][]byte) ([][]byte, error) { | ||
| half := len(data) / 2 | ||
| return leopard.Decode(data[:half], data[half:]) | ||
| } | ||
| func (l *leoRSCodec) Encode(data [][]byte) ([][]byte, error) { | ||
| dataLen := len(data) | ||
| enc, err := l.loadOrInitEncoder(dataLen) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| func (l leoRSFF8Codec) maxChunks() int { | ||
| return 128 * 128 | ||
| } | ||
| shards := make([][]byte, dataLen*2) | ||
| copy(shards, data) | ||
| for i := dataLen; i < len(shards); i++ { | ||
| shards[i] = make([]byte, len(data[0])) | ||
| } | ||
|
|
||
| func newLeoRSFF8Codec() leoRSFF8Codec { | ||
| return leoRSFF8Codec{} | ||
| if err := enc.Encode(shards); err != nil { | ||
| return nil, err | ||
| } | ||
| return shards[dataLen:], nil | ||
| } | ||
|
|
||
| type leoRSFF16Codec struct{} | ||
|
|
||
| func (leo leoRSFF16Codec) Encode(data [][]byte) ([][]byte, error) { | ||
| return leopard.Encode(data) | ||
| func (l *leoRSCodec) Decode(data [][]byte) ([][]byte, error) { | ||
| half := len(data) / 2 | ||
| enc, err := l.loadOrInitEncoder(half) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| err = enc.Reconstruct(data) | ||
| return data, err | ||
| } | ||
|
|
||
| func (leo leoRSFF16Codec) Decode(data [][]byte) ([][]byte, error) { | ||
| half := len(data) / 2 | ||
| return leopard.Decode(data[:half], data[half:]) | ||
| func (l *leoRSCodec) loadOrInitEncoder(dataLen int) (reedsolomon.Encoder, error) { | ||
| enc, ok := l.encCache.Load(dataLen) | ||
| if !ok { | ||
| var err error | ||
| enc, err = reedsolomon.New(dataLen, dataLen, reedsolomon.WithLeopardGF(true)) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| l.encCache.Store(dataLen, enc) | ||
| } | ||
| return enc.(reedsolomon.Encoder), nil | ||
|
|
||
| } | ||
|
|
||
| func (leo leoRSFF16Codec) maxChunks() int { | ||
| func (l *leoRSCodec) maxChunks() int { | ||
| return 32768 * 32768 | ||
| } | ||
|
|
||
| func newLeoRSFF16Codec() leoRSFF16Codec { | ||
| return leoRSFF16Codec{} | ||
| func NewLeoRSCodec() *leoRSCodec { | ||
| return &leoRSCodec{} | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters