diff --git a/internal/commands/table.go b/internal/commands/table.go index 6053b778..227ddece 100644 --- a/internal/commands/table.go +++ b/internal/commands/table.go @@ -4559,4 +4559,184 @@ var redisCommands = map[string]redisCommand{ }, }, }, + "BF.ADD": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.CARD": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.EXISTS": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.INFO": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.INSERT": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.LOADCHUNK": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.MADD": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.MEXISTS": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.RESERVE": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, + "BF.SCANDUMP": { + "BLOOM FILTER", + []keySpec{ + { + "index", + 1, + "", + 0, + "range", + 0, + 1, + 0, + 0, + 0, + 0, + }, + }, + }, } diff --git a/internal/config/config.go b/internal/config/config.go index 10554621..f55643db 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -39,6 +39,10 @@ type AdvancedOptions struct { AwsPSync string `mapstructure:"aws_psync" default:""` // 10.0.0.1:6379@nmfu2sl5osync,10.0.0.1:6379@xhma21xfkssync } +type ModuleOptions struct { + TargetMBbloomVersion int `mapstructure:"target_mbbloom_version" default:"0"` // v1.0.0 <=> 10000 +} + func (opt *AdvancedOptions) GetPSyncCommand(address string) string { items := strings.Split(opt.AwsPSync, ",") for _, item := range items { @@ -53,6 +57,7 @@ func (opt *AdvancedOptions) GetPSyncCommand(address string) string { type ShakeOptions struct { Function string `mapstructure:"function" default:""` Advanced AdvancedOptions + Module ModuleOptions } var Opt ShakeOptions diff --git a/internal/rdb/types/mbbloom.go b/internal/rdb/types/mbbloom.go index d73879cd..5704c2a0 100644 --- a/internal/rdb/types/mbbloom.go +++ b/internal/rdb/types/mbbloom.go @@ -1,13 +1,14 @@ package types import ( + "RedisShake/internal/config" "RedisShake/internal/rdb/structure" "io" "strconv" "unsafe" ) -// BloomObject for MBbloom-- +// BloomObject for MBbloom-- at https://github.com/RedisBloom/RedisBloom type BloomObject struct { encver int key string @@ -56,23 +57,6 @@ type dumpedChainLink struct { n2 uint8 } -type dumpedChainHeaderV3 struct { - size uint64 - nfilters uint32 - options uint32 -} - -type dumpedChainLinkV3 struct { - bytes uint64 - bits uint64 - size uint64 - err float64 - bpe float64 - hashes uint32 - entries uint32 - n2 uint8 -} - const ( BF_MIN_OPTIONS_ENC = 2 BF_MIN_GROWTH_ENC = 4 @@ -143,10 +127,16 @@ func readDouble(rd io.Reader) float64 { func (o *BloomObject) Rewrite() []RedisCmd { var cs []RedisCmd var h string - if o.encver < BF_MIN_GROWTH_ENC { - h = getEncodedHeaderV3(&o.sb) + if ver := config.Opt.Module.TargetMBbloomVersion; ver > 20200 { + h = getEncodedHeader(&o.sb, true, true) + } else if ver == 20200 { + h = getEncodedHeader(&o.sb, true, false) + } else if ver >= 10000 { + h = getEncodedHeader(&o.sb, false, false) + } else if o.encver < BF_MIN_GROWTH_ENC { + h = getEncodedHeader(&o.sb, false, false) } else { - h = getEncodedHeader(&o.sb) + h = getEncodedHeader(&o.sb, true, true) } cmd := RedisCmd{"BF.LOADCHUNK", o.key, "1", h} cs = append(cs, cmd) @@ -162,36 +152,25 @@ func (o *BloomObject) Rewrite() []RedisCmd { return cs } -func getEncodedHeader(sb *chain) string { - h := make([]byte, DUMPED_CHAIN_LINK_SIZE*sb.nfilters+DUMPED_CHAIN_HEADER_SIZE) +func getEncodedHeader(sb *chain, withGrowth, bigEntries bool) string { + var hs uint64 = DUMPED_CHAIN_HEADER_SIZE_V3 + if withGrowth { + hs = DUMPED_CHAIN_HEADER_SIZE + } + var ls uint64 = DUMPED_CHAIN_LINK_SIZE_V3 + if bigEntries { + ls = DUMPED_CHAIN_LINK_SIZE + } + h := make([]byte, hs+ls*sb.nfilters) ph := (*dumpedChainHeader)(unsafe.Pointer(&h[0])) ph.size = sb.size ph.nfilters = uint32(sb.nfilters) ph.options = uint32(sb.options) - ph.growth = uint32(sb.growth) - for i := uint64(0); i < sb.nfilters; i++ { - pl := (*dumpedChainLink)(unsafe.Add(unsafe.Pointer(&h[0]), DUMPED_CHAIN_HEADER_SIZE+DUMPED_CHAIN_LINK_SIZE*i)) - sl := sb.filters[i] - pl.bytes = uint64(len(sl.inner.bf)) - pl.bits = sl.inner.bits - pl.size = sl.size - pl.err = sl.inner.err - pl.hashes = uint32(sl.inner.hashes) - pl.bpe = sl.inner.bpe - *(*uint64)(unsafe.Pointer(&pl.entries)) = sl.inner.entries - pl.n2 = uint8(sl.inner.n2) + if withGrowth { + ph.growth = uint32(sb.growth) } - return *(*string)(unsafe.Pointer(&h)) -} - -func getEncodedHeaderV3(sb *chain) string { - h := make([]byte, DUMPED_CHAIN_LINK_SIZE_V3*sb.nfilters+DUMPED_CHAIN_HEADER_SIZE_V3) - ph := (*dumpedChainHeaderV3)(unsafe.Pointer(&h[0])) - ph.size = sb.size - ph.nfilters = uint32(sb.nfilters) - ph.options = uint32(sb.options) for i := uint64(0); i < sb.nfilters; i++ { - pl := (*dumpedChainLinkV3)(unsafe.Add(unsafe.Pointer(&h[0]), DUMPED_CHAIN_HEADER_SIZE_V3+DUMPED_CHAIN_LINK_SIZE_V3*i)) + pl := (*dumpedChainLink)(unsafe.Add(unsafe.Pointer(&h[0]), hs+ls*i)) sl := sb.filters[i] pl.bytes = uint64(len(sl.inner.bf)) pl.bits = sl.inner.bits @@ -199,8 +178,13 @@ func getEncodedHeaderV3(sb *chain) string { pl.err = sl.inner.err pl.hashes = uint32(sl.inner.hashes) pl.bpe = sl.inner.bpe - pl.entries = uint32(sl.inner.entries) - pl.n2 = uint8(sl.inner.n2) + if bigEntries { + *(*uint64)(unsafe.Pointer(&pl.entries)) = sl.inner.entries + pl.n2 = uint8(sl.inner.n2) + } else { + pl.entries = uint32(sl.inner.entries) + *(*uint8)(unsafe.Pointer(&pl.enthigh)) = uint8(sl.inner.n2) + } } return *(*string)(unsafe.Pointer(&h)) } diff --git a/shake.toml b/shake.toml index 64f40c02..233596d7 100644 --- a/shake.toml +++ b/shake.toml @@ -63,3 +63,7 @@ target_redis_proto_max_bulk_len = 512_000_000 # If the source is Elasticache or MemoryDB, you can set this item. aws_psync = "" # example: aws_psync = "10.0.0.1:6379@nmfu2sl5osync,10.0.0.1:6379@xhma21xfkssync" + +[module] +# The data format for BF.LOADCHUNK is not compatible in different versions. v2.6.3 <=> 20603 +target_mbbloom_version = 20603