diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d33f3d3..ceb9f91 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,6 +11,6 @@ jobs: steps: - uses: actions/checkout@v3 - name: golangci-lint - uses: golangci/golangci-lint-action@v3.2.0 + uses: golangci/golangci-lint-action@v6.1.1 with: - version: v1.45.2 + version: v1.61.0 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d1b3920..f01dca9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,16 +8,16 @@ jobs: test: strategy: matrix: - go-version: [1.17.x, 1.18.x] + go-version: [1.23.x] platform: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: - name: Install Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Test run: | go version diff --git a/.golangci.yml b/.golangci.yml index 13e02bd..b4711b1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,107 +1,90 @@ linters-settings: - depguard: - list-type: blacklist - packages: - # logging is allowed only by logutils.Log, logrus - # is allowed to use only in logutils package - - github.com/sirupsen/logrus - packages-with-error-message: - - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" dupl: threshold: 100 funlen: lines: 100 - statements: 50 + statements: 60 goconst: min-len: 2 min-occurrences: 2 gocritic: enabled-tags: - - diagnostic - - experimental - - opinionated - - performance - - style + - diagnostic + - experimental + - opinionated + - performance + - style disabled-checks: - - dupImport # https://github.com/go-critic/go-critic/issues/845 - - ifElseChain - - octalLiteral - - whyNoLint - - wrapperFunc + - dupImport # https://github.com/go-critic/go-critic/issues/845 + - ifElseChain + - octalLiteral + - whyNoLint + - wrapperFunc gocyclo: min-complexity: 20 # increased from 15 to get us going, but not make things worse goimports: local-prefixes: github.com/golangci/golangci-lint - golint: - min-confidence: 0 - gomnd: - settings: - mnd: - # don't include the "operation" and "assign" - checks: argument,case,condition,return + mnd: + # don't include the "operation" and "assign" + checks: + - argument + - case + - condition + - return govet: - check-shadowing: true settings: printf: funcs: - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf lll: line-length: 140 - maligned: - suggest-new: true misspell: locale: US nolintlint: - allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space) allow-unused: false # report any unused nolint directives require-explanation: false # don't require an explanation for nolint directives require-specific: false # don't require nolint directives to be specific about which linter is being skipped + gosec: + excludes: + - G115 # integer overflow conversion (TODO: verify these) linters: # please, do not use `enable-all`: it's deprecated and will be removed soon. # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint disable-all: true enable: - - bodyclose - - deadcode - - depguard - - dogsled - - dupl - - errcheck - - funlen - - gochecknoinits - - goconst - - gocritic - - gocyclo - - gofmt - - goimports - - golint - - gomnd - - goprintffuncname - - gosec - - gosimple - - govet - - ineffassign - - interfacer - - lll - - misspell - - nakedret - - nolintlint - - rowserrcheck - - scopelint - - staticcheck - - structcheck - - stylecheck - - typecheck - - unconvert - - unparam - - unused - - varcheck - - whitespace - + - bodyclose + - dogsled + - dupl + - errcheck + - funlen + - gochecknoinits + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - mnd + - goprintffuncname + - gosec + - gosimple + - govet + - ineffassign + - lll + - misspell + - nakedret + - nolintlint + - rowserrcheck + - staticcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - whitespace # don't enable: # - asciicheck # - gochecknoglobals @@ -118,31 +101,17 @@ linters: issues: # Excluding configuration per-path, per-linter, per-text and per-source exclude-rules: - - path: _test\.go - linters: - - gomnd - - path: cmd/ice/cmd - linters: - - gochecknoinits - - path: sizes.go - linters: - - gochecknoinits - - # https://github.com/go-critic/go-critic/issues/926 - - linters: - - gocritic - text: "unnecessaryDefer:" - -run: - skip-dirs: - - test/testdata_etc - - internal/cache - - internal/renameio - - internal/robustio + - path: _test\.go + linters: + - mnd + - path: cmd/ice/cmd + linters: + - gochecknoinits + - path: sizes.go + linters: + - gochecknoinits -# golangci.com configuration -# https://github.com/golangci/golangci/wiki/Configuration -service: - golangci-lint-version: 1.23.x # use the fixed version to not introduce new linters unexpectedly - prepare: - - echo "here I can run custom commands, but no preparation needed for this repo" + # https://github.com/go-critic/go-critic/issues/926 + - linters: + - gocritic + text: "unnecessaryDefer:" diff --git a/contentcoder.go b/contentcoder.go index 3dc7ebc..3a4e874 100644 --- a/contentcoder.go +++ b/contentcoder.go @@ -176,7 +176,6 @@ func (c *chunkedContentCoder) Add(docNum uint64, vals []byte) error { // // | ..... data ..... | chunk offsets (varints) // | position of chunk offsets (uint64) | number of offsets (uint64) | -// func (c *chunkedContentCoder) Write() (int, error) { var tw int diff --git a/contentcoder_test.go b/contentcoder_test.go index 8a4ac79..a827a44 100644 --- a/contentcoder_test.go +++ b/contentcoder_test.go @@ -34,10 +34,10 @@ func TestChunkedContentCoder(t *testing.T) { vals: [][]byte{[]byte("bluge")}, // 1 chunk, chunk-0 length 11(b), value expected: []byte{ - 0x1, 0x0, 0x5, 0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x0, 0x29, 0x0, 0x0, - 'b', 'l', 'u', 'g', 'e', - 0x7e, 0xde, 0xed, 0x4a, 0x15, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, - 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, + 0x1, 0x0, 0x5, 0x5, 0x10, + 'b', 'l', 'u', 'g', 'e', 0xa, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, }, }, { @@ -49,13 +49,9 @@ func TestChunkedContentCoder(t *testing.T) { []byte("scorch"), }, - expected: []byte{ - 0x1, 0x0, 0x6, 0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x0, 0x31, 0x0, 0x0, - 0x75, 0x70, 0x73, 0x69, 0x64, 0x65, 0x35, 0x89, 0x5a, 0xd, - 0x1, 0x1, 0x6, 0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x0, 0x31, 0x0, 0x0, - 0x73, 0x63, 0x6f, 0x72, 0x63, 0x68, 0xc4, 0x46, 0x89, 0x39, 0x16, 0x2c, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, - }, + expected: []byte{0x1, 0x0, 0x6, 0x6, 0x14, 0x75, 0x70, 0x73, 0x69, 0x64, 0x65, 0x1, + 0x1, 0x6, 0x6, 0x14, 0x73, 0x63, 0x6f, 0x72, 0x63, 0x68, 0xb, 0x16, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2}, }, } diff --git a/documentcoder_test.go b/documentcoder_test.go index 3067a29..d630ef3 100644 --- a/documentcoder_test.go +++ b/documentcoder_test.go @@ -19,11 +19,8 @@ func TestChunkedDocumentCoder(t *testing.T) { docNums: []uint64{0}, metas: [][]byte{{0}}, datas: [][]byte{[]byte("bluge")}, - expected: []byte{ - 0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x0, 0x41, - 0x0, 0x0, 0x1, 0x5, 0x0, 0x62, 0x6c, 0x75, 0x67, 0x65, 0x2b, 0x30, 0x97, 0x33, 0x0, 0x15, 0x15, - 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3, - }, + expected: []byte{0x8, 0x1c, 0x1, 0x5, 0x0, 0x62, 0x6c, 0x75, 0x67, 0x65, + 0x0, 0xa, 0xa, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x3}, expectedChunkNum: 3, // left, chunk, right }, { @@ -31,14 +28,9 @@ func TestChunkedDocumentCoder(t *testing.T) { docNums: []uint64{0, 1}, metas: [][]byte{{0}, {1}}, datas: [][]byte{[]byte("upside"), []byte("scorch")}, - expected: []byte{ - 0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x0, 0x49, - 0x0, 0x0, 0x1, 0x6, 0x0, 0x75, 0x70, 0x73, 0x69, 0x64, 0x65, - 0x36, 0x6e, 0x7e, 0x39, 0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x0, 0x49, - 0x0, 0x0, 0x1, 0x6, 0x1, 0x73, 0x63, 0x6f, 0x72, 0x63, 0x68, - 0x8f, 0x83, 0xa3, 0x37, 0x0, 0x16, 0x2c, 0x2c, - 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x4, - }, + expected: []byte{0x9, 0x20, 0x1, 0x6, 0x0, 0x75, 0x70, 0x73, 0x69, 0x64, + 0x65, 0x9, 0x20, 0x1, 0x6, 0x1, 0x73, 0x63, 0x6f, 0x72, 0x63, 0x68, + 0x0, 0xb, 0x16, 0x16, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x4}, expectedChunkNum: 4, // left, chunk, chunk, right }, } diff --git a/go.mod b/go.mod index 9bb0122..54bdd01 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,24 @@ module github.com/blugelabs/ice -go 1.16 +go 1.23 + +toolchain go1.23.1 require ( - github.com/RoaringBitmap/roaring v0.9.4 + github.com/RoaringBitmap/roaring v1.9.4 github.com/blevesearch/mmap-go v1.0.4 - github.com/blevesearch/vellum v1.0.7 + github.com/blevesearch/vellum v1.0.10 github.com/blugelabs/bluge_segment_api v0.2.0 - github.com/klauspost/compress v1.15.2 - github.com/spf13/cobra v0.0.5 + github.com/klauspost/compress v1.17.11 + github.com/spf13/cobra v1.8.1 +) + +require ( + github.com/bits-and-blooms/bitset v1.14.3 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mschoch/smat v0.2.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + golang.org/x/sys v0.26.0 // indirect ) replace github.com/blugelabs/bluge_segment_api => github.com/zinclabs/bluge_segment_api v1.0.0 diff --git a/go.sum b/go.sum index 65548cf..8e6285b 100644 --- a/go.sum +++ b/go.sum @@ -1,58 +1,42 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/RoaringBitmap/roaring v0.9.4 h1:ckvZSX5gwCRaJYBNe7syNawCU5oruY9gQmjXlp4riwo= github.com/RoaringBitmap/roaring v0.9.4/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/RoaringBitmap/roaring v1.9.4 h1:yhEIoH4YezLYT04s1nHehNO64EKFTop/wBhxv2QzDdQ= +github.com/RoaringBitmap/roaring v1.9.4/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= -github.com/blevesearch/mmap-go v1.0.3/go.mod h1:pYvKl/grLQrBxuaRYgoTssa4rVujYYeenDp++2E+yvs= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.14.3 h1:Gd2c8lSNf9pKXom5JtD7AaKO8o7fGQ2LtFj1436qilA= +github.com/bits-and-blooms/bitset v1.14.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ= -github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= +github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/klauspost/compress v1.15.2 h1:3WH+AG7s2+T8o3nrM/8u2rdqUEcQhmga7smjrT41nAw= -github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/zinclabs/bluge_segment_api v1.0.0 h1:GJvPxdzR7KjwdxmcKleQLvtIYi/J7Q7ehRlZqgGayzg= github.com/zinclabs/bluge_segment_api v1.0.0/go.mod h1:mYfPVUdXLZ4iXsicXMER+RcI/avwphjMOi8nhN9HDLA= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/helper_test.go b/helper_test.go index 87ad4e8..3539358 100644 --- a/helper_test.go +++ b/helper_test.go @@ -17,7 +17,6 @@ package ice import ( "bufio" "fmt" - "io/ioutil" "math" "os" "testing" @@ -31,7 +30,7 @@ import ( // ice no longer knows about files itself func setupTestDir(t *testing.T) (path string, cleanup func()) { - path, err := ioutil.TempDir("", "ice-test") + path, err := os.MkdirTemp("", "ice-test") if err != nil { t.Fatalf("error creating tmp dir: %v", err) } diff --git a/intcoder.go b/intcoder.go index 4968873..f861f2f 100644 --- a/intcoder.go +++ b/intcoder.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/binary" "io" + "unsafe" "github.com/blugelabs/ice/compress" ) @@ -28,6 +29,8 @@ import ( // file) const termNotEncoded = 0 +var intSize = int(unsafe.Sizeof(uint64(0))) + type chunkedIntCoder struct { final []byte chunkSize uint64 @@ -47,7 +50,7 @@ func newChunkedIntCoder(chunkSize, maxDocNum uint64) *chunkedIntCoder { rv := &chunkedIntCoder{ chunkSize: chunkSize, chunkLens: make([]uint64, total), - final: make([]byte, 0, 64), + final: make([]byte, 0, intSize), } return rv diff --git a/intcoder_test.go b/intcoder_test.go index 7f91e1b..74d3a70 100644 --- a/intcoder_test.go +++ b/intcoder_test.go @@ -36,10 +36,7 @@ func TestChunkIntCoder(t *testing.T) { {3}, }, // 1 chunk, chunk-0 length 1, value 3, but compressed - expected: []byte{ - 0x1, 0xe, 0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x0, 0x9, - 0x0, 0x0, 0x3, 0xb6, 0x4b, 0x1f, 0xbc, - }, + expected: []byte{0x1, 0x3, 0x1, 0x0, 0x3}, }, { maxDocNum: 1, @@ -50,11 +47,7 @@ func TestChunkIntCoder(t *testing.T) { {7}, }, // 2 chunks, chunk-0 offset 1, chunk-1 offset 2, value 3, value 7, but compressed - expected: []byte{ - 0x2, 0xe, 0x1c, 0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x0, 0x9, - 0x0, 0x0, 0x3, 0xb6, 0x4b, 0x1f, 0xbc, 0x28, 0xb5, 0x2f, 0xfd, 0x4, 0x0, 0x9, - 0x0, 0x0, 0x7, 0xb7, 0xbb, 0x58, 0xe8, - }, + expected: []byte{0x2, 0x3, 0x6, 0x1, 0x0, 0x3, 0x1, 0x0, 0x7}, }, } @@ -73,7 +66,7 @@ func TestChunkIntCoder(t *testing.T) { t.Fatalf("error writing: %v", err) } if !reflect.DeepEqual(test.expected, actual.Bytes()) { - t.Errorf("got % x, expected % x", actual.Bytes(), test.expected) + t.Errorf("got %#v, expected %#v", actual.Bytes(), test.expected) } } } diff --git a/intdecoder.go b/intdecoder.go index 101f3b3..bc8947c 100644 --- a/intdecoder.go +++ b/intdecoder.go @@ -121,7 +121,7 @@ func (d *chunkedIntDecoder) reset() { } func (d *chunkedIntDecoder) isNil() bool { - return d.curChunkBytes == nil || len(d.curChunkBytes) == 0 + return len(d.curChunkBytes) == 0 } func (d *chunkedIntDecoder) readUvarint() (uint64, error) { diff --git a/merge.go b/merge.go index e3f7578..4b1a8ed 100644 --- a/merge.go +++ b/merge.go @@ -535,6 +535,7 @@ func writeDvLocs(w *countHashWriter, bufMaxVarintLen64 []byte, fieldDvLocsStart, return fieldDvLocsOffset, nil } +// nolint: gocritic, nolintlint func setupActiveForField(segments []*Segment, dropsIn []*roaring.Bitmap, newDocNumsIn [][]uint64, closeCh chan struct{}, fieldName string) (newDocNums [][]uint64, drops []*roaring.Bitmap, dicts []*Dictionary, itrs []vellum.Iterator, segmentsInFocus []*Segment, err error) { diff --git a/merge_test.go b/merge_test.go index 6d0451c..fc2aa56 100644 --- a/merge_test.go +++ b/merge_test.go @@ -135,6 +135,8 @@ func testMergeWithEmptySegments(t *testing.T, before bool, numEmptySegments int) segsToMerge = append(segsToMerge, seg) } + var closeFuncs []closeFunc + for i := 0; i < numEmptySegments; i++ { fname := fmt.Sprintf("segment-empty-%d.ice", i) segPath := filepath.Join(path, fname) @@ -146,15 +148,17 @@ func testMergeWithEmptySegments(t *testing.T, before bool, numEmptySegments int) if err != nil { t.Fatalf("error opening segment: %v", err) } - defer func(emptyClose closeFunc) { + closeFuncs = append(closeFuncs, emptyClose) + segsToMerge = append(segsToMerge, emptyFileSegment) + } + defer func() { + for _, emptyClose := range closeFuncs { cerr := emptyClose() if cerr != nil { - t.Fatalf("error closing segment: %v", err) + t.Fatalf("error closing segment: %v", cerr) } - }(emptyClose) - - segsToMerge = append(segsToMerge, emptyFileSegment) - } + } + }() if !before { segsToMerge = append(segsToMerge, seg) @@ -207,6 +211,7 @@ func testMergeWithSelf(t *testing.T, segCur *Segment, expectedCount uint64) { // trying merging the segment with itself for a few rounds var diffs []string + var closeFuncs []closeFunc for i := 0; i < 10; i++ { fname := fmt.Sprintf("segment-self-%d.ice", i) @@ -225,12 +230,7 @@ func testMergeWithSelf(t *testing.T, segCur *Segment, expectedCount uint64) { if err != nil { t.Fatalf("error opening merged segment: %v", err) } - defer func(close closeFunc) { - cerr := close() - if cerr != nil { - t.Fatalf("error closing segment: %v", err) - } - }(closeNew) + closeFuncs = append(closeFuncs, closeNew) if segNew.Count() != expectedCount { t.Fatalf("wrong count") @@ -246,6 +246,14 @@ func testMergeWithSelf(t *testing.T, segCur *Segment, expectedCount uint64) { segCur = segNew } + defer func() { + for _, closeF := range closeFuncs { + cerr := closeF() + if cerr != nil { + t.Fatalf("error closing segment: %v", cerr) + } + } + }() if len(diffs) > 0 { t.Errorf("mismatches after repeated self-merging: %v", strings.Join(diffs, "\n")) @@ -602,6 +610,7 @@ func testMergeWithUpdates(t *testing.T, segmentDocIds [][]string, docsToDrop []* defer cleanup() var segsToMerge []segment.Segment + var closeFuncs []closeFunc // convert segmentDocIds to segsToMerge for i, docIds := range segmentDocIds { @@ -618,15 +627,18 @@ func testMergeWithUpdates(t *testing.T, segmentDocIds [][]string, docsToDrop []* if err != nil { t.Fatalf("error opening segment: %v", err) } - defer func(closeF closeFunc) { + closeFuncs = append(closeFuncs, closeF) + segsToMerge = append(segsToMerge, seg) + } + + defer func() { + for _, closeF := range closeFuncs { cerr := closeF() if cerr != nil { - t.Fatalf("error closing segment: %v", err) + t.Fatalf("error closing segment: %v", cerr) } - }(closeF) - - segsToMerge = append(segsToMerge, seg) - } + } + }() testMergeAndDropSegments(t, segsToMerge, docsToDrop, expectedNumDocs) } diff --git a/new.go b/new.go index 046a4a3..2760333 100644 --- a/new.go +++ b/new.go @@ -868,20 +868,19 @@ func (s *interim) writeDictsTermField(docTermMap [][]byte, dict map[string]uint6 return nil } -func (s *interim) calcTimestamp() (int64, int64) { - min := int64(math.MaxInt64) - max := int64(0) +func (s *interim) calcTimestamp() (minVal, maxVal int64) { + minVal = int64(math.MaxInt64) for _, v := range s.results { t := v.Timestamp() - if t < min { - min = t + if t < minVal { + minVal = t } - if t > max { - max = t + if t > maxVal { + maxVal = t } } - if min == math.MaxInt64 { - min = 0 + if minVal == math.MaxInt64 { + minVal = 0 } - return min, max + return minVal, maxVal } diff --git a/posting.go b/posting.go index 404cfbb..e9f5a70 100644 --- a/posting.go +++ b/posting.go @@ -26,13 +26,13 @@ import ( // FST or vellum value (uint64) encoding is determined by the top two // highest-order or most significant bits... // -// encoding : MSB -// name : 63 62 61...to...bit #0 (LSB) -// ----------+---+---+--------------------------------------------------- -// general : 0 | 0 | 62-bits of postingsOffset. -// ~ : 0 | 1 | reserved for future. -// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. -// ~ : 1 | 1 | reserved for future. +// encoding : MSB +// name : 63 62 61...to...bit #0 (LSB) +// ----------+---+---+--------------------------------------------------- +// general : 0 | 0 | 62-bits of postingsOffset. +// ~ : 0 | 1 | reserved for future. +// 1-hit : 1 | 0 | 31-bits of positive float31 norm | 31-bits docNum. +// ~ : 1 | 1 | reserved for future. // // Encoding "general" is able to handle all cases, where the // postingsOffset points to more information about the postings for diff --git a/segment.go b/segment.go index 6ed538d..200bec5 100644 --- a/segment.go +++ b/segment.go @@ -88,7 +88,7 @@ func (s *Segment) Version() uint32 { return s.footer.version } -func (s *Segment) Timestamp() (int64, int64) { +func (s *Segment) Timestamp() (startTime, endTime int64) { if s.footer != nil { return int64(s.footer.docTimeMin), int64(s.footer.docTimeMax) }