nsqd: Switch to Klaus Post's flate compression library

This is a dropin replacement for the standard library flate compression.

apps/nsq_to_file/file_logger.go

@@ -1,7 +1,6 @@
 package main
 
 import (
- "compress/gzip"
  "errors"
  "fmt"
  "io"
@@ -11,6 +10,7 @@ import (
  "strings"
  "time"
 
+ "github.com/klauspost/compress/gzip"
  "github.com/nsqio/go-nsq"
  "github.com/nsqio/nsq/internal/lg"
 )
@@ -331,7 +331,7 @@ func (f *FileLogger) updateFile() {
  } else {
  openFlag |= os.O_APPEND
  }
- f.out, err = os.OpenFile(absFilename, openFlag, 0666)
+ f.out, err = os.OpenFile(absFilename, openFlag, 0o666)
  if err != nil {
  if os.IsExist(err) {
  f.logf(lg.WARN, "[%s/%s] working file already exists: %s", f.topic, f.opts.Channel, absFilename)
@@ -369,7 +369,7 @@ func (f *FileLogger) updateFile() {
 func makeDirFromPath(logf lg.AppLogFunc, path string) error {
  dir, _ := filepath.Split(path)
  if dir != "" {
- return os.MkdirAll(dir, 0770)
+ return os.MkdirAll(dir, 0o770)
  }
  return nil
 }

go.mod

@@ -11,6 +11,7 @@ require (
  github.com/golang/snappy v0.0.4
  github.com/judwhite/go-svc v1.2.1
  github.com/julienschmidt/httprouter v1.3.0
+ github.com/klauspost/compress v1.17.8
  github.com/mreiferson/go-options v1.0.0
  github.com/nsqio/go-diskqueue v1.1.0
  github.com/nsqio/go-nsq v1.1.0

go.sum

@@ -1,5 +1,3 @@
-github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
-github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
 github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=
@@ -18,6 +16,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
+github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/mreiferson/go-options v1.0.0 h1:RMLidydGlDWpL+lQTXo0bVIf/XT2CTq7AEJMoz5/VWs=
 github.com/mreiferson/go-options v1.0.0/go.mod h1:zHtCks/HQvOt8ATyfwVe3JJq2PPuImzXINPRTC03+9w=
 github.com/mreiferson/go-svc v1.2.2-0.20210815184239-7a96e00010f6 h1:NbuBXARvEXrYZ1SzN53ZpObeuwGhl1zvs/C+kzCggrQ=
@@ -32,8 +32,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70 h1:SeSEfdIxyvwGJliREIJhRPPXvW6sDlLT+UQ3B0hD0NA=
-golang.org/x/sys v0.0.0-20211023085530-d6a326fbbf70/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

internal/http_api/compress.go

@@ -7,11 +7,12 @@
 package http_api
 
 import (
- "compress/flate"
- "compress/gzip"
  "io"
  "net/http"
  "strings"
+
+ "github.com/klauspost/compress/flate"
+ "github.com/klauspost/compress/gzip"
 )
 
 type compressResponseWriter struct {

nsqd/client_v2.go

@@ -2,7 +2,6 @@ package nsqd
 
 import (
  "bufio"
- "compress/flate"
  "crypto/tls"
  "fmt"
  "net"
@@ -12,6 +11,7 @@ import (
  "time"
 
  "github.com/golang/snappy"
+ "github.com/klauspost/compress/flate"
  "github.com/nsqio/nsq/internal/auth"
 )
 

nsqd/protocol_v2_test.go

@@ -3,7 +3,6 @@ package nsqd
 import (
  "bufio"
  "bytes"
- "compress/flate"
  "crypto/tls"
  _ "embed"
  "encoding/json"
@@ -25,6 +24,7 @@ import (
  "time"
 
  "github.com/golang/snappy"
+ "github.com/klauspost/compress/flate"
  "github.com/nsqio/go-nsq"
  "github.com/nsqio/nsq/internal/protocol"
  "github.com/nsqio/nsq/internal/test"
@@ -968,7 +968,6 @@ func TestTLSAuthRequire(t *testing.T) {
  t.Logf("frameType: %d, data: %s", frameType, data)
  test.Equal(t, frameTypeResponse, frameType)
  test.Equal(t, []byte("OK"), data)
-
 }
 
 func TestTLSAuthRequireVerify(t *testing.T) {
@@ -1494,7 +1493,8 @@ func TestClientAuth(t *testing.T) {
 }
 
 func runAuthTest(t *testing.T, authResponse string, authSecret string, authError string,
- authSuccess string, tlsEnabled bool, commonName string) {
+ authSuccess string, tlsEnabled bool, commonName string,
+) {
  var err error
  var expectedRemoteIP string
  expectedTLS := "false"

nsqd/protocol_v2_unixsocket_test.go

@@ -3,7 +3,6 @@ package nsqd
 import (
  "bufio"
  "bytes"
- "compress/flate"
  "crypto/tls"
  "encoding/json"
  "errors"
@@ -21,6 +20,7 @@ import (
  "time"
 
  "github.com/golang/snappy"
+ "github.com/klauspost/compress/flate"
  "github.com/nsqio/go-nsq"
  "github.com/nsqio/nsq/internal/protocol"
  "github.com/nsqio/nsq/internal/test"
@@ -903,7 +903,6 @@ func TestUnixSocketTLSAuthRequire(t *testing.T) {
  t.Logf("frameType: %d, data: %s", frameType, data)
  test.Equal(t, frameTypeResponse, frameType)
  test.Equal(t, []byte("OK"), data)
-
 }
 
 func TestUnixSocketTLSAuthRequireVerify(t *testing.T) {
@@ -1527,18 +1526,23 @@ func benchmarkUnixSocketProtocolV2PubMultiTopic(b *testing.B, numTopics int) {
 func BenchmarkUnixSocketProtocolV2PubMultiTopic1(b *testing.B) {
  benchmarkUnixSocketProtocolV2PubMultiTopic(b, 1)
 }
+
 func BenchmarkUnixSocketkProtocolV2PubMultiTopic2(b *testing.B) {
  benchmarkUnixSocketProtocolV2PubMultiTopic(b, 2)
 }
+
 func BenchmarkUnixSocketProtocolV2PubMultiTopic4(b *testing.B) {
  benchmarkUnixSocketProtocolV2PubMultiTopic(b, 4)
 }
+
 func BenchmarkUnixSocketProtocolV2PubMultiTopic8(b *testing.B) {
  benchmarkUnixSocketProtocolV2PubMultiTopic(b, 8)
 }
+
 func BenchmarkUnixSocketProtocolV2PubMultiTopic16(b *testing.B) {
  benchmarkUnixSocketProtocolV2PubMultiTopic(b, 16)
 }
+
 func BenchmarkUnixSocketProtocolV2PubMultiTopic32(b *testing.B) {
  benchmarkUnixSocketProtocolV2PubMultiTopic(b, 32)
 }
@@ -1624,9 +1628,11 @@ func BenchmarkUnixSocketProtocolV2Pub64k(b *testing.B) { benchmarkUnixSocketProt
 func BenchmarkUnixSocketProtocolV2Pub128k(b *testing.B) {
  benchmarkUnixSocketProtocolV2Pub(b, 128*1024)
 }
+
 func BenchmarkUnixSocketProtocolV2Pub256k(b *testing.B) {
  benchmarkUnixSocketProtocolV2Pub(b, 256*1024)
 }
+
 func BenchmarkUnixSocketProtocolV2Pub512k(b *testing.B) {
  benchmarkUnixSocketProtocolV2Pub(b, 512*1024)
 }
@@ -1723,9 +1729,11 @@ func BenchmarkUnixSocketProtocolV2Sub64k(b *testing.B) { benchmarkUnixSocketProt
 func BenchmarkUnixSocketProtocolV2Sub128k(b *testing.B) {
  benchmarkUnixSocketProtocolV2Sub(b, 128*1024)
 }
+
 func BenchmarkUnixSocketProtocolV2Sub256k(b *testing.B) {
  benchmarkUnixSocketProtocolV2Sub(b, 256*1024)
 }
+
 func BenchmarkUnixSocketProtocolV2Sub512k(b *testing.B) {
  benchmarkUnixSocketProtocolV2Sub(b, 512*1024)
 }
@@ -1775,15 +1783,19 @@ func benchmarkUnixSocketProtocolV2MultiSub(b *testing.B, num int) {
 func BenchmarkUnixSocketProtocolV2MultiSub2(b *testing.B) {
  benchmarkUnixSocketProtocolV2MultiSub(b, 2)
 }
+
 func BenchmarkUnixSocketProtocolV2MultiSub1(b *testing.B) {
  benchmarkUnixSocketProtocolV2MultiSub(b, 1)
 }
+
 func BenchmarkUnixSocketProtocolV2MultiSub4(b *testing.B) {
  benchmarkUnixSocketProtocolV2MultiSub(b, 4)
 }
+
 func BenchmarkUnixSocketProtocolV2MultiSub8(b *testing.B) {
  benchmarkUnixSocketProtocolV2MultiSub(b, 8)
 }
+
 func BenchmarkUnixSocketProtocolV2MultiSub16(b *testing.B) {
  benchmarkUnixSocketProtocolV2MultiSub(b, 16)
 }