From def9ac87baf94721f1c67609a9d4c2c58d507b5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jul 2024 05:56:44 +0000 Subject: [PATCH] Bump github.com/go-testfixtures/testfixtures/v3 from 3.11.0 to 3.12.0 Bumps [github.com/go-testfixtures/testfixtures/v3](https://github.com/go-testfixtures/testfixtures) from 3.11.0 to 3.12.0. - [Release notes](https://github.com/go-testfixtures/testfixtures/releases) - [Changelog](https://github.com/go-testfixtures/testfixtures/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-testfixtures/testfixtures/compare/v3.11.0...v3.12.0) --- updated-dependencies: - dependency-name: github.com/go-testfixtures/testfixtures/v3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 +- go.sum | 8 +- .../ClickHouse/clickhouse-go/v2/CHANGELOG.md | 41 +++++++++++ .../ClickHouse/clickhouse-go/v2/README.md | 5 +- .../ClickHouse/clickhouse-go/v2/clickhouse.go | 4 + .../clickhouse-go/v2/clickhouse_options.go | 3 + .../clickhouse-go/v2/clickhouse_rows.go | 33 +++++---- .../clickhouse-go/v2/clickhouse_std.go | 33 ++++++++- .../clickhouse-go/v2/client_info.go | 2 +- .../ClickHouse/clickhouse-go/v2/conn.go | 7 +- .../ClickHouse/clickhouse-go/v2/conn_batch.go | 7 +- .../ClickHouse/clickhouse-go/v2/conn_http.go | 10 ++- .../clickhouse-go/v2/conn_http_batch.go | 5 ++ .../clickhouse-go/v2/contributors/list | 2 +- .../clickhouse-go/v2/lib/column/date.go | 21 +----- .../clickhouse-go/v2/lib/column/date32.go | 12 --- .../v2/lib/column/date_helpers.go | 47 ------------ .../clickhouse-go/v2/lib/column/datetime.go | 21 +----- .../clickhouse-go/v2/lib/column/datetime64.go | 15 +--- .../clickhouse-go/v2/lib/driver/driver.go | 2 + .../clickhouse-go/v2/lib/proto/block.go | 2 - .../testfixtures/v3/CHANGELOG.md | 13 ++++ .../testfixtures/v3/Dockerfile | 4 +- .../go-testfixtures/testfixtures/v3/README.md | 16 ++++ .../go-testfixtures/testfixtures/v3/helper.go | 4 +- .../go-testfixtures/testfixtures/v3/mysql.go | 25 ++++--- .../testfixtures/v3/postgresql.go | 73 +++++++++++-------- .../testfixtures/v3/testfixtures.go | 24 +++++- vendor/modules.txt | 4 +- 29 files changed, 257 insertions(+), 190 deletions(-) delete mode 100644 vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date_helpers.go diff --git a/go.mod b/go.mod index 848a08fc92..e0ff630caf 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fatih/color v1.17.0 github.com/go-co-op/gocron v1.37.0 github.com/go-gormigrate/gormigrate/v2 v2.1.2 - github.com/go-testfixtures/testfixtures/v3 v3.11.0 + github.com/go-testfixtures/testfixtures/v3 v3.12.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/gorilla/handlers v1.5.2 github.com/gorilla/mux v1.8.1 @@ -44,7 +44,7 @@ require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect github.com/ClickHouse/ch-go v0.61.5 // indirect - github.com/ClickHouse/clickhouse-go/v2 v2.24.0 // indirect + github.com/ClickHouse/clickhouse-go/v2 v2.26.0 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index d18ea68f48..6d7a3be3c7 100644 --- a/go.sum +++ b/go.sum @@ -112,8 +112,8 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= -github.com/ClickHouse/clickhouse-go/v2 v2.24.0 h1:L/n/pVVpk95KtkHOiKuSnO7cu2ckeW4gICbbOh5qs74= -github.com/ClickHouse/clickhouse-go/v2 v2.24.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= +github.com/ClickHouse/clickhouse-go/v2 v2.26.0 h1:j4/y6NYaCcFkJwN/TU700ebW+nmsIy34RmUAAcZKy9w= +github.com/ClickHouse/clickhouse-go/v2 v2.26.0/go.mod h1:iDTViXk2Fgvf1jn2dbJd1ys+fBkdD1UMRnXlwmhijhQ= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= @@ -527,8 +527,8 @@ github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEe github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/go-testfixtures/testfixtures/v3 v3.11.0 h1:XxQr8AnPORcZkyNd7go5UNLPD3dULN8ixYISlzrlfEQ= -github.com/go-testfixtures/testfixtures/v3 v3.11.0/go.mod h1:THmudHF1Ixq++J2/UodcJpxUphfyEd77m83TvDtryqE= +github.com/go-testfixtures/testfixtures/v3 v3.12.0 h1:Ew0+c2o1mXSUqMwjuNup3MK/vw1HkLS3ILljX5C6lVE= +github.com/go-testfixtures/testfixtures/v3 v3.12.0/go.mod h1:13F0m6/DtqqSDso9IAVuhbZ4I7AiRAHrolmDMu9v5vY= github.com/goccy/kpoward v0.1.0 h1:UcrLMG9rq7NwrMiUc0h+qUyIlvqPzqLiPb+zQEqH8cE= github.com/goccy/kpoward v0.1.0/go.mod h1:m13lkcWSvNXtYC9yrXzguwrt/YTDAGioPusndMdQ+eA= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/CHANGELOG.md b/vendor/github.com/ClickHouse/clickhouse-go/v2/CHANGELOG.md index fad2a963da..94d1662d21 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/CHANGELOG.md +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/CHANGELOG.md @@ -1,3 +1,44 @@ +# v2.26.0, 2024-06-25 + +## What's Changed +### Enhancements πŸŽ‰ +* Reintroduce the random connection strategy by @larry-cdn77 in https://github.com/ClickHouse/clickhouse-go/pull/1313 +* Make custom debug log function on-par with the built-in one by @vespian in https://github.com/ClickHouse/clickhouse-go/pull/1317 +* Remove date overflow check as it's normalised by ClickHouse server by @gogingersnap777 in https://github.com/ClickHouse/clickhouse-go/pull/1315 +* Batch: impl `Columns() []column.Interface` method by @egsam98 in https://github.com/ClickHouse/clickhouse-go/pull/1277 +### Fixes πŸ› +* Fix rows.Close do not return too early by @yujiarista in https://github.com/ClickHouse/clickhouse-go/pull/1314 +* Setting `X-Clickhouse-SSL-Certificate-Auth` header correctly given `X-ClickHouse-Key` by @gogingersnap777 in https://github.com/ClickHouse/clickhouse-go/pull/1316 +* Retry on network errors and fix retries on async inserts with `database/sql` interface by @tommyzli in https://github.com/ClickHouse/clickhouse-go/pull/1330 +* BatchInsert parentheses issue fix by @ramzes642 in https://github.com/ClickHouse/clickhouse-go/pull/1327 +### Other Changes πŸ›  +* ClickHouse 24.5 by @jkaflik in https://github.com/ClickHouse/clickhouse-go/pull/1319 +* Align `allow_suspicious_low_cardinality_types` and `allow_suspicious_low_cardinality_types ` settings in tests due to ClickHouse Cloud incompatibility by @jkaflik in https://github.com/ClickHouse/clickhouse-go/pull/1331 +* Use HTTPs scheme in std connection failover tests by @jkaflik in https://github.com/ClickHouse/clickhouse-go/pull/1332 + +## New Contributors +* @larry-cdn77 made their first contribution in https://github.com/ClickHouse/clickhouse-go/pull/1313 +* @vespian made their first contribution in https://github.com/ClickHouse/clickhouse-go/pull/1317 +* @gogingersnap777 made their first contribution in https://github.com/ClickHouse/clickhouse-go/pull/1315 +* @yujiarista made their first contribution in https://github.com/ClickHouse/clickhouse-go/pull/1314 +* @egsam98 made their first contribution in https://github.com/ClickHouse/clickhouse-go/pull/1277 +* @tommyzli made their first contribution in https://github.com/ClickHouse/clickhouse-go/pull/1330 +* @ramzes642 made their first contribution in https://github.com/ClickHouse/clickhouse-go/pull/1327 + +**Full Changelog**: https://github.com/ClickHouse/clickhouse-go/compare/v2.25.0...v2.26.0 + +# v2.25.0, 2024-05-28 + +## What's Changed +### Fixes πŸ› +* Add a compatibility layer for a database/sql driver to work with sql.NullString and ClickHouse nullable column by @jkaflik in https://github.com/ClickHouse/clickhouse-go/pull/1306 +### Other Changes πŸ›  +* Use Go 1.22 in head tests by @jkaflik in https://github.com/ClickHouse/clickhouse-go/pull/1305 +* Skip flaky 1127 test by @jkaflik in https://github.com/ClickHouse/clickhouse-go/pull/1307 + + +**Full Changelog**: https://github.com/ClickHouse/clickhouse-go/compare/v2.24.0...v2.25.0 + # v2.24.0, 2024-05-08 ## What's Changed diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/README.md b/vendor/github.com/ClickHouse/clickhouse-go/v2/README.md index ef08eb0aa6..0cef001afa 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/README.md +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/README.md @@ -151,8 +151,9 @@ conn.SetConnMaxLifetime(time.Hour) * username/password - auth credentials * database - select the current default database * dial_timeout - a duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix such as "300ms", "1s". Valid time units are "ms", "s", "m". (default 30s) -* connection_open_strategy - round_robin/in_order (default in_order). - * round_robin - choose a round-robin server from the set +* connection_open_strategy - random/round_robin/in_order (default in_order). + * random - choose random server from the set + * round_robin - choose a round-robin server from the set * in_order - first live server is chosen in specified order * debug - enable debug output (boolean value) * compress - compress - specify the compression algorithm - β€œnone” (default), `zstd`, `lz4`, `gzip`, `deflate`, `br`. If set to `true`, `lz4` will be used. diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse.go index 78f0a6a721..d7ad40ec18 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "math/rand" "sync/atomic" "time" @@ -233,6 +234,7 @@ func (ch *clickhouse) dial(ctx context.Context) (conn *connect, err error) { } func DefaultDialStrategy(ctx context.Context, connID int, opt *Options, dial Dial) (r DialResult, err error) { + random := rand.Int() for i := range opt.Addr { var num int switch opt.ConnOpenStrategy { @@ -240,6 +242,8 @@ func DefaultDialStrategy(ctx context.Context, connID int, opt *Options, dial Dia num = i case ConnOpenRoundRobin: num = (int(connID) + i) % len(opt.Addr) + case ConnOpenRandom: + num = (random + i) % len(opt.Addr) } if r, err = dial(ctx, opt.Addr[num], opt); err == nil { diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_options.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_options.go index d64b04eb89..57376029db 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_options.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_options.go @@ -87,6 +87,7 @@ type ConnOpenStrategy uint8 const ( ConnOpenInOrder ConnOpenStrategy = iota ConnOpenRoundRobin + ConnOpenRandom ) type Protocol int @@ -265,6 +266,8 @@ func (o *Options) fromDSN(in string) error { o.ConnOpenStrategy = ConnOpenInOrder case "round_robin": o.ConnOpenStrategy = ConnOpenRoundRobin + case "random": + o.ConnOpenStrategy = ConnOpenRandom } case "max_open_conns": maxOpenConns, err := strconv.Atoi(params.Get(v)) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_rows.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_rows.go index 46f8a32d6c..1686e902e5 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_rows.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_rows.go @@ -101,33 +101,40 @@ func (r *rows) Close() error { if r.errors == nil && r.stream == nil { return r.err } - active := 0 - if r.errors != nil { - active++ + + if r.errors == nil { + for range r.stream { + } + return nil } - if r.stream != nil { - active++ + + if r.stream == nil { + for err := range r.errors { + r.err = err + } + return r.err } + + errorsClosed := false + streamClosed := false for { select { case _, ok := <-r.stream: if !ok { - active-- - if active == 0 { - return r.err - } + streamClosed = true } case err, ok := <-r.errors: if err != nil { r.err = err } if !ok { - active-- - if active == 0 { - return r.err - } + errorsClosed = true } } + + if errorsClosed && streamClosed { + return r.err + } } } diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_std.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_std.go index 464691569b..7b18480a2b 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_std.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/clickhouse_std.go @@ -25,6 +25,8 @@ import ( "fmt" "io" "log" + "math/rand" + "net" "os" "reflect" "strings" @@ -81,6 +83,7 @@ func (o *stdConnOpener) Connect(ctx context.Context) (_ driver.Conn, err error) return nil, ErrAcquireConnNoAddress } + random := rand.Int() for i := range o.opt.Addr { var num int switch o.opt.ConnOpenStrategy { @@ -88,6 +91,8 @@ func (o *stdConnOpener) Connect(ctx context.Context) (_ driver.Conn, err error) num = i case ConnOpenRoundRobin: num = (int(connID) + i) % len(o.opt.Addr) + case ConnOpenRandom: + num = (random + i) % len(o.opt.Addr) } if conn, err = dialFunc(ctx, o.opt.Addr[num], connID, o.opt); err == nil { var debugf = func(format string, v ...any) {} @@ -120,7 +125,10 @@ func init() { // isConnBrokenError returns true if the error class indicates that the // db connection is no longer usable and should be marked bad func isConnBrokenError(err error) bool { - if errors.Is(err, io.EOF) || errors.Is(err, syscall.EPIPE) { + if errors.Is(err, io.EOF) || errors.Is(err, syscall.EPIPE) || errors.Is(err, syscall.ECONNRESET) { + return true + } + if _, ok := err.(*net.OpError); ok { return true } return false @@ -272,10 +280,14 @@ func (std *stdDriver) CheckNamedValue(nv *driver.NamedValue) error { return nil var _ driver.NamedValueChecker = (*stdDriver)(nil) func (std *stdDriver) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + var err error if options := queryOptions(ctx); options.async.ok { - return driver.RowsAffected(0), std.conn.asyncInsert(ctx, query, options.async.wait, rebind(args)...) + err = std.conn.asyncInsert(ctx, query, options.async.wait, rebind(args)...) + } else { + err = std.conn.exec(ctx, query, rebind(args)...) } - if err := std.conn.exec(ctx, query, rebind(args)...); err != nil { + + if err != nil { if isConnBrokenError(err) { std.debugf("ExecContext got a fatal error, resetting connection: %v\n", err) return nil, driver.ErrBadConn @@ -434,6 +446,21 @@ func (r *stdRows) Next(dest []driver.Value) error { } dest[i] = v default: + // We don't know what is the destination type at this stage, + // but destination type might be a sql.Null* type that expects to receive a value + // instead of a pointer to a value. ClickHouse-go returns pointers to values for nullable columns. + // + // This is a compatibility layer to make sure that the driver works with the standard library. + // Due to reflection used it has a performance cost. + if nullable { + if value == nil { + dest[i] = nil + continue + } + rv := reflect.ValueOf(value) + value = rv.Elem().Interface() + } + dest[i] = value } } diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/client_info.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/client_info.go index 8b1dd33e43..2bf2145c59 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/client_info.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/client_info.go @@ -29,7 +29,7 @@ const ClientName = "clickhouse-go" const ( ClientVersionMajor = 2 - ClientVersionMinor = 24 + ClientVersionMinor = 26 ClientVersionPatch = 0 ClientTCPProtocolVersion = proto.DBMS_TCP_PROTOCOL_VERSION ) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/conn.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/conn.go index 5c2c34b66a..6d831a0478 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/conn.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/conn.go @@ -58,7 +58,12 @@ func dial(ctx context.Context, addr string, num int, opt *Options) (*connect, er } if opt.Debug { if opt.Debugf != nil { - debugf = opt.Debugf + debugf = func(format string, v ...any) { + opt.Debugf( + "[clickhouse][conn=%d][%s] "+format, + append([]interface{}{num, conn.RemoteAddr()}, v...)..., + ) + } } else { debugf = log.New(os.Stdout, fmt.Sprintf("[clickhouse][conn=%d][%s]", num, conn.RemoteAddr()), 0).Printf } diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_batch.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_batch.go index 26ad4ac024..d5963a4281 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_batch.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_batch.go @@ -22,6 +22,7 @@ import ( "fmt" "os" "regexp" + "slices" "strings" "time" @@ -32,7 +33,7 @@ import ( "github.com/ClickHouse/clickhouse-go/v2/lib/proto" ) -var insertMatch = regexp.MustCompile(`(?i)(INSERT\s+INTO\s+\S+(?:\s*\([^()]*(?:\([^()]*\)[^()]*)*\))?)(?:\s*VALUES)?`) +var insertMatch = regexp.MustCompile(`(?i)(INSERT\s+INTO\s+[^( ]+(?:\s*\([^()]*(?:\([^()]*\)[^()]*)*\))?)(?:\s*VALUES)?`) var columnMatch = regexp.MustCompile(`INSERT INTO .+\s\((?P.+)\)$`) func (c *connect) prepareBatch(ctx context.Context, query string, opts driver.PrepareBatchOptions, release func(*connect, error), acquire func(context.Context) (*connect, error)) (driver.Batch, error) { @@ -319,6 +320,10 @@ func (b *batch) Rows() int { return b.block.Rows() } +func (b *batch) Columns() []column.Interface { + return slices.Clone(b.block.Columns) +} + func (b *batch) closeQuery() error { if err := b.conn.sendData(&proto.Block{}, ""); err != nil { return err diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_http.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_http.go index 2510a62be8..2084fbd957 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_http.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_http.go @@ -123,7 +123,12 @@ func dialHttp(ctx context.Context, addr string, num int, opt *Options) (*httpCon var debugf = func(format string, v ...any) {} if opt.Debug { if opt.Debugf != nil { - debugf = opt.Debugf + debugf = func(format string, v ...any) { + opt.Debugf( + "[clickhouse][conn=%d][%s] "+format, + append([]interface{}{num, addr}, v...)..., + ) + } } else { debugf = log.New(os.Stdout, fmt.Sprintf("[clickhouse][conn=%d][%s]", num, addr), 0).Printf } @@ -161,6 +166,9 @@ func dialHttp(ctx context.Context, addr string, num int, opt *Options) (*httpCon headers["X-ClickHouse-User"] = opt.Auth.Username if len(opt.Auth.Password) > 0 { headers["X-ClickHouse-Key"] = opt.Auth.Password + headers["X-ClickHouse-SSL-Certificate-Auth"] = "off" + } else { + headers["X-ClickHouse-SSL-Certificate-Auth"] = "on" } } diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_http_batch.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_http_batch.go index cb7b6cd9fe..b19fce7d92 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_http_batch.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/conn_http_batch.go @@ -23,6 +23,7 @@ import ( "fmt" "io" "regexp" + "slices" "strings" "github.com/ClickHouse/clickhouse-go/v2/lib/column" @@ -240,4 +241,8 @@ func (b *httpBatch) Rows() int { return b.block.Rows() } +func (b *httpBatch) Columns() []column.Interface { + return slices.Clone(b.block.Columns) +} + var _ driver.Batch = (*httpBatch)(nil) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/contributors/list b/vendor/github.com/ClickHouse/clickhouse-go/v2/contributors/list index deb5b003c1..42a2a485c3 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/contributors/list +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/contributors/list @@ -1 +1 @@ -Mark Roberts +Kuba Kaflik diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date.go index bc4f77db49..16d46e17ed 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date.go @@ -21,9 +21,10 @@ import ( "database/sql" "database/sql/driver" "fmt" - "github.com/ClickHouse/ch-go/proto" "reflect" "time" + + "github.com/ClickHouse/ch-go/proto" ) var ( @@ -101,9 +102,6 @@ func (col *Date) Append(v any) (nulls []uint8, err error) { switch v := v.(type) { case []time.Time: for _, t := range v { - if err := dateOverflow(minDate, maxDate, t, defaultDateFormatNoZone); err != nil { - return nil, err - } col.col.Append(t) } case []*time.Time: @@ -111,9 +109,6 @@ func (col *Date) Append(v any) (nulls []uint8, err error) { for i, v := range v { switch { case v != nil: - if err := dateOverflow(minDate, maxDate, *v, defaultDateFormatNoZone); err != nil { - return nil, err - } col.col.Append(*v) default: nulls[i] = 1 @@ -181,16 +176,10 @@ func (col *Date) Append(v any) (nulls []uint8, err error) { func (col *Date) AppendRow(v any) error { switch v := v.(type) { case time.Time: - if err := dateOverflow(minDate, maxDate, v, defaultDateFormatNoZone); err != nil { - return err - } col.col.Append(v) case *time.Time: switch { case v != nil: - if err := dateOverflow(minDate, maxDate, *v, defaultDateFormatNoZone); err != nil { - return err - } col.col.Append(*v) default: col.col.Append(time.Time{}) @@ -257,12 +246,6 @@ func parseDate(value string, minDate time.Time, maxDate time.Time, location *tim if location == nil { location = time.Local } - - defer func() { - if err == nil { - err = dateOverflow(minDate, maxDate, tv, defaultDateFormatNoZone) - } - }() if tv, err = time.Parse(defaultDateFormatWithZone, value); err == nil { return tv, nil } diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date32.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date32.go index aab70d0cb5..174b3761d3 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date32.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date32.go @@ -92,9 +92,6 @@ func (col *Date32) Append(v any) (nulls []uint8, err error) { switch v := v.(type) { case []time.Time: for _, t := range v { - if err := dateOverflow(minDate32, maxDate32, t, "2006-01-02"); err != nil { - return nil, err - } col.col.Append(t) } case []*time.Time: @@ -102,9 +99,6 @@ func (col *Date32) Append(v any) (nulls []uint8, err error) { for i, v := range v { switch { case v != nil: - if err := dateOverflow(minDate32, maxDate32, *v, "2006-01-02"); err != nil { - return nil, err - } col.col.Append(*v) default: nulls[i] = 1 @@ -172,16 +166,10 @@ func (col *Date32) Append(v any) (nulls []uint8, err error) { func (col *Date32) AppendRow(v any) error { switch v := v.(type) { case time.Time: - if err := dateOverflow(minDate32, maxDate32, v, "2006-01-02"); err != nil { - return err - } col.col.Append(v) case *time.Time: switch { case v != nil: - if err := dateOverflow(minDate32, maxDate32, *v, "2006-01-02"); err != nil { - return err - } col.col.Append(*v) default: col.col.Append(time.Time{}) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date_helpers.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date_helpers.go deleted file mode 100644 index 2bf74ee610..0000000000 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/date_helpers.go +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to ClickHouse, Inc. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. ClickHouse, Inc. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package column - -import ( - "fmt" - "time" -) - -const secInDay = 24 * 60 * 60 - -func dateOverflow(min, max, v time.Time, format string) error { - if !v.IsZero() && (v.Before(min) || v.After(max)) { - return &DateOverflowError{ - Min: min, - Max: max, - Value: v, - Format: format, - } - } - return nil -} - -type DateOverflowError struct { - Min, Max time.Time - Value time.Time - Format string -} - -func (e *DateOverflowError) Error() string { - return fmt.Sprintf("clickhouse: dateTime overflow. must be between %s and %s", e.Min.Format(e.Format), e.Max.Format(e.Format)) -} diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/datetime.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/datetime.go index 3337430a77..027931a89b 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/datetime.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/datetime.go @@ -21,11 +21,12 @@ import ( "database/sql" "database/sql/driver" "fmt" - "github.com/ClickHouse/ch-go/proto" "reflect" "strings" "time" + "github.com/ClickHouse/ch-go/proto" + "github.com/ClickHouse/clickhouse-go/v2/lib/timezone" ) @@ -131,9 +132,6 @@ func (col *DateTime) Append(v any) (nulls []uint8, err error) { case []time.Time: nulls = make([]uint8, len(v)) for i := range v { - if err := dateOverflow(minDateTime, maxDateTime, v[i], defaultDateTimeFormatNoZone); err != nil { - return nil, err - } col.col.Append(v[i]) } @@ -142,9 +140,6 @@ func (col *DateTime) Append(v any) (nulls []uint8, err error) { for i := range v { switch { case v[i] != nil: - if err := dateOverflow(minDateTime, maxDateTime, *v[i], defaultDateTimeFormatNoZone); err != nil { - return nil, err - } col.col.Append(*v[i]) default: nulls[i] = 1 @@ -223,16 +218,10 @@ func (col *DateTime) AppendRow(v any) error { col.col.Append(time.Time{}) } case time.Time: - if err := dateOverflow(minDateTime, maxDateTime, v, defaultDateTimeFormatNoZone); err != nil { - return err - } col.col.Append(v) case *time.Time: switch { case v != nil: - if err := dateOverflow(minDateTime, maxDateTime, *v, defaultDateTimeFormatNoZone); err != nil { - return err - } col.col.Append(*v) default: col.col.Append(time.Time{}) @@ -309,12 +298,6 @@ func (col *DateTime) row(i int) time.Time { } func (col *DateTime) parseDateTime(value string) (tv time.Time, err error) { - defer func() { - if err == nil { - err = dateOverflow(minDateTime, maxDateTime, tv, defaultDateFormatNoZone) - } - }() - if tv, err = time.Parse(defaultDateTimeFormatWithZone, value); err == nil { return tv, nil } diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/datetime64.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/datetime64.go index bd683f855f..518176e2fc 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/datetime64.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/column/datetime64.go @@ -21,13 +21,14 @@ import ( "database/sql" "database/sql/driver" "fmt" - "github.com/ClickHouse/ch-go/proto" "math" "reflect" "strconv" "strings" "time" + "github.com/ClickHouse/ch-go/proto" + "github.com/ClickHouse/clickhouse-go/v2/lib/timezone" ) @@ -152,9 +153,6 @@ func (col *DateTime64) Append(v any) (nulls []uint8, err error) { case []time.Time: nulls = make([]uint8, len(v)) for i := range v { - if err := dateOverflow(minDateTime64, maxDateTime64, v[i], "2006-01-02 15:04:05"); err != nil { - return nil, err - } col.col.Append(v[i]) } case []*time.Time: @@ -162,9 +160,6 @@ func (col *DateTime64) Append(v any) (nulls []uint8, err error) { for i := range v { switch { case v[i] != nil: - if err := dateOverflow(minDateTime64, maxDateTime64, *v[i], "2006-01-02 15:04:05"); err != nil { - return nil, err - } col.col.Append(*v[i]) default: col.col.Append(time.Time{}) @@ -227,16 +222,10 @@ func (col *DateTime64) AppendRow(v any) error { col.col.Append(time.Time{}) } case time.Time: - if err := dateOverflow(minDateTime64, maxDateTime64, v, "2006-01-02 15:04:05"); err != nil { - return err - } col.col.Append(v) case *time.Time: switch { case v != nil: - if err := dateOverflow(minDateTime64, maxDateTime64, *v, "2006-01-02 15:04:05"); err != nil { - return err - } col.col.Append(*v) default: col.col.Append(time.Time{}) diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/driver/driver.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/driver/driver.go index f88bb43e14..a3a9c35ff6 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/driver/driver.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/driver/driver.go @@ -22,6 +22,7 @@ import ( "reflect" "time" + "github.com/ClickHouse/clickhouse-go/v2/lib/column" "github.com/ClickHouse/clickhouse-go/v2/lib/proto" ) @@ -85,6 +86,7 @@ type ( Send() error IsSent() bool Rows() int + Columns() []column.Interface } BatchColumn interface { Append(any) error diff --git a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/proto/block.go b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/proto/block.go index 79a5f13fc2..6debe8ccae 100644 --- a/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/proto/block.go +++ b/vendor/github.com/ClickHouse/clickhouse-go/v2/lib/proto/block.go @@ -301,8 +301,6 @@ func (e *BlockError) Error() string { switch err := e.Err.(type) { case *column.Error: return fmt.Sprintf("clickhouse [%s]: (%s %s) %s", e.Op, e.ColumnName, err.ColumnType, err.Err) - case *column.DateOverflowError: - return fmt.Sprintf("clickhouse: dateTime overflow. %s must be between %s and %s", e.ColumnName, err.Min.Format(err.Format), err.Max.Format(err.Format)) } return fmt.Sprintf("clickhouse [%s]: %s %s", e.Op, e.ColumnName, e.Err) } diff --git a/vendor/github.com/go-testfixtures/testfixtures/v3/CHANGELOG.md b/vendor/github.com/go-testfixtures/testfixtures/v3/CHANGELOG.md index 15d06ae18e..63e0c6a1cc 100644 --- a/vendor/github.com/go-testfixtures/testfixtures/v3/CHANGELOG.md +++ b/vendor/github.com/go-testfixtures/testfixtures/v3/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## v3.12.0 - 2024-07-13 + +- Reset sequences in a single exec to improve performance + ([#208](https://github.com/go-testfixtures/testfixtures/pull/208) by @slsyy) +- Skip checksum calculation when not needed to improve performance + ([#207](https://github.com/go-testfixtures/testfixtures/pull/207) by @slsyy). +- Add `SkipTableChecksumComputation` option + ([#203](https://github.com/go-testfixtures/testfixtures/issues/203), [#206](https://github.com/go-testfixtures/testfixtures/pull/206) by @slsyy) +- PostgreSQL: Run some queries concurrently to improve performance + ([#205](https://github.com/go-testfixtures/testfixtures/pull/205) by @slsyy). +- Optimize Docker image a bit + ([#204](https://github.com/go-testfixtures/testfixtures/pull/204) by @slsyy). + ## v3.11.0 - 2024-05-25 - Add `OVERRIDING SYSTEM VALUE` for `INSERT` statements on PostgreSQL diff --git a/vendor/github.com/go-testfixtures/testfixtures/v3/Dockerfile b/vendor/github.com/go-testfixtures/testfixtures/v3/Dockerfile index 8b5668fee5..0891cc1a59 100644 --- a/vendor/github.com/go-testfixtures/testfixtures/v3/Dockerfile +++ b/vendor/github.com/go-testfixtures/testfixtures/v3/Dockerfile @@ -4,6 +4,8 @@ RUN apk update RUN apk add alpine-sdk WORKDIR /testfixtures -COPY . . +COPY go.mod go.sum ./ RUN go mod download + +COPY . . diff --git a/vendor/github.com/go-testfixtures/testfixtures/v3/README.md b/vendor/github.com/go-testfixtures/testfixtures/v3/README.md index cd42573652..d5f4370c04 100644 --- a/vendor/github.com/go-testfixtures/testfixtures/v3/README.md +++ b/vendor/github.com/go-testfixtures/testfixtures/v3/README.md @@ -288,6 +288,22 @@ testfixtures.New( ) ``` +## Disable checksum computation + +Checksums of each table in a database are computed at the end of each `Load()`, +so subsequent calls to `Load()` do not reload the same data again, if nothing +has changed in between. + +The drawback is that it can be slow for database with many tables. Also, it does not +make sense to compute checksum, if you run `Load()` only once. + +```go +testfixtures.New( + ... + testfixtures.SkipTableChecksumComputation(), +) +``` + ## Sequences For PostgreSQL and MySQL/MariaDB, this package also resets all diff --git a/vendor/github.com/go-testfixtures/testfixtures/v3/helper.go b/vendor/github.com/go-testfixtures/testfixtures/v3/helper.go index 468918f9fe..a52db47168 100644 --- a/vendor/github.com/go-testfixtures/testfixtures/v3/helper.go +++ b/vendor/github.com/go-testfixtures/testfixtures/v3/helper.go @@ -21,7 +21,7 @@ type helper interface { databaseName(queryable) (string, error) tableNames(queryable) ([]string, error) isTableModified(queryable, string) (bool, error) - afterLoad(queryable) error + computeTablesChecksum(queryable) error quoteKeyword(string) string whileInsertOnTable(*sql.Tx, string, func() error) error cleanTableQuery(string) string @@ -70,7 +70,7 @@ func (baseHelper) isTableModified(_ queryable, _ string) (bool, error) { return true, nil } -func (baseHelper) afterLoad(_ queryable) error { +func (baseHelper) computeTablesChecksum(_ queryable) error { return nil } diff --git a/vendor/github.com/go-testfixtures/testfixtures/v3/mysql.go b/vendor/github.com/go-testfixtures/testfixtures/v3/mysql.go index 6f48ef695d..843afecec7 100644 --- a/vendor/github.com/go-testfixtures/testfixtures/v3/mysql.go +++ b/vendor/github.com/go-testfixtures/testfixtures/v3/mysql.go @@ -3,6 +3,7 @@ package testfixtures import ( "database/sql" "fmt" + "strings" ) type mySQL struct { @@ -103,31 +104,37 @@ func (h *mySQL) disableReferentialIntegrity(db *sql.DB, loadFn loadFunction) (er } func (h *mySQL) resetSequences(db *sql.DB) error { + if len(h.tables) == 0 { + return nil + } + resetSequencesTo := h.resetSequencesTo if resetSequencesTo == 0 { resetSequencesTo = 10000 } + b := strings.Builder{} for _, t := range h.tables { - if _, err := db.Exec(fmt.Sprintf("ALTER TABLE %s AUTO_INCREMENT = %d", h.quoteKeyword(t), resetSequencesTo)); err != nil { - return err - } + b.WriteString(fmt.Sprintf("ALTER TABLE %s AUTO_INCREMENT = %d;", h.quoteKeyword(t), resetSequencesTo)) } - return nil + _, err := db.Exec(b.String()) + return err } func (h *mySQL) isTableModified(q queryable, tableName string) (bool, error) { + oldChecksum, found := h.tablesChecksum[tableName] + if !found { + return true, nil + } + checksum, err := h.getChecksum(q, tableName) if err != nil { return true, err } - - oldChecksum := h.tablesChecksum[tableName] - - return oldChecksum == 0 || checksum != oldChecksum, nil + return checksum != oldChecksum, nil } -func (h *mySQL) afterLoad(q queryable) error { +func (h *mySQL) computeTablesChecksum(q queryable) error { if h.tablesChecksum != nil { return nil } diff --git a/vendor/github.com/go-testfixtures/testfixtures/v3/postgresql.go b/vendor/github.com/go-testfixtures/testfixtures/v3/postgresql.go index 320a8c3555..97812b3f63 100644 --- a/vendor/github.com/go-testfixtures/testfixtures/v3/postgresql.go +++ b/vendor/github.com/go-testfixtures/testfixtures/v3/postgresql.go @@ -7,6 +7,8 @@ import ( "strconv" "strings" "sync" + + "golang.org/x/sync/errgroup" ) type postgreSQL struct { @@ -35,30 +37,33 @@ type pgConstraint struct { } func (h *postgreSQL) init(db *sql.DB) error { - var err error - - h.tables, err = h.tableNames(db) - if err != nil { + var grp errgroup.Group + grp.Go(func() error { + var err error + h.tables, err = h.tableNames(db) return err - } - - h.sequences, err = h.getSequences(db) - if err != nil { + }) + grp.Go(func() error { + var err error + h.sequences, err = h.getSequences(db) return err - } - - h.nonDeferrableConstraints, err = h.getNonDeferrableConstraints(db) - if err != nil { + }) + grp.Go(func() error { + var err error + h.nonDeferrableConstraints, err = h.getNonDeferrableConstraints(db) return err - } - - h.constraints, err = h.getConstraints(db) - if err != nil { + }) + grp.Go(func() error { + var err error + h.constraints, err = h.getConstraints(db) return err - } - - h.version, err = h.getMajorVersion(db) - if err != nil { + }) + grp.Go(func() error { + var err error + h.version, err = h.getMajorVersion(db) + return err + }) + if err := grp.Wait(); err != nil { return err } @@ -334,32 +339,38 @@ func (h *postgreSQL) disableReferentialIntegrity(db *sql.DB, loadFn loadFunction } func (h *postgreSQL) resetSequences(db *sql.DB) error { + if len(h.sequences) == 0 { + return nil + } + resetSequencesTo := h.resetSequencesTo if resetSequencesTo == 0 { resetSequencesTo = 10000 } + b := strings.Builder{} for _, sequence := range h.sequences { - _, err := db.Exec(fmt.Sprintf("SELECT SETVAL('%s', %d)", sequence, resetSequencesTo)) - if err != nil { - return err - } + b.WriteString(fmt.Sprintf("SELECT SETVAL('%s', %d);", sequence, resetSequencesTo)) } - return nil + + _, err := db.Exec(b.String()) + return err } func (h *postgreSQL) isTableModified(q queryable, tableName string) (bool, error) { + oldChecksum, found := h.tablesChecksum[tableName] + if !found { + return true, nil + } + checksum, err := h.getChecksum(q, tableName) if err != nil { - return false, err + return true, err } - - oldChecksum := h.tablesChecksum[tableName] - - return oldChecksum == "" || checksum != oldChecksum, nil + return checksum != oldChecksum, nil } -func (h *postgreSQL) afterLoad(q queryable) error { +func (h *postgreSQL) computeTablesChecksum(q queryable) error { if h.tablesChecksum != nil { return nil } diff --git a/vendor/github.com/go-testfixtures/testfixtures/v3/testfixtures.go b/vendor/github.com/go-testfixtures/testfixtures/v3/testfixtures.go index 5eb1d0dbca..d95b421139 100644 --- a/vendor/github.com/go-testfixtures/testfixtures/v3/testfixtures.go +++ b/vendor/github.com/go-testfixtures/testfixtures/v3/testfixtures.go @@ -23,9 +23,10 @@ type Loader struct { helper helper fixturesFiles []*fixtureFile - skipCleanup bool - skipTestDatabaseCheck bool - location *time.Location + skipCleanup bool + skipChecksumComputation bool + skipTestDatabaseCheck bool + location *time.Location template bool templateFuncs template.FuncMap @@ -229,6 +230,16 @@ func DangerousSkipCleanupFixtureTables() func(*Loader) error { } } +// SkipTableChecksumComputation will make Loader not compute table checksum at the end of the Loader.Load. +// This may improve performance. It may also slow down subsequent calls to Loader.Load, because checksums are used +// to not reload unchanged tables. +func SkipTableChecksumComputation() func(*Loader) error { + return func(l *Loader) error { + l.skipChecksumComputation = true + return nil + } +} + // Directory informs Loader to load YAML files from a given directory. func Directory(dir string) func(*Loader) error { return func(l *Loader) error { @@ -457,7 +468,12 @@ func (l *Loader) Load() error { if err != nil { return err } - return l.helper.afterLoad(l.db) + if !l.skipChecksumComputation { + if err := l.helper.computeTablesChecksum(l.db); err != nil { + return err + } + } + return nil } // InsertError will be returned if any error happens on database while diff --git a/vendor/modules.txt b/vendor/modules.txt index cdc40b6eb6..1d60471120 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -16,7 +16,7 @@ github.com/AlecAivazis/survey/v2/terminal ## explicit; go 1.21 github.com/ClickHouse/ch-go/compress github.com/ClickHouse/ch-go/proto -# github.com/ClickHouse/clickhouse-go/v2 v2.24.0 +# github.com/ClickHouse/clickhouse-go/v2 v2.26.0 ## explicit; go 1.21 github.com/ClickHouse/clickhouse-go/v2 github.com/ClickHouse/clickhouse-go/v2/contributors @@ -154,7 +154,7 @@ github.com/go-openapi/jsonreference/internal # github.com/go-openapi/swag v0.22.8 ## explicit; go 1.19 github.com/go-openapi/swag -# github.com/go-testfixtures/testfixtures/v3 v3.11.0 +# github.com/go-testfixtures/testfixtures/v3 v3.12.0 ## explicit; go 1.21 github.com/go-testfixtures/testfixtures/v3 # github.com/gogo/protobuf v1.3.2