Skip to content

Commit

Permalink
Merge pull request #7335 from dolthub/db/sysbench
Browse files Browse the repository at this point in the history 
[no-release-notes] sysbench runner postgres and doltgres support
  • Loading branch information
@coffeegoddd
coffeegoddd authored Jan 17, 2024
2 parents 755dcbb + d974a9d commit c01f8d8
Show file tree
Hide file tree
Showing 9 changed files with 474 additions and 25 deletions.
1 change: 1 addition & 0 deletions go/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ require (
github.com/klauspost/compress v1.10.5 // indirect
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
github.com/lestrrat-go/strftime v1.0.4 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,8 @@ github.com/lestrrat-go/strftime v1.0.4/go.mod h1:E1nN3pCbtMSu1yjSVeyuRFVm/U0xoR7
github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E=
github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
Expand Down
105 changes: 85 additions & 20 deletions go/performance/utils/sysbench_runner/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,10 @@ import (
)

const (
Dolt ServerType = "dolt"
MySql ServerType = "mysql"
Dolt ServerType = "dolt"
Doltgres ServerType = "doltgres"
Postgres ServerType = "postgres"
MySql ServerType = "mysql"

CsvFormat = "csv"
JsonFormat = "json"
Expand All @@ -39,30 +41,32 @@ const (
defaultHost = "127.0.0.1"
defaultPort = 3306

defaultSocket = "/var/run/mysqld/mysqld.sock"
defaultMysqlSocket = "/var/run/mysqld/mysqld.sock"

tcpProtocol = "tcp"
unixProtocol = "unix"

sysbenchUsername = "sysbench"
sysbenchUserLocal = "'sysbench'@'localhost'"
sysbenchPassLocal = "sysbenchpass"
)

var (
ErrTestNameNotDefined = errors.New("test name not defined")
ErrNoServersDefined = errors.New("servers not defined")
ErrTooManyServersDefined = errors.New("too many servers defined, two max")
ErrUnsupportedConnectionProtocol = errors.New("unsupported connection protocol")
)

var defaultSysbenchParams = []string{
"--db-driver=mysql",
"--db-ps-mode=disable",
"--rand-type=uniform",
fmt.Sprintf("--mysql-db=%s", dbName),
}

var defaultDoltServerParams = []string{"sql-server"}
var defaultMysqlServerParams = []string{"--user=mysql"}
var defaultDoltgresServerParams = []string{}
var defaultPostgresServerParams = []string{}

var defaultSysbenchTests = []*ConfigTest{
NewConfigTest("oltp_read_only", []string{}, false),
Expand All @@ -77,7 +81,7 @@ var defaultSysbenchTests = []*ConfigTest{
NewConfigTest("oltp_update_non_index", []string{}, false),
}

var defaultLuaScripts = map[string]string{
var defaultDoltLuaScripts = map[string]string{
"covering_index_scan.lua": "covering_index_scan.lua",
"groupby_scan.lua": "groupby_scan.lua",
"index_join.lua": "index_join.lua",
Expand All @@ -89,6 +93,20 @@ var defaultLuaScripts = map[string]string{
"types_table_scan.lua": "types_table_scan.lua",
}

// todo: check expressions need to be supported in doltgres for these
// todo: postgres does not have geometry types also
var defaultDoltgresLuaScripts = map[string]string{
//"covering_index_scan_postgres.lua": "covering_index_scan_postgres.lua",
//"groupby_scan_postgres.lua": "groupby_scan_postgres.lua",
//"index_join_postgres.lua": "index_join_postgres.lua",
//"index_join_scan_postgres.lua": "index_join_scan_postgres.lua",
//"index_scan_postgres.lua": "index_scan_postgres.lua",
//"oltp_delete_insert_postgres.lua": "oltp_delete_insert_postgres.lua",
//"table_scan_postgres.lua": "table_scan_postgres.lua",
//"types_delete_insert_postgres.lua": "types_delete_insert_postgres.lua",
//"types_table_scan_postgres.lua": "types_table_scan_postgres.lua",
}

type ServerType string

// Test is a single sysbench test
Expand Down Expand Up @@ -192,17 +210,32 @@ func (ct *ConfigTest) GetTests(serverConfig *ServerConfig, testIdFunc func() str
func fromConfigTestParams(ct *ConfigTest, serverConfig *ServerConfig) []string {
params := make([]string, 0)
params = append(params, defaultSysbenchParams...)
params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host))
if serverConfig.Port != 0 {
params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port))
if serverConfig.Server == MySql || serverConfig.Server == Dolt {
params = append(params, fmt.Sprintf("--mysql-db=%s", dbName))
params = append(params, "--db-driver=mysql")
params = append(params, fmt.Sprintf("--mysql-host=%s", serverConfig.Host))
if serverConfig.Port != 0 {
params = append(params, fmt.Sprintf("--mysql-port=%d", serverConfig.Port))
}
} else if serverConfig.Server == Doltgres || serverConfig.Server == Postgres {
params = append(params, "--db-driver=pgsql")
params = append(params, fmt.Sprintf("--pgsql-db=%s", dbName))
params = append(params, fmt.Sprintf("--pgsql-host=%s", serverConfig.Host))
if serverConfig.Port != 0 {
params = append(params, fmt.Sprintf("--pgsql-port=%d", serverConfig.Port))
}
}

// handle sysbench user for local mysql server
if serverConfig.Server == MySql && serverConfig.Host == defaultHost {
params = append(params, "--mysql-user=sysbench")
params = append(params, fmt.Sprintf("--mysql-password=%s", sysbenchPassLocal))
} else {
} else if serverConfig.Server == Dolt {
params = append(params, "--mysql-user=root")
} else if serverConfig.Server == Doltgres {
params = append(params, "--pgsql-user=doltgres")
} else if serverConfig.Server == Postgres {
params = append(params, "--pgsql-user=postgres")
}

params = append(params, ct.Options...)
Expand Down Expand Up @@ -233,6 +266,9 @@ type ServerConfig struct {
// ServerExec is the path to a server executable
ServerExec string

// InitExec is the path to the server init db executable
InitExec string

// ServerArgs are the args used to start a server
ServerArgs []string

Expand All @@ -259,10 +295,14 @@ func (sc *ServerConfig) GetServerArgs() []string {
defaultParams = defaultDoltServerParams
} else if sc.Server == MySql {
defaultParams = defaultMysqlServerParams
} else if sc.Server == Doltgres {
defaultParams = defaultDoltgresServerParams
} else if sc.Server == Postgres {
defaultParams = defaultPostgresServerParams
}

params = append(params, defaultParams...)
if sc.Server == Dolt {
if sc.Server == Dolt || sc.Server == Doltgres {
params = append(params, fmt.Sprintf("--host=%s", sc.Host))
}
if sc.Port != 0 {
Expand Down Expand Up @@ -308,6 +348,9 @@ func (c *Config) Validate() error {
if len(c.Servers) < 1 {
return ErrNoServersDefined
}
if len(c.Servers) > 2 {
return ErrTooManyServersDefined
}
err := c.setDefaults()
if err != nil {
return err
Expand All @@ -319,7 +362,7 @@ func (c *Config) Validate() error {
func (c *Config) validateServerConfigs() error {
portMap := make(map[int]ServerType)
for _, s := range c.Servers {
if s.Server != Dolt && s.Server != MySql {
if s.Server != Dolt && s.Server != MySql && s.Server != Doltgres && s.Server != Postgres {
return fmt.Errorf("unsupported server type: %s", s.Server)
}

Expand All @@ -344,14 +387,31 @@ func (c *Config) validateServerConfigs() error {
return err
}

err = CheckExec(s)
err = CheckExec(s.ServerExec, "server exec")
if err != nil {
return err
}

if s.Server == Postgres {
err = CheckExec(s.InitExec, "initdb exec")
if err != nil {
return err
}
}
}

return nil
}

func (c *Config) Contains(st ServerType) bool {
for _, s := range c.Servers {
if s.Server == st {
return true
}
}
return false
}

func ValidateRequiredFields(server, version, format string) error {
if server == "" {
return getMustSupplyError("server")
Expand Down Expand Up @@ -413,18 +473,17 @@ func CheckUpdatePortMap(serverConfig *ServerConfig, portMap map[int]ServerType)
}

// CheckExec verifies the binary exists
func CheckExec(serverConfig *ServerConfig) error {
if serverConfig.ServerExec == "" {
return getMustSupplyError("server exec")
func CheckExec(path, messageIfMissing string) error {
if path == "" {
return getMustSupplyError(messageIfMissing)
}
abs, err := filepath.Abs(serverConfig.ServerExec)
abs, err := filepath.Abs(path)
if err != nil {
return err
}
if _, err := os.Stat(abs); os.IsNotExist(err) {
return fmt.Errorf("server exec not found: %s", abs)
return fmt.Errorf("exec not found: %s", abs)
}
serverConfig.ServerExec = abs
return nil
}

Expand Down Expand Up @@ -480,7 +539,13 @@ func getDefaultTests(config *Config) ([]*ConfigTest, error) {
defaultTests := make([]*ConfigTest, 0)
defaultTests = append(defaultTests, defaultSysbenchTests...)
if config.ScriptDir != "" {
luaScriptTests, err := getLuaScriptTestsFromDir(config.ScriptDir, defaultLuaScripts)
var luaScriptTests []*ConfigTest
var err error
if !config.Contains(Doltgres) && !config.Contains(Postgres) {
luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltLuaScripts)
} else {
luaScriptTests, err = getLuaScriptTestsFromDir(config.ScriptDir, defaultDoltgresLuaScripts)
}
if err != nil {
return nil, err
}
Expand Down
6 changes: 3 additions & 3 deletions go/performance/utils/sysbench_runner/dolt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func BenchmarkDolt(ctx context.Context, config *Config, serverConfig *ServerConf
withKeyCtx, cancel := context.WithCancel(ctx)
gServer, serverCtx := errgroup.WithContext(withKeyCtx)

server := getDoltServer(serverCtx, serverConfig, testRepo, serverParams)
server := getServer(serverCtx, serverConfig, testRepo, serverParams)

// handle user interrupt
quit := make(chan os.Signal, 1)
Expand Down Expand Up @@ -200,8 +200,8 @@ func checkSetDoltConfig(ctx context.Context, serverExec, key, val string) error
return nil
}

// getDoltServer returns a exec.Cmd for a dolt server
func getDoltServer(ctx context.Context, config *ServerConfig, testRepo string, params []string) *exec.Cmd {
// getServer returns a exec.Cmd for a dolt server
func getServer(ctx context.Context, config *ServerConfig, testRepo string, params []string) *exec.Cmd {
server := ExecCommand(ctx, config.ServerExec, params...)
server.Dir = testRepo
return server
Expand Down
Loading

0 comments on commit c01f8d8

Please sign in to comment.