Skip to content

Commit

Permalink
Merge pull request #6607 from dolthub/aaron/dolt-clustering-standby-i…
Browse files Browse the repository at this point in the history
…s-read-only

sqle: cluster: Set the engine to read-only when a replica is in standby mode. Set it back to read-write when it becomes primary.
  • Loading branch information
reltuk authored Sep 1, 2023
2 parents a7bdff5 + 4e35318 commit 51d30f7
Show file tree
Hide file tree
Showing 7 changed files with 233 additions and 93 deletions.
12 changes: 11 additions & 1 deletion go/cmd/dolt/commands/engine/sqlengine.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,14 +130,24 @@ func NewSqlEngine(

config.ClusterController.RegisterStoredProcedures(pro)
pro.InitDatabaseHook = cluster.NewInitDatabaseHook(config.ClusterController, bThreads, pro.InitDatabaseHook)
config.ClusterController.ManageDatabaseProvider(pro)

// Create the engine
engine := gms.New(analyzer.NewBuilder(pro).WithParallelism(parallelism).Build(), &gms.Config{
IsReadOnly: config.IsReadOnly,
IsServerLocked: config.IsServerLocked,
}).WithBackgroundThreads(bThreads)

config.ClusterController.SetIsStandbyCallback(func(isStandby bool) {
pro.SetIsStandby(isStandby)

// Standbys are read only, primarys are not.
// We only change this here if the server was not forced read
// only by its startup config.
if !config.IsReadOnly {
engine.ReadOnly.Store(isStandby)
}
})

// Load in privileges from file, if it exists
var persister cluster.MySQLDbPersister
persister = mysql_file_handler.NewPersister(config.PrivFilePath, config.DoltCfgDirPath)
Expand Down
1 change: 1 addition & 0 deletions go/libraries/doltcore/sqle/cluster/assume_role.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ func newAssumeRoleProcedure(controller *Controller) sql.ExternalStoredProcedureD
}
return sql.RowsToRowIter(sql.Row{0}), nil
},
ReadOnly: true,
}
}

Expand Down
25 changes: 12 additions & 13 deletions go/libraries/doltcore/sqle/cluster/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,10 @@ type Controller struct {
cinterceptor clientinterceptor
lgr *logrus.Logger

provider dbProvider
iterSessions IterSessions
killQuery func(uint32)
killConnection func(uint32) error
standbyCallback IsStandbyCallback
iterSessions IterSessions
killQuery func(uint32)
killConnection func(uint32) error

jwks *jwtauth.MultiJWKS
tlsCfg *tls.Config
Expand All @@ -92,11 +92,10 @@ type sqlvars interface {
GetGlobal(name string) (sql.SystemVariable, interface{}, bool)
}

// We can manage certain aspects of the exposed databases on the server through
// this.
type dbProvider interface {
SetIsStandby(bool)
}
// Our IsStandbyCallback gets called with |true| or |false| when the server
// becomes a standby or a primary respectively. Standby replicas should be read
// only.
type IsStandbyCallback func(bool)

type procedurestore interface {
Register(sql.ExternalStoredProcedureDetails)
Expand Down Expand Up @@ -230,13 +229,13 @@ func (c *Controller) ApplyStandbyReplicationConfig(ctx context.Context, bt *sql.

type IterSessions func(func(sql.Session) (bool, error)) error

func (c *Controller) ManageDatabaseProvider(p dbProvider) {
func (c *Controller) SetIsStandbyCallback(callback IsStandbyCallback) {
if c == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
c.provider = p
c.standbyCallback = callback
c.setProviderIsStandby(c.role != RolePrimary)
}

Expand Down Expand Up @@ -701,8 +700,8 @@ func (c *Controller) killRunningQueries(saveConnID int) {

// called with c.mu held
func (c *Controller) setProviderIsStandby(standby bool) {
if c.provider != nil {
c.provider.SetIsStandby(standby)
if c.standbyCallback != nil {
c.standbyCallback(standby)
}
}

Expand Down
2 changes: 1 addition & 1 deletion go/libraries/doltcore/sqle/dprocedures/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ var DoltProcedures = []sql.ExternalStoredProcedureDetails{
{Name: "dolt_fetch", Schema: int64Schema("success"), Function: doltFetch},

// dolt_gc is enabled behind a feature flag for now, see dolt_gc.go
{Name: "dolt_gc", Schema: int64Schema("success"), Function: doltGC},
{Name: "dolt_gc", Schema: int64Schema("success"), Function: doltGC, ReadOnly: true},

{Name: "dolt_merge", Schema: doltMergeSchema, Function: doltMerge},
{Name: "dolt_pull", Schema: int64Schema("fast_forward", "conflicts"), Function: doltPull},
Expand Down
4 changes: 4 additions & 0 deletions integration-tests/go-sql-server-driver/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,3 +54,7 @@ func TestOriginal(t *testing.T) {
func TestTLS(t *testing.T) {
RunTestsFile(t, "tests/sql-server-tls.yaml")
}

func TestClusterReadOnly(t *testing.T) {
RunTestsFile(t, "tests/sql-server-cluster-read-only.yaml")
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
tests:
- name: users and grants cannot be run on standby
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
port: 3309
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:3852/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: 3851
server:
args: ["--config", "server.yaml"]
port: 3309
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
port: 3310
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:3851/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: 3852
server:
args: ["--config", "server.yaml"]
port: 3310
connections:
- on: server2
queries:
- exec: 'CREATE USER "brian"@"%" IDENTIFIED BY "brianspassword"'
error_match: 'database server is set to read only mode'
- exec: 'GRANT ALL ON *.* TO "aaron"@"%"'
error_match: 'database server is set to read only mode'
- name: create database cannot be run on standby
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
port: 3309
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:3852/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: 3851
server:
args: ["--config", "server.yaml"]
port: 3309
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
port: 3310
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:3851/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: 3852
server:
args: ["--config", "server.yaml"]
port: 3310
connections:
- on: server2
queries:
- exec: 'CREATE DATABASE my_db'
error_match: 'database server is set to read only mode'
- name: drop database cannot be run on standby
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
port: 3309
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:3852/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: 3851
server:
args: ["--config", "server.yaml"]
port: 3309
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
port: 3310
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:3851/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: 3852
server:
args: ["--config", "server.yaml"]
port: 3310
connections:
- on: server1
queries:
- exec: 'SET @@PERSIST.dolt_cluster_ack_writes_timeout_secs = 10'
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: 'CREATE TABLE vals (i INT PRIMARY KEY)'
- exec: 'INSERT INTO vals VALUES (0),(1),(2),(3),(4)'
- on: server2
queries:
- exec: 'DROP DATABASE repo1'
error_match: 'database server is set to read only mode'
- name: when a server becomes primary it accepts writes
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
port: 3309
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:3852/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: 3851
server:
args: ["--config", "server.yaml"]
port: 3309
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
port: 3310
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:3851/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: 3852
server:
args: ["--config", "server.yaml"]
port: 3310
connections:
- on: server2
queries:
- exec: 'CALL DOLT_ASSUME_CLUSTER_ROLE("primary", 2)'
- on: server2
queries:
- exec: 'SET @@PERSIST.dolt_cluster_ack_writes_timeout_secs = 10'
- exec: 'CREATE DATABASE repo1'
- exec: 'USE repo1'
- exec: 'CREATE TABLE vals (i INT PRIMARY KEY)'
- exec: 'INSERT INTO vals VALUES (0),(1),(2),(3),(4)'
- on: server1
queries:
- exec: 'USE repo1'
- query: 'SELECT COUNT(*) FROM vals'
result:
columns: ["COUNT(*)"]
rows:
- [5]
Original file line number Diff line number Diff line change
Expand Up @@ -65,81 +65,3 @@ tests:
result:
columns: ["count(*)"]
rows: [["15"]]
- name: users and grants applied to standby do not replicate
### TODO: This test should not be possible; being able to run create user on a standby is a bug.
multi_repos:
- name: server1
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
port: 3309
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:3852/{database}
bootstrap_role: primary
bootstrap_epoch: 1
remotesapi:
port: 3851
server:
args: ["--config", "server.yaml"]
port: 3309
- name: server2
with_files:
- name: server.yaml
contents: |
log_level: trace
listener:
host: 0.0.0.0
port: 3310
cluster:
standby_remotes:
- name: standby
remote_url_template: http://localhost:3851/{database}
bootstrap_role: standby
bootstrap_epoch: 1
remotesapi:
port: 3852
server:
args: ["--config", "server.yaml"]
port: 3310
connections:
- on: server1
queries:
- exec: 'SET @@PERSIST.dolt_cluster_ack_writes_timeout_secs = 10'
- exec: 'create database repo1'
- exec: "use repo1"
- exec: 'create table vals (i int primary key)'
- exec: 'insert into vals values (0),(1),(2),(3),(4)'
- exec: 'create user "aaron"@"%" IDENTIFIED BY "aaronspassword"'
- exec: 'grant ALL ON *.* to "aaron"@"%"'
- exec: 'insert into vals values (5),(6),(7),(8),(9)'
- on: server1
user: 'aaron'
password: 'aaronspassword'
queries:
- exec: "use repo1"
- exec: 'insert into vals values (10),(11),(12),(13),(14)'
- on: server2
user: 'aaron'
password: 'aaronspassword'
queries:
- exec: "use repo1"
- query: 'select count(*) from vals'
result:
columns: ["count(*)"]
rows: [["15"]]
- exec: 'create user "brian"@"%" IDENTIFIED BY "brianspassword"'
- exec: 'grant ALL ON *.* to "brian"@"%"'
- exec: 'select sleep(1) from dual'
- on: server1
user: 'aaron'
password: 'aaronspassword'
queries:
- query: "select count(*) from mysql.user where User = 'brian'"
result:
columns: ["count(*)"]
rows: [["0"]]

0 comments on commit 51d30f7

Please sign in to comment.