diff --git a/go/cmd/dolt/commands/engine/sqlengine.go b/go/cmd/dolt/commands/engine/sqlengine.go index afbd05a985..3738408071 100644 --- a/go/cmd/dolt/commands/engine/sqlengine.go +++ b/go/cmd/dolt/commands/engine/sqlengine.go @@ -130,7 +130,6 @@ func NewSqlEngine( config.ClusterController.RegisterStoredProcedures(pro) pro.InitDatabaseHook = cluster.NewInitDatabaseHook(config.ClusterController, bThreads, pro.InitDatabaseHook) - config.ClusterController.ManageDatabaseProvider(pro) // Create the engine engine := gms.New(analyzer.NewBuilder(pro).WithParallelism(parallelism).Build(), &gms.Config{ @@ -138,6 +137,17 @@ func NewSqlEngine( IsServerLocked: config.IsServerLocked, }).WithBackgroundThreads(bThreads) + config.ClusterController.SetIsStandbyCallback(func(isStandby bool) { + pro.SetIsStandby(isStandby) + + // Standbys are read only, primarys are not. + // We only change this here if the server was not forced read + // only by its startup config. + if !config.IsReadOnly { + engine.ReadOnly.Store(isStandby) + } + }) + // Load in privileges from file, if it exists var persister cluster.MySQLDbPersister persister = mysql_file_handler.NewPersister(config.PrivFilePath, config.DoltCfgDirPath) diff --git a/go/libraries/doltcore/sqle/cluster/assume_role.go b/go/libraries/doltcore/sqle/cluster/assume_role.go index 3f53865b96..d18ce8a1fe 100644 --- a/go/libraries/doltcore/sqle/cluster/assume_role.go +++ b/go/libraries/doltcore/sqle/cluster/assume_role.go @@ -56,6 +56,7 @@ func newAssumeRoleProcedure(controller *Controller) sql.ExternalStoredProcedureD } return sql.RowsToRowIter(sql.Row{0}), nil }, + ReadOnly: true, } } diff --git a/go/libraries/doltcore/sqle/cluster/controller.go b/go/libraries/doltcore/sqle/cluster/controller.go index 9e39e1ff49..4a10498fc6 100644 --- a/go/libraries/doltcore/sqle/cluster/controller.go +++ b/go/libraries/doltcore/sqle/cluster/controller.go @@ -71,10 +71,10 @@ type Controller struct { cinterceptor clientinterceptor lgr *logrus.Logger - provider dbProvider - iterSessions IterSessions - killQuery func(uint32) - killConnection func(uint32) error + standbyCallback IsStandbyCallback + iterSessions IterSessions + killQuery func(uint32) + killConnection func(uint32) error jwks *jwtauth.MultiJWKS tlsCfg *tls.Config @@ -92,11 +92,10 @@ type sqlvars interface { GetGlobal(name string) (sql.SystemVariable, interface{}, bool) } -// We can manage certain aspects of the exposed databases on the server through -// this. -type dbProvider interface { - SetIsStandby(bool) -} +// Our IsStandbyCallback gets called with |true| or |false| when the server +// becomes a standby or a primary respectively. Standby replicas should be read +// only. +type IsStandbyCallback func(bool) type procedurestore interface { Register(sql.ExternalStoredProcedureDetails) @@ -230,13 +229,13 @@ func (c *Controller) ApplyStandbyReplicationConfig(ctx context.Context, bt *sql. type IterSessions func(func(sql.Session) (bool, error)) error -func (c *Controller) ManageDatabaseProvider(p dbProvider) { +func (c *Controller) SetIsStandbyCallback(callback IsStandbyCallback) { if c == nil { return } c.mu.Lock() defer c.mu.Unlock() - c.provider = p + c.standbyCallback = callback c.setProviderIsStandby(c.role != RolePrimary) } @@ -701,8 +700,8 @@ func (c *Controller) killRunningQueries(saveConnID int) { // called with c.mu held func (c *Controller) setProviderIsStandby(standby bool) { - if c.provider != nil { - c.provider.SetIsStandby(standby) + if c.standbyCallback != nil { + c.standbyCallback(standby) } } diff --git a/go/libraries/doltcore/sqle/dprocedures/init.go b/go/libraries/doltcore/sqle/dprocedures/init.go index 5948171cc8..0897d04d8f 100644 --- a/go/libraries/doltcore/sqle/dprocedures/init.go +++ b/go/libraries/doltcore/sqle/dprocedures/init.go @@ -34,7 +34,7 @@ var DoltProcedures = []sql.ExternalStoredProcedureDetails{ {Name: "dolt_fetch", Schema: int64Schema("success"), Function: doltFetch}, // dolt_gc is enabled behind a feature flag for now, see dolt_gc.go - {Name: "dolt_gc", Schema: int64Schema("success"), Function: doltGC}, + {Name: "dolt_gc", Schema: int64Schema("success"), Function: doltGC, ReadOnly: true}, {Name: "dolt_merge", Schema: doltMergeSchema, Function: doltMerge}, {Name: "dolt_pull", Schema: int64Schema("fast_forward", "conflicts"), Function: doltPull}, diff --git a/integration-tests/go-sql-server-driver/main_test.go b/integration-tests/go-sql-server-driver/main_test.go index cace631a5e..0af5489b42 100644 --- a/integration-tests/go-sql-server-driver/main_test.go +++ b/integration-tests/go-sql-server-driver/main_test.go @@ -54,3 +54,7 @@ func TestOriginal(t *testing.T) { func TestTLS(t *testing.T) { RunTestsFile(t, "tests/sql-server-tls.yaml") } + +func TestClusterReadOnly(t *testing.T) { + RunTestsFile(t, "tests/sql-server-cluster-read-only.yaml") +} diff --git a/integration-tests/go-sql-server-driver/tests/sql-server-cluster-read-only.yaml b/integration-tests/go-sql-server-driver/tests/sql-server-cluster-read-only.yaml new file mode 100644 index 0000000000..8e78106578 --- /dev/null +++ b/integration-tests/go-sql-server-driver/tests/sql-server-cluster-read-only.yaml @@ -0,0 +1,204 @@ +tests: +- name: users and grants cannot be run on standby + multi_repos: + - name: server1 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:3852/{database} + bootstrap_role: primary + bootstrap_epoch: 1 + remotesapi: + port: 3851 + server: + args: ["--config", "server.yaml"] + port: 3309 + - name: server2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:3851/{database} + bootstrap_role: standby + bootstrap_epoch: 1 + remotesapi: + port: 3852 + server: + args: ["--config", "server.yaml"] + port: 3310 + connections: + - on: server2 + queries: + - exec: 'CREATE USER "brian"@"%" IDENTIFIED BY "brianspassword"' + error_match: 'database server is set to read only mode' + - exec: 'GRANT ALL ON *.* TO "aaron"@"%"' + error_match: 'database server is set to read only mode' +- name: create database cannot be run on standby + multi_repos: + - name: server1 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:3852/{database} + bootstrap_role: primary + bootstrap_epoch: 1 + remotesapi: + port: 3851 + server: + args: ["--config", "server.yaml"] + port: 3309 + - name: server2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:3851/{database} + bootstrap_role: standby + bootstrap_epoch: 1 + remotesapi: + port: 3852 + server: + args: ["--config", "server.yaml"] + port: 3310 + connections: + - on: server2 + queries: + - exec: 'CREATE DATABASE my_db' + error_match: 'database server is set to read only mode' +- name: drop database cannot be run on standby + multi_repos: + - name: server1 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:3852/{database} + bootstrap_role: primary + bootstrap_epoch: 1 + remotesapi: + port: 3851 + server: + args: ["--config", "server.yaml"] + port: 3309 + - name: server2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:3851/{database} + bootstrap_role: standby + bootstrap_epoch: 1 + remotesapi: + port: 3852 + server: + args: ["--config", "server.yaml"] + port: 3310 + connections: + - on: server1 + queries: + - exec: 'SET @@PERSIST.dolt_cluster_ack_writes_timeout_secs = 10' + - exec: 'CREATE DATABASE repo1' + - exec: 'USE repo1' + - exec: 'CREATE TABLE vals (i INT PRIMARY KEY)' + - exec: 'INSERT INTO vals VALUES (0),(1),(2),(3),(4)' + - on: server2 + queries: + - exec: 'DROP DATABASE repo1' + error_match: 'database server is set to read only mode' +- name: when a server becomes primary it accepts writes + multi_repos: + - name: server1 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3309 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:3852/{database} + bootstrap_role: primary + bootstrap_epoch: 1 + remotesapi: + port: 3851 + server: + args: ["--config", "server.yaml"] + port: 3309 + - name: server2 + with_files: + - name: server.yaml + contents: | + log_level: trace + listener: + host: 0.0.0.0 + port: 3310 + cluster: + standby_remotes: + - name: standby + remote_url_template: http://localhost:3851/{database} + bootstrap_role: standby + bootstrap_epoch: 1 + remotesapi: + port: 3852 + server: + args: ["--config", "server.yaml"] + port: 3310 + connections: + - on: server2 + queries: + - exec: 'CALL DOLT_ASSUME_CLUSTER_ROLE("primary", 2)' + - on: server2 + queries: + - exec: 'SET @@PERSIST.dolt_cluster_ack_writes_timeout_secs = 10' + - exec: 'CREATE DATABASE repo1' + - exec: 'USE repo1' + - exec: 'CREATE TABLE vals (i INT PRIMARY KEY)' + - exec: 'INSERT INTO vals VALUES (0),(1),(2),(3),(4)' + - on: server1 + queries: + - exec: 'USE repo1' + - query: 'SELECT COUNT(*) FROM vals' + result: + columns: ["COUNT(*)"] + rows: + - [5] diff --git a/integration-tests/go-sql-server-driver/tests/sql-server-cluster-users-and-grants.yaml b/integration-tests/go-sql-server-driver/tests/sql-server-cluster-users-and-grants.yaml index faa003a243..ece050818f 100644 --- a/integration-tests/go-sql-server-driver/tests/sql-server-cluster-users-and-grants.yaml +++ b/integration-tests/go-sql-server-driver/tests/sql-server-cluster-users-and-grants.yaml @@ -65,81 +65,3 @@ tests: result: columns: ["count(*)"] rows: [["15"]] -- name: users and grants applied to standby do not replicate -### TODO: This test should not be possible; being able to run create user on a standby is a bug. - multi_repos: - - name: server1 - with_files: - - name: server.yaml - contents: | - log_level: trace - listener: - host: 0.0.0.0 - port: 3309 - cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:3852/{database} - bootstrap_role: primary - bootstrap_epoch: 1 - remotesapi: - port: 3851 - server: - args: ["--config", "server.yaml"] - port: 3309 - - name: server2 - with_files: - - name: server.yaml - contents: | - log_level: trace - listener: - host: 0.0.0.0 - port: 3310 - cluster: - standby_remotes: - - name: standby - remote_url_template: http://localhost:3851/{database} - bootstrap_role: standby - bootstrap_epoch: 1 - remotesapi: - port: 3852 - server: - args: ["--config", "server.yaml"] - port: 3310 - connections: - - on: server1 - queries: - - exec: 'SET @@PERSIST.dolt_cluster_ack_writes_timeout_secs = 10' - - exec: 'create database repo1' - - exec: "use repo1" - - exec: 'create table vals (i int primary key)' - - exec: 'insert into vals values (0),(1),(2),(3),(4)' - - exec: 'create user "aaron"@"%" IDENTIFIED BY "aaronspassword"' - - exec: 'grant ALL ON *.* to "aaron"@"%"' - - exec: 'insert into vals values (5),(6),(7),(8),(9)' - - on: server1 - user: 'aaron' - password: 'aaronspassword' - queries: - - exec: "use repo1" - - exec: 'insert into vals values (10),(11),(12),(13),(14)' - - on: server2 - user: 'aaron' - password: 'aaronspassword' - queries: - - exec: "use repo1" - - query: 'select count(*) from vals' - result: - columns: ["count(*)"] - rows: [["15"]] - - exec: 'create user "brian"@"%" IDENTIFIED BY "brianspassword"' - - exec: 'grant ALL ON *.* to "brian"@"%"' - - exec: 'select sleep(1) from dual' - - on: server1 - user: 'aaron' - password: 'aaronspassword' - queries: - - query: "select count(*) from mysql.user where User = 'brian'" - result: - columns: ["count(*)"] - rows: [["0"]]