From d7378ab9fc4df204a048ceb55e24cdf7c8475d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Sun, 2 Dec 2018 06:42:36 +0000 Subject: [PATCH 01/12] added support for dialects to separate all SQL-flavours/DB-specific SQL commands --- README.md | 4 ++ db/db.go | 108 +++++++++++++++++++++----------------------- db/db_dialect.go | 31 +++++++++++++ db/db_mysql.go | 28 +++--------- db/db_postgresql.go | 28 +++--------- db/db_test.go | 8 ++-- 6 files changed, 105 insertions(+), 102 deletions(-) create mode 100644 db/db_dialect.go diff --git a/README.md b/README.md index 5c484a7..4d419f7 100644 --- a/README.md +++ b/README.md @@ -88,12 +88,16 @@ Currently migrator supports the following databases: * PostgreSQL - schema-based multi-tenant database, with transactions spanning DDL statements, driver used: https://github.com/lib/pq * PostgreSQL - original PostgreSQL server + * Amazon RDS PostgreSQL - PostgreSQL-compatible relational database built for the cloud * Amazon Aurora PostgreSQL - PostgreSQL-compatible relational database built for the cloud + * Google CloudSQL PostgreSQL - PostgreSQL-compatible relational database built for the cloud * MySQL - database-based multi-tenant database, transactions do not span DDL statements, driver used: https://github.com/go-sql-driver/mysql * MySQL - original MySQL server * MariaDB - enhanced near linearly scalable multi-master MySQL * Percona - an enhanced drop-in replacement for MySQL + * Amazon RDS MySQL - MySQL-compatible relational database built for the cloud * Amazon Aurora MySQL - MySQL-compatible relational database built for the cloud + * Google CloudSQL MySQL - MySQL-compatible relational database built for the cloud # Do you speak docker? diff --git a/db/db.go b/db/db.go index 9de8348..54bded7 100644 --- a/db/db.go +++ b/db/db.go @@ -20,20 +20,21 @@ type Connector interface { GetSchemaPlaceHolder() string // rename to GetDBMigrations, change type to DBMigration GetMigrations() []types.MigrationDB - GetMigrationInsertSql() string ApplyMigrations(migrations []types.Migration) Dispose() } // BaseConnector struct is a base struct for implementing DB specific dialects type BaseConnector struct { - Config *config.Config - DB *sql.DB + Config *config.Config + Dialect Dialect + DB *sql.DB } // CreateConnector constructs Connector instance based on the passed Config func CreateConnector(config *config.Config) Connector { - var bc = BaseConnector{config, nil} + dialect := CreateDialect(config) + bc := BaseConnector{config, dialect, nil} var connector Connector switch config.Driver { @@ -170,22 +171,57 @@ func (bc *BaseConnector) GetMigrations() []types.MigrationDB { // ApplyMigrations applies passed migrations func (bc *BaseConnector) ApplyMigrations(migrations []types.Migration) { - log.Panic("ApplyMigrations() must be overwritten by specific connector") + if len(migrations) == 0 { + return + } + + tenants := bc.GetTenants() + + tx, err := bc.DB.Begin() + if err != nil { + log.Panicf("Could not start DB transaction: %v", err) + } + + bc.applyMigrationsInTx(tx, tenants, migrations) + + tx.Commit() } // AddTenantAndApplyMigrations adds new tenant and applies all existing tenant migrations func (bc *BaseConnector) AddTenantAndApplyMigrations(tenant string, migrations []types.Migration) { - log.Panic("AddTenantAndApplyMigrations() must be overwritten by specific conector") -} + tenantInsertSql := bc.GetTenantInsertSql() -func (bc *BaseConnector) GetMigrationInsertSql() string { - log.Panic("GetMigrationInsertSQL() must be overwritten by specific connector") - return "" + tx, err := bc.DB.Begin() + if err != nil { + log.Panicf("Could not start DB transaction: %v", err) + } + + insert, err := bc.DB.Prepare(tenantInsertSql) + if err != nil { + log.Panicf("Could not create prepared statement: %v", err) + } + + _, err = tx.Stmt(insert).Exec(tenant) + if err != nil { + tx.Rollback() + log.Panicf("Failed to add tenant entry, transaction rollback was called: %v", err) + } + + bc.applyMigrationsInTx(tx, []string{tenant}, migrations) + + tx.Commit() } func (bc *BaseConnector) GetTenantInsertSql() string { - log.Panic("GetTenantInsertSQL() must be overwritten by specific connector") - return "" + var tenantInsertSql string + // if set explicitly in config use it + // otherwise use default value provided by Dialect implementation + if bc.Config.TenantInsertSql != "" { + tenantInsertSql = bc.Config.TenantInsertSql + } else { + tenantInsertSql = bc.Dialect.GetTenantInsertSql() + } + return tenantInsertSql } // GetSchemaPlaceHolder returns a schema placeholder which is @@ -200,30 +236,12 @@ func (bc *BaseConnector) GetSchemaPlaceHolder() string { return schemaPlaceHolder } -// virtual methods do not exist in golang -// this method is called from ApplyMigrations -func (bc *BaseConnector) doApplyMigrations(migrations []types.Migration, insertMigrationSQL string) { - - if len(migrations) == 0 { - return - } - - tenants := bc.GetTenants() - - tx, err := bc.DB.Begin() - if err != nil { - log.Panicf("Could not start DB transaction: %v", err) - } - - bc.doApplyMigrationsInTx(tx, tenants, migrations, insertMigrationSQL) - - tx.Commit() -} - -func (bc *BaseConnector) doApplyMigrationsInTx(tx *sql.Tx, tenants []string, migrations []types.Migration, insertMigrationSQL string) { +func (bc *BaseConnector) applyMigrationsInTx(tx *sql.Tx, tenants []string, migrations []types.Migration) { schemaPlaceHolder := bc.GetSchemaPlaceHolder() - insert, err := bc.DB.Prepare(insertMigrationSQL) + insertMigrationSql := bc.Dialect.GetMigrationInsertSql() + + insert, err := bc.DB.Prepare(insertMigrationSql) if err != nil { log.Panicf("Could not create prepared statement: %v", err) } @@ -256,25 +274,3 @@ func (bc *BaseConnector) doApplyMigrationsInTx(tx *sql.Tx, tenants []string, mig } } - -func (bc *BaseConnector) doAddTenantAndApplyMigrations(tenant string, migrations []types.Migration, insertTenantSQL string, insertMigrationSQL string) { - tx, err := bc.DB.Begin() - if err != nil { - log.Panicf("Could not start DB transaction: %v", err) - } - - insert, err := bc.DB.Prepare(insertTenantSQL) - if err != nil { - log.Panicf("Could not create prepared statement: %v", err) - } - - _, err = tx.Stmt(insert).Exec(tenant) - if err != nil { - tx.Rollback() - log.Panicf("Failed to add tenant entry, transaction rollback was called: %v", err) - } - - bc.doApplyMigrationsInTx(tx, []string{tenant}, migrations, insertMigrationSQL) - - tx.Commit() -} diff --git a/db/db_dialect.go b/db/db_dialect.go new file mode 100644 index 0000000..07f5ca1 --- /dev/null +++ b/db/db_dialect.go @@ -0,0 +1,31 @@ +package db + +import ( + "github.com/lukaszbudnik/migrator/config" + "log" +) + +type Dialect interface { + GetTenantInsertSql() string + GetMigrationInsertSql() string +} + +type BaseDialect struct { +} + +// CreateDialect constructs Dialect instance based on the passed Config +func CreateDialect(config *config.Config) Dialect { + + var dialect Dialect + + switch config.Driver { + case "mysql": + dialect = &mySQLDialect{} + case "postgres": + dialect = &postgreSQLDialect{} + default: + log.Panicf("Failed to create Connector: %q is an unknown driver.", config.Driver) + } + + return dialect +} diff --git a/db/db_mysql.go b/db/db_mysql.go index f9e981a..9328f75 100644 --- a/db/db_mysql.go +++ b/db/db_mysql.go @@ -2,7 +2,6 @@ package db import ( "fmt" - "github.com/lukaszbudnik/migrator/types" // blank import for MySQL driver _ "github.com/go-sql-driver/mysql" ) @@ -11,32 +10,19 @@ type mySQLConnector struct { BaseConnector } +type mySQLDialect struct { + BaseDialect +} + const ( insertMigrationMySQLDialectSql = "insert into %v (name, source_dir, file, type, db_schema) values (?, ?, ?, ?, ?)" defaultInsertTenantMySQLDialectSql = "insert into %v (name) values (?)" ) -func (mc *mySQLConnector) ApplyMigrations(migrations []types.Migration) { - insertMigrationSql := mc.GetMigrationInsertSql() - mc.doApplyMigrations(migrations, insertMigrationSql) -} - -func (mc *mySQLConnector) AddTenantAndApplyMigrations(tenant string, migrations []types.Migration) { - insertMigrationSql := mc.GetMigrationInsertSql() - insertTenantSql := mc.GetTenantInsertSql() - mc.doAddTenantAndApplyMigrations(tenant, migrations, insertTenantSql, insertMigrationSql) -} - -func (mc *mySQLConnector) GetMigrationInsertSql() string { +func (md *mySQLDialect) GetMigrationInsertSql() string { return fmt.Sprintf(insertMigrationMySQLDialectSql, migrationsTableName) } -func (mc *mySQLConnector) GetTenantInsertSql() string { - var tenantsInsertSql string - if mc.Config.TenantInsertSql != "" { - tenantsInsertSql = mc.Config.TenantInsertSql - } else { - tenantsInsertSql = fmt.Sprintf(defaultInsertTenantMySQLDialectSql, defaultTenantsTableName) - } - return tenantsInsertSql +func (md *mySQLDialect) GetTenantInsertSql() string { + return fmt.Sprintf(defaultInsertTenantMySQLDialectSql, defaultTenantsTableName) } diff --git a/db/db_postgresql.go b/db/db_postgresql.go index 94ec57c..f550257 100644 --- a/db/db_postgresql.go +++ b/db/db_postgresql.go @@ -2,7 +2,6 @@ package db import ( "fmt" - "github.com/lukaszbudnik/migrator/types" // blank import for PostgreSQL driver _ "github.com/lib/pq" ) @@ -11,32 +10,19 @@ type postgreSQLConnector struct { BaseConnector } +type postgreSQLDialect struct { + BaseDialect +} + const ( insertMigrationPostgreSQLDialectSql = "insert into %v (name, source_dir, file, type, db_schema) values ($1, $2, $3, $4, $5)" defaultInsertTenantPostgreSQLDialectSql = "insert into %v (name) values ($1)" ) -func (pc *postgreSQLConnector) ApplyMigrations(migrations []types.Migration) { - insertMigrationSql := pc.GetMigrationInsertSql() - pc.doApplyMigrations(migrations, insertMigrationSql) -} - -func (pc *postgreSQLConnector) AddTenantAndApplyMigrations(tenant string, migrations []types.Migration) { - insertMigrationSql := pc.GetMigrationInsertSql() - insertTenantSql := pc.GetTenantInsertSql() - pc.doAddTenantAndApplyMigrations(tenant, migrations, insertTenantSql, insertMigrationSql) -} - -func (pc *postgreSQLConnector) GetMigrationInsertSql() string { +func (pd *postgreSQLDialect) GetMigrationInsertSql() string { return fmt.Sprintf(insertMigrationPostgreSQLDialectSql, migrationsTableName) } -func (pc *postgreSQLConnector) GetTenantInsertSql() string { - var tenantsInsertSql string - if pc.Config.TenantInsertSql != "" { - tenantsInsertSql = pc.Config.TenantInsertSql - } else { - tenantsInsertSql = fmt.Sprintf(defaultInsertTenantPostgreSQLDialectSql, defaultTenantsTableName) - } - return tenantsInsertSql +func (pd *postgreSQLDialect) GetTenantInsertSql() string { + return fmt.Sprintf(defaultInsertTenantPostgreSQLDialectSql, defaultTenantsTableName) } diff --git a/db/db_test.go b/db/db_test.go index bc3e873..0c1d6e7 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -240,9 +240,9 @@ func TestMySQLGetMigrationInsertSql(t *testing.T) { config.Driver = "mysql" - connector := CreateConnector(config) + dialect := CreateDialect(config) - insertMigrationSQL := connector.GetMigrationInsertSql() + insertMigrationSQL := dialect.GetMigrationInsertSql() assert.Equal(t, "insert into public.migrator_migrations (name, source_dir, file, type, db_schema) values (?, ?, ?, ?, ?)", insertMigrationSQL) } @@ -253,9 +253,9 @@ func TestPostgreSQLGetMigrationInsertSql(t *testing.T) { config.Driver = "postgres" - connector := CreateConnector(config) + dialect := CreateDialect(config) - insertMigrationSQL := connector.GetMigrationInsertSql() + insertMigrationSQL := dialect.GetMigrationInsertSql() assert.Equal(t, "insert into public.migrator_migrations (name, source_dir, file, type, db_schema) values ($1, $2, $3, $4, $5)", insertMigrationSQL) } From f0b1605c10bdd516d0dcbd445f552643bc3b7a1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Sun, 2 Dec 2018 06:44:22 +0000 Subject: [PATCH 02/12] gofmt fixes for server module --- server/server.go | 40 ++++++++++++++++++++-------------------- server/server_test.go | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/server/server.go b/server/server.go index 85df6de..decfbb8 100644 --- a/server/server.go +++ b/server/server.go @@ -1,13 +1,13 @@ package server import ( - "io/ioutil" "encoding/json" "fmt" "github.com/lukaszbudnik/migrator/config" "github.com/lukaszbudnik/migrator/core" "github.com/lukaszbudnik/migrator/db" "github.com/lukaszbudnik/migrator/loader" + "io/ioutil" "log" "net/http" "strings" @@ -69,7 +69,7 @@ func migrationsHandler(w http.ResponseWriter, r *http.Request, config *config.Co w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(migrationsApplied) default: - http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) + http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) } } @@ -78,26 +78,26 @@ func tenantsHandler(w http.ResponseWriter, r *http.Request, config *config.Confi switch r.Method { case http.MethodGet: - tenants := core.LoadDBTenants(config, createConnector) - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(tenants) + tenants := core.LoadDBTenants(config, createConnector) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(tenants) case http.MethodPost: - body, err := ioutil.ReadAll(r.Body) - if err != nil { - http.Error(w, "500 internal server error", http.StatusInternalServerError) - return - } - var param tenantParam - err = json.Unmarshal(body, ¶m) - if err != nil || param.Name == "" { - http.Error(w, "400 bad request", http.StatusBadRequest) - return - } - migrationsApplied := core.AddTenant(param.Name, config, createConnector, createLoader) - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(migrationsApplied) + body, err := ioutil.ReadAll(r.Body) + if err != nil { + http.Error(w, "500 internal server error", http.StatusInternalServerError) + return + } + var param tenantParam + err = json.Unmarshal(body, ¶m) + if err != nil || param.Name == "" { + http.Error(w, "400 bad request", http.StatusBadRequest) + return + } + migrationsApplied := core.AddTenant(param.Name, config, createConnector, createLoader) + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(migrationsApplied) default: - http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) + http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) } } diff --git a/server/server_test.go b/server/server_test.go index 7b5357f..effeda3 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -83,7 +83,7 @@ func TestServerTenantsPost(t *testing.T) { json := []byte(`{"name": "new_tenant"}`) req, _ := http.NewRequest("POST", "http://example.com/tenants", bytes.NewBuffer(json)) - req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Type", "application/json") w := httptest.NewRecorder() handler := makeHandler(tenantsHandler, config, createMockedConnector, createMockedDiskLoader) From cf8514b562b119049a86c388c1b6e36c3c6237e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Mon, 3 Dec 2018 07:07:38 +0000 Subject: [PATCH 03/12] exctracted create tables sqls to dialects, made DB tests more dynamic and not fail on re-runs (hardcoded expected values made dynamic) --- db/db.go | 36 ++++++++++-------------------------- db/db_dialect.go | 26 ++++++++++++++++++++++++++ db/db_test.go | 14 +++++++++++--- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/db/db.go b/db/db.go index 54bded7..1dd5b43 100644 --- a/db/db.go +++ b/db/db.go @@ -50,28 +50,10 @@ func CreateConnector(config *config.Config) Connector { } const ( - migrationsTableName = "public.migrator_migrations" - defaultTenantsTableName = "public.migrator_tenants" - createMigrationsTable = ` - create table if not exists %v ( - id serial primary key, - name varchar(200) not null, - source_dir varchar(200) not null, - file varchar(200) not null, - type int not null, - db_schema varchar(200) not null, - created timestamp default now() - ) - ` - createDefaultTenantsTable = ` - create table if not exists %v ( - id serial primary key, - name varchar(200) not null, - created timestamp default now() - ) - ` + migrationsTableName = "public.migrator_migrations" + defaultTenantsTableName = "public.migrator_tenants" selectMigrations = "select name, source_dir, file, type, db_schema, created from %v order by name, source_dir" - defaulttenantsSqlPattern = "select name from %v" + defaultTenantsSqlPattern = "select name from %v" defaultSchemaPlaceHolder = "{schema}" ) @@ -86,15 +68,15 @@ func (bc *BaseConnector) Init() { } bc.DB = db - defaulttenantsSql := fmt.Sprintf(defaulttenantsSqlPattern, defaultTenantsTableName) - if bc.Config.TenantSelectSql != "" && bc.Config.TenantSelectSql != defaulttenantsSql { + defaultTenantsSql := fmt.Sprintf(defaultTenantsSqlPattern, defaultTenantsTableName) + if bc.Config.TenantSelectSql != "" && bc.Config.TenantSelectSql != defaultTenantsSql { + createDefaultTenantsTable := bc.Dialect.GetCreateTenantsTableSql() createTableQuery := fmt.Sprintf(createDefaultTenantsTable, defaultTenantsTableName) if _, err := bc.DB.Query(createTableQuery); err != nil { log.Panicf("Could not create default tenants table: %v", err) } } - } // Dispose closes all resources allocated by connector @@ -110,7 +92,7 @@ func (bc *BaseConnector) GetTenantSelectSql() string { if bc.Config.TenantSelectSql != "" { tenantSelectSql = bc.Config.TenantSelectSql } else { - tenantSelectSql = fmt.Sprintf(defaulttenantsSqlPattern, defaultTenantsTableName) + tenantSelectSql = fmt.Sprintf(defaultTenantsSqlPattern, defaultTenantsTableName) } return tenantSelectSql } @@ -137,7 +119,9 @@ func (bc *BaseConnector) GetTenants() []string { // GetMigrations returns a list of all applied DB migrations func (bc *BaseConnector) GetMigrations() []types.MigrationDB { - createTableQuery := fmt.Sprintf(createMigrationsTable, migrationsTableName) + createMigrationsTableSql := bc.Dialect.GetCreateMigrationsTableSql() + + createTableQuery := fmt.Sprintf(createMigrationsTableSql, migrationsTableName) if _, err := bc.DB.Query(createTableQuery); err != nil { log.Panicf("Could not create migrations table: %v", err) } diff --git a/db/db_dialect.go b/db/db_dialect.go index 07f5ca1..1ecae58 100644 --- a/db/db_dialect.go +++ b/db/db_dialect.go @@ -8,11 +8,37 @@ import ( type Dialect interface { GetTenantInsertSql() string GetMigrationInsertSql() string + GetCreateTenantsTableSql() string + GetCreateMigrationsTableSql() string } type BaseDialect struct { } +func (bd *BaseDialect) GetCreateTenantsTableSql() string { + return ` + create table if not exists %v ( + id serial primary key, + name varchar(200) not null, + created timestamp default now() + ); + ` +} + +func (bd *BaseDialect) GetCreateMigrationsTableSql() string { + return ` + create table if not exists %v ( + id serial primary key, + name varchar(200) not null, + source_dir varchar(200) not null, + file varchar(200) not null, + type int not null, + db_schema varchar(200) not null, + created timestamp default now() + ); + ` +} + // CreateDialect constructs Dialect instance based on the passed Config func CreateDialect(config *config.Config) Dialect { diff --git a/db/db_test.go b/db/db_test.go index 0c1d6e7..f6f84a0 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -91,8 +91,11 @@ func TestDBGetTenants(t *testing.T) { tenants := connector.GetTenants() - assert.Len(t, tenants, 3) - assert.Equal(t, []string{"abc", "def", "xyz"}, tenants) + // todo more than 3 and contains abc, def, xyz + assert.True(t, len(tenants) >= 3) + assert.Contains(t, tenants, "abc") + assert.Contains(t, tenants, "def") + assert.Contains(t, tenants, "xyz") } func TestDBApplyMigrations(t *testing.T) { @@ -103,6 +106,9 @@ func TestDBApplyMigrations(t *testing.T) { connector.Init() defer connector.Dispose() + tenants := connector.GetTenants() + lenTenants := len(tenants) + dbMigrationsBefore := connector.GetMigrations() lenBefore := len(dbMigrationsBefore) @@ -128,7 +134,9 @@ func TestDBApplyMigrations(t *testing.T) { dbMigrationsAfter := connector.GetMigrations() lenAfter := len(dbMigrationsAfter) - assert.Equal(t, 8, lenAfter-lenBefore) + // 2 tenant migrations * no of tenants + 2 public + expected := lenTenants*2 + 2 + assert.Equal(t, expected, lenAfter-lenBefore) } func TestDBApplyMigrationsEmptyMigrationArray(t *testing.T) { From 314d046833f6c38a3eaa82de92cbb040af502f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Mon, 3 Dec 2018 18:28:55 +0100 Subject: [PATCH 04/12] work in progress, added dialect for MSSQL, docker containers --- README.md | 4 +- db/db.go | 18 ++---- db/db_dialect.go | 10 +-- db/db_mssql.go | 56 ++++++++++++++++ db/db_mysql.go | 4 -- db/db_postgresql.go | 4 -- db/db_test.go | 64 +++++++++++++++++-- docker/create-and-setup-container.sh | 5 +- .../mssql-create-and-setup-container.sh | 28 ++++++++ test/create-test-tenants-mssql.sql | 24 +++++++ test/migrator-mssql.yaml | 9 +++ 11 files changed, 193 insertions(+), 33 deletions(-) create mode 100644 db/db_mssql.go create mode 100644 docker/scripts/mssql-create-and-setup-container.sh create mode 100644 test/create-test-tenants-mssql.sql create mode 100644 test/migrator-mssql.yaml diff --git a/README.md b/README.md index 4d419f7..9a47478 100644 --- a/README.md +++ b/README.md @@ -112,9 +112,9 @@ To find out more about migrator docker please visit: https://github.com/lukaszbu PostgreSQL, MySQL, MariaDB, and Percona: ``` -$ docker/create-and-setup-container.sh [postgresql|mysql|mariadb|percona] +$ docker/create-and-setup-container.sh [postgres|mysql|mariadb|percona] $ ./coverage.sh -$ docker/destroy-container.sh [postgresql|mysql|mariadb|percona] +$ docker/destroy-container.sh [postgres|mysql|mariadb|percona] ``` Or see `.travis.yml` to see how it's done on Travis. diff --git a/db/db.go b/db/db.go index 1dd5b43..79176bc 100644 --- a/db/db.go +++ b/db/db.go @@ -34,22 +34,16 @@ type BaseConnector struct { // CreateConnector constructs Connector instance based on the passed Config func CreateConnector(config *config.Config) Connector { dialect := CreateDialect(config) - bc := BaseConnector{config, dialect, nil} - var connector Connector - - switch config.Driver { - case "mysql": - connector = &mySQLConnector{bc} - case "postgres": - connector = &postgreSQLConnector{bc} - default: - log.Panicf("Failed to create Connector: %q is an unknown driver.", config.Driver) - } - + connector := &BaseConnector{config, dialect, nil} return connector } const ( + migratorSchema = "public" + migratorTenantsTable = "migrator_tenants" + migratorMigrationsTable = "migrator_migrations" + + // deprecated migrationsTableName = "public.migrator_migrations" defaultTenantsTableName = "public.migrator_tenants" selectMigrations = "select name, source_dir, file, type, db_schema, created from %v order by name, source_dir" diff --git a/db/db_dialect.go b/db/db_dialect.go index 1ecae58..8cbfbf5 100644 --- a/db/db_dialect.go +++ b/db/db_dialect.go @@ -8,15 +8,15 @@ import ( type Dialect interface { GetTenantInsertSql() string GetMigrationInsertSql() string - GetCreateTenantsTableSql() string - GetCreateMigrationsTableSql() string + GetCreateTenantsTableSql() string + GetCreateMigrationsTableSql() string } type BaseDialect struct { } func (bd *BaseDialect) GetCreateTenantsTableSql() string { - return ` + return ` create table if not exists %v ( id serial primary key, name varchar(200) not null, @@ -26,7 +26,7 @@ func (bd *BaseDialect) GetCreateTenantsTableSql() string { } func (bd *BaseDialect) GetCreateMigrationsTableSql() string { - return ` + return ` create table if not exists %v ( id serial primary key, name varchar(200) not null, @@ -47,6 +47,8 @@ func CreateDialect(config *config.Config) Dialect { switch config.Driver { case "mysql": dialect = &mySQLDialect{} + case "sqlserver": + dialect = &msSQLDialect{} case "postgres": dialect = &postgreSQLDialect{} default: diff --git a/db/db_mssql.go b/db/db_mssql.go new file mode 100644 index 0000000..85090f0 --- /dev/null +++ b/db/db_mssql.go @@ -0,0 +1,56 @@ +package db + +import ( + "fmt" + // blank import for MSSQL driver + _ "github.com/denisenkom/go-mssqldb" +) + +type msSQLDialect struct { + BaseDialect +} + +const ( + insertMigrationMSSQLDialectSql = "insert into %v (name, source_dir, file, type, db_schema) values (@p1, @p2, @p3, @p4, @p5)" + defaultInsertTenantMSSQLDialectSql = "insert into %v (name) values (@p1)" +) + +func (md *msSQLDialect) GetMigrationInsertSql() string { + return fmt.Sprintf(insertMigrationMySQLDialectSql, migrationsTableName) +} + +func (md *msSQLDialect) GetTenantInsertSql() string { + return fmt.Sprintf(defaultInsertTenantMySQLDialectSql, defaultTenantsTableName) +} + +func (md *msSQLDialect) GetCreateTenantsTableSql() string { + createTenantsTableSql := ` +IF NOT EXISTS (select * from information_schema.tables where table_schema = '%v' and table_name = '%v') +BEGIN + create table [%v].%%v ( + id int identity (1,1) primary key, + name varchar(200) not null, + created datetime default CURRENT_TIMESTAMP + ); +END +` + return fmt.Sprintf(createTenantsTableSql, migratorSchema, migratorTenantsTable, migratorSchema) +} + +func (md *msSQLDialect) GetCreateMigrationsTableSql() string { + createMigrationsTableSql := ` +IF NOT EXISTS (select * from information_schema.tables where table_schema = '%v' and table_name = '%v') +BEGIN + create table [%v].%%v ( + id int identity (1,1) primary key, + name varchar(200) not null, + source_dir varchar(200) not null, + file varchar(200) not null, + type int not null, + db_schema varchar(200) not null, + created datetime default CURRENT_TIMESTAMP + ); +END +` + return fmt.Sprintf(createMigrationsTableSql, migratorSchema, migratorMigrationsTable, migratorSchema) +} diff --git a/db/db_mysql.go b/db/db_mysql.go index 9328f75..0e382ae 100644 --- a/db/db_mysql.go +++ b/db/db_mysql.go @@ -6,10 +6,6 @@ import ( _ "github.com/go-sql-driver/mysql" ) -type mySQLConnector struct { - BaseConnector -} - type mySQLDialect struct { BaseDialect } diff --git a/db/db_postgresql.go b/db/db_postgresql.go index f550257..7711493 100644 --- a/db/db_postgresql.go +++ b/db/db_postgresql.go @@ -6,10 +6,6 @@ import ( _ "github.com/lib/pq" ) -type postgreSQLConnector struct { - BaseConnector -} - type postgreSQLDialect struct { BaseDialect } diff --git a/db/db_test.go b/db/db_test.go index f6f84a0..fda9002 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -23,20 +23,20 @@ func TestDBCreateConnectorPanicUnknownDriver(t *testing.T) { }, "Should panic because of unknown driver") } -func TestDBCreateConnectorPostgresDriver(t *testing.T) { +func TestDBCreateDialectPostgreSqlDriver(t *testing.T) { config := &config.Config{} config.Driver = "postgres" - connector := CreateConnector(config) + connector := CreateDialect(config) connectorName := reflect.TypeOf(connector).String() - assert.Equal(t, "*db.postgreSQLConnector", connectorName) + assert.Equal(t, "*db.postgreSQLDialect", connectorName) } -func TestDBCreateConnectorMysqlDriver(t *testing.T) { +func TestDBCreateDialectMysqlDriver(t *testing.T) { config := &config.Config{} config.Driver = "mysql" - connector := CreateConnector(config) + connector := CreateDialect(config) connectorName := reflect.TypeOf(connector).String() - assert.Equal(t, "*db.mySQLConnector", connectorName) + assert.Equal(t, "*db.mySQLDialect", connectorName) } func TestDBConnectorInitPanicConnectionError(t *testing.T) { @@ -305,3 +305,55 @@ func TestGetTenantInsertSqlOverride(t *testing.T) { assert.Equal(t, "insert into XXX", tenantInsertSql) } + +func TestMSSQLDialectGetCreateTenantsTableSql(t *testing.T) { + config, err := config.FromFile("../test/migrator.yaml") + assert.Nil(t, err) + + config.Driver = "mssql" + + dialect := CreateDialect(config) + + createTenantsTableSql := dialect.GetCreateTenantsTableSql() + + expected := ` +IF NOT EXISTS (select * from information_schema.tables where table_schema = 'public' and table_name = 'migrator_tenants') +BEGIN + create table [public].%v ( + id int identity (1,1) primary key, + name varchar(200) not null, + created datetime default CURRENT_TIMESTAMP + ); +END +` + + assert.Equal(t, expected, createTenantsTableSql) +} + +func TestMSSQLDialectGetCreateMigrationsTableSql(t *testing.T) { + config, err := config.FromFile("../test/migrator.yaml") + assert.Nil(t, err) + + config.Driver = "mssql" + + dialect := CreateDialect(config) + + createMigrationsTableSql := dialect.GetCreateMigrationsTableSql() + + expected := ` +IF NOT EXISTS (select * from information_schema.tables where table_schema = 'public' and table_name = 'migrator_migrations') +BEGIN + create table [public].%v ( + id int identity (1,1) primary key, + name varchar(200) not null, + source_dir varchar(200) not null, + file varchar(200) not null, + type int not null, + db_schema varchar(200) not null, + created datetime default CURRENT_TIMESTAMP + ); +END +` + + assert.Equal(t, expected, createMigrationsTableSql) +} diff --git a/docker/create-and-setup-container.sh b/docker/create-and-setup-container.sh index d5b3d14..20a1756 100755 --- a/docker/create-and-setup-container.sh +++ b/docker/create-and-setup-container.sh @@ -11,12 +11,15 @@ for SCRIPT in scripts/*.sh; do done case $CONTAINER_TYPE in - postgresql ) + postgres ) postgresql_start $CONTAINER_TYPE ;; mysql|mariadb|percona ) mysql_start $CONTAINER_TYPE ;; + mssql ) + mssql_start $CONTAINER_TYPE + ;; * ) >&2 echo "Unknown container type $CONTAINER_TYPE" exit 1 diff --git a/docker/scripts/mssql-create-and-setup-container.sh b/docker/scripts/mssql-create-and-setup-container.sh new file mode 100644 index 0000000..9166194 --- /dev/null +++ b/docker/scripts/mssql-create-and-setup-container.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +function mssql_start() { + name=mssql + + ip=127.0.0.1 + port=11433 + database=migratortest + password=YourStrongPassw0rd + + docker run -e 'ACCEPT_EULA=Y' -e "SA_PASSWORD=$password" \ + -p $port:1433 --name "migrator-$name" \ + -d mcr.microsoft.com/mssql/server:2017-latest + + sleep 10 + + running=$(docker inspect -f {{.State.Running}} "migrator-$name") + + if [[ "true" == "$running" ]]; then + sqlcmd -S "$ip,$port" -U SA -P $password -Q "CREATE DATABASE $database" + sqlcmd -S "$ip,$port" -U SA -P $password -d $database -i ../test/create-test-tenants-mssql.sql + + cat ../test/migrator-mssql.yaml | sed "s/A/sqlserver:\/\/SA:$password@$ip:$port\/?database=$database/g" > ../test/migrator.yaml + else + >&2 echo "Could not setup mssql-$name" + exit 1 + fi +} diff --git a/test/create-test-tenants-mssql.sql b/test/create-test-tenants-mssql.sql new file mode 100644 index 0000000..d9603b4 --- /dev/null +++ b/test/create-test-tenants-mssql.sql @@ -0,0 +1,24 @@ +create schema [abc] +GO +create schema [def] +GO +create schema [xyz] +GO +create schema [migrator] +GO + +IF NOT EXISTS (select * from information_schema.tables where table_schema = 'migrator' and table_name = 'migrator_tenants') +BEGIN + create table [migrator].migrator_tenants ( + id int identity (1,1) primary key, + name varchar(200) not null, + created datetime default CURRENT_TIMESTAMP + ); + + insert into [migrator].migrator_tenants (name) values ('abc'); + insert into [migrator].migrator_tenants (name) values ('def'); + insert into [migrator].migrator_tenants (name) values ('xyz'); + +END + +GO diff --git a/test/migrator-mssql.yaml b/test/migrator-mssql.yaml new file mode 100644 index 0000000..ce80444 --- /dev/null +++ b/test/migrator-mssql.yaml @@ -0,0 +1,9 @@ +baseDir: test/migrations +driver: sqlserver +dataSource: "A" +singleSchemas: + - public + - ref + - config +tenantSchemas: + - tenants From 26ae2f47f97a8cd351fd99a9fd77583a39906e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Tue, 4 Dec 2018 18:49:14 +0100 Subject: [PATCH 05/12] implemented support for MSSQL, had to make test migration more universal due to syntax differences --- db/db.go | 53 ++++----- db/db_dialect.go | 62 ++++++---- db/db_mssql.go | 18 +-- db/db_mysql.go | 8 +- db/db_postgresql.go | 8 +- db/db_test.go | 145 ++++++++++++++++++----- test/create-test-tenants.sql | 10 +- test/migrations/public/201602160001.sql | 2 +- test/migrations/tenants/201602160002.sql | 2 +- 9 files changed, 204 insertions(+), 104 deletions(-) diff --git a/db/db.go b/db/db.go index 79176bc..90af44d 100644 --- a/db/db.go +++ b/db/db.go @@ -2,7 +2,6 @@ package db import ( "database/sql" - "fmt" "github.com/lukaszbudnik/migrator/config" "github.com/lukaszbudnik/migrator/types" "log" @@ -39,15 +38,9 @@ func CreateConnector(config *config.Config) Connector { } const ( - migratorSchema = "public" - migratorTenantsTable = "migrator_tenants" - migratorMigrationsTable = "migrator_migrations" - - // deprecated - migrationsTableName = "public.migrator_migrations" - defaultTenantsTableName = "public.migrator_tenants" - selectMigrations = "select name, source_dir, file, type, db_schema, created from %v order by name, source_dir" - defaultTenantsSqlPattern = "select name from %v" + migratorSchema = "migrator" + migratorTenantsTable = "migrator_tenants" + migratorMigrationsTable = "migrator_migrations" defaultSchemaPlaceHolder = "{schema}" ) @@ -62,15 +55,26 @@ func (bc *BaseConnector) Init() { } bc.DB = db - defaultTenantsSql := fmt.Sprintf(defaultTenantsSqlPattern, defaultTenantsTableName) - if bc.Config.TenantSelectSql != "" && bc.Config.TenantSelectSql != defaultTenantsSql { - createDefaultTenantsTable := bc.Dialect.GetCreateTenantsTableSql() - createTableQuery := fmt.Sprintf(createDefaultTenantsTable, defaultTenantsTableName) + tx, err := bc.DB.Begin() + if err != nil { + log.Panicf("Could not start DB transaction: %v", err) + } + + // make sure migrations table exists + createMigrationsTable := bc.Dialect.GetCreateMigrationsTableSql() + if _, err := bc.DB.Query(createMigrationsTable); err != nil { + log.Panicf("Could not create migrations table: %v", err) + } - if _, err := bc.DB.Query(createTableQuery); err != nil { + // if using default migrator tenants table make sure it exists + if bc.Config.TenantSelectSql == "" { + createTenantsTable := bc.Dialect.GetCreateTenantsTableSql() + if _, err := bc.DB.Query(createTenantsTable); err != nil { log.Panicf("Could not create default tenants table: %v", err) } } + + tx.Commit() } // Dispose closes all resources allocated by connector @@ -86,7 +90,7 @@ func (bc *BaseConnector) GetTenantSelectSql() string { if bc.Config.TenantSelectSql != "" { tenantSelectSql = bc.Config.TenantSelectSql } else { - tenantSelectSql = fmt.Sprintf(defaultTenantsSqlPattern, defaultTenantsTableName) + tenantSelectSql = bc.Dialect.GetTenantSelectSql() } return tenantSelectSql } @@ -113,14 +117,7 @@ func (bc *BaseConnector) GetTenants() []string { // GetMigrations returns a list of all applied DB migrations func (bc *BaseConnector) GetMigrations() []types.MigrationDB { - createMigrationsTableSql := bc.Dialect.GetCreateMigrationsTableSql() - - createTableQuery := fmt.Sprintf(createMigrationsTableSql, migrationsTableName) - if _, err := bc.DB.Query(createTableQuery); err != nil { - log.Panicf("Could not create migrations table: %v", err) - } - - query := fmt.Sprintf(selectMigrations, migrationsTableName) + query := bc.Dialect.GetMigrationSelectSql() rows, err := bc.DB.Query(query) if err != nil { @@ -132,15 +129,15 @@ func (bc *BaseConnector) GetMigrations() []types.MigrationDB { var ( name string sourceDir string - file string + filename string migrationType types.MigrationType schema string created time.Time ) - if err := rows.Scan(&name, &sourceDir, &file, &migrationType, &schema, &created); err != nil { + if err := rows.Scan(&name, &sourceDir, &filename, &migrationType, &schema, &created); err != nil { log.Panicf("Could not read DB migration: %v", err) } - mdef := types.MigrationDefinition{name, sourceDir, file, migrationType} + mdef := types.MigrationDefinition{name, sourceDir, filename, migrationType} dbMigrations = append(dbMigrations, types.MigrationDB{mdef, schema, created}) } @@ -240,7 +237,7 @@ func (bc *BaseConnector) applyMigrationsInTx(tx *sql.Tx, tenants []string, migra _, err = tx.Exec(contents) if err != nil { tx.Rollback() - log.Panicf("SQL failed, transaction rollback was called: %v", err) + log.Panicf("SQL failed, transaction rollback was called: %v %v", err, contents) } _, err = tx.Stmt(insert).Exec(m.Name, m.SourceDir, m.File, m.MigrationType, s) diff --git a/db/db_dialect.go b/db/db_dialect.go index 8cbfbf5..748c498 100644 --- a/db/db_dialect.go +++ b/db/db_dialect.go @@ -1,42 +1,60 @@ package db import ( + "fmt" "github.com/lukaszbudnik/migrator/config" "log" ) type Dialect interface { GetTenantInsertSql() string + GetTenantSelectSql() string GetMigrationInsertSql() string - GetCreateTenantsTableSql() string - GetCreateMigrationsTableSql() string + GetMigrationSelectSql() string + GetCreateTenantsTableSql() string + GetCreateMigrationsTableSql() string } type BaseDialect struct { } +const ( + selectMigrations = "select name, source_dir as sd, filename, type, db_schema, created from %v.%v order by name, source_dir" + selectTenants = "select name from %v.%v" +) + func (bd *BaseDialect) GetCreateTenantsTableSql() string { - return ` - create table if not exists %v ( - id serial primary key, - name varchar(200) not null, - created timestamp default now() - ); - ` + createTenantsTableSql := ` +create table if not exists %v.%v ( + id serial primary key, + name varchar(200) not null, + created timestamp default now() +) +` + return fmt.Sprintf(createTenantsTableSql, migratorSchema, migratorTenantsTable) } func (bd *BaseDialect) GetCreateMigrationsTableSql() string { - return ` - create table if not exists %v ( - id serial primary key, - name varchar(200) not null, - source_dir varchar(200) not null, - file varchar(200) not null, - type int not null, - db_schema varchar(200) not null, - created timestamp default now() - ); - ` + createMigrationsTableSql := ` +create table if not exists %v.%v ( + id serial primary key, + name varchar(200) not null, + source_dir varchar(200) not null, + filename varchar(200) not null, + type int not null, + db_schema varchar(200) not null, + created timestamp default now() +) +` + return fmt.Sprintf(createMigrationsTableSql, migratorSchema, migratorMigrationsTable) +} + +func (bd *BaseDialect) GetTenantSelectSql() string { + return fmt.Sprintf(selectTenants, migratorSchema, migratorTenantsTable) +} + +func (bd *BaseDialect) GetMigrationSelectSql() string { + return fmt.Sprintf(selectMigrations, migratorSchema, migratorMigrationsTable) } // CreateDialect constructs Dialect instance based on the passed Config @@ -47,8 +65,8 @@ func CreateDialect(config *config.Config) Dialect { switch config.Driver { case "mysql": dialect = &mySQLDialect{} - case "sqlserver": - dialect = &msSQLDialect{} + case "sqlserver": + dialect = &msSQLDialect{} case "postgres": dialect = &postgreSQLDialect{} default: diff --git a/db/db_mssql.go b/db/db_mssql.go index 85090f0..b828433 100644 --- a/db/db_mssql.go +++ b/db/db_mssql.go @@ -11,46 +11,46 @@ type msSQLDialect struct { } const ( - insertMigrationMSSQLDialectSql = "insert into %v (name, source_dir, file, type, db_schema) values (@p1, @p2, @p3, @p4, @p5)" - defaultInsertTenantMSSQLDialectSql = "insert into %v (name) values (@p1)" + insertMigrationMSSQLDialectSql = "insert into %v.%v (name, source_dir, filename, type, db_schema) values (@p1, @p2, @p3, @p4, @p5)" + insertTenantMSSQLDialectSql = "insert into %v.%v (name) values (@p1)" ) func (md *msSQLDialect) GetMigrationInsertSql() string { - return fmt.Sprintf(insertMigrationMySQLDialectSql, migrationsTableName) + return fmt.Sprintf(insertMigrationMSSQLDialectSql, migratorSchema, migratorMigrationsTable) } func (md *msSQLDialect) GetTenantInsertSql() string { - return fmt.Sprintf(defaultInsertTenantMySQLDialectSql, defaultTenantsTableName) + return fmt.Sprintf(insertTenantMSSQLDialectSql, migratorSchema, migratorTenantsTable) } func (md *msSQLDialect) GetCreateTenantsTableSql() string { createTenantsTableSql := ` IF NOT EXISTS (select * from information_schema.tables where table_schema = '%v' and table_name = '%v') BEGIN - create table [%v].%%v ( + create table [%v].%v ( id int identity (1,1) primary key, name varchar(200) not null, created datetime default CURRENT_TIMESTAMP ); END ` - return fmt.Sprintf(createTenantsTableSql, migratorSchema, migratorTenantsTable, migratorSchema) + return fmt.Sprintf(createTenantsTableSql, migratorSchema, migratorTenantsTable, migratorSchema, migratorTenantsTable) } func (md *msSQLDialect) GetCreateMigrationsTableSql() string { createMigrationsTableSql := ` IF NOT EXISTS (select * from information_schema.tables where table_schema = '%v' and table_name = '%v') BEGIN - create table [%v].%%v ( + create table [%v].%v ( id int identity (1,1) primary key, name varchar(200) not null, source_dir varchar(200) not null, - file varchar(200) not null, + filename varchar(200) not null, type int not null, db_schema varchar(200) not null, created datetime default CURRENT_TIMESTAMP ); END ` - return fmt.Sprintf(createMigrationsTableSql, migratorSchema, migratorMigrationsTable, migratorSchema) + return fmt.Sprintf(createMigrationsTableSql, migratorSchema, migratorMigrationsTable, migratorSchema, migratorMigrationsTable) } diff --git a/db/db_mysql.go b/db/db_mysql.go index 0e382ae..5cf0b01 100644 --- a/db/db_mysql.go +++ b/db/db_mysql.go @@ -11,14 +11,14 @@ type mySQLDialect struct { } const ( - insertMigrationMySQLDialectSql = "insert into %v (name, source_dir, file, type, db_schema) values (?, ?, ?, ?, ?)" - defaultInsertTenantMySQLDialectSql = "insert into %v (name) values (?)" + insertMigrationMySQLDialectSql = "insert into %v.%v (name, source_dir, filename, type, db_schema) values (?, ?, ?, ?, ?)" + insertTenantMySQLDialectSql = "insert into %v.%v (name) values (?)" ) func (md *mySQLDialect) GetMigrationInsertSql() string { - return fmt.Sprintf(insertMigrationMySQLDialectSql, migrationsTableName) + return fmt.Sprintf(insertMigrationMySQLDialectSql, migratorSchema, migratorMigrationsTable) } func (md *mySQLDialect) GetTenantInsertSql() string { - return fmt.Sprintf(defaultInsertTenantMySQLDialectSql, defaultTenantsTableName) + return fmt.Sprintf(insertTenantMySQLDialectSql, migratorSchema, migratorTenantsTable) } diff --git a/db/db_postgresql.go b/db/db_postgresql.go index 7711493..6bb2305 100644 --- a/db/db_postgresql.go +++ b/db/db_postgresql.go @@ -11,14 +11,14 @@ type postgreSQLDialect struct { } const ( - insertMigrationPostgreSQLDialectSql = "insert into %v (name, source_dir, file, type, db_schema) values ($1, $2, $3, $4, $5)" - defaultInsertTenantPostgreSQLDialectSql = "insert into %v (name) values ($1)" + insertMigrationPostgreSQLDialectSql = "insert into %v.%v (name, source_dir, filename, type, db_schema) values ($1, $2, $3, $4, $5)" + insertTenantPostgreSQLDialectSql = "insert into %v.%v (name) values ($1)" ) func (pd *postgreSQLDialect) GetMigrationInsertSql() string { - return fmt.Sprintf(insertMigrationPostgreSQLDialectSql, migrationsTableName) + return fmt.Sprintf(insertMigrationPostgreSQLDialectSql, migratorSchema, migratorMigrationsTable) } func (pd *postgreSQLDialect) GetTenantInsertSql() string { - return fmt.Sprintf(defaultInsertTenantPostgreSQLDialectSql, defaultTenantsTableName) + return fmt.Sprintf(insertTenantPostgreSQLDialectSql, migratorSchema, migratorTenantsTable) } diff --git a/db/db_test.go b/db/db_test.go index fda9002..f19ab97 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -9,7 +9,6 @@ import ( "github.com/lukaszbudnik/migrator/config" "github.com/lukaszbudnik/migrator/types" "github.com/stretchr/testify/assert" - "reflect" "testing" "time" ) @@ -26,17 +25,22 @@ func TestDBCreateConnectorPanicUnknownDriver(t *testing.T) { func TestDBCreateDialectPostgreSqlDriver(t *testing.T) { config := &config.Config{} config.Driver = "postgres" - connector := CreateDialect(config) - connectorName := reflect.TypeOf(connector).String() - assert.Equal(t, "*db.postgreSQLDialect", connectorName) + dialect := CreateDialect(config) + assert.IsType(t, &postgreSQLDialect{}, dialect) } func TestDBCreateDialectMysqlDriver(t *testing.T) { config := &config.Config{} config.Driver = "mysql" - connector := CreateDialect(config) - connectorName := reflect.TypeOf(connector).String() - assert.Equal(t, "*db.mySQLDialect", connectorName) + dialect := CreateDialect(config) + assert.IsType(t, &mySQLDialect{}, dialect) +} + +func TestDBCreateDialectMSSQLDriver(t *testing.T) { + config := &config.Config{} + config.Driver = "sqlserver" + dialect := CreateDialect(config) + assert.IsType(t, &msSQLDialect{}, dialect) } func TestDBConnectorInitPanicConnectionError(t *testing.T) { @@ -114,28 +118,34 @@ func TestDBApplyMigrations(t *testing.T) { p1 := time.Now().UnixNano() p2 := time.Now().UnixNano() + p3 := time.Now().UnixNano() t1 := time.Now().UnixNano() t2 := time.Now().UnixNano() + t3 := time.Now().UnixNano() publicdef1 := types.MigrationDefinition{fmt.Sprintf("%v.sql", p1), "public", fmt.Sprintf("public/%v.sql", p1), types.MigrationTypeSingleSchema} publicdef2 := types.MigrationDefinition{fmt.Sprintf("%v.sql", p2), "public", fmt.Sprintf("public/%v.sql", p2), types.MigrationTypeSingleSchema} - public1 := types.Migration{publicdef1, "create table if not exists {schema}.modules ( k int, v text )"} - public2 := types.Migration{publicdef2, "insert into {schema}.modules values ( 123, '123' )"} + publicdef3 := types.MigrationDefinition{fmt.Sprintf("%v.sql", p3), "public", fmt.Sprintf("public/%v.sql", p3), types.MigrationTypeSingleSchema} + public1 := types.Migration{publicdef1, "drop table if exists modules"} + public2 := types.Migration{publicdef2, "create table modules ( k int, v text )"} + public3 := types.Migration{publicdef3, "insert into modules values ( 123, '123' )"} tenantdef1 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t1), "tenants", fmt.Sprintf("tenants/%v.sql", t1), types.MigrationTypeTenantSchema} tenantdef2 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t2), "tenants", fmt.Sprintf("tenants/%v.sql", t2), types.MigrationTypeTenantSchema} - tenant1 := types.Migration{tenantdef1, "create table if not exists {schema}.settings (k int, v text) "} - tenant2 := types.Migration{tenantdef2, "insert into {schema}.settings values (456, '456') "} + tenantdef3 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t3), "tenants", fmt.Sprintf("tenants/%v.sql", t2), types.MigrationTypeTenantSchema} + tenant1 := types.Migration{tenantdef1, "drop table if exists {schema}.settings"} + tenant2 := types.Migration{tenantdef2, "create table {schema}.settings (k int, v text)"} + tenant3 := types.Migration{tenantdef3, "insert into {schema}.settings values (456, '456') "} - migrationsToApply := []types.Migration{public1, public2, tenant1, tenant2} + migrationsToApply := []types.Migration{public1, public2, public3, tenant1, tenant2, tenant3} connector.ApplyMigrations(migrationsToApply) dbMigrationsAfter := connector.GetMigrations() lenAfter := len(dbMigrationsAfter) - // 2 tenant migrations * no of tenants + 2 public - expected := lenTenants*2 + 2 + // 3 tenant migrations * no of tenants + 3 public + expected := lenTenants*3 + 3 assert.Equal(t, expected, lenAfter-lenBefore) } @@ -169,7 +179,7 @@ func TestGetTenantsSqlDefault(t *testing.T) { tenantSelectSql := connector.GetTenantSelectSql() - assert.Equal(t, "select name from public.migrator_tenants", tenantSelectSql) + assert.Equal(t, "select name from migrator.migrator_tenants", tenantSelectSql) } func TestGetTenantsSqlOverride(t *testing.T) { @@ -222,15 +232,18 @@ func TestAddTenantAndApplyMigrations(t *testing.T) { t1 := time.Now().UnixNano() t2 := time.Now().UnixNano() t3 := time.Now().UnixNano() + t4 := time.Now().UnixNano() tenantdef1 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t1), "tenants", fmt.Sprintf("tenants/%v.sql", t1), types.MigrationTypeTenantSchema} tenantdef2 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t2), "tenants", fmt.Sprintf("tenants/%v.sql", t2), types.MigrationTypeTenantSchema} tenantdef3 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t3), "tenants", fmt.Sprintf("tenants/%v.sql", t3), types.MigrationTypeTenantSchema} + tenantdef4 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t4), "tenants", fmt.Sprintf("tenants/%v.sql", t4), types.MigrationTypeTenantSchema} tenant1 := types.Migration{tenantdef1, "create schema {schema}"} - tenant2 := types.Migration{tenantdef2, "create table if not exists {schema}.settings (k int, v text) "} - tenant3 := types.Migration{tenantdef3, "insert into {schema}.settings values (456, '456') "} + tenant2 := types.Migration{tenantdef2, "drop table if exists {schema}.settings"} + tenant3 := types.Migration{tenantdef3, "create table {schema}.settings (k int, v text) "} + tenant4 := types.Migration{tenantdef4, "insert into {schema}.settings values (456, '456') "} - migrationsToApply := []types.Migration{tenant1, tenant2, tenant3} + migrationsToApply := []types.Migration{tenant1, tenant2, tenant3, tenant4} unique_tenant := fmt.Sprintf("new_test_tenant_%v", time.Now().UnixNano()) @@ -239,7 +252,7 @@ func TestAddTenantAndApplyMigrations(t *testing.T) { dbMigrationsAfter := connector.GetMigrations() lenAfter := len(dbMigrationsAfter) - assert.Equal(t, 3, lenAfter-lenBefore) + assert.Equal(t, 4, lenAfter-lenBefore) } func TestMySQLGetMigrationInsertSql(t *testing.T) { @@ -252,7 +265,7 @@ func TestMySQLGetMigrationInsertSql(t *testing.T) { insertMigrationSQL := dialect.GetMigrationInsertSql() - assert.Equal(t, "insert into public.migrator_migrations (name, source_dir, file, type, db_schema) values (?, ?, ?, ?, ?)", insertMigrationSQL) + assert.Equal(t, "insert into migrator.migrator_migrations (name, source_dir, filename, type, db_schema) values (?, ?, ?, ?, ?)", insertMigrationSQL) } func TestPostgreSQLGetMigrationInsertSql(t *testing.T) { @@ -265,7 +278,20 @@ func TestPostgreSQLGetMigrationInsertSql(t *testing.T) { insertMigrationSQL := dialect.GetMigrationInsertSql() - assert.Equal(t, "insert into public.migrator_migrations (name, source_dir, file, type, db_schema) values ($1, $2, $3, $4, $5)", insertMigrationSQL) + assert.Equal(t, "insert into migrator.migrator_migrations (name, source_dir, filename, type, db_schema) values ($1, $2, $3, $4, $5)", insertMigrationSQL) +} + +func TestMSSQLGetMigrationInsertSql(t *testing.T) { + config, err := config.FromFile("../test/migrator.yaml") + assert.Nil(t, err) + + config.Driver = "sqlserver" + + dialect := CreateDialect(config) + + insertMigrationSQL := dialect.GetMigrationInsertSql() + + assert.Equal(t, "insert into migrator.migrator_migrations (name, source_dir, filename, type, db_schema) values (@p1, @p2, @p3, @p4, @p5)", insertMigrationSQL) } func TestMySQLGetTenantInsertSqlDefault(t *testing.T) { @@ -278,7 +304,7 @@ func TestMySQLGetTenantInsertSqlDefault(t *testing.T) { tenantInsertSql := connector.GetTenantInsertSql() - assert.Equal(t, "insert into public.migrator_tenants (name) values (?)", tenantInsertSql) + assert.Equal(t, "insert into migrator.migrator_tenants (name) values (?)", tenantInsertSql) } func TestPostgreSQLGetTenantInsertSqlDefault(t *testing.T) { @@ -291,7 +317,20 @@ func TestPostgreSQLGetTenantInsertSqlDefault(t *testing.T) { tenantInsertSql := connector.GetTenantInsertSql() - assert.Equal(t, "insert into public.migrator_tenants (name) values ($1)", tenantInsertSql) + assert.Equal(t, "insert into migrator.migrator_tenants (name) values ($1)", tenantInsertSql) +} + +func TestMSSQLGetTenantInsertSqlDefault(t *testing.T) { + config, err := config.FromFile("../test/migrator.yaml") + assert.Nil(t, err) + + config.Driver = "sqlserver" + connector := CreateConnector(config) + defer connector.Dispose() + + tenantInsertSql := connector.GetTenantInsertSql() + + assert.Equal(t, "insert into migrator.migrator_tenants (name) values (@p1)", tenantInsertSql) } func TestGetTenantInsertSqlOverride(t *testing.T) { @@ -310,16 +349,16 @@ func TestMSSQLDialectGetCreateTenantsTableSql(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) - config.Driver = "mssql" + config.Driver = "sqlserver" dialect := CreateDialect(config) createTenantsTableSql := dialect.GetCreateTenantsTableSql() expected := ` -IF NOT EXISTS (select * from information_schema.tables where table_schema = 'public' and table_name = 'migrator_tenants') +IF NOT EXISTS (select * from information_schema.tables where table_schema = 'migrator' and table_name = 'migrator_tenants') BEGIN - create table [public].%v ( + create table [migrator].migrator_tenants ( id int identity (1,1) primary key, name varchar(200) not null, created datetime default CURRENT_TIMESTAMP @@ -334,20 +373,20 @@ func TestMSSQLDialectGetCreateMigrationsTableSql(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) - config.Driver = "mssql" + config.Driver = "sqlserver" dialect := CreateDialect(config) createMigrationsTableSql := dialect.GetCreateMigrationsTableSql() expected := ` -IF NOT EXISTS (select * from information_schema.tables where table_schema = 'public' and table_name = 'migrator_migrations') +IF NOT EXISTS (select * from information_schema.tables where table_schema = 'migrator' and table_name = 'migrator_migrations') BEGIN - create table [public].%v ( + create table [migrator].migrator_migrations ( id int identity (1,1) primary key, name varchar(200) not null, source_dir varchar(200) not null, - file varchar(200) not null, + filename varchar(200) not null, type int not null, db_schema varchar(200) not null, created datetime default CURRENT_TIMESTAMP @@ -357,3 +396,49 @@ END assert.Equal(t, expected, createMigrationsTableSql) } + +func TestBaseDialectGetCreateTenantsTableSql(t *testing.T) { + config, err := config.FromFile("../test/migrator.yaml") + assert.Nil(t, err) + + config.Driver = "postgres" + + dialect := CreateDialect(config) + + createTenantsTableSql := dialect.GetCreateTenantsTableSql() + + expected := ` +create table if not exists migrator.migrator_tenants ( + id serial primary key, + name varchar(200) not null, + created timestamp default now() +) +` + + assert.Equal(t, expected, createTenantsTableSql) +} + +func TestBaseDialectGetCreateMigrationsTableSql(t *testing.T) { + config, err := config.FromFile("../test/migrator.yaml") + assert.Nil(t, err) + + config.Driver = "postgres" + + dialect := CreateDialect(config) + + createMigrationsTableSql := dialect.GetCreateMigrationsTableSql() + + expected := ` +create table if not exists migrator.migrator_migrations ( + id serial primary key, + name varchar(200) not null, + source_dir varchar(200) not null, + filename varchar(200) not null, + type int not null, + db_schema varchar(200) not null, + created timestamp default now() +) +` + + assert.Equal(t, expected, createMigrationsTableSql) +} diff --git a/test/create-test-tenants.sql b/test/create-test-tenants.sql index e6339ec..56979e0 100644 --- a/test/create-test-tenants.sql +++ b/test/create-test-tenants.sql @@ -1,14 +1,14 @@ -create schema if not exists public; +create schema migrator; -create table if not exists public.migrator_tenants ( +create table migrator.migrator_tenants ( id serial primary key, name varchar(200) not null, created timestamp default now() ); -insert into public.migrator_tenants (name) values ('abc'); -insert into public.migrator_tenants (name) values ('def'); -insert into public.migrator_tenants (name) values ('xyz'); +insert into migrator.migrator_tenants (name) values ('abc'); +insert into migrator.migrator_tenants (name) values ('def'); +insert into migrator.migrator_tenants (name) values ('xyz'); create schema abc; create schema def; diff --git a/test/migrations/public/201602160001.sql b/test/migrations/public/201602160001.sql index e43d3fc..8176f8c 100644 --- a/test/migrations/public/201602160001.sql +++ b/test/migrations/public/201602160001.sql @@ -1,4 +1,4 @@ -create table {schema}.config ( +create table config ( id integer, k varchar(100), v varchar(100), diff --git a/test/migrations/tenants/201602160002.sql b/test/migrations/tenants/201602160002.sql index fa0d78a..b54adb3 100644 --- a/test/migrations/tenants/201602160002.sql +++ b/test/migrations/tenants/201602160002.sql @@ -2,4 +2,4 @@ create schema if not exists {schema}; create table {schema}.module (id integer, id_config integer); -alter table {schema}.module add foreign key (id_config) references public.config(id); +alter table {schema}.module add foreign key (id_config) references config(id); From 855cd7fedb82b6867f5e627b83ddcab956ce6d18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Wed, 5 Dec 2018 18:45:07 +0100 Subject: [PATCH 06/12] further refactoring and moving more functionality to Dialect, full MSSQL support --- README.md | 35 +++++++++++------ db/db.go | 12 ++++++ db/db_dialect.go | 5 +++ db/db_mssql.go | 10 +++++ db/db_test.go | 48 ++++++++++++++++++++---- test/migrations/tenants/201602160002.sql | 2 - 6 files changed, 90 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9a47478..fe33711 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Migrator [![Build Status](https://travis-ci.org/lukaszbudnik/migrator.svg?branch=master)](https://travis-ci.org/lukaszbudnik/migrator) -Fast and lightweight DB migration & evolution tool written in go. +Super fast and lightweight DB migration & evolution tool written in go. migrator manages all the DB changes for you and completely eliminates manual and error-prone administrative tasks. migrator not only supports single schemas, but also comes with a multi-tenant support. @@ -27,11 +27,12 @@ Migrator requires a simple `migrator.yaml` file: ``` baseDir: test/migrations driver: postgres +# dataSource format is specific to DB go driver implementation - see below 'Supported databases' dataSource: "user=postgres dbname=migrator_test host=192.168.99.100 port=55432 sslmode=disable" # override only if you have a specific way of determining tenants, default is: -tenantSelectSql: "select name from public.migrator_tenants" +tenantSelectSql: "select name from migrator.migrator_tenants" # override only if you have a specific way of creating tenants, default is: -tenantInsertSql: "insert into public.migrator_tenants (name) values ($1)" +tenantInsertSql: "insert into migrator.migrator_tenants (name) values ($1)" # override only if you have a specific schema placeholder, default is: schemaPlaceHolder: {schema} singleSchemas: @@ -56,6 +57,14 @@ create table if not exists {schema}.modules ( k int, v text ); insert into {schema}.modules values ( 123, '123' ); ``` +# DB Schemas + +When using migrator please remember about these: + +* migrator creates `migrator` schema (where `migrator_migrations` and `migrator_tenants` tables reside) automatically +* when adding a new tenant migrator creates a new schema automatically +* single schemas are not created automatically, for this you must add initial migration with `create schema` SQL command (see example above) + # Server mode When migrator is run with `-mode server` it starts a HTTP service and exposes simple REST API which you can use to invoke migrator actions remotely. @@ -84,7 +93,7 @@ Port is configurable in `migrator.yaml` and defaults to 8080. Should you need HT # Supported databases -Currently migrator supports the following databases: +Currently migrator supports the following databases and their flavours: * PostgreSQL - schema-based multi-tenant database, with transactions spanning DDL statements, driver used: https://github.com/lib/pq * PostgreSQL - original PostgreSQL server @@ -98,6 +107,8 @@ Currently migrator supports the following databases: * Amazon RDS MySQL - MySQL-compatible relational database built for the cloud * Amazon Aurora MySQL - MySQL-compatible relational database built for the cloud * Google CloudSQL MySQL - MySQL-compatible relational database built for the cloud +* Microsoft SQL Server - a relational database management system developed by Microsoft, driver used: https://github.com/denisenkom/go-mssqldb + * Microsoft SQL Server - original Microsoft SQL Server # Do you speak docker? @@ -109,15 +120,15 @@ To find out more about migrator docker please visit: https://github.com/lukaszbu # Running unit & integration tests -PostgreSQL, MySQL, MariaDB, and Percona: +PostgreSQL, MySQL, MariaDB, Percona, and MSSQL: ``` -$ docker/create-and-setup-container.sh [postgres|mysql|mariadb|percona] +$ docker/create-and-setup-container.sh [postgres|mysql|mariadb|percona|mssql] $ ./coverage.sh -$ docker/destroy-container.sh [postgres|mysql|mariadb|percona] +$ docker/destroy-container.sh [postgres|mysql|mariadb|percona|mssql] ``` -Or see `.travis.yml` to see how it's done on Travis. +Or see `.travis.yml` to see how it's done on Travis. Note: MSSQL is not supported on Travis. # Customisation @@ -138,7 +149,7 @@ tenantInsertSql: insert into global.customers (name, active, date_added) values As a benchmarks I used 2 migrations frameworks: -* proprietary Ruby framework - used in my company +* proprietary Ruby framework - used at my company * flyway - leading market feature rich DB migration framework: https://flywaydb.org There is a performance test generator shipped with migrator (`test/performance/generate-test-migrations.sh`). In order to generate flyway-compatible migrations you need to pass `-f` param (see script for details). @@ -154,7 +165,7 @@ Migrator is the undisputed winner. The Ruby framework has an undesired functionality of making a DB call each time to check if given migration was already applied. Migrator fetches all applied migrations at once and compares them in memory. This is the primary reason why migrator is so much better in the second test. -flyway results are... dramatic. I was so shocked that I had to re-run flyway as well as all other tests. Yes, flyway is almost 15 times slower than migrator in the first test. In the second test flyway was faster than Ruby. Still a couple orders of magnitude slower than migrator. +flyway results are... very surprising. I was so shocked that I had to re-run flyway as well as all other tests. Yes, flyway is 15 times slower than migrator in the first test. In the second test flyway was faster than Ruby. Still a couple orders of magnitude slower than migrator. The other thing to consider is the fact that migrator is written in go which is known to be much faster than Ruby and Java. @@ -164,7 +175,7 @@ To install migrator use: `go get github.com/lukaszbudnik/migrator` -Migrator supports the following Go versions: 1.8, 1.9, 1.10, 1.11 (all built on Travis). +Migrator supports the following Go versions: 1.8, 1.9, 1.10, 1.11, and tip (all built on Travis). # Code Style @@ -178,7 +189,7 @@ $ go tool vet -v . # License -Copyright 2016 Łukasz Budnik +Copyright 2016-2018 Łukasz Budnik Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at diff --git a/db/db.go b/db/db.go index 90af44d..9ac0763 100644 --- a/db/db.go +++ b/db/db.go @@ -60,6 +60,12 @@ func (bc *BaseConnector) Init() { log.Panicf("Could not start DB transaction: %v", err) } + // make sure migrator schema exists + createSchema := bc.Dialect.GetCreateSchemaSql(migratorSchema) + if _, err := bc.DB.Query(createSchema); err != nil { + log.Panicf("Could not create migrator schema: %v", err) + } + // make sure migrations table exists createMigrationsTable := bc.Dialect.GetCreateMigrationsTableSql() if _, err := bc.DB.Query(createMigrationsTable); err != nil { @@ -171,6 +177,12 @@ func (bc *BaseConnector) AddTenantAndApplyMigrations(tenant string, migrations [ log.Panicf("Could not start DB transaction: %v", err) } + createSchema := bc.Dialect.GetCreateSchemaSql(tenant) + if _, err = tx.Exec(createSchema); err != nil { + tx.Rollback() + log.Panicf("Create schema failed, transaction rollback was called: %v", err) + } + insert, err := bc.DB.Prepare(tenantInsertSql) if err != nil { log.Panicf("Could not create prepared statement: %v", err) diff --git a/db/db_dialect.go b/db/db_dialect.go index 748c498..b128c54 100644 --- a/db/db_dialect.go +++ b/db/db_dialect.go @@ -13,6 +13,7 @@ type Dialect interface { GetMigrationSelectSql() string GetCreateTenantsTableSql() string GetCreateMigrationsTableSql() string + GetCreateSchemaSql(string) string } type BaseDialect struct { @@ -57,6 +58,10 @@ func (bd *BaseDialect) GetMigrationSelectSql() string { return fmt.Sprintf(selectMigrations, migratorSchema, migratorMigrationsTable) } +func (bd *BaseDialect) GetCreateSchemaSql(schema string) string { + return fmt.Sprintf("create schema if not exists %v", schema) +} + // CreateDialect constructs Dialect instance based on the passed Config func CreateDialect(config *config.Config) Dialect { diff --git a/db/db_mssql.go b/db/db_mssql.go index b828433..4b112f5 100644 --- a/db/db_mssql.go +++ b/db/db_mssql.go @@ -54,3 +54,13 @@ END ` return fmt.Sprintf(createMigrationsTableSql, migratorSchema, migratorMigrationsTable, migratorSchema, migratorMigrationsTable) } + +func (md *msSQLDialect) GetCreateSchemaSql(schema string) string { + createSchemaSql := ` +IF NOT EXISTS (select * from information_schema.schemata where schema_name = '%v') +BEGIN + EXEC sp_executesql N'create schema %v'; +END +` + return fmt.Sprintf(createSchemaSql, schema, schema) +} diff --git a/db/db_test.go b/db/db_test.go index f19ab97..fef2618 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -232,18 +232,15 @@ func TestAddTenantAndApplyMigrations(t *testing.T) { t1 := time.Now().UnixNano() t2 := time.Now().UnixNano() t3 := time.Now().UnixNano() - t4 := time.Now().UnixNano() tenantdef1 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t1), "tenants", fmt.Sprintf("tenants/%v.sql", t1), types.MigrationTypeTenantSchema} tenantdef2 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t2), "tenants", fmt.Sprintf("tenants/%v.sql", t2), types.MigrationTypeTenantSchema} tenantdef3 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t3), "tenants", fmt.Sprintf("tenants/%v.sql", t3), types.MigrationTypeTenantSchema} - tenantdef4 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t4), "tenants", fmt.Sprintf("tenants/%v.sql", t4), types.MigrationTypeTenantSchema} - tenant1 := types.Migration{tenantdef1, "create schema {schema}"} - tenant2 := types.Migration{tenantdef2, "drop table if exists {schema}.settings"} - tenant3 := types.Migration{tenantdef3, "create table {schema}.settings (k int, v text) "} - tenant4 := types.Migration{tenantdef4, "insert into {schema}.settings values (456, '456') "} + tenant1 := types.Migration{tenantdef1, "drop table if exists {schema}.settings"} + tenant2 := types.Migration{tenantdef2, "create table {schema}.settings (k int, v text) "} + tenant3 := types.Migration{tenantdef3, "insert into {schema}.settings values (456, '456') "} - migrationsToApply := []types.Migration{tenant1, tenant2, tenant3, tenant4} + migrationsToApply := []types.Migration{tenant1, tenant2, tenant3} unique_tenant := fmt.Sprintf("new_test_tenant_%v", time.Now().UnixNano()) @@ -252,7 +249,7 @@ func TestAddTenantAndApplyMigrations(t *testing.T) { dbMigrationsAfter := connector.GetMigrations() lenAfter := len(dbMigrationsAfter) - assert.Equal(t, 4, lenAfter-lenBefore) + assert.Equal(t, 3, lenAfter-lenBefore) } func TestMySQLGetMigrationInsertSql(t *testing.T) { @@ -442,3 +439,38 @@ create table if not exists migrator.migrator_migrations ( assert.Equal(t, expected, createMigrationsTableSql) } + +func TestBaseDialectGetCreateSchemaSql(t *testing.T) { + config, err := config.FromFile("../test/migrator.yaml") + assert.Nil(t, err) + + config.Driver = "postgres" + + dialect := CreateDialect(config) + + createSchemaSql := dialect.GetCreateSchemaSql("abc") + + expected := "create schema if not exists abc" + + assert.Equal(t, expected, createSchemaSql) +} + +func TestMSSQLDialectGetCreateSchemaSql(t *testing.T) { + config, err := config.FromFile("../test/migrator.yaml") + assert.Nil(t, err) + + config.Driver = "sqlserver" + + dialect := CreateDialect(config) + + createSchemaSql := dialect.GetCreateSchemaSql("def") + + expected := ` +IF NOT EXISTS (select * from information_schema.schemata where schema_name = 'def') +BEGIN + EXEC sp_executesql N'create schema def'; +END +` + + assert.Equal(t, expected, createSchemaSql) +} diff --git a/test/migrations/tenants/201602160002.sql b/test/migrations/tenants/201602160002.sql index b54adb3..20ac502 100644 --- a/test/migrations/tenants/201602160002.sql +++ b/test/migrations/tenants/201602160002.sql @@ -1,5 +1,3 @@ -create schema if not exists {schema}; - create table {schema}.module (id integer, id_config integer); alter table {schema}.module add foreign key (id_config) references config(id); From 1b195c5f308efbfa59b80f274788f7680061b0f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Wed, 5 Dec 2018 22:16:41 +0100 Subject: [PATCH 07/12] contains fixes for: vet, lint, fmt, increased test coverage --- README.md | 24 +++--- config/config.go | 12 +-- config/config_test.go | 10 +-- core/core.go | 68 +++++++++------- core/core_mocks.go | 10 +-- core/core_test.go | 40 ++++++++-- coverage.sh | 4 +- db/db.go | 71 ++++++++--------- db/db_dialect.go | 71 ++++++++++------- db/db_mssql.go | 60 ++++++++------ db/db_mysql.go | 14 ++-- db/db_postgresql.go | 14 ++-- db/db_test.go | 146 +++++++++++++++++----------------- loader/disk_loader.go | 8 +- loader/disk_loader_test.go | 4 +- loader/loader.go | 2 +- migrations/migrations.go | 1 + migrations/migrations_test.go | 75 +++++++++-------- migrator.go | 27 ++++--- server/server.go | 6 +- server/server_mocks.go | 20 ++--- test/migrator-overrides.yaml | 4 +- test/migrator-test-envs.yaml | 4 +- test/migrator-test.yaml | 2 +- utils/utils.go | 6 +- utils/utils_test.go | 14 ++-- vet.sh | 16 ++++ 27 files changed, 406 insertions(+), 327 deletions(-) create mode 100755 vet.sh diff --git a/README.md b/README.md index fe33711..67bfbd1 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,9 @@ driver: postgres # dataSource format is specific to DB go driver implementation - see below 'Supported databases' dataSource: "user=postgres dbname=migrator_test host=192.168.99.100 port=55432 sslmode=disable" # override only if you have a specific way of determining tenants, default is: -tenantSelectSql: "select name from migrator.migrator_tenants" +tenantSelectSQL: "select name from migrator.migrator_tenants" # override only if you have a specific way of creating tenants, default is: -tenantInsertSql: "insert into migrator.migrator_tenants (name) values ($1)" +tenantInsertSQL: "insert into migrator.migrator_tenants (name) values ($1)" # override only if you have a specific schema placeholder, default is: schemaPlaceHolder: {schema} singleSchemas: @@ -47,7 +47,7 @@ port: 8080 slackWebHook: https://hooks.slack.com/services/TTT/BBB/XXX ``` -Migrator will scan all directories under `baseDir` directory. Migrations listed under `singleSchemas` directories will be applied once. Migrations listed under `tenantSchemas` directories will be applied for all tenants fetched using `tenantSelectSql`. +Migrator will scan all directories under `baseDir` directory. Migrations listed under `singleSchemas` directories will be applied once. Migrations listed under `tenantSchemas` directories will be applied for all tenants fetched using `tenantSelectSQL`. SQL migrations in both `singleSchemas` and `tenantsSchemas` can use `{schema}` placeholder which will be automatically replaced by migrator with a current schema. For example: @@ -135,14 +135,14 @@ Or see `.travis.yml` to see how it's done on Travis. Note: MSSQL is not supporte If you have an existing way of storing information about your tenants you can configure migrator to use it. In the config file you need to provide 2 parameters: -* `tenantSelectSql` - a select statement which returns names of the tenants -* `tenantInsertSql` - an insert statement which creates a new tenant entry, this is called as a prepared statement and is called with the name of the tenant as a parameter; should your table require additional columns you need to provide default values for them +* `tenantSelectSQL` - a select statement which returns names of the tenants +* `tenantInsertSQL` - an insert statement which creates a new tenant entry, this is called as a prepared statement and is called with the name of the tenant as a parameter; should your table require additional columns you need to provide default values for them Here is an example: ``` -tenantSelectSql: select name from global.customers -tenantInsertSql: insert into global.customers (name, active, date_added) values (?, true, NOW()) +tenantSelectSQL: select name from global.customers +tenantInsertSQL: insert into global.customers (name, active, date_added) values (?, true, NOW()) ``` # Performance @@ -179,12 +179,14 @@ Migrator supports the following Go versions: 1.8, 1.9, 1.10, 1.11, and tip (all # Code Style -If you would like to send me a pull request please always add unit/integration tests. Code should be formatted & checked using the following commands: +If you would like to send me a pull request please always add unit/integration tests. Code should be formatted, checked, and tested using the following commands: ``` -$ gofmt -s -w . -$ golint ./... -$ go tool vet -v . +go clean -testcache +gofmt -s -w . +golint ./... +./vet.sh +./coverage.sh ``` # License diff --git a/config/config.go b/config/config.go index eaf3265..2e2dbf0 100644 --- a/config/config.go +++ b/config/config.go @@ -14,8 +14,8 @@ type Config struct { BaseDir string `yaml:"baseDir" validate:"nonzero"` Driver string `yaml:"driver" validate:"nonzero"` DataSource string `yaml:"dataSource" validate:"nonzero"` - TenantSelectSql string `yaml:"tenantSelectSql"` - TenantInsertSql string `yaml:"tenantInsertSql"` + TenantSelectSQL string `yaml:"tenantSelectSQL"` + TenantInsertSQL string `yaml:"tenantInsertSQL"` SchemaPlaceHolder string `yaml:"schemaPlaceHolder"` SingleSchemas []string `yaml:"singleSchemas" validate:"min=1"` TenantSchemas []string `yaml:"tenantSchemas"` @@ -69,11 +69,11 @@ func substituteEnvVariables(config *Config) { if strings.HasPrefix(config.DataSource, "$") { config.DataSource = os.Getenv(config.DataSource[1:]) } - if strings.HasPrefix(config.TenantSelectSql, "$") { - config.TenantSelectSql = os.Getenv(config.TenantSelectSql[1:]) + if strings.HasPrefix(config.TenantSelectSQL, "$") { + config.TenantSelectSQL = os.Getenv(config.TenantSelectSQL[1:]) } - if strings.HasPrefix(config.TenantInsertSql, "$") { - config.TenantInsertSql = os.Getenv(config.TenantInsertSql[1:]) + if strings.HasPrefix(config.TenantInsertSQL, "$") { + config.TenantInsertSQL = os.Getenv(config.TenantInsertSQL[1:]) } if strings.HasPrefix(config.Port, "$") { config.Port = os.Getenv(config.Port[1:]) diff --git a/config/config_test.go b/config/config_test.go index 94fe48f..012e55f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -20,7 +20,7 @@ func TestFromFile(t *testing.T) { config, err := FromFile("../test/migrator-test.yaml") assert.Nil(t, err) assert.Equal(t, "test/migrations", config.BaseDir) - assert.Equal(t, "select name from public.migrator_tenants", config.TenantSelectSql) + assert.Equal(t, "select name from migrator.migrator_tenants", config.TenantSelectSQL) assert.Equal(t, "postgres", config.Driver) assert.Equal(t, "user=postgres dbname=migrator_test host=192.168.99.100 port=55432 sslmode=disable", config.DataSource) assert.Equal(t, []string{"tenants"}, config.TenantSchemas) @@ -33,8 +33,8 @@ func TestWithEnvFromFile(t *testing.T) { config, err := FromFile("../test/migrator-test-envs.yaml") assert.Nil(t, err) assert.Equal(t, os.Getenv("HOME"), config.BaseDir) - assert.Equal(t, os.Getenv("PATH"), config.TenantSelectSql) - assert.Equal(t, os.Getenv("GOPATH"), config.TenantInsertSql) + assert.Equal(t, os.Getenv("PATH"), config.TenantSelectSQL) + assert.Equal(t, os.Getenv("GOPATH"), config.TenantInsertSQL) assert.Equal(t, os.Getenv("PWD"), config.Driver) assert.Equal(t, os.Getenv("TERM"), config.DataSource) assert.Equal(t, os.Getenv("_"), config.Port) @@ -50,8 +50,8 @@ func TestConfigString(t *testing.T) { expected := `baseDir: /opt/app/migrations driver: postgres dataSource: user=p dbname=db host=localhost -tenantSelectSql: select abc -tenantInsertSql: insert into table +tenantSelectSQL: select abc +tenantInsertSQL: insert into table schemaPlaceHolder: :tenant singleSchemas: - ref diff --git a/core/core.go b/core/core.go index 911cbaa..28186ed 100644 --- a/core/core.go +++ b/core/core.go @@ -22,30 +22,36 @@ const ( AddTenantAction = "addTenant" // PrintConfigAction is an action which prints contents of config PrintConfigAction = "config" - // ListDBMigrationsAction is an action which lists migrations recorded in DB - ListDBMigrationsAction = "dbMigrations" - // ListDBTenantsAction is an action which list tenants for multi-tenant schemas - ListDBTenantsAction = "dbTenants" - // ListDiskMigrationsAction is an action which lists migrations stored on disk - ListDiskMigrationsAction = "diskMigrations" + // GetDBMigrationsAction is an action which lists migrations recorded in DB + GetDBMigrationsAction = "dbMigrations" + // GetDBTenantsAction is an action which list tenants for multi-tenant schemas + GetDBTenantsAction = "dbTenants" + // GetDiskMigrationsAction is an action which lists migrations stored on disk + GetDiskMigrationsAction = "diskMigrations" // ServerMode is a mode for using migrator HTTP server ServerMode = "server" // ToolMode is a mode for using migrator command line ToolMode = "tool" ) -// LoadDiskMigrations is a function which loads all migrations from disk as defined in config passed as first argument +// ExecuteFlags is used to group flags passed to migrator when running in tool mode +type ExecuteFlags struct { + Action string + Tenant string +} + +// GetDiskMigrations is a function which loads all migrations from disk as defined in config passed as first argument // and using loader created by a function passed as second argument -func LoadDiskMigrations(config *config.Config, createLoader func(*config.Config) loader.Loader) []types.Migration { +func GetDiskMigrations(config *config.Config, createLoader func(*config.Config) loader.Loader) []types.Migration { loader := createLoader(config) - diskMigrations := loader.GetMigrations() + diskMigrations := loader.GetDiskMigrations() log.Printf("Read disk migrations: %d", len(diskMigrations)) return diskMigrations } -// LoadDBTenants is a function which loads all tenants for multi-tenant schemas from DB as defined in config passed as first argument +// GetDBTenants is a function which loads all tenants for multi-tenant schemas from DB as defined in config passed as first argument // and using connector created by a function passed as second argument -func LoadDBTenants(config *config.Config, createConnector func(*config.Config) db.Connector) []string { +func GetDBTenants(config *config.Config, createConnector func(*config.Config) db.Connector) []string { connector := createConnector(config) connector.Init() defer connector.Dispose() @@ -54,13 +60,13 @@ func LoadDBTenants(config *config.Config, createConnector func(*config.Config) d return dbTenants } -// LoadDBMigrations is a function which loads all DB migrations for multi-tenant schemas from DB as defined in config passed as first argument +// GetDBMigrations is a function which loads all DB migrations for multi-tenant schemas from DB as defined in config passed as first argument // and using connector created by a function passed as second argument -func LoadDBMigrations(config *config.Config, createConnector func(*config.Config) db.Connector) []types.MigrationDB { +func GetDBMigrations(config *config.Config, createConnector func(*config.Config) db.Connector) []types.MigrationDB { connector := createConnector(config) connector.Init() defer connector.Dispose() - dbMigrations := connector.GetMigrations() + dbMigrations := connector.GetDBMigrations() log.Printf("Read DB migrations: %d", len(dbMigrations)) return dbMigrations } @@ -68,8 +74,8 @@ func LoadDBMigrations(config *config.Config, createConnector func(*config.Config // ApplyMigrations is a function which applies disk migrations to DB as defined in config passed as first argument // and using connector created by a function passed as second argument and disk loader created by a function passed as third argument func ApplyMigrations(config *config.Config, createConnector func(*config.Config) db.Connector, createLoader func(*config.Config) loader.Loader) []types.Migration { - diskMigrations := LoadDiskMigrations(config, createLoader) - dbMigrations := LoadDBMigrations(config, createConnector) + diskMigrations := GetDiskMigrations(config, createLoader) + dbMigrations := GetDBMigrations(config, createConnector) migrationsToApply := migrations.ComputeMigrationsToApply(diskMigrations, dbMigrations) log.Printf("Found migrations to apply: %d", len(migrationsToApply)) @@ -88,16 +94,17 @@ func ApplyMigrations(config *config.Config, createConnector func(*config.Config) return migrationsToApply } +// AddTenant creates new tenant in DB and applies all tenant migrations func AddTenant(tenant string, config *config.Config, createConnector func(*config.Config) db.Connector, createLoader func(*config.Config) loader.Loader) []types.Migration { - diskMigrations := LoadDiskMigrations(config, createLoader) + diskMigrations := GetDiskMigrations(config, createLoader) // filter only tenant schemas // var migrationsToApply []types.Migration migrationsToApply := migrations.FilterTenantMigrations(diskMigrations) log.Printf("Found migrations to apply: %d", len(migrationsToApply)) - createAndApplyMigrationsForTenant(tenant, migrationsToApply, config, createConnector) + doAddTenantAndApplyMigrations(tenant, migrationsToApply, config, createConnector) notifier := notifications.CreateNotifier(config) text := fmt.Sprintf("Tenant %q added, migrations applied: %d", tenant, len(migrationsToApply)) @@ -114,30 +121,33 @@ func AddTenant(tenant string, config *config.Config, createConnector func(*confi // ExecuteMigrator is a function which executes actions on resources defined in config passed as first argument action defined as second argument // and using connector created by a function passed as third argument and disk loader created by a function passed as fourth argument -func ExecuteMigrator(config *config.Config, action string, tenant string) { +func ExecuteMigrator(config *config.Config, executeFlags ExecuteFlags) { + doExecuteMigrator(config, executeFlags, db.CreateConnector, loader.CreateLoader) +} - switch action { +func doExecuteMigrator(config *config.Config, executeFlags ExecuteFlags, createConnector func(*config.Config) db.Connector, createLoader func(*config.Config) loader.Loader) { + switch executeFlags.Action { case PrintConfigAction: log.Printf("Configuration file ==>\n%v\n", config) - case ListDiskMigrationsAction: - diskMigrations := LoadDiskMigrations(config, loader.CreateLoader) + case GetDiskMigrationsAction: + diskMigrations := GetDiskMigrations(config, createLoader) if len(diskMigrations) > 0 { log.Printf("List of disk migrations\n%v", utils.MigrationArrayToString(diskMigrations)) } - case ListDBMigrationsAction: - dbMigrations := LoadDBMigrations(config, db.CreateConnector) + case GetDBMigrationsAction: + dbMigrations := GetDBMigrations(config, createConnector) if len(dbMigrations) > 0 { log.Printf("List of db migrations\n%v", utils.MigrationDBArrayToString(dbMigrations)) } case AddTenantAction: - AddTenant(tenant, config, db.CreateConnector, loader.CreateLoader) - case ListDBTenantsAction: - dbTenants := LoadDBTenants(config, db.CreateConnector) + AddTenant(executeFlags.Tenant, config, createConnector, createLoader) + case GetDBTenantsAction: + dbTenants := GetDBTenants(config, createConnector) if len(dbTenants) > 0 { log.Printf("List of db tenants\n%v", utils.TenantArrayToString(dbTenants)) } case ApplyAction: - migrationsApplied := ApplyMigrations(config, db.CreateConnector, loader.CreateLoader) + migrationsApplied := ApplyMigrations(config, createConnector, createLoader) if len(migrationsApplied) > 0 { log.Printf("List of migrations applied\n%v", utils.MigrationArrayToString(migrationsApplied)) } @@ -151,7 +161,7 @@ func doApplyMigrations(migrationsToApply []types.Migration, config *config.Confi connector.ApplyMigrations(migrationsToApply) } -func createAndApplyMigrationsForTenant(tenant string, migrationsToApply []types.Migration, config *config.Config, createConnector func(*config.Config) db.Connector) { +func doAddTenantAndApplyMigrations(tenant string, migrationsToApply []types.Migration, config *config.Config, createConnector func(*config.Config) db.Connector) { connector := createConnector(config) connector.Init() defer connector.Dispose() diff --git a/core/core_mocks.go b/core/core_mocks.go index 301645c..803fff3 100644 --- a/core/core_mocks.go +++ b/core/core_mocks.go @@ -10,7 +10,7 @@ import ( type mockedDiskLoader struct { } -func (m *mockedDiskLoader) GetMigrations() []types.Migration { +func (m *mockedDiskLoader) GetDiskMigrations() []types.Migration { // returns empty array return []types.Migration{} } @@ -32,15 +32,15 @@ func (m *mockedConnector) GetSchemaPlaceHolder() string { return "" } -func (m *mockedConnector) GetTenantSelectSql() string { +func (m *mockedConnector) GetTenantSelectSQL() string { return "" } -func (m *mockedConnector) GetMigrationInsertSql() string { +func (m *mockedConnector) GetMigrationInsertSQL() string { return "" } -func (m *mockedConnector) GetTenantInsertSql() string { +func (m *mockedConnector) GetTenantInsertSQL() string { return "" } @@ -52,7 +52,7 @@ func (m *mockedConnector) GetTenants() []string { func (m *mockedConnector) AddTenantAndApplyMigrations(string, []types.Migration) { } -func (m *mockedConnector) GetMigrations() []types.MigrationDB { +func (m *mockedConnector) GetDBMigrations() []types.MigrationDB { // returns empty array return []types.MigrationDB{} } diff --git a/core/core_test.go b/core/core_test.go index 2f1042b..25112e8 100644 --- a/core/core_test.go +++ b/core/core_test.go @@ -12,31 +12,55 @@ import ( "testing" ) -var ( +const ( unknownAction = "unknown" configFile = "../test/migrator.yaml" ) -func TestLoadDiskMigrations(t *testing.T) { +func TestPrintConfig(t *testing.T) { config, err := config.FromFile(configFile) assert.Nil(t, err) - LoadDiskMigrations(config, createMockedDiskLoader) + executeFlags := ExecuteFlags{} + executeFlags.Action = PrintConfigAction + doExecuteMigrator(config, executeFlags, createMockedConnector, createMockedDiskLoader) } -func TestLoadDBTenants(t *testing.T) { +func TestGetDiskMigrations(t *testing.T) { config, err := config.FromFile(configFile) assert.Nil(t, err) - LoadDBTenants(config, createMockedConnector) + executeFlags := ExecuteFlags{} + executeFlags.Action = GetDiskMigrationsAction + doExecuteMigrator(config, executeFlags, createMockedConnector, createMockedDiskLoader) } -func TestLoadDBMigrations(t *testing.T) { +func TestGetDBTenants(t *testing.T) { config, err := config.FromFile(configFile) assert.Nil(t, err) - LoadDBMigrations(config, createMockedConnector) + executeFlags := ExecuteFlags{} + executeFlags.Action = GetDBTenantsAction + doExecuteMigrator(config, executeFlags, createMockedConnector, createMockedDiskLoader) +} + +func TestGetDBMigrations(t *testing.T) { + config, err := config.FromFile(configFile) + assert.Nil(t, err) + executeFlags := ExecuteFlags{} + executeFlags.Action = GetDBMigrationsAction + doExecuteMigrator(config, executeFlags, createMockedConnector, createMockedDiskLoader) } func TestApplyMigrations(t *testing.T) { config, err := config.FromFile(configFile) assert.Nil(t, err) - ApplyMigrations(config, createMockedConnector, createMockedDiskLoader) + executeFlags := ExecuteFlags{} + executeFlags.Action = ApplyAction + doExecuteMigrator(config, executeFlags, createMockedConnector, createMockedDiskLoader) +} + +func TestAddTenant(t *testing.T) { + config, err := config.FromFile(configFile) + assert.Nil(t, err) + executeFlags := ExecuteFlags{} + executeFlags.Action = AddTenantAction + doExecuteMigrator(config, executeFlags, createMockedConnector, createMockedDiskLoader) } diff --git a/coverage.sh b/coverage.sh index 794b0c6..4fb75ef 100755 --- a/coverage.sh +++ b/coverage.sh @@ -1,10 +1,10 @@ #!/bin/bash # when called with no arguments calls tests for all packages -if [[ -z $1 ]]; then +if [[ -z "$1" ]]; then packages=`go list -f "{{.Name}}" ./...` else - packages=$1 + packages="$1" fi echo "mode: set" > coverage-all.txt diff --git a/db/db.go b/db/db.go index 9ac0763..3a7b56a 100644 --- a/db/db.go +++ b/db/db.go @@ -12,13 +12,12 @@ import ( // Connector interface abstracts all DB operations performed by migrator type Connector interface { Init() - GetTenantSelectSql() string - GetTenantInsertSql() string + GetTenantSelectSQL() string + GetTenantInsertSQL() string AddTenantAndApplyMigrations(string, []types.Migration) GetTenants() []string GetSchemaPlaceHolder() string - // rename to GetDBMigrations, change type to DBMigration - GetMigrations() []types.MigrationDB + GetDBMigrations() []types.MigrationDB ApplyMigrations(migrations []types.Migration) Dispose() } @@ -61,20 +60,20 @@ func (bc *BaseConnector) Init() { } // make sure migrator schema exists - createSchema := bc.Dialect.GetCreateSchemaSql(migratorSchema) + createSchema := bc.Dialect.GetCreateSchemaSQL(migratorSchema) if _, err := bc.DB.Query(createSchema); err != nil { log.Panicf("Could not create migrator schema: %v", err) } // make sure migrations table exists - createMigrationsTable := bc.Dialect.GetCreateMigrationsTableSql() + createMigrationsTable := bc.Dialect.GetCreateMigrationsTableSQL() if _, err := bc.DB.Query(createMigrationsTable); err != nil { log.Panicf("Could not create migrations table: %v", err) } // if using default migrator tenants table make sure it exists - if bc.Config.TenantSelectSql == "" { - createTenantsTable := bc.Dialect.GetCreateTenantsTableSql() + if bc.Config.TenantSelectSQL == "" { + createTenantsTable := bc.Dialect.GetCreateTenantsTableSQL() if _, err := bc.DB.Query(createTenantsTable); err != nil { log.Panicf("Could not create default tenants table: %v", err) } @@ -90,23 +89,22 @@ func (bc *BaseConnector) Dispose() { } } -// GettenantsSql returns SQL to be executed to list all DB tenants -func (bc *BaseConnector) GetTenantSelectSql() string { - var tenantSelectSql string - if bc.Config.TenantSelectSql != "" { - tenantSelectSql = bc.Config.TenantSelectSql +// GetTenantSelectSQL returns SQL to be executed to list all DB tenants +func (bc *BaseConnector) GetTenantSelectSQL() string { + var tenantSelectSQL string + if bc.Config.TenantSelectSQL != "" { + tenantSelectSQL = bc.Config.TenantSelectSQL } else { - tenantSelectSql = bc.Dialect.GetTenantSelectSql() + tenantSelectSQL = bc.Dialect.GetTenantSelectSQL() } - return tenantSelectSql + return tenantSelectSQL } -// GetTenants returns a list of all DB tenants as specified by -// defaultSelectTenants or the value specified in config +// GetTenants returns a list of all DB tenants func (bc *BaseConnector) GetTenants() []string { - tenantSelectSql := bc.GetTenantSelectSql() + tenantSelectSQL := bc.GetTenantSelectSQL() - rows, err := bc.DB.Query(tenantSelectSql) + rows, err := bc.DB.Query(tenantSelectSQL) if err != nil { log.Panicf("Could not query tenants: %v", err) } @@ -121,9 +119,9 @@ func (bc *BaseConnector) GetTenants() []string { return tenants } -// GetMigrations returns a list of all applied DB migrations -func (bc *BaseConnector) GetMigrations() []types.MigrationDB { - query := bc.Dialect.GetMigrationSelectSql() +// GetDBMigrations returns a list of all applied DB migrations +func (bc *BaseConnector) GetDBMigrations() []types.MigrationDB { + query := bc.Dialect.GetMigrationSelectSQL() rows, err := bc.DB.Query(query) if err != nil { @@ -143,8 +141,8 @@ func (bc *BaseConnector) GetMigrations() []types.MigrationDB { if err := rows.Scan(&name, &sourceDir, &filename, &migrationType, &schema, &created); err != nil { log.Panicf("Could not read DB migration: %v", err) } - mdef := types.MigrationDefinition{name, sourceDir, filename, migrationType} - dbMigrations = append(dbMigrations, types.MigrationDB{mdef, schema, created}) + mdef := types.MigrationDefinition{Name: name, SourceDir: sourceDir, File: filename, MigrationType: migrationType} + dbMigrations = append(dbMigrations, types.MigrationDB{MigrationDefinition: mdef, Schema: schema, Created: created}) } return dbMigrations @@ -170,20 +168,20 @@ func (bc *BaseConnector) ApplyMigrations(migrations []types.Migration) { // AddTenantAndApplyMigrations adds new tenant and applies all existing tenant migrations func (bc *BaseConnector) AddTenantAndApplyMigrations(tenant string, migrations []types.Migration) { - tenantInsertSql := bc.GetTenantInsertSql() + tenantInsertSQL := bc.GetTenantInsertSQL() tx, err := bc.DB.Begin() if err != nil { log.Panicf("Could not start DB transaction: %v", err) } - createSchema := bc.Dialect.GetCreateSchemaSql(tenant) + createSchema := bc.Dialect.GetCreateSchemaSQL(tenant) if _, err = tx.Exec(createSchema); err != nil { tx.Rollback() log.Panicf("Create schema failed, transaction rollback was called: %v", err) } - insert, err := bc.DB.Prepare(tenantInsertSql) + insert, err := bc.DB.Prepare(tenantInsertSQL) if err != nil { log.Panicf("Could not create prepared statement: %v", err) } @@ -199,16 +197,18 @@ func (bc *BaseConnector) AddTenantAndApplyMigrations(tenant string, migrations [ tx.Commit() } -func (bc *BaseConnector) GetTenantInsertSql() string { - var tenantInsertSql string +// GetTenantInsertSQL returns tenant insert SQL statement from configuration file +// or, if absent, returns default Dialect-specific migrator tenant insert SQL +func (bc *BaseConnector) GetTenantInsertSQL() string { + var tenantInsertSQL string // if set explicitly in config use it // otherwise use default value provided by Dialect implementation - if bc.Config.TenantInsertSql != "" { - tenantInsertSql = bc.Config.TenantInsertSql + if bc.Config.TenantInsertSQL != "" { + tenantInsertSQL = bc.Config.TenantInsertSQL } else { - tenantInsertSql = bc.Dialect.GetTenantInsertSql() + tenantInsertSQL = bc.Dialect.GetTenantInsertSQL() } - return tenantInsertSql + return tenantInsertSQL } // GetSchemaPlaceHolder returns a schema placeholder which is @@ -226,9 +226,8 @@ func (bc *BaseConnector) GetSchemaPlaceHolder() string { func (bc *BaseConnector) applyMigrationsInTx(tx *sql.Tx, tenants []string, migrations []types.Migration) { schemaPlaceHolder := bc.GetSchemaPlaceHolder() - insertMigrationSql := bc.Dialect.GetMigrationInsertSql() - - insert, err := bc.DB.Prepare(insertMigrationSql) + insertMigrationSQL := bc.Dialect.GetMigrationInsertSQL() + insert, err := bc.DB.Prepare(insertMigrationSQL) if err != nil { log.Panicf("Could not create prepared statement: %v", err) } diff --git a/db/db_dialect.go b/db/db_dialect.go index b128c54..60f6c1c 100644 --- a/db/db_dialect.go +++ b/db/db_dialect.go @@ -6,60 +6,73 @@ import ( "log" ) +// Dialect returns SQL statements for given DB type Dialect interface { - GetTenantInsertSql() string - GetTenantSelectSql() string - GetMigrationInsertSql() string - GetMigrationSelectSql() string - GetCreateTenantsTableSql() string - GetCreateMigrationsTableSql() string - GetCreateSchemaSql(string) string + GetTenantInsertSQL() string + GetTenantSelectSQL() string + GetMigrationInsertSQL() string + GetMigrationSelectSQL() string + GetCreateTenantsTableSQL() string + GetCreateMigrationsTableSQL() string + GetCreateSchemaSQL(string) string } +// BaseDialect struct is used to provide default Dialect interface implementation type BaseDialect struct { } const ( - selectMigrations = "select name, source_dir as sd, filename, type, db_schema, created from %v.%v order by name, source_dir" - selectTenants = "select name from %v.%v" -) - -func (bd *BaseDialect) GetCreateTenantsTableSql() string { - createTenantsTableSql := ` + selectMigrationsSQL = "select name, source_dir as sd, filename, type, db_schema, created from %v.%v order by name, source_dir" + selectTenantsSQL = "select name from %v.%v" + createMigrationsTableSQL = ` create table if not exists %v.%v ( id serial primary key, name varchar(200) not null, + source_dir varchar(200) not null, + filename varchar(200) not null, + type int not null, + db_schema varchar(200) not null, created timestamp default now() ) ` - return fmt.Sprintf(createTenantsTableSql, migratorSchema, migratorTenantsTable) -} - -func (bd *BaseDialect) GetCreateMigrationsTableSql() string { - createMigrationsTableSql := ` + createTenantsTableSQL = ` create table if not exists %v.%v ( id serial primary key, name varchar(200) not null, - source_dir varchar(200) not null, - filename varchar(200) not null, - type int not null, - db_schema varchar(200) not null, created timestamp default now() ) ` - return fmt.Sprintf(createMigrationsTableSql, migratorSchema, migratorMigrationsTable) + createSchemaSQL = "create schema if not exists %v" +) + +// GetCreateTenantsTableSQL returns migrator's default create tenants table SQL statement. +// This SQL is used by both MySQL and PostgreSQL. +func (bd *BaseDialect) GetCreateTenantsTableSQL() string { + return fmt.Sprintf(createTenantsTableSQL, migratorSchema, migratorTenantsTable) +} + +// GetCreateMigrationsTableSQL returns migrator's create migrations table SQL statement. +// This SQL is used by both MySQL and PostgreSQL. +func (bd *BaseDialect) GetCreateMigrationsTableSQL() string { + return fmt.Sprintf(createMigrationsTableSQL, migratorSchema, migratorMigrationsTable) } -func (bd *BaseDialect) GetTenantSelectSql() string { - return fmt.Sprintf(selectTenants, migratorSchema, migratorTenantsTable) +// GetTenantSelectSQL returns migrator's default tenant select SQL statement. +// This SQL is used by all MySQL, PostgreSQL, and MS SQL. +func (bd *BaseDialect) GetTenantSelectSQL() string { + return fmt.Sprintf(selectTenantsSQL, migratorSchema, migratorTenantsTable) } -func (bd *BaseDialect) GetMigrationSelectSql() string { - return fmt.Sprintf(selectMigrations, migratorSchema, migratorMigrationsTable) +// GetMigrationSelectSQL returns migrator's migrations select SQL statement. +// This SQL is used by all MySQL, PostgreSQL, MS SQL. +func (bd *BaseDialect) GetMigrationSelectSQL() string { + return fmt.Sprintf(selectMigrationsSQL, migratorSchema, migratorMigrationsTable) } -func (bd *BaseDialect) GetCreateSchemaSql(schema string) string { - return fmt.Sprintf("create schema if not exists %v", schema) +// GetCreateSchemaSQL returns create schema SQL statement. +// This SQL is used by both MySQL and PostgreSQL. +func (bd *BaseDialect) GetCreateSchemaSQL(schema string) string { + return fmt.Sprintf(createSchemaSQL, schema) } // CreateDialect constructs Dialect instance based on the passed Config diff --git a/db/db_mssql.go b/db/db_mssql.go index 4b112f5..df28aca 100644 --- a/db/db_mssql.go +++ b/db/db_mssql.go @@ -11,20 +11,9 @@ type msSQLDialect struct { } const ( - insertMigrationMSSQLDialectSql = "insert into %v.%v (name, source_dir, filename, type, db_schema) values (@p1, @p2, @p3, @p4, @p5)" - insertTenantMSSQLDialectSql = "insert into %v.%v (name) values (@p1)" -) - -func (md *msSQLDialect) GetMigrationInsertSql() string { - return fmt.Sprintf(insertMigrationMSSQLDialectSql, migratorSchema, migratorMigrationsTable) -} - -func (md *msSQLDialect) GetTenantInsertSql() string { - return fmt.Sprintf(insertTenantMSSQLDialectSql, migratorSchema, migratorTenantsTable) -} - -func (md *msSQLDialect) GetCreateTenantsTableSql() string { - createTenantsTableSql := ` + insertMigrationMSSQLDialectSQL = "insert into %v.%v (name, source_dir, filename, type, db_schema) values (@p1, @p2, @p3, @p4, @p5)" + insertTenantMSSQLDialectSQL = "insert into %v.%v (name) values (@p1)" + createTenantsTableMSSQLDialectSQL = ` IF NOT EXISTS (select * from information_schema.tables where table_schema = '%v' and table_name = '%v') BEGIN create table [%v].%v ( @@ -34,11 +23,7 @@ BEGIN ); END ` - return fmt.Sprintf(createTenantsTableSql, migratorSchema, migratorTenantsTable, migratorSchema, migratorTenantsTable) -} - -func (md *msSQLDialect) GetCreateMigrationsTableSql() string { - createMigrationsTableSql := ` + createMigrationsTableMSSQLDialectSQL = ` IF NOT EXISTS (select * from information_schema.tables where table_schema = '%v' and table_name = '%v') BEGIN create table [%v].%v ( @@ -52,15 +37,38 @@ BEGIN ); END ` - return fmt.Sprintf(createMigrationsTableSql, migratorSchema, migratorMigrationsTable, migratorSchema, migratorMigrationsTable) -} - -func (md *msSQLDialect) GetCreateSchemaSql(schema string) string { - createSchemaSql := ` + createSchemaMSSQLDialectSQL = ` IF NOT EXISTS (select * from information_schema.schemata where schema_name = '%v') BEGIN - EXEC sp_executesql N'create schema %v'; + EXEC sp_executesql N'create schema %v'; END ` - return fmt.Sprintf(createSchemaSql, schema, schema) +) + +// GetMigrationInsertSQL returns MS SQL-specific migration insert SQL statement +func (md *msSQLDialect) GetMigrationInsertSQL() string { + return fmt.Sprintf(insertMigrationMSSQLDialectSQL, migratorSchema, migratorMigrationsTable) +} + +// GetTenantInsertSQL returns MS SQL-specific migrator's default tenant insert SQL statement +func (md *msSQLDialect) GetTenantInsertSQL() string { + return fmt.Sprintf(insertTenantMSSQLDialectSQL, migratorSchema, migratorTenantsTable) +} + +// GetCreateTenantsTableSQL returns migrator's default create tenants table SQL statement. +// This SQL is used by MS SQL. +func (md *msSQLDialect) GetCreateTenantsTableSQL() string { + return fmt.Sprintf(createTenantsTableMSSQLDialectSQL, migratorSchema, migratorTenantsTable, migratorSchema, migratorTenantsTable) +} + +// GetCreateMigrationsTableSQL returns migrator's create migrations table SQL statement. +// This SQL is used by both MS SQL. +func (md *msSQLDialect) GetCreateMigrationsTableSQL() string { + return fmt.Sprintf(createMigrationsTableMSSQLDialectSQL, migratorSchema, migratorMigrationsTable, migratorSchema, migratorMigrationsTable) +} + +// GetCreateSchemaSQL returns create schema SQL statement. +// This SQL is used by both MySQL and PostgreSQL. +func (md *msSQLDialect) GetCreateSchemaSQL(schema string) string { + return fmt.Sprintf(createSchemaMSSQLDialectSQL, schema, schema) } diff --git a/db/db_mysql.go b/db/db_mysql.go index 5cf0b01..5f57ab3 100644 --- a/db/db_mysql.go +++ b/db/db_mysql.go @@ -11,14 +11,16 @@ type mySQLDialect struct { } const ( - insertMigrationMySQLDialectSql = "insert into %v.%v (name, source_dir, filename, type, db_schema) values (?, ?, ?, ?, ?)" - insertTenantMySQLDialectSql = "insert into %v.%v (name) values (?)" + insertMigrationMySQLDialectSQL = "insert into %v.%v (name, source_dir, filename, type, db_schema) values (?, ?, ?, ?, ?)" + insertTenantMySQLDialectSQL = "insert into %v.%v (name) values (?)" ) -func (md *mySQLDialect) GetMigrationInsertSql() string { - return fmt.Sprintf(insertMigrationMySQLDialectSql, migratorSchema, migratorMigrationsTable) +// GetMigrationInsertSQL returns MySQL-specific migration insert SQL statement +func (md *mySQLDialect) GetMigrationInsertSQL() string { + return fmt.Sprintf(insertMigrationMySQLDialectSQL, migratorSchema, migratorMigrationsTable) } -func (md *mySQLDialect) GetTenantInsertSql() string { - return fmt.Sprintf(insertTenantMySQLDialectSql, migratorSchema, migratorTenantsTable) +// GetTenantInsertSQL returns MySQL-specific migrator's default tenant insert SQL statement +func (md *mySQLDialect) GetTenantInsertSQL() string { + return fmt.Sprintf(insertTenantMySQLDialectSQL, migratorSchema, migratorTenantsTable) } diff --git a/db/db_postgresql.go b/db/db_postgresql.go index 6bb2305..edf359a 100644 --- a/db/db_postgresql.go +++ b/db/db_postgresql.go @@ -11,14 +11,16 @@ type postgreSQLDialect struct { } const ( - insertMigrationPostgreSQLDialectSql = "insert into %v.%v (name, source_dir, filename, type, db_schema) values ($1, $2, $3, $4, $5)" - insertTenantPostgreSQLDialectSql = "insert into %v.%v (name) values ($1)" + insertMigrationPostgreSQLDialectSQL = "insert into %v.%v (name, source_dir, filename, type, db_schema) values ($1, $2, $3, $4, $5)" + insertTenantPostgreSQLDialectSQL = "insert into %v.%v (name) values ($1)" ) -func (pd *postgreSQLDialect) GetMigrationInsertSql() string { - return fmt.Sprintf(insertMigrationPostgreSQLDialectSql, migratorSchema, migratorMigrationsTable) +// GetMigrationInsertSQL returns PostgreSQL-specific migration insert SQL statement +func (pd *postgreSQLDialect) GetMigrationInsertSQL() string { + return fmt.Sprintf(insertMigrationPostgreSQLDialectSQL, migratorSchema, migratorMigrationsTable) } -func (pd *postgreSQLDialect) GetTenantInsertSql() string { - return fmt.Sprintf(insertTenantPostgreSQLDialectSql, migratorSchema, migratorTenantsTable) +// GetTenantInsertSQL returns PostgreSQL-specific migrator's default tenant insert SQL statement +func (pd *postgreSQLDialect) GetTenantInsertSQL() string { + return fmt.Sprintf(insertTenantPostgreSQLDialectSQL, migratorSchema, migratorTenantsTable) } diff --git a/db/db_test.go b/db/db_test.go index fef2618..562e943 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -22,7 +22,7 @@ func TestDBCreateConnectorPanicUnknownDriver(t *testing.T) { }, "Should panic because of unknown driver") } -func TestDBCreateDialectPostgreSqlDriver(t *testing.T) { +func TestDBCreateDialectPostgreSQLDriver(t *testing.T) { config := &config.Config{} config.Driver = "postgres" dialect := CreateDialect(config) @@ -59,7 +59,7 @@ func TestDBGetTenantsPanicSQLSyntaxError(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) - config.TenantSelectSql = "sadfdsfdsf" + config.TenantSelectSQL = "sadfdsfdsf" connector := CreateConnector(config) connector.Init() assert.Panics(t, func() { @@ -76,8 +76,8 @@ func TestDBApplyMigrationsPanicSQLSyntaxError(t *testing.T) { connector := CreateConnector(config) connector.Init() defer connector.Dispose() - m := types.MigrationDefinition{"201602220002.sql", "error", "error/201602220002.sql", types.MigrationTypeTenantSchema} - ms := []types.Migration{{m, "createtablexyx ( idint primary key (id) )"}} + m := types.MigrationDefinition{Name: "201602220002.sql", SourceDir: "error", File: "error/201602220002.sql", MigrationType: types.MigrationTypeTenantSchema} + ms := []types.Migration{{MigrationDefinition: m, Contents: "createtablexyx ( idint primary key (id) )"}} assert.Panics(t, func() { connector.ApplyMigrations(ms) @@ -113,7 +113,7 @@ func TestDBApplyMigrations(t *testing.T) { tenants := connector.GetTenants() lenTenants := len(tenants) - dbMigrationsBefore := connector.GetMigrations() + dbMigrationsBefore := connector.GetDBMigrations() lenBefore := len(dbMigrationsBefore) p1 := time.Now().UnixNano() @@ -123,25 +123,25 @@ func TestDBApplyMigrations(t *testing.T) { t2 := time.Now().UnixNano() t3 := time.Now().UnixNano() - publicdef1 := types.MigrationDefinition{fmt.Sprintf("%v.sql", p1), "public", fmt.Sprintf("public/%v.sql", p1), types.MigrationTypeSingleSchema} - publicdef2 := types.MigrationDefinition{fmt.Sprintf("%v.sql", p2), "public", fmt.Sprintf("public/%v.sql", p2), types.MigrationTypeSingleSchema} - publicdef3 := types.MigrationDefinition{fmt.Sprintf("%v.sql", p3), "public", fmt.Sprintf("public/%v.sql", p3), types.MigrationTypeSingleSchema} - public1 := types.Migration{publicdef1, "drop table if exists modules"} - public2 := types.Migration{publicdef2, "create table modules ( k int, v text )"} - public3 := types.Migration{publicdef3, "insert into modules values ( 123, '123' )"} + publicdef1 := types.MigrationDefinition{Name: fmt.Sprintf("%v.sql", p1), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p1), MigrationType: types.MigrationTypeSingleSchema} + publicdef2 := types.MigrationDefinition{Name: fmt.Sprintf("%v.sql", p2), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p2), MigrationType: types.MigrationTypeSingleSchema} + publicdef3 := types.MigrationDefinition{Name: fmt.Sprintf("%v.sql", p3), SourceDir: "public", File: fmt.Sprintf("public/%v.sql", p3), MigrationType: types.MigrationTypeSingleSchema} + public1 := types.Migration{MigrationDefinition: publicdef1, Contents: "drop table if exists modules"} + public2 := types.Migration{MigrationDefinition: publicdef2, Contents: "create table modules ( k int, v text )"} + public3 := types.Migration{MigrationDefinition: publicdef3, Contents: "insert into modules values ( 123, '123' )"} - tenantdef1 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t1), "tenants", fmt.Sprintf("tenants/%v.sql", t1), types.MigrationTypeTenantSchema} - tenantdef2 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t2), "tenants", fmt.Sprintf("tenants/%v.sql", t2), types.MigrationTypeTenantSchema} - tenantdef3 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t3), "tenants", fmt.Sprintf("tenants/%v.sql", t2), types.MigrationTypeTenantSchema} - tenant1 := types.Migration{tenantdef1, "drop table if exists {schema}.settings"} - tenant2 := types.Migration{tenantdef2, "create table {schema}.settings (k int, v text)"} - tenant3 := types.Migration{tenantdef3, "insert into {schema}.settings values (456, '456') "} + tenantdef1 := types.MigrationDefinition{Name: fmt.Sprintf("%v.sql", t1), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t1), MigrationType: types.MigrationTypeTenantSchema} + tenantdef2 := types.MigrationDefinition{Name: fmt.Sprintf("%v.sql", t2), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t2), MigrationType: types.MigrationTypeTenantSchema} + tenantdef3 := types.MigrationDefinition{Name: fmt.Sprintf("%v.sql", t3), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t2), MigrationType: types.MigrationTypeTenantSchema} + tenant1 := types.Migration{MigrationDefinition: tenantdef1, Contents: "drop table if exists {schema}.settings"} + tenant2 := types.Migration{MigrationDefinition: tenantdef2, Contents: "create table {schema}.settings (k int, v text)"} + tenant3 := types.Migration{MigrationDefinition: tenantdef3, Contents: "insert into {schema}.settings values (456, '456') "} migrationsToApply := []types.Migration{public1, public2, public3, tenant1, tenant2, tenant3} connector.ApplyMigrations(migrationsToApply) - dbMigrationsAfter := connector.GetMigrations() + dbMigrationsAfter := connector.GetDBMigrations() lenAfter := len(dbMigrationsAfter) // 3 tenant migrations * no of tenants + 3 public @@ -157,41 +157,41 @@ func TestDBApplyMigrationsEmptyMigrationArray(t *testing.T) { connector.Init() defer connector.Dispose() - dbMigrationsBefore := connector.GetMigrations() + dbMigrationsBefore := connector.GetDBMigrations() lenBefore := len(dbMigrationsBefore) migrationsToApply := []types.Migration{} connector.ApplyMigrations(migrationsToApply) - dbMigrationsAfter := connector.GetMigrations() + dbMigrationsAfter := connector.GetDBMigrations() lenAfter := len(dbMigrationsAfter) assert.Equal(t, lenAfter, lenBefore) } -func TestGetTenantsSqlDefault(t *testing.T) { +func TestGetTenantsSQLDefault(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) connector := CreateConnector(config) defer connector.Dispose() - tenantSelectSql := connector.GetTenantSelectSql() + tenantSelectSQL := connector.GetTenantSelectSQL() - assert.Equal(t, "select name from migrator.migrator_tenants", tenantSelectSql) + assert.Equal(t, "select name from migrator.migrator_tenants", tenantSelectSQL) } -func TestGetTenantsSqlOverride(t *testing.T) { +func TestGetTenantsSQLOverride(t *testing.T) { config, err := config.FromFile("../test/migrator-overrides.yaml") assert.Nil(t, err) connector := CreateConnector(config) defer connector.Dispose() - tenantSelectSql := connector.GetTenantSelectSql() + tenantSelectSQL := connector.GetTenantSelectSQL() - assert.Equal(t, "select somename from someschema.sometable", tenantSelectSql) + assert.Equal(t, "select somename from someschema.sometable", tenantSelectSQL) } func TestGetSchemaPlaceHolderDefault(t *testing.T) { @@ -226,33 +226,33 @@ func TestAddTenantAndApplyMigrations(t *testing.T) { connector.Init() defer connector.Dispose() - dbMigrationsBefore := connector.GetMigrations() + dbMigrationsBefore := connector.GetDBMigrations() lenBefore := len(dbMigrationsBefore) t1 := time.Now().UnixNano() t2 := time.Now().UnixNano() t3 := time.Now().UnixNano() - tenantdef1 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t1), "tenants", fmt.Sprintf("tenants/%v.sql", t1), types.MigrationTypeTenantSchema} - tenantdef2 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t2), "tenants", fmt.Sprintf("tenants/%v.sql", t2), types.MigrationTypeTenantSchema} - tenantdef3 := types.MigrationDefinition{fmt.Sprintf("%v.sql", t3), "tenants", fmt.Sprintf("tenants/%v.sql", t3), types.MigrationTypeTenantSchema} - tenant1 := types.Migration{tenantdef1, "drop table if exists {schema}.settings"} - tenant2 := types.Migration{tenantdef2, "create table {schema}.settings (k int, v text) "} - tenant3 := types.Migration{tenantdef3, "insert into {schema}.settings values (456, '456') "} + tenantdef1 := types.MigrationDefinition{Name: fmt.Sprintf("%v.sql", t1), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t1), MigrationType: types.MigrationTypeTenantSchema} + tenantdef2 := types.MigrationDefinition{Name: fmt.Sprintf("%v.sql", t2), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t2), MigrationType: types.MigrationTypeTenantSchema} + tenantdef3 := types.MigrationDefinition{Name: fmt.Sprintf("%v.sql", t3), SourceDir: "tenants", File: fmt.Sprintf("tenants/%v.sql", t3), MigrationType: types.MigrationTypeTenantSchema} + tenant1 := types.Migration{MigrationDefinition: tenantdef1, Contents: "drop table if exists {schema}.settings"} + tenant2 := types.Migration{MigrationDefinition: tenantdef2, Contents: "create table {schema}.settings (k int, v text) "} + tenant3 := types.Migration{MigrationDefinition: tenantdef3, Contents: "insert into {schema}.settings values (456, '456') "} migrationsToApply := []types.Migration{tenant1, tenant2, tenant3} - unique_tenant := fmt.Sprintf("new_test_tenant_%v", time.Now().UnixNano()) + uniqueTenant := fmt.Sprintf("new_test_tenant_%v", time.Now().UnixNano()) - connector.AddTenantAndApplyMigrations(unique_tenant, migrationsToApply) + connector.AddTenantAndApplyMigrations(uniqueTenant, migrationsToApply) - dbMigrationsAfter := connector.GetMigrations() + dbMigrationsAfter := connector.GetDBMigrations() lenAfter := len(dbMigrationsAfter) assert.Equal(t, 3, lenAfter-lenBefore) } -func TestMySQLGetMigrationInsertSql(t *testing.T) { +func TestMySQLGetMigrationInsertSQL(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -260,12 +260,12 @@ func TestMySQLGetMigrationInsertSql(t *testing.T) { dialect := CreateDialect(config) - insertMigrationSQL := dialect.GetMigrationInsertSql() + insertMigrationSQL := dialect.GetMigrationInsertSQL() assert.Equal(t, "insert into migrator.migrator_migrations (name, source_dir, filename, type, db_schema) values (?, ?, ?, ?, ?)", insertMigrationSQL) } -func TestPostgreSQLGetMigrationInsertSql(t *testing.T) { +func TestPostgreSQLGetMigrationInsertSQL(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -273,12 +273,12 @@ func TestPostgreSQLGetMigrationInsertSql(t *testing.T) { dialect := CreateDialect(config) - insertMigrationSQL := dialect.GetMigrationInsertSql() + insertMigrationSQL := dialect.GetMigrationInsertSQL() assert.Equal(t, "insert into migrator.migrator_migrations (name, source_dir, filename, type, db_schema) values ($1, $2, $3, $4, $5)", insertMigrationSQL) } -func TestMSSQLGetMigrationInsertSql(t *testing.T) { +func TestMSSQLGetMigrationInsertSQL(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -286,12 +286,12 @@ func TestMSSQLGetMigrationInsertSql(t *testing.T) { dialect := CreateDialect(config) - insertMigrationSQL := dialect.GetMigrationInsertSql() + insertMigrationSQL := dialect.GetMigrationInsertSQL() assert.Equal(t, "insert into migrator.migrator_migrations (name, source_dir, filename, type, db_schema) values (@p1, @p2, @p3, @p4, @p5)", insertMigrationSQL) } -func TestMySQLGetTenantInsertSqlDefault(t *testing.T) { +func TestMySQLGetTenantInsertSQLDefault(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -299,12 +299,12 @@ func TestMySQLGetTenantInsertSqlDefault(t *testing.T) { connector := CreateConnector(config) defer connector.Dispose() - tenantInsertSql := connector.GetTenantInsertSql() + tenantInsertSQL := connector.GetTenantInsertSQL() - assert.Equal(t, "insert into migrator.migrator_tenants (name) values (?)", tenantInsertSql) + assert.Equal(t, "insert into migrator.migrator_tenants (name) values (?)", tenantInsertSQL) } -func TestPostgreSQLGetTenantInsertSqlDefault(t *testing.T) { +func TestPostgreSQLGetTenantInsertSQLDefault(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -312,12 +312,12 @@ func TestPostgreSQLGetTenantInsertSqlDefault(t *testing.T) { connector := CreateConnector(config) defer connector.Dispose() - tenantInsertSql := connector.GetTenantInsertSql() + tenantInsertSQL := connector.GetTenantInsertSQL() - assert.Equal(t, "insert into migrator.migrator_tenants (name) values ($1)", tenantInsertSql) + assert.Equal(t, "insert into migrator.migrator_tenants (name) values ($1)", tenantInsertSQL) } -func TestMSSQLGetTenantInsertSqlDefault(t *testing.T) { +func TestMSSQLGetTenantInsertSQLDefault(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -325,24 +325,24 @@ func TestMSSQLGetTenantInsertSqlDefault(t *testing.T) { connector := CreateConnector(config) defer connector.Dispose() - tenantInsertSql := connector.GetTenantInsertSql() + tenantInsertSQL := connector.GetTenantInsertSQL() - assert.Equal(t, "insert into migrator.migrator_tenants (name) values (@p1)", tenantInsertSql) + assert.Equal(t, "insert into migrator.migrator_tenants (name) values (@p1)", tenantInsertSQL) } -func TestGetTenantInsertSqlOverride(t *testing.T) { +func TestGetTenantInsertSQLOverride(t *testing.T) { config, err := config.FromFile("../test/migrator-overrides.yaml") assert.Nil(t, err) connector := CreateConnector(config) defer connector.Dispose() - tenantInsertSql := connector.GetTenantInsertSql() + tenantInsertSQL := connector.GetTenantInsertSQL() - assert.Equal(t, "insert into XXX", tenantInsertSql) + assert.Equal(t, "insert into someschema.sometable (somename) values ($1)", tenantInsertSQL) } -func TestMSSQLDialectGetCreateTenantsTableSql(t *testing.T) { +func TestMSSQLDialectGetCreateTenantsTableSQL(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -350,7 +350,7 @@ func TestMSSQLDialectGetCreateTenantsTableSql(t *testing.T) { dialect := CreateDialect(config) - createTenantsTableSql := dialect.GetCreateTenantsTableSql() + createTenantsTableSQL := dialect.GetCreateTenantsTableSQL() expected := ` IF NOT EXISTS (select * from information_schema.tables where table_schema = 'migrator' and table_name = 'migrator_tenants') @@ -363,10 +363,10 @@ BEGIN END ` - assert.Equal(t, expected, createTenantsTableSql) + assert.Equal(t, expected, createTenantsTableSQL) } -func TestMSSQLDialectGetCreateMigrationsTableSql(t *testing.T) { +func TestMSSQLDialectGetCreateMigrationsTableSQL(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -374,7 +374,7 @@ func TestMSSQLDialectGetCreateMigrationsTableSql(t *testing.T) { dialect := CreateDialect(config) - createMigrationsTableSql := dialect.GetCreateMigrationsTableSql() + createMigrationsTableSQL := dialect.GetCreateMigrationsTableSQL() expected := ` IF NOT EXISTS (select * from information_schema.tables where table_schema = 'migrator' and table_name = 'migrator_migrations') @@ -391,10 +391,10 @@ BEGIN END ` - assert.Equal(t, expected, createMigrationsTableSql) + assert.Equal(t, expected, createMigrationsTableSQL) } -func TestBaseDialectGetCreateTenantsTableSql(t *testing.T) { +func TestBaseDialectGetCreateTenantsTableSQL(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -402,7 +402,7 @@ func TestBaseDialectGetCreateTenantsTableSql(t *testing.T) { dialect := CreateDialect(config) - createTenantsTableSql := dialect.GetCreateTenantsTableSql() + createTenantsTableSQL := dialect.GetCreateTenantsTableSQL() expected := ` create table if not exists migrator.migrator_tenants ( @@ -412,10 +412,10 @@ create table if not exists migrator.migrator_tenants ( ) ` - assert.Equal(t, expected, createTenantsTableSql) + assert.Equal(t, expected, createTenantsTableSQL) } -func TestBaseDialectGetCreateMigrationsTableSql(t *testing.T) { +func TestBaseDialectGetCreateMigrationsTableSQL(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -423,7 +423,7 @@ func TestBaseDialectGetCreateMigrationsTableSql(t *testing.T) { dialect := CreateDialect(config) - createMigrationsTableSql := dialect.GetCreateMigrationsTableSql() + createMigrationsTableSQL := dialect.GetCreateMigrationsTableSQL() expected := ` create table if not exists migrator.migrator_migrations ( @@ -437,10 +437,10 @@ create table if not exists migrator.migrator_migrations ( ) ` - assert.Equal(t, expected, createMigrationsTableSql) + assert.Equal(t, expected, createMigrationsTableSQL) } -func TestBaseDialectGetCreateSchemaSql(t *testing.T) { +func TestBaseDialectGetCreateSchemaSQL(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -448,14 +448,14 @@ func TestBaseDialectGetCreateSchemaSql(t *testing.T) { dialect := CreateDialect(config) - createSchemaSql := dialect.GetCreateSchemaSql("abc") + createSchemaSQL := dialect.GetCreateSchemaSQL("abc") expected := "create schema if not exists abc" - assert.Equal(t, expected, createSchemaSql) + assert.Equal(t, expected, createSchemaSQL) } -func TestMSSQLDialectGetCreateSchemaSql(t *testing.T) { +func TestMSSQLDialectGetCreateSchemaSQL(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) @@ -463,14 +463,14 @@ func TestMSSQLDialectGetCreateSchemaSql(t *testing.T) { dialect := CreateDialect(config) - createSchemaSql := dialect.GetCreateSchemaSql("def") + createSchemaSQL := dialect.GetCreateSchemaSQL("def") expected := ` IF NOT EXISTS (select * from information_schema.schemata where schema_name = 'def') BEGIN - EXEC sp_executesql N'create schema def'; + EXEC sp_executesql N'create schema def'; END ` - assert.Equal(t, expected, createSchemaSql) + assert.Equal(t, expected, createSchemaSQL) } diff --git a/loader/disk_loader.go b/loader/disk_loader.go index 676dc7c..8d9c30c 100644 --- a/loader/disk_loader.go +++ b/loader/disk_loader.go @@ -16,8 +16,8 @@ type DiskLoader struct { Config *config.Config } -// GetMigrations loads all migrations from disk -func (dl *DiskLoader) GetMigrations() []types.Migration { +// GetDiskMigrations returns all migrations from disk +func (dl *DiskLoader) GetDiskMigrations() []types.Migration { dirs, err := ioutil.ReadDir(dl.Config.BaseDir) if err != nil { log.Panicf("Could not read migration base dir: %v", err) @@ -69,12 +69,12 @@ func (dl *DiskLoader) readMigrationsFromSchemaDirs(migrations map[string][]types } for _, file := range files { if !file.IsDir() { - mdef := types.MigrationDefinition{file.Name(), sourceDir, filepath.Join(sourceDir, file.Name()), migrationType} + mdef := types.MigrationDefinition{Name: file.Name(), SourceDir: sourceDir, File: filepath.Join(sourceDir, file.Name()), MigrationType: migrationType} contents, err := ioutil.ReadFile(filepath.Join(dl.Config.BaseDir, mdef.File)) if err != nil { log.Panicf("Could not read migration contents: %v", err) } - m := types.Migration{mdef, string(contents)} + m := types.Migration{MigrationDefinition: mdef, Contents: string(contents)} e, ok := migrations[m.Name] if ok { e = append(e, m) diff --git a/loader/disk_loader_test.go b/loader/disk_loader_test.go index 92e7735..a0f7621 100644 --- a/loader/disk_loader_test.go +++ b/loader/disk_loader_test.go @@ -14,7 +14,7 @@ func TestDiskPanicReadDiskMigrationsNonExistingBaseDir(t *testing.T) { loader := CreateLoader(&config) assert.Panics(t, func() { - loader.GetMigrations() + loader.GetDiskMigrations() }, "Should panic because of non-existing base dir") } @@ -27,7 +27,7 @@ func TestDiskGetDiskMigrations(t *testing.T) { config.TenantSchemas = []string{"tenants"} loader := CreateLoader(&config) - migrations := loader.GetMigrations() + migrations := loader.GetDiskMigrations() assert.Len(t, migrations, 6) diff --git a/loader/loader.go b/loader/loader.go index 33d6dda..93cc635 100644 --- a/loader/loader.go +++ b/loader/loader.go @@ -7,7 +7,7 @@ import ( // Loader interface abstracts all loading operations performed by migrator type Loader interface { - GetMigrations() []types.Migration + GetDiskMigrations() []types.Migration } // CreateLoader abstracts all loading operations performed by migrator diff --git a/migrations/migrations.go b/migrations/migrations.go index 21c517a..197d274 100644 --- a/migrations/migrations.go +++ b/migrations/migrations.go @@ -45,6 +45,7 @@ func ComputeMigrationsToApply(diskMigrations []types.Migration, dbMigrations []t return out } +// FilterTenantMigrations returns only migrations which are of type MigrationTypeTenantSchema func FilterTenantMigrations(diskMigrations []types.Migration) []types.Migration { filteredTenantMigrations := []types.Migration{} for _, m := range diskMigrations { diff --git a/migrations/migrations_test.go b/migrations/migrations_test.go index 9cdee09..2492ff5 100644 --- a/migrations/migrations_test.go +++ b/migrations/migrations_test.go @@ -8,17 +8,16 @@ import ( ) func TestMigrationsFlattenMigrationDBs1(t *testing.T) { + m1 := types.MigrationDefinition{Name: "001.sql", SourceDir: "public", File: "public/001.sql", MigrationType: types.MigrationTypeSingleSchema} + db1 := types.MigrationDB{MigrationDefinition: m1, Schema: "public", Created: time.Now()} - m1 := types.MigrationDefinition{"001.sql", "public", "public/001.sql", types.MigrationTypeSingleSchema} - db1 := types.MigrationDB{m1, "public", time.Now()} + m2 := types.MigrationDefinition{Name: "002.sql", SourceDir: "tenants", File: "tenants/002.sql", MigrationType: types.MigrationTypeTenantSchema} + db2 := types.MigrationDB{MigrationDefinition: m2, Schema: "abc", Created: time.Now()} - m2 := types.MigrationDefinition{"002.sql", "tenants", "tenants/002.sql", types.MigrationTypeTenantSchema} - db2 := types.MigrationDB{m2, "abc", time.Now()} + db3 := types.MigrationDB{MigrationDefinition: m2, Schema: "def", Created: time.Now()} - db3 := types.MigrationDB{m2, "def", time.Now()} - - m4 := types.MigrationDefinition{"003.sql", "ref", "ref/003.sql", types.MigrationTypeSingleSchema} - db4 := types.MigrationDB{m4, "ref", time.Now()} + m4 := types.MigrationDefinition{Name: "003.sql", SourceDir: "ref", File: "ref/003.sql", MigrationType: types.MigrationTypeSingleSchema} + db4 := types.MigrationDB{MigrationDefinition: m4, Schema: "ref", Created: time.Now()} dbs := []types.MigrationDB{db1, db2, db3, db4} @@ -30,13 +29,13 @@ func TestMigrationsFlattenMigrationDBs1(t *testing.T) { func TestMigrationsFlattenMigrationDBs2(t *testing.T) { - m2 := types.MigrationDefinition{"002.sql", "tenants", "tenants/002.sql", types.MigrationTypeTenantSchema} - db2 := types.MigrationDB{m2, "abc", time.Now()} + m2 := types.MigrationDefinition{Name: "002.sql", SourceDir: "tenants", File: "tenants/002.sql", MigrationType: types.MigrationTypeTenantSchema} + db2 := types.MigrationDB{MigrationDefinition: m2, Schema: "abc", Created: time.Now()} - db3 := types.MigrationDB{m2, "def", time.Now()} + db3 := types.MigrationDB{MigrationDefinition: m2, Schema: "def", Created: time.Now()} - m4 := types.MigrationDefinition{"003.sql", "ref", "ref/003.sql", types.MigrationTypeSingleSchema} - db4 := types.MigrationDB{m4, "ref", time.Now()} + m4 := types.MigrationDefinition{Name: "003.sql", SourceDir: "ref", File: "ref/003.sql", MigrationType: types.MigrationTypeSingleSchema} + db4 := types.MigrationDB{MigrationDefinition: m4, Schema: "ref", Created: time.Now()} dbs := []types.MigrationDB{db2, db3, db4} @@ -47,13 +46,13 @@ func TestMigrationsFlattenMigrationDBs2(t *testing.T) { } func TestComputeMigrationsToApply(t *testing.T) { - mdef1 := types.MigrationDefinition{"a", "a", "a", types.MigrationTypeSingleSchema} - mdef2 := types.MigrationDefinition{"b", "b", "b", types.MigrationTypeTenantSchema} - mdef3 := types.MigrationDefinition{"c", "c", "c", types.MigrationTypeTenantSchema} - mdef4 := types.MigrationDefinition{"d", "d", "d", types.MigrationTypeSingleSchema} + mdef1 := types.MigrationDefinition{Name: "a", SourceDir: "a", File: "a", MigrationType: types.MigrationTypeSingleSchema} + mdef2 := types.MigrationDefinition{Name: "b", SourceDir: "b", File: "b", MigrationType: types.MigrationTypeTenantSchema} + mdef3 := types.MigrationDefinition{Name: "c", SourceDir: "c", File: "c", MigrationType: types.MigrationTypeTenantSchema} + mdef4 := types.MigrationDefinition{Name: "d", SourceDir: "d", File: "d", MigrationType: types.MigrationTypeSingleSchema} - diskMigrations := []types.Migration{{mdef1, ""}, {mdef2, ""}, {mdef3, ""}, {mdef4, ""}} - dbMigrations := []types.MigrationDB{{mdef1, "a", time.Now()}, {mdef2, "abc", time.Now()}, {mdef2, "def", time.Now()}} + diskMigrations := []types.Migration{{MigrationDefinition: mdef1, Contents: ""}, {MigrationDefinition: mdef2, Contents: ""}, {MigrationDefinition: mdef3, Contents: ""}, {MigrationDefinition: mdef4, Contents: ""}} + dbMigrations := []types.MigrationDB{{MigrationDefinition: mdef1, Schema: "a", Created: time.Now()}, {MigrationDefinition: mdef2, Schema: "abc", Created: time.Now()}, {MigrationDefinition: mdef2, Schema: "def", Created: time.Now()}} migrations := ComputeMigrationsToApply(diskMigrations, dbMigrations) assert.Len(t, migrations, 2) @@ -63,18 +62,18 @@ func TestComputeMigrationsToApply(t *testing.T) { } func TestFilterTenantMigrations(t *testing.T) { - mdef1 := types.MigrationDefinition{"20181111", "tenants", "tenants/20181111", types.MigrationTypeTenantSchema} - mdef2 := types.MigrationDefinition{"20181111", "public", "public/20181111", types.MigrationTypeSingleSchema} - mdef3 := types.MigrationDefinition{"20181112", "public", "public/20181112", types.MigrationTypeSingleSchema} + mdef1 := types.MigrationDefinition{Name: "20181111", SourceDir: "tenants", File: "tenants/20181111", MigrationType: types.MigrationTypeTenantSchema} + mdef2 := types.MigrationDefinition{Name: "20181111", SourceDir: "public", File: "public/20181111", MigrationType: types.MigrationTypeSingleSchema} + mdef3 := types.MigrationDefinition{Name: "20181112", SourceDir: "public", File: "public/20181112", MigrationType: types.MigrationTypeSingleSchema} - dev1 := types.MigrationDefinition{"20181119", "tenants", "tenants/20181119", types.MigrationTypeTenantSchema} - dev1p1 := types.MigrationDefinition{"201811190", "public", "public/201811190", types.MigrationTypeSingleSchema} - dev1p2 := types.MigrationDefinition{"20181191", "public", "public/201811191", types.MigrationTypeSingleSchema} + dev1 := types.MigrationDefinition{Name: "20181119", SourceDir: "tenants", File: "tenants/20181119", MigrationType: types.MigrationTypeTenantSchema} + dev1p1 := types.MigrationDefinition{Name: "201811190", SourceDir: "public", File: "public/201811190", MigrationType: types.MigrationTypeSingleSchema} + dev1p2 := types.MigrationDefinition{Name: "20181191", SourceDir: "public", File: "public/201811191", MigrationType: types.MigrationTypeSingleSchema} - dev2 := types.MigrationDefinition{"20181120", "tenants", "tenants/20181120", types.MigrationTypeTenantSchema} - dev2p := types.MigrationDefinition{"20181120", "public", "public/20181120", types.MigrationTypeSingleSchema} + dev2 := types.MigrationDefinition{Name: "20181120", SourceDir: "tenants", File: "tenants/20181120", MigrationType: types.MigrationTypeTenantSchema} + dev2p := types.MigrationDefinition{Name: "20181120", SourceDir: "public", File: "public/20181120", MigrationType: types.MigrationTypeSingleSchema} - diskMigrations := []types.Migration{{mdef1, ""}, {mdef2, ""}, {mdef3, ""}, {dev1, ""}, {dev1p1, ""}, {dev1p2, ""}, {dev2, ""}, {dev2p, ""}} + diskMigrations := []types.Migration{{MigrationDefinition: mdef1, Contents: ""}, {MigrationDefinition: mdef2, Contents: ""}, {MigrationDefinition: mdef3, Contents: ""}, {MigrationDefinition: dev1, Contents: ""}, {MigrationDefinition: dev1p1, Contents: ""}, {MigrationDefinition: dev1p2, Contents: ""}, {MigrationDefinition: dev2, Contents: ""}, {MigrationDefinition: dev2p, Contents: ""}} migrations := FilterTenantMigrations(diskMigrations) assert.Len(t, migrations, 3) @@ -97,19 +96,19 @@ func TestComputeMigrationsToApplyDifferentTimestamps(t *testing.T) { // migrator should detect dev1 migrations // previous implementation relied only on counts and such migration was not applied - mdef1 := types.MigrationDefinition{"20181111", "tenants", "tenants/20181111", types.MigrationTypeTenantSchema} - mdef2 := types.MigrationDefinition{"20181111", "public", "public/20181111", types.MigrationTypeSingleSchema} - mdef3 := types.MigrationDefinition{"20181112", "public", "public/20181112", types.MigrationTypeSingleSchema} + mdef1 := types.MigrationDefinition{Name: "20181111", SourceDir: "tenants", File: "tenants/20181111", MigrationType: types.MigrationTypeTenantSchema} + mdef2 := types.MigrationDefinition{Name: "20181111", SourceDir: "public", File: "public/20181111", MigrationType: types.MigrationTypeSingleSchema} + mdef3 := types.MigrationDefinition{Name: "20181112", SourceDir: "public", File: "public/20181112", MigrationType: types.MigrationTypeSingleSchema} - dev1 := types.MigrationDefinition{"20181119", "tenants", "tenants/20181119", types.MigrationTypeTenantSchema} - dev1p1 := types.MigrationDefinition{"201811190", "public", "public/201811190", types.MigrationTypeSingleSchema} - dev1p2 := types.MigrationDefinition{"20181191", "public", "public/201811191", types.MigrationTypeSingleSchema} + dev1 := types.MigrationDefinition{Name: "20181119", SourceDir: "tenants", File: "tenants/20181119", MigrationType: types.MigrationTypeTenantSchema} + dev1p1 := types.MigrationDefinition{Name: "201811190", SourceDir: "public", File: "public/201811190", MigrationType: types.MigrationTypeSingleSchema} + dev1p2 := types.MigrationDefinition{Name: "20181191", SourceDir: "public", File: "public/201811191", MigrationType: types.MigrationTypeSingleSchema} - dev2 := types.MigrationDefinition{"20181120", "tenants", "tenants/20181120", types.MigrationTypeTenantSchema} - dev2p := types.MigrationDefinition{"20181120", "public", "public/20181120", types.MigrationTypeSingleSchema} + dev2 := types.MigrationDefinition{Name: "20181120", SourceDir: "tenants", File: "tenants/20181120", MigrationType: types.MigrationTypeTenantSchema} + dev2p := types.MigrationDefinition{Name: "20181120", SourceDir: "public", File: "public/20181120", MigrationType: types.MigrationTypeSingleSchema} - diskMigrations := []types.Migration{{mdef1, ""}, {mdef2, ""}, {mdef3, ""}, {dev1, ""}, {dev1p1, ""}, {dev1p2, ""}, {dev2, ""}, {dev2p, ""}} - dbMigrations := []types.MigrationDB{{mdef1, "abc", time.Now()}, {mdef1, "def", time.Now()}, {mdef2, "public", time.Now()}, {mdef3, "public", time.Now()}, {dev2, "abc", time.Now()}, {dev2, "def", time.Now()}, {dev2p, "public", time.Now()}} + diskMigrations := []types.Migration{{MigrationDefinition: mdef1, Contents: ""}, {MigrationDefinition: mdef2, Contents: ""}, {MigrationDefinition: mdef3, Contents: ""}, {MigrationDefinition: dev1, Contents: ""}, {MigrationDefinition: dev1p1, Contents: ""}, {MigrationDefinition: dev1p2, Contents: ""}, {MigrationDefinition: dev2, Contents: ""}, {MigrationDefinition: dev2p, Contents: ""}} + dbMigrations := []types.MigrationDB{{MigrationDefinition: mdef1, Schema: "abc", Created: time.Now()}, {MigrationDefinition: mdef1, Schema: "def", Created: time.Now()}, {MigrationDefinition: mdef2, Schema: "public", Created: time.Now()}, {MigrationDefinition: mdef3, Schema: "public", Created: time.Now()}, {MigrationDefinition: dev2, Schema: "abc", Created: time.Now()}, {MigrationDefinition: dev2, Schema: "def", Created: time.Now()}, {MigrationDefinition: dev2p, Schema: "public", Created: time.Now()}} migrations := ComputeMigrationsToApply(diskMigrations, dbMigrations) assert.Len(t, migrations, 3) diff --git a/migrator.go b/migrator.go index 86583f5..337074e 100644 --- a/migrator.go +++ b/migrator.go @@ -16,18 +16,21 @@ func main() { log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) - validActions := []string{core.ApplyAction, core.AddTenantAction, core.PrintConfigAction, core.ListDiskMigrationsAction, core.ListDBTenantsAction, core.ListDBMigrationsAction} + validActions := []string{core.ApplyAction, core.AddTenantAction, core.PrintConfigAction, core.GetDiskMigrationsAction, core.GetDBTenantsAction, core.GetDBMigrationsAction} validModes := []string{core.ToolMode, core.ServerMode} flag := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) buf := new(bytes.Buffer) flag.SetOutput(buf) - configFile := flag.String("configFile", core.DefaultConfigFile, "path to migrator configuration yaml file") - mode := flag.String("mode", core.ToolMode, fmt.Sprintf("migrator mode to run: %q", validModes)) + var configFile string + var mode string + executeFlags := core.ExecuteFlags{} + flag.StringVar(&configFile, "configFile", core.DefaultConfigFile, "path to migrator configuration yaml file") + flag.StringVar(&mode, "mode", core.ToolMode, fmt.Sprintf("migrator mode to run: %q", validModes)) // below flags apply only when run in tool mode - action := flag.String("action", core.ApplyAction, fmt.Sprintf("when run in tool mode, action to execute, valid actions are: %q", validActions)) - tenant := flag.String("tenant", "", fmt.Sprintf("when run in tool mode and action set to %q, specifies new tenant name", core.AddTenantAction)) + flag.StringVar(&executeFlags.Action, "action", core.ApplyAction, fmt.Sprintf("when run in tool mode, action to execute, valid actions are: %q", validActions)) + flag.StringVar(&executeFlags.Tenant, "tenant", "", fmt.Sprintf("when run in tool mode and action set to %q, specifies new tenant name", core.AddTenantAction)) err := flag.Parse(os.Args[1:]) if err != nil { @@ -35,31 +38,31 @@ func main() { os.Exit(1) } - if !utils.Contains(validModes, mode) { - log.Printf("Invalid mode: %v", *mode) + if !utils.Contains(validModes, &mode) { + log.Printf("Invalid mode: %v", mode) flag.Usage() log.Fatal(buf) os.Exit(1) } - config, err := config.FromFile(*configFile) + config, err := config.FromFile(configFile) if err != nil { log.Fatalf("Error reading config file: %v", err) } - if *mode == "server" { + if mode == "server" { server.Start(config) } else { - if !utils.Contains(validActions, action) { - log.Printf("Invalid action: %v", *action) + if !utils.Contains(validActions, &executeFlags.Action) { + log.Printf("Invalid action: %v", executeFlags.Action) flag.Usage() log.Fatal(buf) os.Exit(1) } - core.ExecuteMigrator(config, *action, *tenant) + core.ExecuteMigrator(config, executeFlags) } } diff --git a/server/server.go b/server/server.go index decfbb8..dc9c7ed 100644 --- a/server/server.go +++ b/server/server.go @@ -52,7 +52,7 @@ func diskMigrationsHandler(w http.ResponseWriter, r *http.Request, config *confi http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) return } - diskMigrations := core.LoadDiskMigrations(config, createLoader) + diskMigrations := core.GetDiskMigrations(config, createLoader) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(diskMigrations) } @@ -61,7 +61,7 @@ func migrationsHandler(w http.ResponseWriter, r *http.Request, config *config.Co switch r.Method { case http.MethodGet: - dbMigrations := core.LoadDBMigrations(config, createConnector) + dbMigrations := core.GetDBMigrations(config, createConnector) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(dbMigrations) case http.MethodPost: @@ -78,7 +78,7 @@ func tenantsHandler(w http.ResponseWriter, r *http.Request, config *config.Confi switch r.Method { case http.MethodGet: - tenants := core.LoadDBTenants(config, createConnector) + tenants := core.GetDBTenants(config, createConnector) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(tenants) case http.MethodPost: diff --git a/server/server_mocks.go b/server/server_mocks.go index 993c649..f6e17ca 100644 --- a/server/server_mocks.go +++ b/server/server_mocks.go @@ -11,10 +11,10 @@ import ( type mockedDiskLoader struct { } -func (m *mockedDiskLoader) GetMigrations() []types.Migration { - m1 := types.MigrationDefinition{"201602220000.sql", "source", "source/201602220000.sql", types.MigrationTypeSingleSchema} - m2 := types.MigrationDefinition{"201602220001.sql", "source", "source/201602220001.sql", types.MigrationTypeSingleSchema} - return []types.Migration{{m1, "select abc"}, {m2, "select def"}} +func (m *mockedDiskLoader) GetDiskMigrations() []types.Migration { + m1 := types.MigrationDefinition{Name: "201602220000.sql", SourceDir: "source", File: "source/201602220000.sql", MigrationType: types.MigrationTypeSingleSchema} + m2 := types.MigrationDefinition{Name: "201602220001.sql", SourceDir: "source", File: "source/201602220001.sql", MigrationType: types.MigrationTypeSingleSchema} + return []types.Migration{{MigrationDefinition: m1, Contents: "select abc"}, {MigrationDefinition: m2, Contents: "select def"}} } func createMockedDiskLoader(config *config.Config) loader.Loader { @@ -34,15 +34,15 @@ func (m *mockedConnector) GetSchemaPlaceHolder() string { return "" } -func (m *mockedConnector) GetTenantSelectSql() string { +func (m *mockedConnector) GetTenantSelectSQL() string { return "" } -func (m *mockedConnector) GetMigrationInsertSql() string { +func (m *mockedConnector) GetMigrationInsertSQL() string { return "" } -func (m *mockedConnector) GetTenantInsertSql() string { +func (m *mockedConnector) GetTenantInsertSQL() string { return "" } @@ -53,10 +53,10 @@ func (m *mockedConnector) GetTenants() []string { return []string{"a", "b", "c"} } -func (m *mockedConnector) GetMigrations() []types.MigrationDB { - m1 := types.MigrationDefinition{"201602220000.sql", "source", "source/201602220000.sql", types.MigrationTypeSingleSchema} +func (m *mockedConnector) GetDBMigrations() []types.MigrationDB { + m1 := types.MigrationDefinition{Name: "201602220000.sql", SourceDir: "source", File: "source/201602220000.sql", MigrationType: types.MigrationTypeSingleSchema} d1 := time.Date(2016, 02, 22, 16, 41, 1, 123, time.UTC) - ms := []types.MigrationDB{{m1, "source", d1}} + ms := []types.MigrationDB{{MigrationDefinition: m1, Schema: "source", Created: d1}} return ms } diff --git a/test/migrator-overrides.yaml b/test/migrator-overrides.yaml index 429c71c..fd58d55 100644 --- a/test/migrator-overrides.yaml +++ b/test/migrator-overrides.yaml @@ -1,8 +1,8 @@ baseDir: test/migrations driver: postgres dataSource: "user=postgres dbname=A host=B port=C sslmode=disable" -tenantSelectSql: select somename from someschema.sometable -tenantInsertSql: insert into XXX +tenantSelectSQL: select somename from someschema.sometable +tenantInsertSQL: insert into someschema.sometable (somename) values ($1) schemaPlaceHolder: "[schema]" singleSchemas: - public diff --git a/test/migrator-test-envs.yaml b/test/migrator-test-envs.yaml index f3c5994..59df598 100644 --- a/test/migrator-test-envs.yaml +++ b/test/migrator-test-envs.yaml @@ -3,8 +3,8 @@ baseDir: $HOME driver: $PWD dataSource: $TERM # override only if you have own specific way of determining tenants -tenantSelectSql: $PATH -tenantInsertSql: $GOPATH +tenantSelectSQL: $PATH +tenantInsertSQL: $GOPATH schemaPlaceHolder: $USER port: $_ slackWebHook: $SHLVL diff --git a/test/migrator-test.yaml b/test/migrator-test.yaml index d2e3e7b..50c78fd 100644 --- a/test/migrator-test.yaml +++ b/test/migrator-test.yaml @@ -3,7 +3,7 @@ baseDir: test/migrations driver: postgres dataSource: "user=postgres dbname=migrator_test host=192.168.99.100 port=55432 sslmode=disable" # override only if you have own specific way of determining tenants -tenantSelectSql: "select name from public.migrator_tenants" +tenantSelectSQL: "select name from migrator.migrator_tenants" schemaPlaceHolder: "{schema}" port: 8811 singleSchemas: diff --git a/utils/utils.go b/utils/utils.go index a3a27c2..c57dbdd 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -30,7 +30,7 @@ func MigrationDBToString(m *types.MigrationDB) string { return fmt.Sprintf("| %-10s | %-20s | %-30s | %-10s | %-20s | %4d |", m.SourceDir, m.Name, m.File, m.Schema, created, m.MigrationType) } -// MigrationArrayString creates a string representation of Migration array +// MigrationArrayToString creates a string representation of Migration array func MigrationArrayToString(migrations []types.Migration) string { var buffer bytes.Buffer @@ -55,7 +55,7 @@ func MigrationArrayToString(migrations []types.Migration) string { return buffer.String() } -// MigrationDBArrayString creates a string representation of MigrationDB array +// MigrationDBArrayToString creates a string representation of MigrationDB array func MigrationDBArrayToString(migrations []types.MigrationDB) string { var buffer bytes.Buffer @@ -80,7 +80,7 @@ func MigrationDBArrayToString(migrations []types.MigrationDB) string { return buffer.String() } -// TenantArrayString creates a string representation of Tenant array +// TenantArrayToString creates a string representation of Tenant array func TenantArrayToString(dbTenants []string) string { var buffer bytes.Buffer diff --git a/utils/utils_test.go b/utils/utils_test.go index d159512..75f3c03 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -46,10 +46,10 @@ func TestTenantArrayToString(t *testing.T) { func TestMigrationArrayToString(t *testing.T) { - m1 := types.MigrationDefinition{"201602220000.sql", "source", "source/201602220000.sql", types.MigrationTypeSingleSchema} - m2 := types.MigrationDefinition{"201602220001.sql", "tenants", "tenants/201602220001.sql", types.MigrationTypeTenantSchema} - m3 := types.MigrationDefinition{"201602220002.sql", "tenants", "tenants/201602220002.sql", types.MigrationTypeTenantSchema} - var ms = []types.Migration{{m1, ""}, {m2, ""}, {m3, ""}} + m1 := types.MigrationDefinition{Name: "201602220000.sql", SourceDir: "source", File: "source/201602220000.sql", MigrationType: types.MigrationTypeSingleSchema} + m2 := types.MigrationDefinition{Name: "201602220001.sql", SourceDir: "tenants", File: "tenants/201602220001.sql", MigrationType: types.MigrationTypeTenantSchema} + m3 := types.MigrationDefinition{Name: "201602220002.sql", SourceDir: "tenants", File: "tenants/201602220002.sql", MigrationType: types.MigrationTypeTenantSchema} + var ms = []types.Migration{{MigrationDefinition: m1, Contents: ""}, {MigrationDefinition: m2, Contents: ""}, {MigrationDefinition: m3, Contents: ""}} expected := `+---------------------------------------------------------------------------+ | SourceDir | Name | File | Type | @@ -77,11 +77,11 @@ func TestMigrationArrayToStringEmpty(t *testing.T) { } func TestMigrationDBArrayToString(t *testing.T) { - m1 := types.MigrationDefinition{"201602220000.sql", "source", "source/201602220000.sql", types.MigrationTypeSingleSchema} - m2 := types.MigrationDefinition{"201602220001.sql", "tenants", "tenants/201602220001.sql", types.MigrationTypeTenantSchema} + m1 := types.MigrationDefinition{Name: "201602220000.sql", SourceDir: "source", File: "source/201602220000.sql", MigrationType: types.MigrationTypeSingleSchema} + m2 := types.MigrationDefinition{Name: "201602220001.sql", SourceDir: "tenants", File: "tenants/201602220001.sql", MigrationType: types.MigrationTypeTenantSchema} d1 := time.Date(2016, 02, 22, 16, 41, 1, 123, time.UTC) d2 := time.Date(2016, 02, 22, 16, 41, 2, 456, time.UTC) - var ms = []types.MigrationDB{{m1, "source", d1}, {m2, "abc", d2}, {m2, "def", d2}} + var ms = []types.MigrationDB{{MigrationDefinition: m1, Schema: "source", Created: d1}, {MigrationDefinition: m2, Schema: "abc", Created: d2}, {MigrationDefinition: m2, Schema: "def", Created: d2}} expected := `+---------------------------------------------------------------------------------------------------------------+ | SourceDir | Name | File | Schema | Created | Type | diff --git a/vet.sh b/vet.sh new file mode 100755 index 0000000..ef43253 --- /dev/null +++ b/vet.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +# when called with no arguments calls tests for all packages +if [[ -z "$1" ]]; then + packages=`go list -f "{{.Name}}" ./...` +else + packages="$1" +fi + +for package in $packages +do + if [[ "main" == "$package" ]]; then + continue + fi + go vet ./$package +done From 45cd329722ced87fc259d2cff9c08757ec2c188e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Wed, 5 Dec 2018 23:18:18 +0100 Subject: [PATCH 08/12] increasing coverage in server and core modules --- core/core_mocks.go | 16 ++++--- server/server.go | 8 ++-- server/server_test.go | 107 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 112 insertions(+), 19 deletions(-) diff --git a/core/core_mocks.go b/core/core_mocks.go index 803fff3..1d56b4d 100644 --- a/core/core_mocks.go +++ b/core/core_mocks.go @@ -5,14 +5,16 @@ import ( "github.com/lukaszbudnik/migrator/db" "github.com/lukaszbudnik/migrator/loader" "github.com/lukaszbudnik/migrator/types" + "time" ) type mockedDiskLoader struct { } func (m *mockedDiskLoader) GetDiskMigrations() []types.Migration { - // returns empty array - return []types.Migration{} + m1 := types.MigrationDefinition{Name: "201602220000.sql", SourceDir: "source", File: "source/201602220000.sql", MigrationType: types.MigrationTypeSingleSchema} + m2 := types.MigrationDefinition{Name: "201602220001.sql", SourceDir: "source", File: "source/201602220001.sql", MigrationType: types.MigrationTypeSingleSchema} + return []types.Migration{{MigrationDefinition: m1, Contents: "select abc"}, {MigrationDefinition: m2, Contents: "select def"}} } func createMockedDiskLoader(config *config.Config) loader.Loader { @@ -45,16 +47,18 @@ func (m *mockedConnector) GetTenantInsertSQL() string { } func (m *mockedConnector) GetTenants() []string { - // returns empty array - return []string{} + return []string{"a", "b", "c"} } func (m *mockedConnector) AddTenantAndApplyMigrations(string, []types.Migration) { } func (m *mockedConnector) GetDBMigrations() []types.MigrationDB { - // returns empty array - return []types.MigrationDB{} + m1 := types.MigrationDefinition{Name: "201602220000.sql", SourceDir: "source", File: "source/201602220000.sql", MigrationType: types.MigrationTypeSingleSchema} + d1 := time.Date(2016, 02, 22, 16, 41, 1, 123, time.UTC) + ms := []types.MigrationDB{{MigrationDefinition: m1, Schema: "source", Created: d1}} + + return ms } func (m *mockedConnector) ApplyMigrations(migrations []types.Migration) { diff --git a/server/server.go b/server/server.go index dc9c7ed..eae0a8e 100644 --- a/server/server.go +++ b/server/server.go @@ -21,7 +21,7 @@ type tenantParam struct { Name string `json:"name"` } -func getDefaultPort(config *config.Config) string { +func getPort(config *config.Config) string { if len(strings.TrimSpace(config.Port)) == 0 { return defaultPort } @@ -39,7 +39,7 @@ func makeHandler(handler func(w http.ResponseWriter, r *http.Request, config *co } func configHandler(w http.ResponseWriter, r *http.Request, config *config.Config, createConnector func(*config.Config) db.Connector, createLoader func(*config.Config) loader.Loader) { - if r.Method != "GET" { + if r.Method != http.MethodGet { http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) return } @@ -48,7 +48,7 @@ func configHandler(w http.ResponseWriter, r *http.Request, config *config.Config } func diskMigrationsHandler(w http.ResponseWriter, r *http.Request, config *config.Config, createConnector func(*config.Config) db.Connector, createLoader func(*config.Config) loader.Loader) { - if r.Method != "GET" { + if r.Method != http.MethodGet { http.Error(w, "405 method not allowed", http.StatusMethodNotAllowed) return } @@ -113,7 +113,7 @@ func registerHandlers(config *config.Config, createConnector func(*config.Config // and using connector created by a function passed as second argument and disk loader created by a function passed as third argument func Start(config *config.Config) { registerHandlers(config, db.CreateConnector, loader.CreateLoader) - port := getDefaultPort(config) + port := getPort(config) log.Printf("Migrator web server starting on port %s", port) http.ListenAndServe(":"+port, nil) } diff --git a/server/server_test.go b/server/server_test.go index effeda3..661dbd6 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -24,13 +24,13 @@ var ( func TestGetDefaultPort(t *testing.T) { config, err := config.FromFile(configFile) assert.Nil(t, err) - assert.Equal(t, "8080", getDefaultPort(config)) + assert.Equal(t, "8080", getPort(config)) } func TestGetDefaultPortOverrides(t *testing.T) { config, err := config.FromFile(configFileOverrides) assert.Nil(t, err) - assert.Equal(t, "8811", getDefaultPort(config)) + assert.Equal(t, "8811", getPort(config)) } func TestRegisterHandlers(t *testing.T) { @@ -40,7 +40,7 @@ func TestRegisterHandlers(t *testing.T) { } func TestServerDefaultHandler(t *testing.T) { - req, _ := http.NewRequest("GET", "http://example.com/qwq", nil) + req, _ := http.NewRequest(http.MethodGet, "http://example.com/qwq", nil) w := httptest.NewRecorder() defaultHandler(w, req) @@ -52,7 +52,7 @@ func TestServerConfig(t *testing.T) { config, err := config.FromFile(configFile) assert.Nil(t, err) - req, _ := http.NewRequest("GET", "http://example.com/", nil) + req, _ := http.NewRequest(http.MethodGet, "http://example.com/", nil) w := httptest.NewRecorder() handler := makeHandler(configHandler, config, nil, nil) @@ -66,7 +66,7 @@ func TestServerTenantsGet(t *testing.T) { config, err := config.FromFile(configFile) assert.Nil(t, err) - req, _ := http.NewRequest("GET", "http://example.com/tenants", nil) + req, _ := http.NewRequest(http.MethodGet, "http://example.com/tenants", nil) w := httptest.NewRecorder() handler := makeHandler(tenantsHandler, config, createMockedConnector, nil) @@ -82,7 +82,7 @@ func TestServerTenantsPost(t *testing.T) { assert.Nil(t, err) json := []byte(`{"name": "new_tenant"}`) - req, _ := http.NewRequest("POST", "http://example.com/tenants", bytes.NewBuffer(json)) + req, _ := http.NewRequest(http.MethodPost, "http://example.com/tenants", bytes.NewBuffer(json)) req.Header.Set("Content-Type", "application/json") w := httptest.NewRecorder() @@ -94,11 +94,26 @@ func TestServerTenantsPost(t *testing.T) { assert.Equal(t, `[{"Name":"201602220000.sql","SourceDir":"source","File":"source/201602220000.sql","MigrationType":1,"Contents":"select abc"},{"Name":"201602220001.sql","SourceDir":"source","File":"source/201602220001.sql","MigrationType":1,"Contents":"select def"}]`, strings.TrimSpace(w.Body.String())) } +func TestServerTenantsPostBadRequest(t *testing.T) { + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + // empty JSON payload + json := []byte("") + req, _ := http.NewRequest(http.MethodPost, "http://example.com/tenants", bytes.NewBuffer(json)) + + w := httptest.NewRecorder() + handler := makeHandler(tenantsHandler, config, createMockedConnector, createMockedDiskLoader) + handler(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) +} + func TestServerDiskMigrationsGet(t *testing.T) { config, err := config.FromFile(configFile) assert.Nil(t, err) - req, _ := http.NewRequest("GET", "http://example.com/diskMigrations", nil) + req, _ := http.NewRequest(http.MethodGet, "http://example.com/diskMigrations", nil) w := httptest.NewRecorder() handler := makeHandler(diskMigrationsHandler, config, nil, createMockedDiskLoader) @@ -113,7 +128,7 @@ func TestServerMigrationsGet(t *testing.T) { config, err := config.FromFile(configFile) assert.Nil(t, err) - req, _ := http.NewRequest("GET", "http://example.com/migrations", nil) + req, _ := http.NewRequest(http.MethodGet, "http://example.com/migrations", nil) w := httptest.NewRecorder() handler := makeHandler(migrationsHandler, config, createMockedConnector, nil) @@ -128,7 +143,7 @@ func TestServerMigrationsPost(t *testing.T) { config, err := config.FromFile(configFile) assert.Nil(t, err) - req, _ := http.NewRequest("POST", "http://example.com/migrations", nil) + req, _ := http.NewRequest(http.MethodPost, "http://example.com/migrations", nil) w := httptest.NewRecorder() handler := makeHandler(migrationsHandler, config, createMockedConnector, createMockedDiskLoader) @@ -138,3 +153,77 @@ func TestServerMigrationsPost(t *testing.T) { assert.Equal(t, "application/json", w.HeaderMap["Content-Type"][0]) assert.Equal(t, `[{"Name":"201602220001.sql","SourceDir":"source","File":"source/201602220001.sql","MigrationType":1,"Contents":"select def"}]`, strings.TrimSpace(w.Body.String())) } + +func TestServerMigrationsMethodNotAllowed(t *testing.T) { + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + httpMethods := []string{http.MethodHead, http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodConnect, http.MethodOptions, http.MethodTrace} + + for _, httpMethod := range httpMethods { + req, _ := http.NewRequest(httpMethod, "http://example.com/migrations", nil) + + w := httptest.NewRecorder() + handler := makeHandler(migrationsHandler, config, createMockedConnector, createMockedDiskLoader) + handler(w, req) + + assert.Equal(t, http.StatusMethodNotAllowed, w.Code) + } + +} + +func TestServerTenantMethodNotAllowed(t *testing.T) { + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + httpMethods := []string{http.MethodHead, http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodConnect, http.MethodOptions, http.MethodTrace} + + for _, httpMethod := range httpMethods { + req, _ := http.NewRequest(httpMethod, "http://example.com/tenants", nil) + + w := httptest.NewRecorder() + handler := makeHandler(tenantsHandler, config, createMockedConnector, createMockedDiskLoader) + handler(w, req) + + assert.Equal(t, http.StatusMethodNotAllowed, w.Code) + } + +} + +func TestServerDiskMigrationsMethodNotAllowed(t *testing.T) { + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + httpMethods := []string{http.MethodHead, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodConnect, http.MethodOptions, http.MethodTrace} + + for _, httpMethod := range httpMethods { + + req, _ := http.NewRequest(httpMethod, "http://example.com/diskMigrations", nil) + + w := httptest.NewRecorder() + handler := makeHandler(diskMigrationsHandler, config, createMockedConnector, createMockedDiskLoader) + handler(w, req) + + assert.Equal(t, http.StatusMethodNotAllowed, w.Code) + } + +} + +func TestServerConfigMethodNotAllowed(t *testing.T) { + config, err := config.FromFile(configFile) + assert.Nil(t, err) + + httpMethods := []string{http.MethodHead, http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodConnect, http.MethodOptions, http.MethodTrace} + + for _, httpMethod := range httpMethods { + + req, _ := http.NewRequest(httpMethod, "http://example.com/", nil) + + w := httptest.NewRecorder() + handler := makeHandler(configHandler, config, createMockedConnector, createMockedDiskLoader) + handler(w, req) + + assert.Equal(t, http.StatusMethodNotAllowed, w.Code) + } + +} From 78899cce9f93691273f35f60da080b4a5c3c2f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Wed, 5 Dec 2018 23:42:11 +0100 Subject: [PATCH 09/12] adding integration to coverall service --- .travis.yml | 5 ++++- README.md | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0daffc9..b507023 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,9 @@ go: - "1.11" - "tip" +before_install: + - go get github.com/mattn/goveralls + before_script: - sh -c "if [ '$DB' = 'postgresql' ]; then psql -U postgres -c 'create database migrator_test'; fi" - sh -c "if [ '$DB' = 'postgresql' ]; then psql -U postgres -d migrator_test -f test/create-test-tenants.sql; fi" @@ -23,4 +26,4 @@ before_script: - sh -c "cp test/migrator-$DB.yaml.travis test/migrator.yaml" script: - - ./coverage.sh + - $GOPATH/bin/goveralls -service=travis-ci diff --git a/README.md b/README.md index 67bfbd1..439851a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Migrator [![Build Status](https://travis-ci.org/lukaszbudnik/migrator.svg?branch=master)](https://travis-ci.org/lukaszbudnik/migrator) +# Migrator [![Build Status](https://travis-ci.org/lukaszbudnik/migrator.svg?branch=master)](https://travis-ci.org/lukaszbudnik/migrator) [![Coverage Status](https://coveralls.io/repos/github/lukaszbudnik/migrator/badge.svg?branch=master)](https://coveralls.io/github/lukaszbudnik/migrator?branch=master) Super fast and lightweight DB migration & evolution tool written in go. From 2ff5b86d44e4948a7d0690659381316dfdda5f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Thu, 6 Dec 2018 00:03:39 +0100 Subject: [PATCH 10/12] checking coverall integration and failing postgres tests --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index b507023..1bb56bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,4 +26,5 @@ before_script: - sh -c "cp test/migrator-$DB.yaml.travis test/migrator.yaml" script: + - ./coverage.sh - $GOPATH/bin/goveralls -service=travis-ci From b5a772597f0b0dc91b9a2aab3c2696aa2b94f5aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Thu, 6 Dec 2018 01:14:54 +0100 Subject: [PATCH 11/12] solving postgres 9.3 issues on Travis-CI --- .travis.yml | 4 +++- README.md | 6 +++--- db/db_test.go | 3 ++- docker/scripts/mssql-create-and-setup-container.sh | 2 +- docker/scripts/mysql-create-and-setup-container.sh | 2 +- test/migrator-mysql.yaml.travis | 2 +- test/migrator-postgresql.yaml | 2 +- test/migrator-postgresql.yaml.travis | 2 +- 8 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1bb56bd..43b398e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ language: go +addons: + postgresql: "9.3" + env: - DB=postgresql - DB=mysql @@ -26,5 +29,4 @@ before_script: - sh -c "cp test/migrator-$DB.yaml.travis test/migrator.yaml" script: - - ./coverage.sh - $GOPATH/bin/goveralls -service=travis-ci diff --git a/README.md b/README.md index 439851a..fa39e26 100644 --- a/README.md +++ b/README.md @@ -95,19 +95,19 @@ Port is configurable in `migrator.yaml` and defaults to 8080. Should you need HT Currently migrator supports the following databases and their flavours: -* PostgreSQL - schema-based multi-tenant database, with transactions spanning DDL statements, driver used: https://github.com/lib/pq +* PostgreSQL 9.3+ - schema-based multi-tenant database, with transactions spanning DDL statements, driver used: https://github.com/lib/pq * PostgreSQL - original PostgreSQL server * Amazon RDS PostgreSQL - PostgreSQL-compatible relational database built for the cloud * Amazon Aurora PostgreSQL - PostgreSQL-compatible relational database built for the cloud * Google CloudSQL PostgreSQL - PostgreSQL-compatible relational database built for the cloud -* MySQL - database-based multi-tenant database, transactions do not span DDL statements, driver used: https://github.com/go-sql-driver/mysql +* MySQL 5.6+ - database-based multi-tenant database, transactions do not span DDL statements, driver used: https://github.com/go-sql-driver/mysql * MySQL - original MySQL server * MariaDB - enhanced near linearly scalable multi-master MySQL * Percona - an enhanced drop-in replacement for MySQL * Amazon RDS MySQL - MySQL-compatible relational database built for the cloud * Amazon Aurora MySQL - MySQL-compatible relational database built for the cloud * Google CloudSQL MySQL - MySQL-compatible relational database built for the cloud -* Microsoft SQL Server - a relational database management system developed by Microsoft, driver used: https://github.com/denisenkom/go-mssqldb +* Microsoft SQL Server 2017 - a relational database management system developed by Microsoft, driver used: https://github.com/denisenkom/go-mssqldb * Microsoft SQL Server - original Microsoft SQL Server # Do you speak docker? diff --git a/db/db_test.go b/db/db_test.go index 562e943..9f7dc5a 100644 --- a/db/db_test.go +++ b/db/db_test.go @@ -9,6 +9,7 @@ import ( "github.com/lukaszbudnik/migrator/config" "github.com/lukaszbudnik/migrator/types" "github.com/stretchr/testify/assert" + "strings" "testing" "time" ) @@ -47,7 +48,7 @@ func TestDBConnectorInitPanicConnectionError(t *testing.T) { config, err := config.FromFile("../test/migrator.yaml") assert.Nil(t, err) - config.DataSource = "" + config.DataSource = strings.Replace(config.DataSource, "127.0.0.1", "1.0.0.1", -1) connector := CreateConnector(config) assert.Panics(t, func() { diff --git a/docker/scripts/mssql-create-and-setup-container.sh b/docker/scripts/mssql-create-and-setup-container.sh index 9166194..6c0c40f 100644 --- a/docker/scripts/mssql-create-and-setup-container.sh +++ b/docker/scripts/mssql-create-and-setup-container.sh @@ -20,7 +20,7 @@ function mssql_start() { sqlcmd -S "$ip,$port" -U SA -P $password -Q "CREATE DATABASE $database" sqlcmd -S "$ip,$port" -U SA -P $password -d $database -i ../test/create-test-tenants-mssql.sql - cat ../test/migrator-mssql.yaml | sed "s/A/sqlserver:\/\/SA:$password@$ip:$port\/?database=$database/g" > ../test/migrator.yaml + cat ../test/migrator-mssql.yaml | sed "s/A/sqlserver:\/\/SA:$password@$ip:$port\/?database=$database&connection+timeout=1/g" > ../test/migrator.yaml else >&2 echo "Could not setup mssql-$name" exit 1 diff --git a/docker/scripts/mysql-create-and-setup-container.sh b/docker/scripts/mysql-create-and-setup-container.sh index fa67304..eca48a4 100644 --- a/docker/scripts/mysql-create-and-setup-container.sh +++ b/docker/scripts/mysql-create-and-setup-container.sh @@ -22,7 +22,7 @@ function mysql_start() { if [[ "true" == "$running" ]]; then mysql -u root -h $ip -P $port -e "create database $database" mysql -u root -h $ip -P $port -D $database < ../test/create-test-tenants.sql - cat ../test/migrator-mysql.yaml | sed "s/A/root:@tcp($ip:$port)\/$database?parseTime=true/g" > ../test/migrator.yaml + cat ../test/migrator-mysql.yaml | sed "s/A/root:@tcp($ip:$port)\/$database?parseTime=true&timeout=1s/g" > ../test/migrator.yaml else >&2 echo "Could not setup mysql-$name" exit 1 diff --git a/test/migrator-mysql.yaml.travis b/test/migrator-mysql.yaml.travis index 9179e91..a7c570c 100644 --- a/test/migrator-mysql.yaml.travis +++ b/test/migrator-mysql.yaml.travis @@ -1,6 +1,6 @@ baseDir: test/migrations driver: mysql -dataSource: "root:@tcp(127.0.0.1:3306)/migrator_test?parseTime=true" +dataSource: "root:@tcp(127.0.0.1:3306)/migrator_test?parseTime=true&timeout=1s" singleSchemas: - public - ref diff --git a/test/migrator-postgresql.yaml b/test/migrator-postgresql.yaml index e8e0181..8f3cfea 100644 --- a/test/migrator-postgresql.yaml +++ b/test/migrator-postgresql.yaml @@ -1,6 +1,6 @@ baseDir: test/migrations driver: postgres -dataSource: "user=postgres dbname=A host=B port=C sslmode=disable" +dataSource: "user=postgres dbname=A host=B port=C sslmode=disable connect_timeout=1" singleSchemas: - public - ref diff --git a/test/migrator-postgresql.yaml.travis b/test/migrator-postgresql.yaml.travis index 5f7257c..dd53f60 100644 --- a/test/migrator-postgresql.yaml.travis +++ b/test/migrator-postgresql.yaml.travis @@ -1,7 +1,7 @@ # migrator configuration baseDir: test/migrations driver: postgres -dataSource: "user=postgres dbname=migrator_test host=127.0.0.1 port=5432 sslmode=disable" +dataSource: "user=postgres dbname=migrator_test host=127.0.0.1 port=5432 sslmode=disable connect_timeout=1" singleSchemas: - public - ref From 83c273a176f8c69ade316670b7c8b47ce7a7c601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Budnik?= Date: Thu, 6 Dec 2018 19:07:33 +0100 Subject: [PATCH 12/12] merged migrator-docker into migration main project, reworked existing test docker scripts, added better guidance for contributing, new scripts fmt-lint-vet.sh and ultimate-coverage.sh which loops through 5 docker containers and executes all scripts --- DOCKER.md | 47 +++++++++++++++++++ Dockerfile | 18 +++++++ README.md | 41 +++++++--------- coverage.sh | 4 +- docker-entrypoint.sh | 16 +++++++ .../mssql-create-and-setup-container.sh | 28 ----------- .../mysql-create-and-setup-container.sh | 31 ------------ .../postgresql-create-and-setup-container.sh | 29 ------------ fmt-lint-vet.sh | 14 ++++++ .../docker}/create-and-setup-container.sh | 4 +- {docker => test/docker}/destroy-container.sh | 4 +- .../docker}/scripts/destroy-container.sh | 0 .../mssql-create-and-setup-container.sh | 32 +++++++++++++ .../mysql-create-and-setup-container.sh | 32 +++++++++++++ .../postgresql-create-and-setup-container.sh | 31 ++++++++++++ ultimate-coverage.sh | 15 ++++++ vet.sh | 16 ------- 17 files changed, 232 insertions(+), 130 deletions(-) create mode 100644 DOCKER.md create mode 100644 Dockerfile create mode 100755 docker-entrypoint.sh delete mode 100644 docker/scripts/mssql-create-and-setup-container.sh delete mode 100644 docker/scripts/mysql-create-and-setup-container.sh delete mode 100644 docker/scripts/postgresql-create-and-setup-container.sh create mode 100755 fmt-lint-vet.sh rename {docker => test/docker}/create-and-setup-container.sh (90%) rename {docker => test/docker}/destroy-container.sh (78%) rename {docker => test/docker}/scripts/destroy-container.sh (100%) create mode 100644 test/docker/scripts/mssql-create-and-setup-container.sh create mode 100644 test/docker/scripts/mysql-create-and-setup-container.sh create mode 100644 test/docker/scripts/postgresql-create-and-setup-container.sh create mode 100755 ultimate-coverage.sh delete mode 100755 vet.sh diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 0000000..1dfc989 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,47 @@ +# migrator docker + +To run migrator docker you need to: + +* pull `lukasz/migrator` from docker cloud +* mount a volume with migrations under `/data` +* (optional) specify location of migrator configuration file via environmental variable `MIGRATOR_YAML`, defaults to `/data/migrator.yaml` + +To run migrator as a service: + +```bash +docker run -p 8080:8080 -v /Users/lukasz/migrator-test:/data -e MIGRATOR_YAML=/data/m.yaml -d --link migrator-postgres lukasz/migrator +Starting migrator using config file: /data/m.yaml +2016/08/04 06:24:58 Read config file ==> OK +2016/08/04 06:24:58 Migrator web server starting on port 8080... +``` + +To run migrator in interactive terminal mode: + +```bash +docker run -it -v /Users/lukasz/migrator-test:/data --entrypoint sh --link migrator-postgres lukasz/migrator +``` + +# History and releases + +Here is a short history of migrator docker images: + +1. initial release in 2016 - migrator on debian:jessie - 603MB +2. v1.0 - migrator v1.0 on golang:1.11.2-alpine3.8 - 346MB +3. v1.0-mini - migrator v1.0 multi-stage build with final image on alpine:3.8 - 13.4MB +4. v2.0 - migrator v2.0 - 14.8MB + +Starting with v2.0 all migrator images by default use multi-stage builds. For migrator v1.0 you have to explicitly use `v1.0-mini` tag in order to enjoy an ultra lightweight migrator image. Still, I recommend using latest and greatest. + +Finally, starting with v2.0 migrator-docker project was merged into migrator main project. New version of docker image is built automatically every time a new release is created. + +To view all available docker containers see [lukasz/migrator/tags](https://cloud.docker.com/repository/docker/lukasz/migrator/tags). + +# License + +Copyright 2016-2018 Łukasz Budnik + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f40a6c0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.11.2-alpine3.8 as builder + +MAINTAINER Łukasz Budnik lukasz.budnik@gmail.com + +# install migrator +RUN apk add git +RUN go get github.com/lukaszbudnik/migrator + +FROM alpine:3.8 +COPY --from=builder /go/bin/migrator /bin + +VOLUME ["/data"] + +# copy and register entrypoint script +COPY docker-entrypoint.sh / +ENTRYPOINT ["/docker-entrypoint.sh"] + +EXPOSE 8080 diff --git a/README.md b/README.md index fa39e26..07fc2c8 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ When using migrator please remember about these: * migrator creates `migrator` schema (where `migrator_migrations` and `migrator_tenants` tables reside) automatically * when adding a new tenant migrator creates a new schema automatically -* single schemas are not created automatically, for this you must add initial migration with `create schema` SQL command (see example above) +* single schemas are not created automatically, for this you must add initial migration with `create schema` SQL statement (see example above) # Server mode @@ -114,21 +114,9 @@ Currently migrator supports the following databases and their flavours: Yes, there is an official docker image available on docker hub. -migrator docker image is ultra lightweight and has a size of only 13.4MB. Ideal for micro-services deployments! +migrator docker image is ultra lightweight and has a size of approx. 15MB. Ideal for micro-services deployments! -To find out more about migrator docker please visit: https://github.com/lukaszbudnik/migrator-docker. - -# Running unit & integration tests - -PostgreSQL, MySQL, MariaDB, Percona, and MSSQL: - -``` -$ docker/create-and-setup-container.sh [postgres|mysql|mariadb|percona|mssql] -$ ./coverage.sh -$ docker/destroy-container.sh [postgres|mysql|mariadb|percona|mssql] -``` - -Or see `.travis.yml` to see how it's done on Travis. Note: MSSQL is not supported on Travis. +To find out more about migrator docker container see [DOCKER.md](DOCKER.md) for more details. # Customisation @@ -173,22 +161,29 @@ The other thing to consider is the fact that migrator is written in go which is To install migrator use: -`go get github.com/lukaszbudnik/migrator` +``` +go get github.com/lukaszbudnik/migrator +cd migrator +./setup.sh +``` Migrator supports the following Go versions: 1.8, 1.9, 1.10, 1.11, and tip (all built on Travis). -# Code Style +# Contributing, code style, running unit & integration tests + +Contributions are most welcomed. -If you would like to send me a pull request please always add unit/integration tests. Code should be formatted, checked, and tested using the following commands: +If you would like to help me and implement a new feature, enhance existing one, or spotted and fixed bug please send me a pull request. + +Code should be formatted, checked, and tested using the following commands: ``` -go clean -testcache -gofmt -s -w . -golint ./... -./vet.sh -./coverage.sh +./fmt-lint-vet.sh +./ultimate-coverage.sh ``` +The `ultimate-coverage.sh` script loops through 5 different containers (3 MySQL flavours, PostgreSQL, and MSSQL) creates db docker container, executes `coverage.sh` script, and finally tears down given db docker container. + # License Copyright 2016-2018 Łukasz Budnik diff --git a/coverage.sh b/coverage.sh index 4fb75ef..e77baf3 100755 --- a/coverage.sh +++ b/coverage.sh @@ -2,13 +2,15 @@ # when called with no arguments calls tests for all packages if [[ -z "$1" ]]; then - packages=`go list -f "{{.Name}}" ./...` + packages=$(go list -f "{{.Name}}" ./...) else packages="$1" fi echo "mode: set" > coverage-all.txt +go clean -testcache + for package in $packages do if [[ "main" == "$package" ]]; then diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..d92247f --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +DEFAULT_YAML_LOCATION="/data/migrator.yaml" + +# if migrator config file is not provided explicitly fallback to default location +if [ -z "$MIGRATOR_YAML" ]; then + MIGRATOR_YAML=$DEFAULT_YAML_LOCATION +fi + +if [ ! -f "$MIGRATOR_YAML" ] ; then + echo "Migrator config file not found. Please use either default location ($DEFAULT_YAML_LOCATION) or provide a custom one using \$MIGRATOR_YAML." + exit 1 +fi + +echo "Starting migrator using config file: $MIGRATOR_YAML" +migrator -configFile "$MIGRATOR_YAML" -mode server diff --git a/docker/scripts/mssql-create-and-setup-container.sh b/docker/scripts/mssql-create-and-setup-container.sh deleted file mode 100644 index 6c0c40f..0000000 --- a/docker/scripts/mssql-create-and-setup-container.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -function mssql_start() { - name=mssql - - ip=127.0.0.1 - port=11433 - database=migratortest - password=YourStrongPassw0rd - - docker run -e 'ACCEPT_EULA=Y' -e "SA_PASSWORD=$password" \ - -p $port:1433 --name "migrator-$name" \ - -d mcr.microsoft.com/mssql/server:2017-latest - - sleep 10 - - running=$(docker inspect -f {{.State.Running}} "migrator-$name") - - if [[ "true" == "$running" ]]; then - sqlcmd -S "$ip,$port" -U SA -P $password -Q "CREATE DATABASE $database" - sqlcmd -S "$ip,$port" -U SA -P $password -d $database -i ../test/create-test-tenants-mssql.sql - - cat ../test/migrator-mssql.yaml | sed "s/A/sqlserver:\/\/SA:$password@$ip:$port\/?database=$database&connection+timeout=1/g" > ../test/migrator.yaml - else - >&2 echo "Could not setup mssql-$name" - exit 1 - fi -} diff --git a/docker/scripts/mysql-create-and-setup-container.sh b/docker/scripts/mysql-create-and-setup-container.sh deleted file mode 100644 index eca48a4..0000000 --- a/docker/scripts/mysql-create-and-setup-container.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -function mysql_start() { - flavour=$1 - name=${flavour//\//-} - - ip=127.0.0.1 - port=33306 - database=migrator_test - - docker run -d \ - --name "migrator-$flavour" \ - -e MYSQL_ALLOW_EMPTY_PASSWORD=yes \ - -p $port:3306 \ - "$flavour" - - # mysql needs a little bit more time - sleep 20 - - running=$(docker inspect -f {{.State.Running}} "migrator-$name") - - if [[ "true" == "$running" ]]; then - mysql -u root -h $ip -P $port -e "create database $database" - mysql -u root -h $ip -P $port -D $database < ../test/create-test-tenants.sql - cat ../test/migrator-mysql.yaml | sed "s/A/root:@tcp($ip:$port)\/$database?parseTime=true&timeout=1s/g" > ../test/migrator.yaml - else - >&2 echo "Could not setup mysql-$name" - exit 1 - fi - -} diff --git a/docker/scripts/postgresql-create-and-setup-container.sh b/docker/scripts/postgresql-create-and-setup-container.sh deleted file mode 100644 index 622dfb4..0000000 --- a/docker/scripts/postgresql-create-and-setup-container.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -function postgresql_start() { - flavour=$1 - name=${flavour//\//-} - - ip=127.0.0.1 - port=55432 - database=migrator_test - - docker run -d \ - --name "migrator-$name" \ - -p $port:5432 \ - "$flavour" \ - - sleep 10 - - running=$(docker inspect -f {{.State.Running}} "migrator-$name") - - if [[ "true" == "$running" ]]; then - psql -U postgres -h $ip -p $port -c "create database $database" - psql -U postgres -h $ip -p $port -d $database -f ../test/create-test-tenants.sql - - cat ../test/migrator-postgresql.yaml | sed "s/dbname=[^ ]* host=[^ ]* port=[^ ]*/dbname=$database host=$ip port=$port/g" > ../test/migrator.yaml - else - >&2 echo "Could not setup postgresql-$name" - exit 1 - fi -} diff --git a/fmt-lint-vet.sh b/fmt-lint-vet.sh new file mode 100755 index 0000000..a20508d --- /dev/null +++ b/fmt-lint-vet.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +gofmt -s -w . +golint ./... + +packages=`go list -f "{{.Name}}" ./...` + +for package in $packages +do + if [[ "main" == "$package" ]]; then + continue + fi + go vet ./$package +done diff --git a/docker/create-and-setup-container.sh b/test/docker/create-and-setup-container.sh similarity index 90% rename from docker/create-and-setup-container.sh rename to test/docker/create-and-setup-container.sh index 20a1756..69ad9f4 100755 --- a/docker/create-and-setup-container.sh +++ b/test/docker/create-and-setup-container.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -set -x +if [[ ! -z "$DEBUG" ]]; then + set -x +fi cd `dirname $0` diff --git a/docker/destroy-container.sh b/test/docker/destroy-container.sh similarity index 78% rename from docker/destroy-container.sh rename to test/docker/destroy-container.sh index 74ccf6c..a34fc37 100755 --- a/docker/destroy-container.sh +++ b/test/docker/destroy-container.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -set -x +if [[ ! -z "$DEBUG" ]]; then + set -x +fi cd `dirname $0` diff --git a/docker/scripts/destroy-container.sh b/test/docker/scripts/destroy-container.sh similarity index 100% rename from docker/scripts/destroy-container.sh rename to test/docker/scripts/destroy-container.sh diff --git a/test/docker/scripts/mssql-create-and-setup-container.sh b/test/docker/scripts/mssql-create-and-setup-container.sh new file mode 100644 index 0000000..a09abca --- /dev/null +++ b/test/docker/scripts/mssql-create-and-setup-container.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +function mssql_start() { + name=mssql + + ip=127.0.0.1 + port=1433 + database=migratortest + password=YourStrongPassw0rd + + docker run -d \ + --name "migrator-$name" \ + -e 'ACCEPT_EULA=Y' -e "SA_PASSWORD=$password" \ + -P \ + mcr.microsoft.com/mssql/server:2017-latest + + sleep 15 + + running=$(docker inspect -f {{.State.Running}} "migrator-$name") + + if [[ "true" == "$running" ]]; then + docker_port=$(docker port "migrator-$name" | grep "^$port/tcp" | awk -F ':' '{print $2}') + + sqlcmd -S "$ip,$docker_port" -U SA -P $password -Q "CREATE DATABASE $database" + sqlcmd -S "$ip,$docker_port" -U SA -P $password -d $database -i ../create-test-tenants-mssql.sql + + cat ../migrator-mssql.yaml | sed "s/A/sqlserver:\/\/SA:$password@$ip:$docker_port\/?database=$database\&connection+timeout=1\&dial+timeout=1/g" > ../migrator.yaml + else + >&2 echo "Could not setup mssql-$name" + exit 1 + fi +} diff --git a/test/docker/scripts/mysql-create-and-setup-container.sh b/test/docker/scripts/mysql-create-and-setup-container.sh new file mode 100644 index 0000000..0048f6c --- /dev/null +++ b/test/docker/scripts/mysql-create-and-setup-container.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +function mysql_start() { + flavour=$1 + name=${flavour//\//-} + + ip=127.0.0.1 + port=3306 + database=migrator_test + + docker run -d \ + --name "migrator-$flavour" \ + -e MYSQL_ALLOW_EMPTY_PASSWORD=yes \ + -P \ + "$flavour" + + sleep 15 + + running=$(docker inspect -f {{.State.Running}} "migrator-$name") + + if [[ "true" == "$running" ]]; then + docker_port=$(docker port "migrator-$name" | grep "^$port/tcp" | awk -F ':' '{print $2}') + + mysql -u root -h $ip -P $docker_port -e "create database $database" + mysql -u root -h $ip -P $docker_port -D $database < ../create-test-tenants.sql + cat ../migrator-mysql.yaml | sed "s/A/root:@tcp($ip:$docker_port)\/$database?parseTime=true\&timeout=1s/g" > ../migrator.yaml + else + >&2 echo "Could not setup mysql-$name" + exit 1 + fi + +} diff --git a/test/docker/scripts/postgresql-create-and-setup-container.sh b/test/docker/scripts/postgresql-create-and-setup-container.sh new file mode 100644 index 0000000..40a3570 --- /dev/null +++ b/test/docker/scripts/postgresql-create-and-setup-container.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +function postgresql_start() { + flavour=$1 + name=${flavour//\//-} + + ip=127.0.0.1 + port=5432 + database=migrator_test + + docker run -d \ + --name "migrator-$flavour" \ + -P \ + "$flavour" + + sleep 10 + + running=$(docker inspect -f {{.State.Running}} "migrator-$name") + + if [[ "true" == "$running" ]]; then + docker_port=$(docker port "migrator-$name" | grep "^$port/tcp" | awk -F ':' '{print $2}') + + psql -U postgres -h $ip -p $docker_port -c "create database $database" + psql -U postgres -h $ip -p $docker_port -d $database -f ../create-test-tenants.sql + + cat ../migrator-postgresql.yaml | sed "s/dbname=[^ ]* host=[^ ]* port=[^ ]*/dbname=$database host=$ip port=$docker_port/g" > ../migrator.yaml + else + >&2 echo "Could not setup postgresql-$name" + exit 1 + fi +} diff --git a/ultimate-coverage.sh b/ultimate-coverage.sh new file mode 100755 index 0000000..02a0e04 --- /dev/null +++ b/ultimate-coverage.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +dbs="postgres mysql mariadb percona mssql" + +for db in $dbs +do + echo "1. destroy $db docker (just in case it's already created)" + ./test/docker/destroy-container.sh $db + echo "2. create and setup $db docker" + ./test/docker/create-and-setup-container.sh $db + echo "3. run all tests" + ./coverage.sh + echo "4. destroy $db docker (cleanup)" + ./test/docker/destroy-container.sh $db +done diff --git a/vet.sh b/vet.sh deleted file mode 100755 index ef43253..0000000 --- a/vet.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# when called with no arguments calls tests for all packages -if [[ -z "$1" ]]; then - packages=`go list -f "{{.Name}}" ./...` -else - packages="$1" -fi - -for package in $packages -do - if [[ "main" == "$package" ]]; then - continue - fi - go vet ./$package -done