Merge pull request #44 from lukaszbudnik/migrator-v2

Migrator v2

.travis.yml

@@ -1,5 +1,8 @@
 language: go
 
+addons:
+  postgresql: "9.3"
+
 env:
   - DB=postgresql
   - DB=mysql
@@ -15,6 +18,9 @@ go:
   - "1.11"
   - "tip"
 
+before_install:
+  - go get github.com/mattn/goveralls
+
 before_script:
   - sh -c "if [ '$DB' = 'postgresql' ]; then psql -U postgres -c 'create database migrator_test'; fi"
   - sh -c "if [ '$DB' = 'postgresql' ]; then psql -U postgres -d migrator_test -f test/create-test-tenants.sql; fi"
@@ -23,4 +29,4 @@ before_script:
   - sh -c "cp test/migrator-$DB.yaml.travis test/migrator.yaml"
 
 script:
-  - ./coverage.sh
+  - $GOPATH/bin/goveralls -service=travis-ci

DOCKER.md

@@ -0,0 +1,47 @@
+# migrator docker
+
+To run migrator docker you need to:
+
+* pull `lukasz/migrator` from docker cloud
+* mount a volume with migrations under `/data`
+* (optional) specify location of migrator configuration file via environmental variable `MIGRATOR_YAML`, defaults to `/data/migrator.yaml`
+
+To run migrator as a service:
+
+```bash
+docker run -p 8080:8080 -v /Users/lukasz/migrator-test:/data -e MIGRATOR_YAML=/data/m.yaml -d --link migrator-postgres lukasz/migrator
+Starting migrator using config file: /data/m.yaml
+2016/08/04 06:24:58 Read config file ==> OK
+2016/08/04 06:24:58 Migrator web server starting on port 8080...
+```
+
+To run migrator in interactive terminal mode:
+
+```bash
+docker run -it -v /Users/lukasz/migrator-test:/data --entrypoint sh --link migrator-postgres lukasz/migrator
+```
+
+# History and releases
+
+Here is a short history of migrator docker images:
+
+1. initial release in 2016 - migrator on debian:jessie - 603MB
+2. v1.0 - migrator v1.0 on golang:1.11.2-alpine3.8 - 346MB
+3. v1.0-mini - migrator v1.0 multi-stage build with final image on alpine:3.8 - 13.4MB
+4. v2.0 - migrator v2.0 - 14.8MB
+
+Starting with v2.0 all migrator images by default use multi-stage builds. For migrator v1.0 you have to explicitly use `v1.0-mini` tag in order to enjoy an ultra lightweight migrator image. Still, I recommend using latest and greatest.
+
+Finally, starting with v2.0 migrator-docker project was merged into migrator main project. New version of docker image is built automatically every time a new release is created.
+
+To view all available docker containers see [lukasz/migrator/tags](https://cloud.docker.com/repository/docker/lukasz/migrator/tags).
+
+# License
+
+Copyright 2016-2018 Łukasz Budnik
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Dockerfile

@@ -0,0 +1,18 @@
+FROM golang:1.11.2-alpine3.8 as builder
+
+MAINTAINER Łukasz Budnik [email protected]
+
+# install migrator
+RUN apk add git
+RUN go get github.com/lukaszbudnik/migrator
+
+FROM alpine:3.8
+COPY --from=builder /go/bin/migrator /bin
+
+VOLUME ["/data"]
+
+# copy and register entrypoint script
+COPY docker-entrypoint.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
+EXPOSE 8080

README.md

@@ -1,6 +1,6 @@
-# Migrator [![Build Status](https://travis-ci.org/lukaszbudnik/migrator.svg?branch=master)](https://travis-ci.org/lukaszbudnik/migrator)
+# Migrator [![Build Status](https://travis-ci.org/lukaszbudnik/migrator.svg?branch=master)](https://travis-ci.org/lukaszbudnik/migrator) [![Coverage Status](https://coveralls.io/repos/github/lukaszbudnik/migrator/badge.svg?branch=master)](https://coveralls.io/github/lukaszbudnik/migrator?branch=master)
 
-Fast and lightweight DB migration & evolution tool written in go.
+Super fast and lightweight DB migration & evolution tool written in go.
 
 migrator manages all the DB changes for you and completely eliminates manual and error-prone administrative tasks. migrator not only supports single schemas, but also comes with a multi-tenant support.
 
@@ -27,11 +27,12 @@ Migrator requires a simple `migrator.yaml` file:
 ```
 baseDir: test/migrations
 driver: postgres
+# dataSource format is specific to DB go driver implementation - see below 'Supported databases'
 dataSource: "user=postgres dbname=migrator_test host=192.168.99.100 port=55432 sslmode=disable"
 # override only if you have a specific way of determining tenants, default is:
-tenantSelectSql: "select name from public.migrator_tenants"
+tenantSelectSQL: "select name from migrator.migrator_tenants"
 # override only if you have a specific way of creating tenants, default is:
-tenantInsertSql: "insert into public.migrator_tenants (name) values ($1)"
+tenantInsertSQL: "insert into migrator.migrator_tenants (name) values ($1)"
 # override only if you have a specific schema placeholder, default is:
 schemaPlaceHolder: {schema}
 singleSchemas:
@@ -46,7 +47,7 @@ port: 8080
 slackWebHook: https://hooks.slack.com/services/TTT/BBB/XXX
 ```
 
-Migrator will scan all directories under `baseDir` directory. Migrations listed under `singleSchemas` directories will be applied once. Migrations listed under `tenantSchemas` directories will be applied for all tenants fetched using `tenantSelectSql`.
+Migrator will scan all directories under `baseDir` directory. Migrations listed under `singleSchemas` directories will be applied once. Migrations listed under `tenantSchemas` directories will be applied for all tenants fetched using `tenantSelectSQL`.
 
 SQL migrations in both `singleSchemas` and `tenantsSchemas` can use `{schema}` placeholder which will be automatically replaced by migrator with a current schema. For example:
 
@@ -56,6 +57,14 @@ create table if not exists {schema}.modules ( k int, v text );
 insert into {schema}.modules values ( 123, '123' );
 ```
 
+# DB Schemas
+
+When using migrator please remember about these:
+
+* migrator creates `migrator` schema (where `migrator_migrations` and `migrator_tenants` tables reside) automatically
+* when adding a new tenant migrator creates a new schema automatically
+* single schemas are not created automatically, for this you must add initial migration with `create schema` SQL statement (see example above)
+
 # Server mode
 
 When migrator is run with `-mode server` it starts a HTTP service and exposes simple REST API which you can use to invoke migrator actions remotely.
@@ -84,57 +93,51 @@ Port is configurable in `migrator.yaml` and defaults to 8080. Should you need HT
 
 # Supported databases
 
-Currently migrator supports the following databases:
+Currently migrator supports the following databases and their flavours:
 
-* PostgreSQL - schema-based multi-tenant database, with transactions spanning DDL statements, driver used: https://github.com/lib/pq
+* PostgreSQL 9.3+ - schema-based multi-tenant database, with transactions spanning DDL statements, driver used: https://github.com/lib/pq
   * PostgreSQL - original PostgreSQL server
+  * Amazon RDS PostgreSQL - PostgreSQL-compatible relational database built for the cloud
   * Amazon Aurora PostgreSQL - PostgreSQL-compatible relational database built for the cloud
-* MySQL - database-based multi-tenant database, transactions do not span DDL statements, driver used: https://github.com/go-sql-driver/mysql
+  * Google CloudSQL PostgreSQL - PostgreSQL-compatible relational database built for the cloud
+* MySQL 5.6+ - database-based multi-tenant database, transactions do not span DDL statements, driver used: https://github.com/go-sql-driver/mysql
   * MySQL - original MySQL server
   * MariaDB - enhanced near linearly scalable multi-master MySQL
   * Percona - an enhanced drop-in replacement for MySQL
+  * Amazon RDS MySQL - MySQL-compatible relational database built for the cloud
   * Amazon Aurora MySQL - MySQL-compatible relational database built for the cloud
+  * Google CloudSQL MySQL - MySQL-compatible relational database built for the cloud
+* Microsoft SQL Server 2017 - a relational database management system developed by Microsoft, driver used: https://github.com/denisenkom/go-mssqldb
+  * Microsoft SQL Server - original Microsoft SQL Server
 
 # Do you speak docker?
 
 Yes, there is an official docker image available on docker hub.
 
-migrator docker image is ultra lightweight and has a size of only 13.4MB. Ideal for micro-services deployments!
-
-To find out more about migrator docker please visit: https://github.com/lukaszbudnik/migrator-docker.
-
-# Running unit & integration tests
-
-PostgreSQL, MySQL, MariaDB, and Percona:
-
-```
-$ docker/create-and-setup-container.sh [postgresql|mysql|mariadb|percona]
-$ ./coverage.sh
-$ docker/destroy-container.sh [postgresql|mysql|mariadb|percona]
-```
+migrator docker image is ultra lightweight and has a size of approx. 15MB. Ideal for micro-services deployments!
 
-Or see `.travis.yml` to see how it's done on Travis.
+To find out more about migrator docker container see [DOCKER.md](DOCKER.md) for more details.
 
 # Customisation
 
 If you have an existing way of storing information about your tenants you can configure migrator to use it.
 In the config file you need to provide 2 parameters:
 
-* `tenantSelectSql` - a select statement which returns names of the tenants
-* `tenantInsertSql` - an insert statement which creates a new tenant entry, this is called as a prepared statement and is called with the name of the tenant as a parameter; should your table require additional columns you need to provide default values for them
+* `tenantSelectSQL` - a select statement which returns names of the tenants
+* `tenantInsertSQL` - an insert statement which creates a new tenant entry, this is called as a prepared statement and is called with the name of the tenant as a parameter; should your table require additional columns you need to provide default values for them
 
 Here is an example:
 
 ```
-tenantSelectSql: select name from global.customers
-tenantInsertSql: insert into global.customers (name, active, date_added) values (?, true, NOW())
+tenantSelectSQL: select name from global.customers
+tenantInsertSQL: insert into global.customers (name, active, date_added) values (?, true, NOW())
 ```
 
 # Performance
 
 As a benchmarks I used 2 migrations frameworks:
 
-* proprietary Ruby framework - used in my company
+* proprietary Ruby framework - used at my company
 * flyway - leading market feature rich DB migration framework: https://flywaydb.org
 
 There is a performance test generator shipped with migrator (`test/performance/generate-test-migrations.sh`). In order to generate flyway-compatible migrations you need to pass `-f` param (see script for details).
@@ -150,31 +153,40 @@ Migrator is the undisputed winner.
 
 The Ruby framework has an undesired functionality of making a DB call each time to check if given migration was already applied. Migrator fetches all applied migrations at once and compares them in memory. This is the primary reason why migrator is so much better in the second test.
 
-flyway results are... dramatic. I was so shocked that I had to re-run flyway as well as all other tests. Yes, flyway is almost 15 times slower than migrator in the first test. In the second test flyway was faster than Ruby. Still a couple orders of magnitude slower than migrator.
+flyway results are... very surprising. I was so shocked that I had to re-run flyway as well as all other tests. Yes, flyway is 15 times slower than migrator in the first test. In the second test flyway was faster than Ruby. Still a couple orders of magnitude slower than migrator.
 
 The other thing to consider is the fact that migrator is written in go which is known to be much faster than Ruby and Java.
 
 # Installation and supported Go versions
 
 To install migrator use:
 
-`go get github.com/lukaszbudnik/migrator`
+```
+go get github.com/lukaszbudnik/migrator
+cd migrator
+./setup.sh
+```
 
-Migrator supports the following Go versions: 1.8, 1.9, 1.10, 1.11 (all built on Travis).
+Migrator supports the following Go versions: 1.8, 1.9, 1.10, 1.11, and tip (all built on Travis).
 
-# Code Style
+# Contributing, code style, running unit & integration tests
 
-If you would like to send me a pull request please always add unit/integration tests. Code should be formatted & checked using the following commands:
+Contributions are most welcomed.
+
+If you would like to help me and implement a new feature, enhance existing one, or spotted and fixed bug please send me a pull request.
+
+Code should be formatted, checked, and tested using the following commands:
 
 ```
-$ gofmt -s -w .
-$ golint ./...
-$ go tool vet -v .
+./fmt-lint-vet.sh
+./ultimate-coverage.sh
 ```
 
+The `ultimate-coverage.sh` script loops through 5 different containers (3 MySQL flavours, PostgreSQL, and MSSQL) creates db docker container, executes `coverage.sh` script, and finally tears down given db docker container.
+
 # License
 
-Copyright 2016 Łukasz Budnik
+Copyright 2016-2018 Łukasz Budnik
 
 Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
 

config/config.go

@@ -14,8 +14,8 @@ type Config struct {
 	BaseDir           string   `yaml:"baseDir" validate:"nonzero"`
 	Driver            string   `yaml:"driver" validate:"nonzero"`
 	DataSource        string   `yaml:"dataSource" validate:"nonzero"`
-	TenantSelectSql   string   `yaml:"tenantSelectSql"`
-	TenantInsertSql   string   `yaml:"tenantInsertSql"`
+	TenantSelectSQL   string   `yaml:"tenantSelectSQL"`
+	TenantInsertSQL   string   `yaml:"tenantInsertSQL"`
 	SchemaPlaceHolder string   `yaml:"schemaPlaceHolder"`
 	SingleSchemas     []string `yaml:"singleSchemas" validate:"min=1"`
 	TenantSchemas     []string `yaml:"tenantSchemas"`
@@ -69,11 +69,11 @@ func substituteEnvVariables(config *Config) {
 	if strings.HasPrefix(config.DataSource, "$") {
 		config.DataSource = os.Getenv(config.DataSource[1:])
 	}
-	if strings.HasPrefix(config.TenantSelectSql, "$") {
-		config.TenantSelectSql = os.Getenv(config.TenantSelectSql[1:])
+	if strings.HasPrefix(config.TenantSelectSQL, "$") {
+		config.TenantSelectSQL = os.Getenv(config.TenantSelectSQL[1:])
 	}
-	if strings.HasPrefix(config.TenantInsertSql, "$") {
-		config.TenantInsertSql = os.Getenv(config.TenantInsertSql[1:])
+	if strings.HasPrefix(config.TenantInsertSQL, "$") {
+		config.TenantInsertSQL = os.Getenv(config.TenantInsertSQL[1:])
 	}
 	if strings.HasPrefix(config.Port, "$") {
 		config.Port = os.Getenv(config.Port[1:])

config/config_test.go

@@ -20,7 +20,7 @@ func TestFromFile(t *testing.T) {
 	config, err := FromFile("../test/migrator-test.yaml")
 	assert.Nil(t, err)
 	assert.Equal(t, "test/migrations", config.BaseDir)
-	assert.Equal(t, "select name from public.migrator_tenants", config.TenantSelectSql)
+	assert.Equal(t, "select name from migrator.migrator_tenants", config.TenantSelectSQL)
 	assert.Equal(t, "postgres", config.Driver)
 	assert.Equal(t, "user=postgres dbname=migrator_test host=192.168.99.100 port=55432 sslmode=disable", config.DataSource)
 	assert.Equal(t, []string{"tenants"}, config.TenantSchemas)
@@ -33,8 +33,8 @@ func TestWithEnvFromFile(t *testing.T) {
 	config, err := FromFile("../test/migrator-test-envs.yaml")
 	assert.Nil(t, err)
 	assert.Equal(t, os.Getenv("HOME"), config.BaseDir)
-	assert.Equal(t, os.Getenv("PATH"), config.TenantSelectSql)
-	assert.Equal(t, os.Getenv("GOPATH"), config.TenantInsertSql)
+	assert.Equal(t, os.Getenv("PATH"), config.TenantSelectSQL)
+	assert.Equal(t, os.Getenv("GOPATH"), config.TenantInsertSQL)
 	assert.Equal(t, os.Getenv("PWD"), config.Driver)
 	assert.Equal(t, os.Getenv("TERM"), config.DataSource)
 	assert.Equal(t, os.Getenv("_"), config.Port)
@@ -50,8 +50,8 @@ func TestConfigString(t *testing.T) {
 	expected := `baseDir: /opt/app/migrations
 driver: postgres
 dataSource: user=p dbname=db host=localhost
-tenantSelectSql: select abc
-tenantInsertSql: insert into table
+tenantSelectSQL: select abc
+tenantInsertSQL: insert into table
 schemaPlaceHolder: :tenant
 singleSchemas:
 - ref