Merge pull request #79 from lukaszbudnik/dev-v4.1.1

migrator v4.1.1

README.md

@@ -74,6 +74,8 @@ Sample HTTP response:
 }
 ```
 
+API v1 was introduced in migrator v4.0. Any non API-breaking changes will be added to v1. Any significant change or an API-breaking change will be added to API v2.
+
 ## GET /v1/config
 
 Returns migrator's config as `application/x-yaml`.
@@ -246,7 +248,8 @@ This operation requires as an input the following JSON payload:
   * `sync` - synchronises all source migrations with internal migrator's table, this action loads and marks all source migrations as applied but does not apply them
   * `dry-run` - instead of calling commit, calls rollback at the end of the operation
 * `response` - controls how much information is returned by migrator, valid values are:
-  * `full` - the response will contain both summary results and a list of applied migrations
+  * `full` - the response will contain both summary results and a list of applied migrations/scripts
+  * `list` - the response will contain both summary results and a list of applied migrations/scripts but without their contents (introduced in migrator `v4.1.1` and a part of API v1; does not break API v1 contract - existing integrations will continue to work)
   * `summary` - the response will contain only summary results
 
 Sample request:

coordinator/coordinator.go

@@ -86,7 +86,8 @@ func (c *coordinator) GetAppliedMigrations() []types.MigrationDB {
 	return c.appliedMigrations
 }
 
-// VerifySourceMigrationsCheckSums verifies if CheckSum of disk and flattened DB migrations match
+// VerifySourceMigrationsCheckSums verifies if CheckSum of source and applied DB migrations match
+// VerifySourceMigrationsCheckSums allows CheckSum of scripts to be different (they are applied every time and are often updated)
 // returns bool indicating if offending (i.e., modified) disk migrations were found
 // if bool is false the function returns a slice of offending migrations
 // if bool is true the slice of effending migrations is empty
@@ -101,6 +102,9 @@ func (c *coordinator) VerifySourceMigrationsCheckSums() (bool, []types.Migration
 	var offendingMigrations []types.Migration
 	var result = true
 	for _, t := range intersect {
+		if t.source.MigrationType == types.MigrationTypeSingleScript || t.source.MigrationType == types.MigrationTypeTenantScript {
+			continue
+		}
 		if t.source.CheckSum != t.applied.CheckSum {
 			offendingMigrations = append(offendingMigrations, t.source)
 			result = false

coordinator/coordinator_mocks.go

@@ -46,6 +46,19 @@ func newBrokenCheckSumMockedDiskLoader(_ context.Context, _ *config.Config) load
 	return new(mockedBrokenCheckSumDiskLoader)
 }
 
+type mockedDifferentScriptCheckSumMockedDiskLoader struct {
+}
+
+func (m *mockedDifferentScriptCheckSumMockedDiskLoader) GetSourceMigrations() []types.Migration {
+	m1 := types.Migration{Name: "201602220000.sql", SourceDir: "source", File: "source/201602220000.sql", MigrationType: types.MigrationTypeSingleMigration, Contents: "select abc"}
+	m2 := types.Migration{Name: "recreate-indexes.sql", SourceDir: "tenants-scripts", File: "tenants-scripts/recreate-indexes.sql", MigrationType: types.MigrationTypeTenantScript, Contents: "select abc", CheckSum: "sha256-1"}
+	return []types.Migration{m1, m2}
+}
+
+func newDifferentScriptCheckSumMockedDiskLoader(_ context.Context, _ *config.Config) loader.Loader {
+	return new(mockedDifferentScriptCheckSumMockedDiskLoader)
+}
+
 type mockedConnector struct {
 }
 
@@ -74,3 +87,20 @@ func (m *mockedConnector) ApplyMigrations(types.MigrationsModeType, []types.Migr
 func newMockedConnector(context.Context, *config.Config) db.Connector {
 	return &mockedConnector{}
 }
+
+type mockedDifferentScriptCheckSumMockedConnector struct {
+	mockedConnector
+}
+
+func (m *mockedDifferentScriptCheckSumMockedConnector) GetAppliedMigrations() []types.MigrationDB {
+	m1 := types.Migration{Name: "201602220000.sql", SourceDir: "source", File: "source/201602220000.sql", MigrationType: types.MigrationTypeSingleMigration, Contents: "select abc"}
+	d1 := time.Date(2016, 02, 22, 16, 41, 1, 123, time.UTC)
+	m2 := types.Migration{Name: "recreate-indexes.sql", SourceDir: "tenants-scripts", File: "tenants-scripts/recreate-indexes.sql", MigrationType: types.MigrationTypeTenantScript, Contents: "select abc", CheckSum: "sha256-2"}
+	d2 := time.Date(2016, 02, 22, 16, 41, 1, 456, time.UTC)
+	ms := []types.MigrationDB{{Migration: m1, Schema: "source", AppliedAt: d1}, {Migration: m2, Schema: "customer1", AppliedAt: d2}}
+	return ms
+}
+
+func newDifferentScriptCheckSumMockedConnector(context.Context, *config.Config) db.Connector {
+	return &mockedDifferentScriptCheckSumMockedConnector{mockedConnector{}}
+}

coordinator/coordinator_test.go

@@ -229,6 +229,14 @@ func TestVerifySourceMigrationsCheckSumsKO(t *testing.T) {
 	assert.Equal(t, coordinator.GetSourceMigrations()[0], offendingMigrations[0])
 }
 
+func TestVerifySourceMigrationsAndScriptsCheckSumsOK(t *testing.T) {
+	coordinator := New(context.TODO(), nil, newDifferentScriptCheckSumMockedConnector, newDifferentScriptCheckSumMockedDiskLoader, newMockedNotifier)
+	defer coordinator.Dispose()
+	verified, offendingMigrations := coordinator.VerifySourceMigrationsCheckSums()
+	assert.True(t, verified)
+	assert.Empty(t, offendingMigrations)
+}
+
 func TestApplyMigrations(t *testing.T) {
 	coordinator := New(context.TODO(), nil, newMockedConnector, newMockedDiskLoader, newMockedNotifier)
 	defer coordinator.Dispose()

server/server.go

@@ -154,9 +154,15 @@ func migrationsPostHandler(c *gin.Context, config *config.Config, newCoordinator
 	common.LogInfo(c.Request.Context(), "Returning applied migrations: %v", len(appliedMigrations))
 
 	var response *migrationsSuccessResponse
-	if request.Response == types.ResponseTypeFull {
+	switch request.Response {
+	case types.ResponseTypeFull:
 		response = &migrationsSuccessResponse{results, appliedMigrations}
-	} else {
+	case types.ResponseTypeList:
+		for i := range appliedMigrations {
+			appliedMigrations[i].Contents = ""
+		}
+		response = &migrationsSuccessResponse{results, appliedMigrations}
+	default:
 		response = &migrationsSuccessResponse{results, nil}
 	}
 
@@ -194,9 +200,15 @@ func tenantsPostHandler(c *gin.Context, config *config.Config, newCoordinator fu
 	common.LogInfo(c.Request.Context(), "Tenant %v added, migrations applied: %v", request.Name, len(appliedMigrations))
 
 	var response *migrationsSuccessResponse
-	if request.Response == types.ResponseTypeFull {
+	switch request.Response {
+	case types.ResponseTypeFull:
+		response = &migrationsSuccessResponse{results, appliedMigrations}
+	case types.ResponseTypeList:
+		for i := range appliedMigrations {
+			appliedMigrations[i].Contents = ""
+		}
 		response = &migrationsSuccessResponse{results, appliedMigrations}
-	} else {
+	default:
 		response = &migrationsSuccessResponse{results, nil}
 	}
 

server/server_mocks.go

@@ -39,7 +39,7 @@ func (m *mockedCoordinator) GetSourceMigrations() []types.Migration {
 }
 
 func (m *mockedCoordinator) GetAppliedMigrations() []types.MigrationDB {
-	m1 := types.Migration{Name: "201602220000.sql", SourceDir: "source", File: "source/201602220000.sql", MigrationType: types.MigrationTypeSingleMigration}
+	m1 := types.Migration{Name: "201602220000.sql", SourceDir: "source", File: "source/201602220000.sql", MigrationType: types.MigrationTypeSingleMigration, Contents: "select abc", CheckSum: "sha256"}
 	d1 := time.Date(2016, 02, 22, 16, 41, 1, 123, time.UTC)
 	ms := []types.MigrationDB{{Migration: m1, Schema: "source", AppliedAt: d1}}
 	return ms

server/server_test.go

@@ -28,6 +28,7 @@ func newTestRequest(method, url string, body io.Reader) (*http.Request, error) {
 
 func testSetupRouter(config *config.Config, newCoordinator func(ctx context.Context, config *config.Config) coordinator.Coordinator) *gin.Engine {
 	versionInfo := &types.VersionInfo{Release: "GitBranch", CommitSha: "GitCommitSha", CommitDate: "2020-01-08T09:56:41+01:00", APIVersions: []string{"v1"}}
+	gin.SetMode(gin.ReleaseMode)
 	return SetupRouter(versionInfo, config, newCoordinator)
 }
 
@@ -109,7 +110,7 @@ func TestAppliedMigrationsRoute(t *testing.T) {
 
 	assert.Equal(t, http.StatusOK, w.Code)
 	assert.Equal(t, "application/json; charset=utf-8", w.HeaderMap["Content-Type"][0])
-	assert.Equal(t, `[{"name":"201602220000.sql","sourceDir":"source","file":"source/201602220000.sql","migrationType":1,"contents":"","checkSum":"","schema":"source","appliedAt":"2016-02-22T16:41:01.000000123Z"}]`, strings.TrimSpace(w.Body.String()))
+	assert.Equal(t, `[{"name":"201602220000.sql","sourceDir":"source","file":"source/201602220000.sql","migrationType":1,"contents":"select abc","checkSum":"sha256","schema":"source","appliedAt":"2016-02-22T16:41:01.000000123Z"}]`, strings.TrimSpace(w.Body.String()))
 }
 
 // section /migrations
@@ -149,6 +150,24 @@ func TestMigrationsPostRouteSummaryResponse(t *testing.T) {
 	assert.NotContains(t, strings.TrimSpace(w.Body.String()), `[{"name":"201602220000.sql","sourceDir":"source","file":"source/201602220000.sql","migrationType":1,"contents":"select abc","checkSum":""},{"name":"201602220001.sql","sourceDir":"source","file":"source/201602220001.sql","migrationType":2,"contents":"select def","checkSum":""}]`)
 }
 
+func TestMigrationsPostRouteListResponse(t *testing.T) {
+	config, err := config.FromFile(configFile)
+	assert.Nil(t, err)
+
+	router := testSetupRouter(config, newMockedCoordinator)
+
+	json := []byte(`{"mode": "apply", "response": "list"}`)
+	req, _ := newTestRequest(http.MethodPost, "/migrations", bytes.NewBuffer(json))
+
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusOK, w.Code)
+	assert.Equal(t, "application/json; charset=utf-8", w.HeaderMap["Content-Type"][0])
+	assert.Contains(t, strings.TrimSpace(w.Body.String()), `"results":`)
+	assert.Contains(t, strings.TrimSpace(w.Body.String()), `[{"name":"201602220000.sql","sourceDir":"source","file":"source/201602220000.sql","migrationType":1,"checkSum":""},{"name":"201602220001.sql","sourceDir":"source","file":"source/201602220001.sql","migrationType":2,"checkSum":""}]`)
+}
+
 func TestMigrationsPostRouteBadRequest(t *testing.T) {
 	config, err := config.FromFile(configFile)
 	assert.Nil(t, err)
@@ -236,6 +255,24 @@ func TestTenantsPostRouteSummaryResponse(t *testing.T) {
 	assert.NotContains(t, strings.TrimSpace(w.Body.String()), `[{"name":"201602220001.sql","sourceDir":"source","file":"source/201602220001.sql","migrationType":2,"contents":"select def","checkSum":""}]`)
 }
 
+func TestTenantsPostRouteListResponse(t *testing.T) {
+	config, err := config.FromFile(configFile)
+	assert.Nil(t, err)
+
+	router := testSetupRouter(config, newMockedCoordinator)
+
+	json := []byte(`{"name": "new_tenant", "response": "list", "mode":"dry-run"}`)
+	req, _ := newTestRequest(http.MethodPost, "/tenants", bytes.NewBuffer(json))
+
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusOK, w.Code)
+	assert.Equal(t, "application/json; charset=utf-8", w.HeaderMap["Content-Type"][0])
+	assert.Contains(t, strings.TrimSpace(w.Body.String()), `"results":`)
+	assert.Contains(t, strings.TrimSpace(w.Body.String()), `[{"name":"201602220001.sql","sourceDir":"source","file":"source/201602220001.sql","migrationType":2,"checkSum":""}]`)
+}
+
 func TestTenantsPostRouteBadRequestError(t *testing.T) {
 	config, err := config.FromFile(configFile)
 	assert.Nil(t, err)

types/types.go

@@ -26,8 +26,10 @@ type MigrationsResponseType string
 const (
 	// ResponseTypeSummary instructs migrator to only return JSON representation of Results struct
 	ResponseTypeSummary MigrationsResponseType = "summary"
-	// ResponseTypeFull instructs migrator to return JSON representation of both Results struct and all applied migrations
+	// ResponseTypeFull instructs migrator to return JSON representation of both Results struct and all applied migrations/scripts
 	ResponseTypeFull MigrationsResponseType = "full"
+	// ResponseTypeList instructs migrator to return JSON representation of both Results struct and all applied migrations/scripts but without their contents
+	ResponseTypeList MigrationsResponseType = "list"
 )
 
 // MigrationsModeType represents mode in which migrations should be applied
@@ -55,7 +57,7 @@ func ValidateMigrationsModeType(fl validator.FieldLevel) bool {
 func ValidateMigrationsResponseType(fl validator.FieldLevel) bool {
 	response, ok := fl.Field().Interface().(MigrationsResponseType)
 	if ok {
-		return response == ResponseTypeSummary || response == ResponseTypeFull
+		return response == ResponseTypeSummary || response == ResponseTypeFull || response == ResponseTypeList
 	}
 	return false
 }
@@ -66,7 +68,7 @@ type Migration struct {
 	SourceDir     string        `json:"sourceDir"`
 	File          string        `json:"file"`
 	MigrationType MigrationType `json:"migrationType"`
-	Contents      string        `json:"contents"`
+	Contents      string        `json:"contents,omitempty"`
 	CheckSum      string        `json:"checkSum"`
 }