Merge pull request eventuate-tram#42 from dartartem/wip-db-id-gen

Added db id migration testing

_build-and-test-all.sh

@@ -27,4 +27,27 @@ ${dockerall}Up
 
 ./gradlew :end-to-end-tests:cleanTest :end-to-end-tests:test
 
+./wait-for-services.sh localhost readers/${READER}/finished "8099"
+
+migration_file="migration_scripts/${DATABASE}/migration.sql"
+
+rm -f $migration_file
+if [ "${DATABASE}" == "mysql" ]; then
+  curl https://raw.githubusercontent.com/eventuate-foundation/eventuate-common/wip-db-id-gen/mysql/4.initialize-database-db-id.sql --output $migration_file --create-dirs
+  cat $migration_file | ./mysql-cli.sh -i
+elif [ "${DATABASE}" == "postgres" ]; then
+  curl https://raw.githubusercontent.com/eventuate-foundation/eventuate-common/wip-db-id-gen/postgres/5.initialize-database-db-id.sql --output $migration_file --create-dirs
+  cat $migration_file | ./postgres-cli.sh -i
+else
+  echo "Unknown Database"
+  exit 99
+fi
+rm -f $migration_file
+
+${dockerall}Up -P envFile=docker-compose-env-files/db-id-gen.env
+
+./gradlew :end-to-end-tests:cleanTest :end-to-end-tests:test
+
+./gradlew -P verifyDbIdMigration=true :migration-tests:cleanTest migration-tests:test
+
 ${dockerall}Down

build-and-test-all-mysql.sh

@@ -3,5 +3,6 @@
 set -e
 
 export DATABASE=mysql
+export READER=MySqlReader
 
 ./_build-and-test-all.sh

build-and-test-all-postgres.sh

@@ -3,6 +3,7 @@
 set -e
 
 export DATABASE=postgres
+export READER=PostgresPollingReader
 
 export SPRING_PROFILES_ACTIVE=postgres
 

build.gradle

@@ -35,11 +35,14 @@ subprojects {
 
 dockerCompose {
     environment.put "EVENTUATE_COMMON_VERSION", eventuateCommonImageVersion
-    environment.put "EVENTUATE_KAFKA_VERSION", eventuateMessagingKafkaImageVersion
     environment.put "EVENTUATE_CDC_VERSION", eventuateCdcImageVersion
     environment.put "EVENTUATE_SAGA_VERSION", eventuateTramSagasImageVersion
     environment.put "EVENTUATE_JAVA_BASE_IMAGE_VERSION", eventuateExamplesBaseImageVersion
 
+    if (project.ext.has("envFile")) {
+        environment.put "ENV_FILE", project.ext.envFile
+    }
+
     mysqlinfrastructure {
         projectName = null
         useComposeFiles = ["docker-compose-mysql.yml"]

docker-compose-env-files/db-id-gen.env

@@ -0,0 +1 @@
+EVENTUATE_OUTBOX_ID=1

docker-compose-mysql.yml

@@ -1,6 +1,8 @@
 version: '3.5'
 services:
   order-service:
+    env_file:
+      - ${ENV_FILE:-docker-compose-env-files/empty.env}
     build:
       context: ./order-service/
       args:
@@ -24,6 +26,8 @@ services:
       SPRING_ZIPKIN_BASE_URL: http://zipkin:9411/
 
   customer-service:
+    env_file:
+      - ${ENV_FILE:-docker-compose-env-files/empty.env}
     build:
       context: ./customer-service/
       args:
@@ -115,6 +119,7 @@ services:
       EVENTUATELOCAL_CDC_OFFSET_STORE_KEY: MySqlBinlog
       EVENTUATELOCAL_CDC_MYSQL_BINLOG_CLIENT_UNIQUE_ID: 1234567890
       EVENTUATELOCAL_CDC_READ_OLD_DEBEZIUM_DB_OFFSET_STORAGE_TOPIC: "false"
+      EVENTUATE_OUTBOX_ID: 1
       JAVA_OPTS: -Xmx64m
 
   zipkin:

docker-compose-postgres.yml

@@ -1,6 +1,8 @@
 version: '3'
 services:
   order-service:
+    env_file:
+      - ${ENV_FILE:-docker-compose-env-files/empty.env}
     build:
       context: ./order-service/
       args:
@@ -21,6 +23,8 @@ services:
       EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181
 
   customer-service:
+    env_file:
+      - ${ENV_FILE:-docker-compose-env-files/empty.env}
     build:
       context: ./customer-service/
       args:
@@ -104,4 +108,5 @@ services:
       EVENTUATELOCAL_ZOOKEEPER_CONNECTION_STRING: zookeeper:2181
       EVENTUATELOCAL_CDC_READER_NAME: PostgresPollingReader
       SPRING_PROFILES_ACTIVE: EventuatePolling
+      EVENTUATE_OUTBOX_ID: 1
       JAVA_OPTS: -Xmx64m

migration-tests/build.gradle

@@ -0,0 +1,18 @@
+dependencies {
+    testCompile "org.springframework.boot:spring-boot-starter-jdbc:${springBootVersion}"
+    testCompile "org.springframework.boot:spring-boot-starter-test:${springBootVersion}"
+
+    testRuntime 'mysql:mysql-connector-java:5.1.36'
+    testRuntime ('org.postgresql:postgresql:9.4-1200-jdbc41') {
+        exclude group: "org.slf4j", module: "slf4j-simple"
+    }
+    testRuntime 'com.microsoft.sqlserver:mssql-jdbc:7.2.1.jre8'
+}
+
+test {
+    if (!project.ext.has("verifyDbIdMigration")) {
+        exclude '**/DbIdMigrationVerificationTest**'
+    }
+
+    forkEvery 1
+}

migration-tests/src/test/java/io/eventuate/examples/tram/sagas/ordersandcustomers/migration/DbIdMigrationVerificationTest.java

@@ -0,0 +1,39 @@
+package io.eventuate.examples.tram.sagas.ordersandcustomers.migration;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import java.util.List;
+import java.util.Map;
+
+@RunWith(SpringJUnit4ClassRunner.class)
+@SpringBootTest(classes = DbIdMigrationVerificationTest.Config.class)
+public class DbIdMigrationVerificationTest {
+
+  @Configuration
+  @EnableAutoConfiguration
+  public static class Config {}
+
+  @Autowired
+  private JdbcTemplate jdbcTemplate;
+
+  @Test
+  //after first call of e2e tests (before migration), messages should have ids, after second call (after migration) don't
+  public void testThatMessagesAreMigrated() {
+    List<Map<String, Object>> messagesWithEmptyId =
+            jdbcTemplate.queryForList("select * from eventuate.message where destination <> 'CDC-IGNORED' and id = ''");
+
+    List<Map<String, Object>> messagesWithNotEmptyId =
+            jdbcTemplate.queryForList("select * from eventuate.message where destination <> 'CDC-IGNORED' and id <> ''");
+
+    Assert.assertTrue(messagesWithEmptyId.size() > 0);
+    Assert.assertEquals(messagesWithEmptyId.size(), messagesWithNotEmptyId.size());
+  }
+}

migration-tests/src/test/resources/application-mssql.properties

@@ -0,0 +1,4 @@
+spring.datasource.url=jdbc:sqlserver://${DOCKER_HOST_IP:localhost}:1433;databaseName=eventuate
+spring.datasource.username=sa
+spring.datasource.password=Eventuate123!
+spring.datasource.driver-class-name=com.microsoft.sqlserver.jdbc.SQLServerDriver

migration-tests/src/test/resources/application-postgres.properties

@@ -0,0 +1,4 @@
+spring.datasource.url=jdbc:postgresql://${DOCKER_HOST_IP:localhost}/eventuate
+spring.datasource.username=eventuate
+spring.datasource.password=eventuate
+spring.datasource.driver-class-name=org.postgresql.Driver

migration-tests/src/test/resources/application.properties

@@ -0,0 +1,4 @@
+spring.datasource.url=jdbc:mysql://${DOCKER_HOST_IP:localhost}/eventuate
+spring.datasource.username=mysqluser
+spring.datasource.password=mysqlpw
+spring.datasource.driver-class-name=com.mysql.jdbc.Driver

settings.gradle

@@ -13,3 +13,5 @@ include 'api-gateway-service-api-web'
 include 'api-gateway-service'
 
 include 'end-to-end-tests'
+
+include 'migration-tests'

wait-for-services.sh

@@ -1,14 +1,23 @@
 #! /bin/bash
 
-host=$1
-ports=$2
+done=false
 
-shift 2
+echo waiting for: $*
+
+host=${1?}
+shift
+health_url=${1?}
+shift
+ports=$*
+
+if [ -z "$ports" ] ; then
+	echo no ports
+	exit 99
+fi
 
-done=false
 while [[ "$done" = false ]]; do
 	for port in $ports; do
-		curl -q http://${host}:${port}/health >& /dev/null
+		curl --fail http://${host}:${port}/${health_url} >& /dev/null
 		if [[ "$?" -eq "0" ]]; then
 			done=true
 		else
@@ -17,7 +26,7 @@ while [[ "$done" = false ]]; do
 		fi
 	done
 	if [[ "$done" = true ]]; then
-		echo services are started
+		echo connected
 		break;
   fi
 	echo -n .