Initial integration tests

.gitignore

@@ -5,6 +5,6 @@ gradle/
 gradlew
 gradlew.bat
 out/
-shared/serde
-shared/input
-shared/output
+integration-test/docker-tmp/serde
+integration-test/docker-tmp/input
+integration-test/docker-tmp/output

build.gradle

@@ -16,8 +16,8 @@
 buildscript {
     ext.kotlin_version = '1.2.61'
     ext.ionjava_version = '1.0.+'
-    ext.hive_version = '1.1.+'
-    ext.hadoop_version = '2.6.+'
+    ext.hive_version = '2.3.2'
+    ext.hadoop_version = '2.7.4'
 
     repositories {
         mavenCentral()
@@ -34,10 +34,7 @@ allprojects {
     }
 
     apply plugin: "kotlin"
-    apply plugin: 'java-library'
 
-    sourceCompatibility = 1.7
-    targetCompatibility = 1.7
 
     configurations {
         // to make compileOnly available to testing scope
@@ -46,7 +43,7 @@ allprojects {
 
     dependencies {
         // using kotlin to make tests less verbose
-        testCompile "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version"
+        testCompile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
         testCompile "org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version"
 
         testCompile "junit:junit:4.+"
@@ -68,13 +65,25 @@ allprojects {
         }
     }
 
-    apply plugin: 'findbugs'
-    tasks.withType(FindBugs) {
-        excludeFilter = file("$rootProject.projectDir/config/findbugs/excludeFilter.xml")
+//    apply plugin: 'findbugs'
+//    tasks.withType(FindBugs) {
+//        excludeFilter = file("$rootProject.projectDir/config/findbugs/excludeFilter.xml")
+//
+//        reports {
+//            xml.enabled = false
+//            html.enabled = true
+//        }
+//    }
+}
 
-        reports {
-            xml.enabled = false
-            html.enabled = true
+// clean all subprojects
+task cleanAll {
+    subprojects.each {
+        it.afterEvaluate {
+            def cleanTask = it.tasks.findByName('clean')
+            if (cleanTask) {
+                dependsOn(cleanTask)
+            }
         }
     }
 }

integration-test/build.gradle

@@ -12,19 +12,20 @@
  * language governing permissions and limitations under the License.
  */
 
-task makeSharedDir {
-    mkdir './shared/serde'
-    mkdir './shared/input'
-    mkdir './shared/output'
+task createTmpDir {
+    doLast {
+        mkdir 'docker-tmp/input'
+        mkdir 'docker-tmp/output'
+    }
 }
 
-task cleanUpShared {
-    delete './shared/serde'
-    delete './shared/input'
-    delete './shared/output'
+task removeTmpDir {
+    doLast {
+        delete 'docker-tmp'
+    }
 }
 
-task startDocker(type: Exec, dependsOn: 'makeSharedDir') {
+task startDocker(type: Exec, dependsOn: 'copySerDeJar') {
     workingDir './docker'
     commandLine 'docker-compose', 'up', '-d'
 }
@@ -35,22 +36,30 @@ task stopDocker(type: Exec) {
 }
 
 task copySerDeJar {
-    dependsOn 'makeSharedDir', ':serde:singleJar'
+    dependsOn ':serde:singleJar'
 
-    copy {
-        from file('../serde/build/libs/serde-all.jar')
-        into file('./shared/serde/')
+    doLast {
+        copy {
+            from file('../serde/build/libs/serde-all.jar')
+            into file('./docker-tmp/serde/')
+        }
     }
 }
 
+task tailHiveLog(type: Exec, dependsOn: 'testSetUp') {
+    workingDir './docker'
+    commandLine 'docker-compose',  'exec',  'hive-server',  'tail -f /tmp/root/'
+}
+
 task testSetUp {
-    dependsOn copySerDeJar
+    dependsOn createTmpDir
     dependsOn startDocker
+    dependsOn copySerDeJar
 }
 
 task testTearDown {
     dependsOn stopDocker
-    dependsOn cleanUpShared
+    dependsOn removeTmpDir
 }
 
 /**

integration-test/docker/beeline

@@ -0,0 +1,2 @@
+docker-compose exec hive-server /opt/hive/bin/beeline -u jdbc:hive2://localhost:10000
+

integration-test/docker/conf/beeline-log4j2.properties

@@ -24,6 +24,7 @@ logger.HiveConnection.name = org.apache.hive.jdbc.HiveConnection
 logger.HiveConnection.level = INFO
 
 # root logger
-rootLogger.level = ${sys:hive.log.level}
+# rootLogger.level = ${sys:hive.log.level}
+rootLogger.level = DEBUG
 rootLogger.appenderRefs = root
 rootLogger.appenderRef.root.ref = ${sys:hive.root.logger}

integration-test/docker/conf/hive-log4j2.properties

@@ -3,7 +3,7 @@ name = HiveLog4j2
 packages = org.apache.hadoop.hive.ql.log
 
 # list of properties
-property.hive.log.level = INFO
+property.hive.log.level = DEBUG
 property.hive.root.logger = DRFA
 property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name}
 property.hive.log.file = hive.log

integration-test/docker/docker-compose.yml

@@ -44,7 +44,9 @@ services:
     container_name: hive-server
     image: bde2020/hive:2.3.2-postgresql-metastore
     volumes:
-      - ../shared:/shared
+      - ../docker-tmp:/docker-tmp
+      - ../test-data:/test-data
+
     env_file:
       - ./hadoop-hive.env
     environment:

integration-test/docker/tailLog

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+docker-compose exec hive-server tail -f /tmp/root/hive.log
+

integration-test/src/test/kotlin/software/amazon/ionhiveserde/integrationtest/AllTests.java

@@ -0,0 +1,29 @@
+package software.amazon.ionhiveserde.integrationtest;
+
+import static software.amazon.ionhiveserde.integrationtest.TestConfigurationKt.getTestSuiteConfiguration;
+import static software.amazon.ionhiveserde.integrationtest.docker.DockerUtilKt.waitForHiveServer;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+
+@RunWith(Suite.class)
+@SuiteClasses({
+    SuccessfulRoundTripTest.class,
+    SuccessfulJdbcTest.class
+})
+public class AllTests {
+
+    @BeforeClass
+    public static void beforeAll() {
+        Base.setConfiguration(getTestSuiteConfiguration());
+        waitForHiveServer();
+    }
+
+    @AfterClass
+    public static void afterAll() {
+        Base.hive().close();
+    }
+}

integration-test/src/test/kotlin/software/amazon/ionhiveserde/integrationtest/Base.kt

@@ -1,15 +1,56 @@
 package software.amazon.ionhiveserde.integrationtest
 
+import org.junit.AfterClass
+import org.junit.Before
 import org.junit.BeforeClass
+import software.amazon.ion.*
+import software.amazon.ion.system.IonSystemBuilder
 import software.amazon.ionhiveserde.integrationtest.docker.waitForHiveServer
 import software.amazon.ionhiveserde.integrationtest.hive.Hive
 
+/**
+ * Base class for all integration tests. Handles the connection lifecycle to reuse the same connection as much as
+ * possible.
+ */
 abstract class Base {
     companion object {
+        @JvmStatic
+        var configuration: TestConfiguration = NonSuiteConfiguration
+
+        private var hive: Hive? = null
+
+        @JvmStatic
+        fun hive(): Hive {
+            if (hive == null || hive!!.isClosed()) {
+                hive = Hive()
+            }
+
+            return hive!!
+        }
+
         @BeforeClass
         @JvmStatic
-        fun beforeClass() {
-            waitForHiveServer()
+        fun beforeAll() {
+            if (configuration.shouldWait) {
+                waitForHiveServer()
+            }
         }
+
+        @AfterClass
+        @JvmStatic
+        fun afterAll() {
+            hive().dropAllTables()
+
+            if (configuration.shouldClose) {
+                hive().close()
+            }
+        }
+    }
+
+    protected val ion: IonSystem = IonSystemBuilder.standard().build()
+
+    @Before
+    fun beforeEach() {
+        hive().dropAllTables()
     }
-}
+}