build.gradle

plugins {
  id 'org.hidetake.ssh' version '1.5.0'

  // required to build scala example
  id 'scala'

  // useful for working on scala example
  id 'eclipse'

  // we need a fat jar
  id "com.github.johnrengelman.shadow" version "1.2.3"
}

// set the dependencies for compiling the groovy script
repositories {
    mavenCentral()
}

dependencies {
	// required to build the spark example
	compile 'org.scala-lang:scala-library:2.10'
    compile 'org.apache.spark:spark-core_2.10:1.6.2'
    compile 'org.apache.spark:spark-streaming_2.10:1.6.2'
    compile 'org.apache.kafka:kafka_2.10:0.9.0.0'
    compile 'org.apache.kafka:kafka-clients:0.9.0.0'
    compile 'org.apache.kafka:kafka-log4j-appender:0.9.0.0'
    compile files('libs/messagehub.login-1.0.0.jar')
}

// load some common helper methods
apply from: "${projectDir}/../../shared/common-helpers.gradle"

// get the cluster connection details
Properties props = new Properties()
props.load(new FileInputStream("$projectDir/../../connection.properties"))

def jsonFile = file('vcap.json')
def mhProps = new groovy.json.JsonSlurper().parseText(jsonFile.text)


task ScalaWordCount {

    dependsOn shadowJar

    doLast {

        def tmpDir = "test-${new Date().getTime()}"

        // ssh plugin documentation: https://gradle-ssh-plugin.github.io/docs/
        
        ssh.run {
            // remotes.bicluster is defined in shared/common-helpers.gradle
            session(remotes.bicluster) {

                try {
                    // initialise kerberos
                    execute "kinit -k -t ${props.username}.keytab ${props.username}@IBM.COM"
                } 
                catch (Exception e) {
                    println "problem running kinit - maybe this is a Basic cluster?"
                }

                // create temp local dir for holding LICENSE file before uploading it to hdfs
                execute "mkdir ${tmpDir}"

                println "Uploading SparkMessageHubScala-all.jar to cluster - this may take some time"

                // upload spark script and text file to process
                put from: "${projectDir}/build/libs/SparkMessageHubScala-all.jar", 
                    into: "${tmpDir}/SparkMessageHubScala-all.jar"

                println "Finished Uploading SparkMessageHubScala-all.jar"

                def clz = "--class \"biginsights.examples.MessageHubConsumer\""

                // FIXME! we are passing passwords as command line arguments which will be visible
                //        to other users.  We could probably bundle the properties in the jar file
                //        and the app read them from there.

                def kafka_brokers = "${mhProps.kafka_brokers_sasl}".minus('[').minus(']').replaceAll(/\s*/, "")

                def args = "--conf spark.driver.args=\"${kafka_brokers} ${mhProps.user} ${mhProps.password} ${mhProps.api_key} ${mhProps.kafka_rest_url} ${mhProps.topic} ${props.username}\""

                def cmd = "spark-submit ${args} --num-executors 3 --executor-cores 1 --executor-memory 1G ${clz} ${tmpDir}/SparkMessageHubScala-all.jar"

                println "Running: ${cmd}"
                execute cmd
            }
        }
    }
}

task PsAll {

    doLast {

        // ssh plugin documentation: https://gradle-ssh-plugin.github.io/docs/
        
        ssh.run {
            // remotes.bicluster is defined in shared/common-helpers.gradle
            session(remotes.bicluster) {
                execute '''ps aux | grep biginsights.examples.MessageHubConsumer'''
            }
        }
    }
}

task KillAll {

    doLast {

        // ssh plugin documentation: https://gradle-ssh-plugin.github.io/docs/
        
        ssh.run {
            // remotes.bicluster is defined in shared/common-helpers.gradle
            session(remotes.bicluster) {
                try {
                    execute '''kill $(ps aux | grep biginsights.examples.MessageHubConsumer | awk '{print $2}')'''
                } catch (Exception e) {
                    println "Ignoring error from kill command"
                }
            }
        }
    }
}

task Example {
	dependsOn ScalaWordCount
}