first commit

MBtech · Dec 4, 2015 · c4ae3f8 · c4ae3f8
commit c4ae3f8
Show file tree

Hide file tree

Showing 29 changed files with 2,659 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+.DS_Store
diff --git a/README.markdown b/README.markdown
@@ -0,0 +1 @@
+# flink-deploy
diff --git a/conf/configuration_example.yaml b/conf/configuration_example.yaml
@@ -0,0 +1,14 @@
+#
+# Amazon EC2 example cluster configuration
+#
+ectwo:
+    - storm-version "0.9.5"					# Version of Storm
+    - zk-version "3.4.6"					# Version of Zookeeper
+    - image "eu-west-1/ami-97344ae0"				# Ubuntu 14.04 LTS AMI
+    - region "eu-west-1"					# Region
+    - m1.medium {ZK, MASTER, UI}           			# Request service
+    - m1.medium {ZK, WORKER}					# Request service
+    - m1.medium {ZK, WORKER}					# Request service
+    - m1.medium {WORKER}					# Request service
+    - remote-exec-preconfig {cd ~, echo hey > hey.txt}
+    - remote-exec-postconfig {}
diff --git a/conf/credential_example.yaml b/conf/credential_example.yaml
@@ -0,0 +1,9 @@
+##
+## Amazon Elastic Compute Cloud
+##
+ec2-identity: "AKIAJD4HTWSE6LB7WZPA"
+ec2-credential: "kAYbUTvUhjWwnt4tnHiaetP5cDG80CUVnDJUVuiR"
+
+# Optional - Amazon Web Services x509 credentials (for configuring ec2-api-tools)
+ec2-x509-certificate-path: ""
+ec2-x509-private-path: ""
diff --git a/pom.xml b/pom.xml
@@ -0,0 +1,138 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+	<modelVersion>4.0.0</modelVersion>
+
+	<groupId>dk.kaspergsm.stormdeploy</groupId>
+	<artifactId>flink-deploy</artifactId>	
+	<version>1</version>
+	<packaging>jar</packaging>
+
+	<name>flink-deploy</name>
+	<url>https://github.com/KasperMadsen/storm-deploy-alternative</url>
+
+	<properties>
+		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+	</properties>
+
+	<dependencies>
+		<dependency>
+			<groupId>org.apache.jclouds</groupId>
+			<artifactId>jclouds-all</artifactId>
+			<version>1.9.1</version>
+		</dependency>
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-log4j12</artifactId>
+			<version>1.7.5</version>
+		</dependency>  
+		<dependency>
+			<groupId>org.apache.jclouds.driver</groupId>
+			<artifactId>jclouds-log4j</artifactId>
+			<version>1.9.1</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.jclouds.driver</groupId>
+			<artifactId>jclouds-slf4j</artifactId>
+			<version>1.9.1</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.jclouds.driver</groupId>
+			<artifactId>jclouds-sshj</artifactId>
+			<version>1.9.1</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.jclouds.driver</groupId>
+			<artifactId>jclouds-bouncycastle</artifactId>
+			<version>1.9.1</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.jclouds.driver</groupId>
+			<artifactId>jclouds-enterprise</artifactId>
+			<version>1.9.1</version>
+		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-exec</artifactId>
+			<version>1.2</version>
+		</dependency>
+ 		<dependency>
+                        <groupId>com.sun</groupId>
+                        <artifactId>tools</artifactId>
+                        <version>1.6.0</version>
+                        <scope>system</scope>
+                        <systemPath>${java.home}/../lib/tools.jar</systemPath>
+                </dependency> 
+	</dependencies>
+
+	<build>
+		<sourceDirectory>src</sourceDirectory>
+		<resources>
+			<resource>
+				<directory>src</directory>
+				<includes>
+					<include>log4j.properties</include>
+				</includes>
+			</resource>
+		</resources>
+
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-shade-plugin</artifactId>
+				<version>2.2</version>
+				<configuration>
+					<transformers>
+						<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+							<manifestEntries>
+								<Main-Class>dk.kaspergsm.stormdeploy.StormDeployAlternative</Main-Class>
+							</manifestEntries>
+						</transformer>
+						<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+					</transformers>
+
+					<filters>
+						<filter>
+							<artifact>*:*</artifact>
+							<excludes>
+								<exclude>META-INF/*.SF</exclude>
+								<exclude>META-INF/*.DSA</exclude>
+								<exclude>META-INF/*.RSA</exclude>
+							</excludes>
+						</filter>
+					</filters>
+				</configuration>
+				<executions>
+			  		<execution>
+			    		<phase>package</phase>
+			    		<goals>
+			      			<goal>shade</goal>
+			    		</goals>
+			  	</execution>
+				</executions>
+		      	</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-compiler-plugin</artifactId>
+				<version>3.1</version>
+				<configuration>
+					<source>1.6</source>
+					<target>1.6</target>
+				</configuration>
+			</plugin>
+
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-eclipse-plugin</artifactId>
+				<version>2.10</version>
+				<configuration>
+					<downloadSources>true</downloadSources>
+					<downloadJavadocs>true</downloadJavadocs>
+				</configuration>
+			</plugin>
+
+		</plugins>
+	</build>
+</project>
+
+
diff --git a/src/log4j.properties b/src/log4j.properties
@@ -0,0 +1,19 @@
+# Root logger option
+log4j.rootLogger=INFO, stdout
+
+# Define loglevels for components
+log4j.logger.net.schmizz.sshj=WARN
+log4j.logger.jclouds.ssh=WARN
+log4j.logger.dk.kaspergsm.stormdeploy.image.MemoryMonitor=INFO, FileAppender
+
+# Direct log messages to stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.out
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} [%p] %m%n
+
+# Direct log messages to file
+log4j.appender.FileAppender=org.pollerosoftware.log4j.additions.LazyFileAppender
+log4j.appender.FileAppender.File=memory-monitor.log
+log4j.appender.FileAppender.layout = org.apache.log4j.PatternLayout
+log4j.appender.FileAppender.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
diff --git a/src/mb/learningcurve/stormdeploy/LaunchNodeThread.java b/src/mb/learningcurve/stormdeploy/LaunchNodeThread.java
@@ -0,0 +1,116 @@
+package mb.learningcurve.stormdeploy;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import org.jclouds.compute.ComputeService;
+import org.jclouds.compute.domain.Hardware;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.compute.options.TemplateOptions;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.StatementList;
+import static org.jclouds.scriptbuilder.domain.Statements.exec;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import mb.learningcurve.stormdeploy.configurations.Zookeeper;
+import mb.learningcurve.stormdeploy.userprovided.Configuration;
+
+/**
+ * Used to launch a new nodes
+ * 
+ * @author Kasper Grud Skat Madsen
+ */
+public class LaunchNodeThread extends Thread {
+	private static Logger log = LoggerFactory.getLogger(LaunchNodeThread.class);
+	private String _instanceType, _clustername, _region, _image, _username;
+	private Set<NodeMetadata> _newNodes = null;
+	private List<Statement> _initScript;
+	private ComputeService _compute;
+	private List<Integer> _nodeids;
+	private List<String> _daemons;
+
+	/**
+	 * @param compute
+	 *            ComputeService from JClouds
+	 * @param instanceType
+	 *            Supported instanceType (e.g. m1.medium on aws-ec2)
+	 * @param image
+	 *            Image to deploy
+	 * @param region
+	 *            Region to deploy into (image must be in this region)
+	 * @param clustername
+	 *            Name of cluster to deploy
+	 * @param nodeids
+	 *            Set of nodeids being launched
+	 * @param daemons
+	 *            Set of daemons to launch on this set of nodes
+	 * @param zkMyId
+	 *            If contain(daemons, zk) then write this zkMyId on init
+	 */
+	public LaunchNodeThread(ComputeService compute, Configuration config, String instanceType, String clustername, List<Integer> nodeids, List<String> daemons, Integer zkMyId) {
+		_region = config.getDeploymentLocation();
+		_username = config.getImageUsername();
+		_image = config.getDeploymentImage();
+		_instanceType = instanceType;
+		_clustername = clustername;
+		_daemons = daemons;
+		_compute = compute;
+		_nodeids = nodeids;
+
+		// Create initScript
+		_initScript = new ArrayList<Statement>();
+		_initScript.add(exec("echo \"" + daemons.toString() + "\" > ~/daemons"));
+		_initScript.add(exec("echo \"" + instanceType.toString() + "\" > ~/.instance-type"));
+		if (zkMyId != null)
+			_initScript.addAll(Zookeeper.writeZKMyIds(_username, zkMyId));
+
+		// Run thread now
+		this.start();
+	}
+
+	@SuppressWarnings("unchecked")
+	@Override
+	public void run() {
+		try {
+			_newNodes = (Set<NodeMetadata>) _compute.createNodesInGroup(
+					_clustername,
+					_nodeids.size(),
+					_compute.templateBuilder()
+							.hardwareId(_instanceType)
+							.locationId(_region)
+							.imageId(_image)
+							.options(
+									new TemplateOptions()
+											.runAsRoot(false)
+											.wrapInInitScript(true)
+											.overrideLoginUser(_username)
+											.inboundPorts(Tools.getPortsToOpen())
+											.userMetadata("daemons", _daemons.toString())
+											.runScript(new StatementList(_initScript))
+											.overrideLoginCredentials(Tools.getPrivateKeyCredentials(_username))
+											.authorizePublicKey(Tools.getPublicKey())).build());
+		} catch (NoSuchElementException ex) {
+			// happens often when hardwareId is not found. List all possible hardware types
+			if (ex.getMessage().toLowerCase().contains("hardwareid") && ex.getMessage().toLowerCase().contains("not found")) {
+				log.error("You have specified unknown hardware profile. Here follows a list of supported profiles: ");
+				Set<? extends Hardware> availableHardware = _compute.listHardwareProfiles();
+				for (Hardware h : availableHardware) {
+					log.info(h.toString());
+				}
+			} else {
+				log.error("Problem: ", ex);	
+			}
+		} catch (Exception ex) {
+			log.error("Problem launching instance", ex);
+		}
+	}
+
+	public List<Integer> getNodeIds() {
+		return _nodeids;
+	}
+
+	public Set<NodeMetadata> getNewNodes() {
+		return _newNodes;
+	}
+}