merging umd-hadoop-dist into umd-hadoop-core

extras, memcached experiment

extras/memcached/bin-ec2/InstallMemcacheOnCluster.sh

@@ -0,0 +1,39 @@
+#! /bin/sh
+
+# checking the number of argument. First argument should be the path of rsa key file
+if [ $# -ne 1 ]
+then
+     echo  "args: [path of the rsa key file]"
+     exit
+fi
+
+# if file does not exist then exit
+if [ ! -f $1 ]
+then
+	echo "$1 does not exist"
+	exit
+fi
+
+
+# getting list of ip addresses 
+LIST_OF_IP_ADDRESSESS='ListOfIpAddressess.txt'
+if [ -f $LIST_OF_IP_ADDRESSESS ]
+then
+	rm $LIST_OF_IP_ADDRESSESS
+fi
+
+ec2-describe-instances | awk '"INSTANCE"==$1 && "running"==$6 {print $4}' > $LIST_OF_IP_ADDRESSESS
+echo  "List of IP Addresses are stored in file ListOfIpAddresses.txt in the current directory"
+
+var=`cat "$LIST_OF_IP_ADDRESSESS"`
+for i in $var; do
+	ip_address=$i
+	echo "Starting memcached server on $ip_address ..."
+	`ssh -i $1 root@"$ip_address" 'killall memcached'`
+	`scp -i $1 dist/memcached root@"$ip_address":`
+	`scp -i $1 dist/libevent-1.4.so.2 root@"$ip_address":`
+	`scp -i $1 dist/start_memcached.sh root@"$ip_address":`
+	`ssh -i $1 root@"$ip_address" 'chmod +x start_memcached.sh'`
+	`ssh -i $1 root@"$ip_address" './start_memcached.sh'`
+	echo " ... Success!"
+done

extras/memcached/bin-ec2/RestartMemcached.sh

@@ -0,0 +1,26 @@
+#! /bin/sh
+
+# checking the number of argument. First argument should be the path of rsa key file
+if [ $# -ne 1 ]
+then
+     echo  "args: [path of the rsa key file]"
+     exit
+fi
+
+# if file does not exist then exit
+if [ ! -f $1 ]
+then
+	echo "$1 does not exist"
+	exit
+fi
+
+LIST_OF_IP_ADDRESSESS='ListOfIpAddressess.txt'
+
+var=`cat "$LIST_OF_IP_ADDRESSESS"`
+for i in $var; do
+	ip_address=$i
+	echo "Restarting memcached server on $ip_address ..."
+	`ssh -i $1 root@"$ip_address" 'killall memcached'`
+	`ssh -i $1 root@"$ip_address" './start_memcached.sh'`
+done
+

extras/memcached/bin-ec2/VerifyMemcachedStatus.sh

@@ -0,0 +1,25 @@
+#! /bin/sh
+
+# checking the number of argument. First argument should be the path of rsa key file
+if [ $# -ne 1 ]
+then
+     echo  "args: [path of the rsa key file]"
+     exit
+fi
+
+# if file does not exist then exit
+if [ ! -f $1 ]
+then
+	echo "$1 does not exist"
+	exit
+fi
+
+LIST_OF_IP_ADDRESSESS='ListOfIpAddressess.txt'
+
+var=`cat "$LIST_OF_IP_ADDRESSESS"`
+for i in $var; do
+	ip_address=$i
+	echo "Verifying memcached on $ip_address ..."
+	echo `ssh -i $1 root@"$ip_address" 'netstat -atpn | grep memcached'`
+done
+

extras/memcached/bin-ec2/dist/start_memcached.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+export LD_LIBRARY_PATH=.
+chmod +x memcached
+./memcached -d -u nobody -m 1024 -p 11211
+

extras/memcached/bin-ec2/notes.txt

@@ -0,0 +1,21 @@
+make install exec_prefix=/fs/clip-qa/jimmylin/memcache/le-install
+
+gcc -g -O2 -o memcached-debug memcached.o slabs.o items.o assoc.o thread.o stats.o ../le-install/lib/libevent-1.4.so.2
+
+hadoop jar hadoop-0.17.0-streaming.jar -input stream.in -output stream.out \
+   -mapper start_memcached.sh  \
+   -file start_memcached.sh -file memcached -file libevent-1.4.so.2 \
+   -jobconf mapred.map.tasks=1
+
+"export LD_LIBRARY_PATH=.; ./memcached -d -u nobody -m 512 127.0.0.1 -p 11211; while [ 1 ] do sleep 5 done;"
+
+hadoop jar hadoop-0.17.0-streaming.jar -input stream.in -output stream.out \
+   -mapper "/lib/ld-linux.so.2 --library-path . ./memcached -u nobody -m 512 -p 11211"  \
+   -file start_memcached.sh -file memcached -file libevent-1.4.so.2 \
+   -jobconf mapred.map.tasks=2
+
+
+scp memcached [email protected]:
+scp libevent-1.4.so.2 [email protected]:
+scp start_memcached.sh [email protected]:
+ssh [email protected] ./start_memcached.sh