Skip to content
This repository has been archived by the owner on Aug 30, 2022. It is now read-only.

Commit

Permalink
Better service management scripts (#532)
Browse files Browse the repository at this point in the history
* Scripts to manage hillview services
  • Loading branch information
Mihai Budiu authored Aug 21, 2019
1 parent 2f6b673 commit 2f570d8
Show file tree
Hide file tree
Showing 20 changed files with 358 additions and 132 deletions.
4 changes: 2 additions & 2 deletions bin/backend-start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,5 @@ source ${mydir}/lib.sh
# If you want GRPC logging uncomment the following line
# LOGGING="-Djava.util.logging.config.file=logging.properties"

cd ${mydir}/../platform/
java ${LOGGING} -server -jar target/hillview-server-jar-with-dependencies.jar 127.0.0.1:3569
cd ${mydir}
java ${LOGGING} -server -jar ../platform/target/hillview-server-jar-with-dependencies.jar 127.0.0.1:3569
2 changes: 1 addition & 1 deletion bin/delete-data.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ def delete_folder(config, folder):
assert isinstance(config, ClusterConfiguration)
message = "Deleting " + folder + " from all hosts"
logger.info(message)
config.run_on_all_workers(lambda rh: delete_remote_folder(rh, folder), True)
config.run_on_all_workers(lambda rh: delete_remote_folder(rh, folder))

def main():
"""Main function"""
Expand Down
64 changes: 40 additions & 24 deletions bin/deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,25 @@

logger = get_logger("deploy")

def generate_script(config, rh, template):
"""Generates a shell script based on a template inserting configuration variables"""
logger.info("Generating script for host " + rh.host + " from " + template)
variables = ""
variables += "SERVICE_DIRECTORY=" + config.service_folder + "\n"
variables += "HEAPSIZE=\"" + rh.heapsize + "\"\n"
variables += "USER=" + rh.user + "\n"
variables += "WORKER_PORT=" + str(config.worker_port) + "\n"
variables += "AGGREGATOR_PORT=" + str(config.aggregator_port) + "\n"
variables += "CLEANUP=" + str(1 if config.cleanup_on_install() else 0) + "\n"
variables += "TOMCAT=" + config.tomcat + "\n"
lines = list(open(template))
filename = template.replace("-template", "")
lines = [variables if "REPLACE_WITH_VARIABLES" in x else x for x in lines]
with open(filename, "w") as f:
for l in lines:
f.write(l)
os.chmod(filename, 0o770)

def prepare_webserver(config):
"""Deploys files needed by the Hillview web server"""
logger.info("Creating web service folder")
Expand All @@ -21,7 +40,6 @@ def prepare_webserver(config):
logger.info(message)
rh.create_remote_folder(config.service_folder)
rh.run_remote_shell_command("chown " + config.get_user() + " " + config.service_folder)
rh.create_remote_folder(config.service_folder + "/hillview")

major = config.tomcat_version[0:config.tomcat_version.find('.')]

Expand Down Expand Up @@ -50,31 +68,30 @@ def prepare_webserver(config):
tmp.close()
rh.copy_file_to_remote(tmp.name, config.service_folder + "/serverlist", "")
os.unlink(tmp.name)
if config.cleanup_on_install():
rh.run_remote_shell_command(
"cd " + config.service_folder + ";" + \
"rm -f hillview-web.log hillview-web.log.* hillview-web.log*.lck")
# link to web server logs
rh.run_remote_shell_command("ln -sf " + config.service_folder + "/hillview-web.log " + \
config.service_folder + "/hillview/hillview-web.log")
generate_script(config, rh, "hillview-webserver-manager-template.sh")
rh.copy_file_to_remote(
"hillview-webserver-manager.sh", config.service_folder, "")
os.unlink("hillview-webserver-manager.sh")

def prepare_worker(config, rh):
"""Prepares files needed by a Hillview worker on a remote machine"""
assert isinstance(config, ClusterConfiguration)
message = "Preparing worker " + str(rh)
logger.info(message)
# rh.run_remote_shell_command("sudo apt-get install libgfortran3")

rh.create_remote_folder(config.service_folder)
rh.run_remote_shell_command("chown " + config.get_user() + " " + config.service_folder)
rh.create_remote_folder(config.service_folder + "/hillview")
rh.create_remote_folder(config.service_folder)
rh.copy_file_to_remote(
config.scriptFolder +
"/../platform/target/hillview-server-jar-with-dependencies.jar",
config.service_folder + "/hillview", "")
if config.cleanup_on_install():
rh.run_remote_shell_command(
"cd " + config.service_folder + "/hillview;"
"rm -f hillview.log hillview.log.* hillview.log*.lck")
config.service_folder, "")
generate_script(config, rh, "hillview-worker-manager-template.sh")
rh.copy_file_to_remote(
"hillview-worker-manager.sh", config.service_folder, "")
rh.copy_file_to_remote("forever.sh", config.service_folder, "")
os.unlink("hillview-worker-manager.sh")

def prepare_aggregator(config, rh):
"""Prepares files needed by a Hillview aggregator on a remote machine"""
Expand All @@ -83,23 +100,22 @@ def prepare_aggregator(config, rh):
logger.info(message)
rh.create_remote_folder(config.service_folder)
rh.run_remote_shell_command("chown " + config.get_user() + " " + config.service_folder)
rh.create_remote_folder(config.service_folder + "/hillview")
rh.create_remote_folder(config.service_folder)
rh.copy_file_to_remote(
config.scriptFolder +
"/../platform/target/hillview-server-jar-with-dependencies.jar",
config.service_folder + "/hillview", "")
if config.cleanup_on_install():
rh.run_remote_shell_command(
"cd " + config.service_folder + "/hillview;"
"rm -f hillview-agg.log hillview-agg.log.* hillview-agg.log*.lck")
config.scriptFolder + "/../platform/target/hillview-server-jar-with-dependencies.jar",
config.service_folder, "")
rh.copy_file_to_remote("forever.sh", config.service_folder, "")
tmp = tempfile.NamedTemporaryFile(mode="w", delete=False)
for h in rh.children:
tmp.write(h + ":" + str(config.worker_port) + "\n")
tmp.close()
rh.copy_file_to_remote(tmp.name, config.service_folder + "/workers", "")
os.unlink(tmp.name)
rh.run_remote_shell_command("ln -sf " + config.service_folder + "/hillview-agg.log " + \
config.service_folder + "/hillview/hillview-agg.log")

generate_script(config, rh, "hillview-aggregator-manager-template.sh")
rh.copy_file_to_remote(
"hillview-aggregator-manager.sh", config.service_folder, "")
os.unlink("hillview-aggregator-manager.sh")

def prepare_workers(config):
"""Prepares all Hillview workers"""
Expand Down
20 changes: 20 additions & 0 deletions bin/forever.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/sh
# Runs another process in a loop

if [ $# -le 1 ]; then
echo "Usage: forever.sh pidfile command"
echo "Re-runs command every time it terminates"
echo "Writes its own pid in pidfile"
exit 1
fi

pidfile=$1
shift

echo "Running $1 forever"
echo $$ >${pidfile}
while /bin/true; do
$*
sleep 2
echo "Restarting..."
done
2 changes: 1 addition & 1 deletion bin/frontend-start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,5 +13,5 @@ export JAVA_OPTS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=
# LOGGING=" -Djava.util.logging.config.file=logging.properties"
export JAVA_OPTS="$JAVA_OPTS$LOGGING"

cd ${mydir}/../web # logs will always be in this folder
cd ${mydir}
../apache-tomcat-${TOMCATVERSION}/bin/catalina.sh run
66 changes: 66 additions & 0 deletions bin/hillview-aggregator-manager-template.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#!/bin/bash
# This is teamplate for a script that manages a hillview aggregator as a Unix service.

#REPLACE_WITH_VARIABLES

FOREVERPID="forever-aggregator.pid"
PIDFILE="hillview-aggregator.pid"

cd ${SERVICE_DIRECTORY}

start() {
if [ "x${CLEANUP}" == "x1" ]; then
rm -f hillview-agg.log hillview-agg.log.* hillview-agg.log*.lck
fi
./forever.sh ${FOREVERPID} nohup java -ea -Dlog4j.configurationFile=./log4j.properties -server -Xmx${HEAPSIZE} -jar ${SERVICE_DIRECTORY}/hillview-server-jar-with-dependencies.jar ${SERVICE_DIRECTORY}/workers 0.0.0.0:${AGGREGATOR_PORT} >nohup.out 2>&1 &
}

killbypid() {
local PIDFILE=$1
if [ -f ${PIDFILE} ]; then
# First try to find the service pid
read LINE < ${SERVICE_DIRECTORY}/${PIDFILE}
if [ -d "/proc/${LINE}" ]; then
echo "Killing $2 process ${LINE}"
kill ${LINE}
fi
rm -f ${SERVICE_DIRECTORY}/${PIDFILE}
return 0
else
return 1
fi
}

stop() {
killbypid ${FOREVERPID} forever
killbypid ${PIDFILE} hillview-server
}

status() {
if [ -f ${PIDFILE} ]; then
read LINE < ${SERVICE_DIRECTORY}/${PIDFILE}
if [ -d "/proc/${LINE}" ]; then
echo "Process seems to be running"
return 0
fi
fi
echo "Could not find running aggregator"
}

case ${1} in
start)
start
;;
stop)
stop
;;
restart)
stop
start
;;
status)
status
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
esac
55 changes: 55 additions & 0 deletions bin/hillview-webserver-manager-template.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
#!/bin/bash
# This is teamplate for a script that manages a hillview web server as a service.

#REPLACE_WITH_VARIABLES

# File storing PID of the hillview worker
PIDFILE="hillview-webserevr.pid"
cd ${SERVICE_DIRECTORY}

start() {
if [ "x${CLEANUP}" == "x1" ]; then
rm -f hillview-web.log hillview-web.log.* hillview-web.log*.lck
fi
export WEB_CLUSTER_DESCRIPTOR=serverlist
nohup ./${TOMCAT}/bin/startup.sh &
}

stop() {
if pgrep -f tomcat; then
${SERVICE_DIRECTORY}"/"${TOMCAT}/bin/shutdown.sh
echo Stopped
else
echo "Web server already stopped"
fi
pkill -f Bootstrap
true
}

status() {
# This assumes there's a single tomcat on the machine, which may not be true...
if ! pgrep -f tomcat; then
echo "Web server not running"
else
echo "Web server running"
fi
true
}

case ${1} in
start)
start
;;
stop)
stop
;;
restart)
stop
start
;;
status)
status
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
esac
79 changes: 79 additions & 0 deletions bin/hillview-worker-manager-template.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
#!/bin/bash
# This is teamplate for a script that manages a hillview worker as a service.

#REPLACE_WITH_VARIABLES

# File storing PID of the hillview worker
PIDFILE="hillview-worker.pid"
# File storing PID of the forever process that runs the hillview worker
FOREVERPID="forever-worker.pid"

cd ${SERVICE_DIRECTORY}

start() {
echo "Starting worker"
if [ "x${CLEANUP}" == "x1" ]; then
rm -f hillview.log hillview.log.* hillview.log*.lck
fi
./forever.sh ${FOREVERPID} nohup java -ea -Dlog4j.configurationFile=./log4j.properties -server -Xmx${HEAPSIZE} -jar ${SERVICE_DIRECTORY}/hillview-server-jar-with-dependencies.jar 0.0.0.0:${WORKER_PORT} >nohup.out 2>&1 &
}

killbypid() {
local PIDFILE=$1
if [ -f ${PIDFILE} ]; then
# First try to find the service pid
read LINE < ${SERVICE_DIRECTORY}/${PIDFILE}
if [ -d "/proc/${LINE}" ]; then
echo "Killing $2 process ${LINE}"
kill ${LINE}
fi
rm -f ${SERVICE_DIRECTORY}/${PIDFILE}
return 0
else
return 1
fi
}

stop() {
killbypid ${FOREVERPID} forever
killbypid ${PIDFILE} hillview-server
if [ $? -ne 0 ]; then
# Kill it by name; may have collateral damage
if pgrep -f hillview-server; then
pkill -f hillview-server
echo "Stopped"
else
echo "Already stopped"
fi
fi
true
}

status() {
if [ -f ${PIDFILE} ]; then
read LINE < ${PIDFILE}
if [ -d "/proc/${LINE}" ]; then
echo "Process seems to be running"
return 0
fi
fi
echo "Could not find running worker"
}

case ${1} in
start)
start
;;
stop)
stop
;;
restart)
stop
start
;;
status)
status
;;
*)
echo "Usage: $0 {start|stop|restart|status}"
esac
Loading

0 comments on commit 2f570d8

Please sign in to comment.