mongodb-backup

#!/bin/bash
#
#  mongodb-backup is a backup script for MongoDB database
#  Version : 0.3.0
#
#  Apache License, Version 2.0
#
#  Copyright (C) 2013 - Twenga SA
#  Copyright (C) 2013 - Michael Lopez <mickael.lopez@gmail.com>
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#
###########################################################################

############################################################################
#      Please edit /etc/mongodb-backup.conf according to your needs       #
###########################################################################

PROGNAME=$(basename $0)
DATE=$(date +%Y-%m-%d)
CONF="/etc/mongodb-backup.conf"
PID_FILE="/var/run/mongodb-backup.pid"
STOP_BALANCER_JS="/usr/local/lib/stopBalancer.js"
DUMP_OPTIONS=""
OPTIONS=""
NB_ERRORS=0
NB_FILE=0
COUNT_PROC=0
declare -a PIDS
declare -A PROCINFO

## Activate job monitoring
set -o monitor

## Check if mongo binary is present
which mongo &> /dev/null ; iIsInstalled=$?

if [ "$iIsInstalled" == 1 ] ; then
	echo -e "\nCan't find mongodb client... aborting!"
	exit 1
fi

## Check if /etc/mongodb-backup.conf is present
if [ ! -s "$CONF" ] ; then
	echo -e "\n$CONF is missing or empty... failed!"
	exit 1
else
	source $CONF
fi	

## Log file 
LOG_FILE="${LOGS_DIR}/mongodb-backup_${DATE}.log"


###########################################################################
#                                Functions                                #
###########################################################################

## Usage function
function usage() {
	echo -e "Usage: $PROGNAME [options] ...\n"
        echo "Options are..."
	echo -e "   -h \t Display this help message"
	echo -e "   -j \t Enable dump journalisation"
	echo -e "   -e \t Display events to STDOUT"
	echo -e "   -m <mail_address> \t Set email reporting address"
	echo -e "   -c <compression_level>\t Override compression level: normal, fast or best (default: normal)"
	echo -e "   -u <username>" 
	echo -e "   -p <password>"
}

## Log events to log file and to stdout
function log {
	echo -e "$(date +%D" "%T) $1" >> $LOG_FILE
	if [ "$DISPLAY_ERRORS" ] ; then
	       echo -e "$1"	
	fi       
}

## Check if a backup is runnning 
function isRunning {
	local iIsRunning
	if [ -f $PID_FILE ] ; then
		PID=$(cat $PID_FILE)
		ps -A | grep $PID &> /dev/null ; iIsRunning=$?
		if [ "$iIsRunning" -eq "0" ] ; then
			return 0
		else
			rm -f $PID_FILE
		fi
	fi
	return 1
}

## Check if host is master of replica set
function isMaster {
	local bIsMaster
	bIsMaster=$(mongo --host $1 --eval "printjson(rs.isMaster())" | grep ismaster | sed 's/[\t",]//g' | awk '{print $3}') 
	echo $bIsMaster
}

## Pick a secondary host of replica set
function pickingSecondary {
	local countMemberInRS sSlaves sSlave sHost=$1

	# Test if the server passed in parameter is alone in replicaSet. If yes return this server
	countMemberInRS=( $(mongo --host $sHost --eval "printjson(rs.isMaster().hosts)" | tail -1 | tr -d "\",[]") )
	if [[ ${#countMemberInRS[@]} != "1" ]] ; then
		sSlaves=$(mongo --host $sHost --eval "printjson(rs.status())" | grep -B3 SECONDARY | grep name | sed 's/[\t",]//g' | awk '{print $3}')
		sSlave=$(shuf -e $sSlaves -n 1)
	else
		sSlave=$sHost
	fi
	echo $sSlave
}

## get script options to prepare command parameters
function prepareJob {
	local iTryConnection
	log "Preparing mongodb backup job..."

	# Compression level selection
	case "${COMPRESS_LVL}" in
		fast)   log "Info : Fast comppression was chosen"
			COMPRESS_LVL="-1"
    		;;
		normal) log  "Info : Normal compression was chosen"
    			COMPRESS_LVL="-5"
    		;;
		best)   log "Info : Best compression was chosen"
    			COMPRESS_LVL="-9"
		;;
		*) log "Info : Default compression was chosen"
    			COMPRESS_LVL="-5"
   		;;
	esac

	# Do we need to use a username/password?
	if [ "$USERNAME" ] ; then 
		OPTIONS="${OPTIONS} --username=$USERNAME --password=$PASSWORD"
	fi

	# Set host parameter if host is different of localhost
        if [ "$MONGOS_HOST" != "localhost" ] ; then
		MONGOS_OPT="${OPTIONS} --host ${MONGOS_HOST}"
	fi

	# Try to connect
	mongo $MONGOS_OPT --eval "printjson()" &> /dev/null ; iTryConnection=$?
	if [ "$iTryConnection" -eq 1 ] ; then
		log "Critical: Unable to connect on ${MONGOS_HOST}. Please check if mongos is running or check 'MONGOS_HOST' option in /etc/mongodb-backup.conf"
		exit 1
	fi

	# Check if SHARD options is correct
	if [ "$NB_SHARD" -ne 0 ] ; then
		for (( n=1; n<=$NB_SHARD; n++ )) ; do
			SHARD_HOSTS[$n]=$(eval "echo \$SHARD${n}_HOST")
			SHARD_NAMES[$n]=$(eval "echo \$SHARD${n}_NAME")

			# Check if value of SHARD_* array is not empty
			if [ "${SHARD_HOSTS[$n]}" == "" ] || [ "${SHARD_NAMES[$n]}" == "" ] ; then
				log "Critical: Wrong shard configuration in /etc/mongodb-backup.conf file. Please check the shard options in file."
				exit 1
			fi
		done
		log "Info : ${NB_SHARD} shard has been selected for backup."
	fi
}

## Last actions of script
function finishUp {
	doCompress
	END_TIME=$(date +%s)
	TIME_DIFF=$(( $END_TIME - $START_TIME ))
	TIME_DIFF=`echo $(($TIME_DIFF / 60 ))`
	log "Finished MongoDB backup in $TIME_DIFF minutes..."
	sendMail
	rm -f $PID_FILE >> $LOG_FILE
	exit 0
}

## Compress backups and log files
function doCompress {
	local i=0
	log "Starting compression..."

	# Compress backups
	cd $BACKUP_DIR
	for DIR in ${DIRECTORIES[@]} ; do
		if [ -d "$DIR" ] ; then
			FILES[$i]=$(tar -cf - $DIR | gzip ${COMPRESS_LVL} - > ${DIR}_${DATE}.tar.gz)
			rm -fr ${DIR}/ &> /dev/null
		else
			log "Error : ${DIR} doesn't exist."
			let NB_ERRORS++
		fi
		let i++
	done
	NB_FILE=${#FILES[@]} 

	# Compress logs
	cd $BACKUP_DIR/logs
	iNbLogsNotCompressed=$(find . -name "*.log" -type f -mtime +0 | wc -l)
	if [ "$iNbLogsNotCompressed" != "0" ] ; then 
		for logFile in $(find . -name "*.log" -type f -mtime +0) ; do
			gzip $logFile
		done
	fi

	checkRetention
}

## Apply retention
function retention {
	local sType=$1 sDir=$2

	if [[ "$sType" == "backup" ]]; then
		sExt="tar.gz"
	else
		sExt=".gz"
	fi

	if [ "$(eval "echo \$MAX_${sType^^}_RETENTION")" -gt 0 ] ; then
		log "Checking $sType retention..."
		toDelete=$(find $sDir -name "*.$sExt" -type f -mtime +$(eval "echo \$MAX_${sType^^}_RETENTION"))
		if [ ! -z "$toDelete" ] ; then
			log "Deleting files older than $(eval "echo \$MAX_${sType^^}_RETENTION") days:"
			log "$(find $sDir -name "*.$sExt" -type f -mtime +$(eval "echo \$MAX_${sType^^}_RETENTION") -exec du -hs {} \;)" 
			find $sDir -name "*.$sExt" -type f -mtime +$(eval "echo \$MAX_${sType^^}_RETENTION") -delete
		fi
	else
		log "Info : ${sType^} retention is disable."
	fi
}

## Check retention by type
function checkRetention {
	# Check backup retention
	retention backup $BACKUP_DIR

	# Check logs retention
	retention log $BACKUP_DIR/logs
}

## eMail reporting
function sendMail {
	local sizeOfBackupDir sizeOfBackup

	if [ "$REPORT_EMAIL" ] && [ -n ${REPORT_EMAIL+x}  ] ; then
		log "Sending email notifications..."
	    	echo "Backup time: $TIME_DIFF minutes" > /tmp/$PROGNAME_mail.tmp
	    	echo -e "disk usage:  (size : usage : avail : percent : parent) \n $(df -h $BACKUP_DIR | tail -1)" >> /tmp/$PROGNAME_mail.tmp 

	       	sizeOfBackupDir=$(du -hs $BACKUP_DIR | awk '{print $1}')
	        sizeOfBackup=$(find $BACKUP_DIR -name "*_${DATE}.tar.gz" -type f -exec du -hs {} \;)
	        echo "Backup host:  $(hostname)" >> /tmp/$PROGNAME_mail.tmp
	        echo "Number of backup files :  $NB_FILE" >> /tmp/$PROGNAME_mail.tmp
	        echo -e "Backup size:\n$sizeOfBackup" >> /tmp/$PROGNAME_mail.tmp
	        echo "Backup directory size : $sizeOfBackupDir" >> /tmp/$PROGNAME_mail.tmp
	        echo -e "\nnumber of erros/warning : $NB_ERRORS \nCheck log file for more details : $LOG_FILE" >> /tmp/$PROGNAME_mail.tmp
	   	
		SUBJECT="[MongoDB Backup] Report -> Host:  $(hostname) - errors: $NB_ERRORS - runtime: $TIME_DIFF minutes"
	      	mail -s "$SUBJECT" "$REPORT_EMAIL" < /tmp/$PROGNAME_mail.tmp
	        if [ $? -eq "0" ] ; then
	        	log "Email notification successfully sent."
	        fi
	      	rm -f /tmp/$PROGNAME_mail.tmp
	fi
}

## Stop balancer to prevent chunk migration during backup
function stopBalancer {
	local bBalancerState

	log "Stopping MongoDB balancer..."
	if [ ! -f "$STOP_BALANCER_JS" ] ; then
		log "Error : ${STOP_BALANCER_JS} doesn't exist !"
		log "Error : Balancer hasn't been stopped."
		let NB_ERRORS++
      	else
		bBalancerState=$(getBalancerState)
		if [ "$bBalancerState" == "true" ] ; then
            		mongo $MONGOS_OPT config $STOP_BALANCER_JS >> $LOG_FILE
         	else
			log "Warning : Balancer is already stopped !"
			let NB_ERRORS++
		fi
	fi
}

## Start balancer
function startBalancer {
	local bBalancerState

	log "Restarting MongoDB balancer..."
	bBalancerState=$(getBalancerState)
        if [ "$bBalancerState" == "false" ] ; then
        	mongo $MONGOS_OPT config --eval "sh.setBalancerState(true)" &> /dev/null
		sleep 10
		bBalancerState=$(getBalancerState)
		if [ "$balancerState" == "false" ] ; then
			log "Error :: Couldn't start the balancer"
			log "Error :: Please check manually !!"
			let NB_ERRORS++
		fi
        else
		log "Warning : Balancer is already running."
		let NB_ERRORS++
	fi
}

## Get balancer status
function getBalancerState {
	mongo $MONGOS_OPT admin --eval "sh.getBalancerState()" | tail -1 
}

## Search in PROCINFO array
function findInArray {
	local sReturnType=$1 sSearch=$2
	[[ "$sReturnType" != "pid" ]] && sSearchType="pid" || sSearchType="name" 
	for (( i=0; i<$(( ${#PROCINFO[@]}/2 )); i++ )) ; do
		if [ ${PROCINFO[$i,$sSearchType]} == $sSearch ] ; then
			echo ${PROCINFO[$i,$sReturnType]}
			return 0
		fi
	done
	return 1
}

## Allow to handle timeout
function timedWait {
	sleep $BACKUP_TIMEOUT &
	iSleepPid=$!
	wait $! >/dev/null 2>&1
}

## Send SIGTERM to remaining childs
function killAllChilds {
	for iPid in "$@"; do
		shift
		log "send SIGTERM to $iPid"
		kill -0 $iPid 2>/dev/null && kill $iPid
	done
}

## Childs process control
function waitChildProcess {
	# Wait for children to exit and indicate whether all exited with 0 status.
	# start the infinite loop
	while :; do
		for iPid in "$@"; do
			shift
			sName=$(findInArray name $iPid) 

			# Test if still present
			if kill -0 "$iPid" 2>/dev/null; then
				# still present, remove the pid from pid list
				set -- "$@" "$iPid"
			# Use wait to get back the return code of the child
			elif wait "$iPid" 2>/dev/null; then
				log "Info: $sName backup completed"
				let COUNT_PROC--
			else
				log "Error: $sName backup failed"
				(( NB_ERRORS++, COUNT_PROC--))
			fi
		done

		# Exit when no child remains
		(("$#" > 0)) || break

		timedWait
		if [ $? -eq 0 ]; then
			# Timeout reached
			log "Timeout... killing remaining child."
			killAllChilds $@
		fi
	done
}

## Called when a child exits (trap)
function childExit {
	kill -0 $iSleepPid 2>/dev/null && kill $iSleepPid
}


###########################################################################
#                              Options checks                             #
###########################################################################

## Get options
while getopts ":hjerm:c:u:p:" opt ; do
        case "$opt" in
		h) usage >&2 ; exit 0;;
		j) DUMP_OPTIONS="--journal" >&2
		;;	
		e) DISPLAY_ERRORS=true >&2
		;;
		m) REPORT_EMAIL=$OPTARG >&2
		;;	
		c) COMPRESS_LVL=$OPTARG >&2
		;;	
		u) USERNAME=$OPTARG >&2
		;;
		p) PASSWORD=$OPTARG >&2 
		;; 	
		\?) echo "Invalid option: -$OPTARG" >&2
		exit 1;;
		:)  echo "Option -$OPTARG requires an argument." >&2
		exit 1;;
	esac
done

## Check if eMail is valid 
if [ "$REPORT_EMAIL" ] && [ -n ${REPORT_EMAIL+x} ] ; then
	CHECK_EMAIL=$(echo $REPORT_EMAIL | egrep "^(([-a-zA-Z0-9\!#\$%\&\'*+/=?^_\`{\|}~])+\.)*[-a-zA-Z0-9\!#\$%\&\'*+/=?^_\`{\|}~]+@\w((-|\w)*\w)*\.(\w((-|\w)*\w)*\.)*\w{2,4}$")
	if [ "x$CHECK_EMAIL" = "x" ]; then 
		echo -e "\nInvalid email format...failed!"
		usage
		exit 1
	fi	
fi

## Check if compression level is correct
if [ "$COMPRESS_LVL" != "normal" ] && [ "$COMPRESS_LVL" != "fast" ] && [ "$COMPRESS_LVL" != "best" ] ; then
	echo -e "\nBad compression level... what did you do ?!"
	echo "Valid values: fast, normal or best"
	usage
	exit 1
fi

# Execute childExit function if child process crash/exit
trap "childExit" CHLD
# Kill all child process if ctrl-c
trap "killAllChilds $PIDS" SIGINT SIGTERM

###########################################################################
#                                   BEGIN                                 #
###########################################################################

isRunning ; iIsRunning=$?

if [ "$iIsRunning" -eq 0 ] ; then
	log "MongoDB backup is already running."
	exit
else
	echo "$$" > $PID_FILE
	START_TIME=$(date +%s)

	if [ ! -d "$BACKUP_DIR" ] ; then
                mkdir -p $LOGS_DIR
        elif [ ! -d $LOGS_DIR ] ; then
                mkdir $LOGS_DIR
        fi

	echo "--------------------------------------------------------------------------------" > $LOG_FILE
	log "MongoDB backup started at $(date +%H:%M:%S)"
	prepareJob
	stopBalancer

	log "Starting backups..."
	mongodump $MONGOS_OPT $DUMP_OPTIONS --db config --out $BACKUP_DIR/config &> /dev/null &	
	PROCINFO[0,pid]=$!
	PROCINFO[0,name]="config"
	DIRECTORIES[0]=${PROCINFO[0,name]}
	PIDS="$PIDS ${PROCINFO[0,pid]}"
	let COUNT_PROC++

	if [ "$NB_SHARD" -ne "0" ] ; then 
		i=1
		for SHARD in ${SHARD_HOSTS[@]} ; do 
			if [[ "$COUNT_PROC" -eq "$MAX_PROC" ]] ; then
				log "Info: Max backup processus exceeded, waiting..."
				waitChildProcess $PIDS
				PIDS=""
			fi

			sTmp=$SHARD
			bIsMaster=$(isMaster $SHARD)
			if [ "$bIsMaster" == "true" ] ; then
				log "${SHARD} is master of replicaSet. Picking a slave host..."
				SHARD=$(pickingSecondary $SHARD)
				[[ "$sTmp" == "$SHARD" ]] && log "${SHARD} is alone in replicaSet. Selects it anyway." || log "${SHARD} has been selected."
			fi
			mongodump --host $SHARD $OPTIONS $DUMP_OPTIONS --oplog --out $BACKUP_DIR/$(eval "echo \$SHARD${i}_NAME") &> /dev/null &	
			PROCINFO[$i,pid]=$!
			PROCINFO[$i,name]=$(eval "echo \$SHARD${i}_NAME")
			DIRECTORIES[$i]=${PROCINFO[$i,name]}
			PIDS="$PIDS ${PROCINFO[$i,pid]}"
			(( i++, COUNT_PROC++ ))
		done
	fi

	# Allow to ease the CPU and let system to consider properly the ongoing process 
	sleep ${WAITALL_DELAY:-1}

	log "Info: Waiting for ending jobs..."
	waitChildProcess $PIDS
	log "Info: Backups completed."

	startBalancer
	finishUp
fi