hdfspipe.in

#!/bin/bash

#############################################
#    Copyright (C) 2012 HPCC Systems.
#
#    All rights reserved. This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with All rights reserved. This program is free software: you can redistribute program.  If not, see <http://www.gnu.org/licenses/>.
#############################################

#DO NOT MODIFY THIS FILE!
#CONFIGURATION CHANGES CAN BE DONE HERE: /etc/HPCCSystems/@HDFSCONN_CONF_FILE@

source @HPCC_ETC_DIR@/init.d/hpcc_common
set_environmentvars

source $configs/@HDFSCONN_CONF_FILE@

#Since batch doesn't allow simple decimal algebra, we have to parse
#the target hadoop version
OLDIFS=$IFS;
IFS='.';
VER_TOKS=( $HADOOP_VERSION )
IFS=$OLDIFS;

CLASSPATH=$CLASSPATH:$HADOOP_CONF_DIR

###################################################
# CLASSPATH set up for libhdfs
# One of the most common problems with libhdfs is the CLASSPATH is not set properly when calling a program that uses libhdfs.
# per Hadoop: http://hadoop.apache.org/docs/r2.3.0/hadoop-project-dist/hadoop-hdfs/LibHdfs.html
# Make sure you set it to all the Hadoop jars needed to run Hadoop itself.
# Currently, there is no way to programmatically generate the classpath, but a good bet is to include all the jar files in
# $HADOOP_PREFIX and $HADOOP_PREFIX/lib as well as the right configuration directory containing hdfs-site.xml
# However as it turns out, the jars are in different subfolders under the share directory
#
###################################################
if [ -n "$TARBALL_HADOOP_LOCATION"  ]; then
    CLASSPATH=$CLASSPATH:$TARBALL_HADOOP_LOCATION/conf

    for f in $TARBALL_HADOOP_LOCATION/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done

    for f in $TARBALL_HADOOP_LOCATION/lib/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done

    if [ ${VER_TOKS[0]} -ge "2" ] && [ ${VER_TOKS[1]} -ge 5 ]; then
        for f in $TARBALL_HADOOP_LOCATION/share/hadoop/common/*.jar ; do
           CLASSPATH=$CLASSPATH:$f;
        done

        for f in $TARBALL_HADOOP_LOCATION/share/hadoop/common/lib/*.jar ; do
           CLASSPATH=$CLASSPATH:$f;
        done

        for f in $TARBALL_HADOOP_LOCATION/share/hadoop/hdfs/*.jar ; do
           CLASSPATH=$CLASSPATH:$f;
        done

        for f in $TARBALL_HADOOP_LOCATION/share/hadoop/hdfs/lib/*.jar ; do
           CLASSPATH=$CLASSPATH:$f;
        done

        for f in $TARBALL_HADOOP_LOCATION/share/hadoop/yarn/*.jar ; do
           CLASSPATH=$CLASSPATH:$f;
        done

        for f in $TARBALL_HADOOP_LOCATION/share/hadoop/yarn/lib/*.jar ; do
           CLASSPATH=$CLASSPATH:$f;
        done

        for f in $TARBALL_HADOOP_LOCATION/share/hadoop/mapreduce/*.jar ; do
           CLASSPATH=$CLASSPATH:$f;
        done

        for f in $TARBALL_HADOOP_LOCATION/share/hadoop/mapreduce/lib/*.jar ; do
           CLASSPATH=$CLASSPATH:$f;
        done
    fi
else
    for f in $HADOOP_SHARE_DIR/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done

    for f in $HADOOP_SHARE_DIR/lib/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done

    for f in $HADOOP_SHARE_DIR/hadoop/hdfs/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done

    for f in $HADOOP_SHARE_DIR/hadoop/hdfs/lib/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done

    for f in $HADOOP_SHARE_DIR/hadoop/common/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done

    for f in $HADOOP_SHARE_DIR/hadoop/common/lib/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done

    for f in $HADOOP_SHARE_DIR/hadoop/tools/lib/*.jar ; do
        CLASSPATH=$CLASSPATH:$f
    done
fi

export CLASSPATH=$CLASSPATH

PID=$$

nodeid=0;
wuid=0;
for ((paramindex=1; paramindex <= $#; paramindex++ ));
 do
   if [ "${!paramindex}" = "-wuid" ];
   then
        ((paramindex++));
        wuid=${!paramindex};
   elif [ "${!paramindex}" = "-nodeid" ];
   then
        ((paramindex++));
        nodeid=${!paramindex};
   fi
done;

#the log variable is read from the HPCC Platform config
LOGS_LOCATION=$log
HDFSCONNLOGLOC=$LOGS_LOCATION/mydataconnectors
LOG=$HDFSCONNLOGLOC/@HDFS_CONNECTOR_TYPE@.$nodeid.$PID.$wuid.log
PIPERRLOG=$HDFSCONNLOGLOC/HPCC-FIFO.err.$PID

if [ -e $HDFSCONNLOGLOC ]
  then
    echo "log file found"           >> $LOG
  else
    mkdir $HDFSCONNLOGLOC
fi
username=`id -nu`
echo "Script starting"               >> $LOG
echo "PID: $PID"                     >> $LOG
echo "User: $username"               >> $LOG
echo 'Incoming params: '             >> $LOG
echo $(printf '%q ' ${@})            >> $LOG
echo "NodeId: $nodeid"               >> $LOG
echo "WUID: $wuid"                   >> $LOG

h2hpid=0;
h2hstatus=0;

TARGETCONNECTORNAME="@HDFSCONN_EXE_NAME@";

if [ "$1" = "" ];
then
    echo "Error: No input params detected!" >> $LOG
    exit 1;
elif [ $1 = "-mf" ];
then
    $TARGETCONNECTORNAME ${@}      2>> $LOG;
    h2hstatus=$?;
    h2hpid=$!;
elif [ $1 = "-si" ];
then
    $TARGETCONNECTORNAME  ${@}      2>> $LOG;
    h2hstatus=$?
    h2hpid=$!;
elif [ $1 = "-so" ];
then
    HPCCTMPFILE=/tmp/HPCCTMPFILE-$nodeid-$PID-$wuid;
    if [ -f $HPCCTMPFILE ]
    then
        rm "$HPCCTMPFILE" 			2>> $LOG
        echo "deleted tmp file" 		>> $LOG
    else
        echo "$HPCCTMPFILE does not exist"    >> $LOG
    fi

    echo "redirecting stdin to tmp file ... "   >> $LOG
    cat /dev/stdin > "$HPCCTMPFILE"

	ls -l "$HPCCTMPFILE"                    >> $LOG

    echo "calling hdfsconnector..."         >> $LOG

    $TARGETCONNECTORNAME ${@} -pipepath $HPCCTMPFILE  	2>> $LOG
    h2hpid=$!;
    h2hstatus=$?;
elif [ $1 = "-sop" ];
then
    pipepath=/tmp/HPCCPIPE-$nodeid-$PID-$wuid;
    mkfifo $pipepath 2> $PIPERRLOG;
    chmod 777 $pipepath 2> $PIPERRLOG;

    echo "mkfifo $pipepath setup ..."           >> $LOG

    if test ! -s "$PIPERRLOG"
    then
        rm -f $PIPERRLOG 2> /dev/null
    else
        echo "  WARNING (hdfsconnector mkfifo) error registered in file: $PIPERRLOG " >> $LOG
        exit 1
    fi
    $TARGETCONNECTORNAME  ${@} -pipepath $pipepath	2>> $LOG &

    h2hpid=$!;

    echo h2h pid: ${h2hpid} >> $LOG
    echo "redirecting stdin to named pipe ... "    >> $LOG

    cat < /dev/stdin > "$pipepath" &
    ls -l "$pipepath"                               >> $LOG
    echo "Waiting on hdfsconnector PID ${h2hpid}"   >> $LOG
    wait ${h2hpid};
    h2hstatus=$?;
    echo "Finished waiting on hdfsconnector exit status: ${h2hstatus}" >> $LOG

    if [ ${h2hstatus} -ne 0 ];
    then
        echo "Closing pipe due to hdfsconnector error" >> $LOG
        exec  <&- 2>> $LOG
        exec $pipepath <&- 2>> $LOG
    fi
    rm -f $pipepath 2>> $LOG
else
    echo "Error: check your params."             >> $LOG;
    h2hstatus=1;
fi

if [ ${h2hstatus} -ne 0 ]
then
    if [ "$TARGETCONNECTORNAME" = "libhdfsconnector" ]
    then
    echo ""                                                                                              >> $LOG
        if [ -z "$TARBALL_HADOOP_LOCATION" ] && [ -z "$HADOOP_SHARE_DIR" ]
        then
           echo "WARNING: both TARBALL_HADOOP_LOCATION and HADOOP_SHARE_DIR are empty"                   >> $LOG
           echo "This could cause incomplete classpath setup which is necessary to use"                  >> $LOG
		fi

		if [ -n "$TARBALL_HADOOP_LOCATION" ] && [ ! -d "$TARBALL_HADOOP_LOCATION" ]
        then
           echo "WARNING: The specified TARBALL_HADOOP_LOCATION value is invalid: $TARBALL_HADOOP_LOCATION"  >> $LOG
           echo "This value might be necessary for running libhdfs."                                     >> $LOG
        fi

        if [ -n "$HADOOP_SHARE_DIR" ] && [ ! -d "$HADOOP_SHARE_DIR" ]
        then
           echo "WARNING: The specified HADOOP_SHARE_DIR value is invalid: $HADOOP_SHARE_DIR"            >> $LOG
           echo "This value might be necessary for running libhdfs."                                     >> $LOG
        fi

        echo "" >> $LOG
        echo "A common error running h2h, is inaccesible pre-requisites."                                >> $LOG

        PATHSEP=":";
        arr=$(echo $LD_LIBRARY_PATH | tr "$PATHSEP" "\n")

        for v in $arr
        do
            if [ -n "$v" ] && [ ! -d "$v" ]
            then
               echo "WARNING: detected an invalid entry in LD_LIBRARY_PATH: $v"                          >> $LOG
               echo "Is this entry necessary to run H2H?"                                                >> $LOG
            fi
        done

        echo ""                                                                                          >> $LOG
        echo "Please verify all configuration entries in $configs/@HDFSCONN_CONF_FILE@"                  >> $LOG

    echo ""                                                                                              >> $LOG
	fi
fi

echo "$TARGETCONNECTORNAME exited with: ${h2hstatus}"            >> $LOG

exit ${h2hstatus};