gpwhere

#!/usr/bin/python -u

# Copyright 2016 Hewlett-Packard Development Company, L.P.
# Use of this script is subject to HP Terms of Use at
# http://www8.hp.com/us/en/privacy/terms-of-use.html.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#    http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import os
import re
import sys
import subprocess
from optparse import OptionParser, OptionGroup

buckets = {}
gran = 1000

def error(text):
    print text
    sys.exit()

def lat_calc(type, lat):

    bucket = int(round(float(lat) * gran))
    #print "TYPE: %s LAT: %s  BKT: %d" % (type, lat, bucket)
    if bucket not in buckets[type]:
	buckets[type][bucket] = 0
    buckets[type][bucket] += 1

def proc_results(results, summary):

    time = ''
    for line in results.split('\n'):
        if debug & 2 and line != '':
            print line
        if line == '':
            break

        # pdsh results start with address in addition to log record so remove
        fields = line.split()
        if options.pdsh:
            del fields[0]

        # looks like the date/time format has changed, not sure which version, but instead
        # of 3 separate fields we now have only 1 so need to use an offset for older vesions.
        offset=0 if len(fields[0])>3 else 2

        if fields[6 + offset] == '--':    # generated by debugging
            continue

        name = fields[1 + offset]
        type = fields[2 + offset][0]
        addr = fields[3 + offset]

        f = name.split('-')
        name = '%s-%s' % (f[2], f[3])

        error = ''
        if re.search('DEBUG', line):
            continue
        if re.search('Quar', line):
            summary[type].append("%s Q" % name)
            continue
        if re.search('ERROR|disconnected', line):
	    error = '???'
            if re.search('100-c', line):
	        error='Expect'
            if re.search('ConnectionTimeout', line):
	        error='T'
            if re.search('disconnected', line):
	        error='D'
            summary[type].append("%s E:%s" % (name, error))
            continue

        time = fields[17 + offset] if type != 'p' else fields[18 + offset]
        if type == 'o':
            mod = '' + error if error == '' else '-' + error
            disk = fields[9 + offset].split('/')[1]
            stat = fields[10 + offset]
            if stat == '404':
                mod += '*'
            
        # really ugly but at least for now we don't care about HEAD
        if (type == 'p' and fields[6 + offset] == 'HEAD') or \
           (type == 'c' and fields[8 + offset] == '"HEAD'):
            continue

        if re.search('ConnectionTimeout', line):
            if re.search('container update', line):
                disk = fields[8 + offset].split('/')[1]
            else:
                disk = fields[7 + offset].split('/')[1]
                error = 'T'
                timeouts.append('%s/%s-%6s' % (name, disk))
                continue

        if re.search('error', line, flags=re.IGNORECASE):
            errors += 1
            continue

        # at least until I understand, ignore object-updater record(s)
        if re.search('object-updater', line):
            continue

        #print "summary[%s] = %s %s" % (type, name, time)
        d = ''
        if type == 'o':
	    d = ':%s%s' % (disk, mod)
        summary[type].append("%s%-10s %s" % (name, d, time))

    return(time)

def proc_txid(txid, end='', tx_time='', tries=''):

    errors = 0
    found = False
    timeouts = []
    summary = {}
    summary['a']=[]
    summary['c']=[]
    summary['o']=[]
    summary['p']=[]

    if options.pdsh:
        errlog = '/tmp/pdsh.err'
        command = 'pdsh -Rssh %s sudo grep %s /var/log/swift/swift.log 2>%s' % (options.pdsh, txid, errlog)
        
        if debug &1:
            print 'Command:', command
        try:
            results = subprocess.check_output(command, shell=True)
            if debug & 2:
                print results
            if results != '':
                found = True
        except:
            pass

        # it looks like when an error is encounted, pdsh prints it on the preceding followed
        # but test like '...exited with exit code xx' which we don't need to see
        with open(errlog, 'r') as f:
            for line in f:
                if re.search('exit code', line):
                    continue
                print line,
        os.remove(errlog)

        time = proc_results(results, summary)

    if options.swift:
        ssh = 'ssh -o StrictHostKeyChecking=no -o BatchMode=yes'
        ssh += ' -o ServerAliveInterval=60' 
        for addr in addrs:
            command = '%s %s sudo grep %s /var/log/swift/swift.log' % (ssh, addr, txid) 
            if debug & 1:        
                print "Command:", command
            try:
                results = subprocess.check_output(command, shell=True)
                if debug & 2:
                    print results
                found = True
            except:
                continue

            time = proc_results(results, summary)

    if not found:
        print "no log records found with txid:", txid
        return

    output = '%s' % txid
    if end != '':
	output += " %s" % end
    if tx_time != '':
        output += '  %s  %s' % (tries, tx_time)
    time += '  '

    last = ''
    for t in ('p', 'a', 'c', 'o'):
        for sum in summary[t]:
            name, time = sum.split()
	    # force disks names to line for when we have more than 9
	    if t == 'o' and re.search('\d+$', name):
		if not re.search('disk\d{2}', name):
		    name += ' '
		name += ' '
	    if output != '':
		if options.vert or (options.vertt and t != last):
	            output += '\n'
	    last = t
            output += "  %s/%s %s" % (t, name, time)

            # only if valid time
            if not re.match('E', time):
                lat_calc(t, time)

    print "%-160s" % output,

    if len(timeouts):
        print "TimeOuts:",
        for disk in timeouts:
            print disk,
    if errors > 0:
        print "Errors:", errors,
        output += " Errors: %s" % errors
    print

parser = OptionParser(add_help_option=False)
parser.add_option('-d', dest='debug', help='debug mask', default='0')
parser.add_option('-h', dest='help', help='display help', action='store_true')
parser.add_option('-l', dest='lat', help='generate latency distribution', action='store_true')
parser.add_option('-s', dest='swift', help='swift server list or file', default='')
parser.add_option('-t', dest='txid',  help='transaction id or file', default='')
parser.add_option('-v', dest='vert',  help='one server per line', action='store_true')
parser.add_option('-V', dest='vertt', help='one server type per line', action='store_true')
parser.add_option('-w', dest='pdsh',  help='psdh -w string OR file', default='')

try:
    (options, args) = parser.parse_args(sys.argv[1:])
except:
    print 'invalid command'
    sys.exit()

if options.help:
    parser.print_help()
    sys.exit()

if options.swift and options.pdsh:
    print "-s and -w are mutually exclusive"
    sys.exit()

if not options.swift and not options.pdsh:
    print "-s OR -w is required"
    sys.exit()

debug = int(options.debug)

if options.txid == '':
    error('-t required')

for t in ('a', 'c', 'o', 'p'):
    buckets[t] = {}

addrs = []
if options.swift:
    if os.path.isfile(options.swift):
        f = open(options.swift, 'r')
        for addr in f:
            addrs.append(addr[:-1])
    else:
        addrs = options.swift.split(',')
else:
    try:
        f = open(options.pdsh, 'r')
        options.pdsh = f.readline()[0:-1]
        f.close()
    except:
        error("can't open: %s" % options.pdsh)

if not os.path.isfile(options.txid):
    proc_txid(options.txid)
    sys.exit()

with open(options.txid, 'r') as f:
    for line in f:
        line = line[0:-1]
        if line == '':
            break

        fields = line.split()

        # if output from --latexc, get time and txid
        if re.match('Host', line):
	    end = fields[5]
            time = fields[9]
            txid = fields[14]
            tries = fields[16]

        # assume we're getting it in --exclog format
        else:
	    end = fields[1]
            time = fields[5]
            txid = fields[10]
            tries = fields[12]
        proc_txid(txid, end, time, tries)

if options.lat:
    for i in ('p', 'a', 'c', 'o'):
	#print "%s: %s" % (i, buckets[i])
	print "Bucket %s" % i,
        for j in sorted(buckets[i]):
            latency = "  %4d:%d" % (j, buckets[i][j])
	    print " %-7s" % latency,
	print