Source

optiCall / opticall / logfileanalyser.py

#!/usr/bin/python

"""
A script to analyse opticall log output
"""


import traceback
import struct
import exceptions


import os
import random

import sys

import histogram



    


def readlogfile(fhandle):
    
    steps_taken = []
    hweps = []
    time_taken = []
    is_rescue = []
    
    rescue_steps_taken = []
    rescue_post_hweps = []
    rescue_time_taken = []
    
    
    
    next = None
    rescued = False
    for line in fhandle:
        
    
        
        #look for the step keyword
        
        
        #next look for the 
        if next == None and line[:11] == "steps taken":
            steps_taken.append(int(line.split()[2]) )
            next = "HWE"
        
        elif next == "HWE":
            hwep = line.split()[2]
            if hwep != 'nan':
                hweps.append(float(hwep) )
            else:
                hweps.append(0.0 )            
            
            next = "POSRES"
        
        elif next == "POSRES" and line[:10] == "Out of HWE":
            rescued = True
            next = "POSRESSTEPS"
        
        elif line[:9] == "line step":
            time_taken.append(float(line.split()[3]) )
            is_rescue.append(rescued)
            next = None
            rescued = False
        
        elif next == "POSRESSTEPS" and line[:18] == "rescue steps taken":
            rescue_steps_taken.append(int(line.split()[3]) )
            
        elif next == "POSRESSTEPS" and line[:16] == "rescue step took":
            rescue_time_taken.append(float(line.split()[3]) )
        
        elif next == "POSRESSTEPS" and line[:10] == "HWEP after":
            hwep = line.split()[4]
            if hwep != 'nan':
                rescue_post_hweps.append(float(hwep) )
            else:
                rescue_post_hweps.append(0.0 )
        
    return steps_taken, hweps, time_taken, is_rescue, rescue_steps_taken, rescue_post_hweps, rescue_time_taken
        
            
    
def filterlist(data,exclude_list, invert=False):    
        
    outputd = []
    for i in xrange(0,len(data)):
        if (exclude_list[i] and not invert) or (not exclude_list[i] and invert) :
            outputd.append(data[i])
    return outputd



if __name__ == '__main__':
    
    
    logfname = sys.argv[1]
    logfhandle = open(logfname,'r')

    steps_taken, hweps, time_taken,is_rescue, rescue_steps_taken, rescue_post_hweps, rescue_time_taken = readlogfile(logfhandle)

    non_resc_steps_taken = filterlist(steps_taken, is_rescue,True)
    non_resc_time_taken = filterlist(time_taken, is_rescue, True)

    resc_time_taken = filterlist(time_taken, is_rescue,False)


    
    #steps taken hist:
    #print "STANDARD CALL STEPS TAKEN HISTOGRAM"
    #histogram.createhistogram(non_resc_steps_taken,buckets=30)
    #print "\n\n\n"
    
    #print "RESCUE CALL STEPS TAKEN HISTOGRAM"
    #histogram.createhistogram(rescue_steps_taken,buckets=30)
    #print "\n\n\n"      
    
    print "STANDARD CALL EXECUTION TIME HISTOGRAM"
    histogram.createhistogram(non_resc_time_taken,buckets=30)
    print "\n\n\n"
    
    print "RESCUE CALL EXECUTION TIME HISTOGRAM"
    histogram.createhistogram(resc_time_taken,buckets=30)
    print "\n\n\n"


    #print "RESCUE CALL HWE P-VALUE HISTOGRAM"
    #histogram.createhistogram(rescue_post_hweps,buckets=30)
    #print "\n\n\n"


    #print "STANDARD CALL HWE P-VALUE HISTOGRAM"
    #histogram.createhistogram(hweps,buckets=30)
    #print "\n\n\n"

    #other stats
    print "SNPS CALLED: " + str(len(time_taken))
    print "RESCUES: " + str((len(rescue_time_taken)))

    suc_rescues = 0
    suc_res_list = []
    for res in rescue_post_hweps:
        if res >= 1e-15:
            suc_rescues += 1
            suc_res_list.append(True)
        else:
            suc_res_list.append(False)


    print "TIME FOR STANDARD CALLS (mins): " + str(sum(non_resc_time_taken)/60)
    print "TIME IN RESCUES (mins): " + str(sum(resc_time_taken)/60) 


    print "SUCCESSFUL RESCUES: " + str((suc_rescues))
    print "PROPORTION OF SUCCESSFUL RESCUES: " + str((suc_rescues/(len(rescue_time_taken)*1.0)))
    

    rescue_time_suc_tuples = []
    for i in xrange(0,len(resc_time_taken)):
        rescue_time_suc_tuples.append((resc_time_taken[i],suc_res_list[i]))
    
    sorted_rescue_time_by_suc = sorted(rescue_time_suc_tuples,key=lambda rescue: rescue[0])
    

    
    
    
    """
    suc = None
    count = 0
    sum = 0
    for item in sorted_rescue_time_by_suc:
        if not(suc == None or suc == item[1]):
            #there's been a switch
            avg_time = (sum*1.0/count)
            result = " unrescued"
            if suc == True:
                result = " rescued"
            print str(count) + result + " SNPs, avg time: " + str(avg_time/60.0)
            count = 0
            sum = 0 
            
        count += 1
        sum += item[0]
        suc = item[1]
    """
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for ProjectModifiedEvent.java.
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.