cluster_tools/computeEnlapsedTime.py

   1 import sys
   2 import os
   3 import datetime
   4 import numpy
   5
   6
   7 def usage():
   8     print("Compute the enlapsed time for simulation with submitted, started and ended time")
   9     print("in ParentFolderName and its subfolder and starting by FileName.")
  10     print("The output is in statJobs.txt.")
  11     print("Usage:")
  12     print("python computeEnlapseTyme.py ParentFolderName FileName")
  13     print("eg: python computeEnlapseTyme.py spect_phs_build gate")
  14     return
  15
  16 def is_number(s):
  17     try:
  18         int(s)
  19         return True
  20     except ValueError:
  21         pass
  22
  23 def createDate(s):
  24     #Split the line to find the different element of the date
  25     words = s.split(' ')
  26     submittedDate = datetime.datetime(year=2000, month=1, day=1, hour=0, minute=0, second=0)
  27     for word in words:
  28         if is_number(word):
  29             #Find the year
  30             if int(word) > 2000:
  31                 submittedDate = submittedDate.replace(year=int(word))
  32             #Find the day
  33             elif int(word) < 32:
  34                 submittedDate = submittedDate.replace(day=int(word))
  35         #Find the month
  36         elif word == "Jan":
  37             submittedDate = submittedDate.replace(month=1)
  38         elif word == "Feb":
  39             submittedDate = submittedDate.replace(month=2)
  40         elif word == "Mar":
  41             submittedDate = submittedDate.replace(month=3)
  42         elif word == "Apr":
  43             submittedDate = submittedDate.replace(month=4)
  44         elif word == "May":
  45             submittedDate = submittedDate.replace(month=5)
  46         elif word == "Jun":
  47             submittedDate = submittedDate.replace(month=6)
  48         elif word == "Jul":
  49             submittedDate = submittedDate.replace(month=7)
  50         elif word == "Aug":
  51             submittedDate = submittedDate.replace(month=8)
  52         elif word == "Sep":
  53             submittedDate = submittedDate.replace(month=9)
  54         elif word == "Oct":
  55             submittedDate = submittedDate.replace(month=10)
  56         elif word == "Nov":
  57             submittedDate = submittedDate.replace(month=11)
  58         elif word == "Dec":
  59             submittedDate = submittedDate.replace(month=12)
  60         #Find the hour:minute:second
  61         elif word.count(':') == 2:
  62             times = word.split(':')
  63             submittedDate = submittedDate.replace(hour=int(times[0]))
  64             submittedDate = submittedDate.replace(minute=int(times[1]))
  65             submittedDate = submittedDate.replace(second=int(times[2]))
  66     return submittedDate
  67
  68 def computeEnlapsedTime():
  69     #Get the folder name ad the filename
  70     folderName = sys.argv[1]
  71     fileName = sys.argv[2]
  72
  73     #Look for all files starting with the fileName
  74     filePaths = []
  75     for root, dirs, files in os.walk(folderName):
  76         for file in files:
  77             if file.startswith(fileName):
  78                 filePaths.append(os.path.join(root, file))
  79
  80     #For each file into filePaths, save the submitted, started end ended time
  81     submittedTime = []
  82     startedTime = []
  83     endedTime = []
  84     for file in filePaths:
  85         for line in open(file):
  86             if "Submitted" in line:
  87                 submittedTime += [createDate(line)]
  88             if "Started" in line:
  89                 startedTime += [createDate(line)]
  90             if "Ended" in line:
  91                 endedTime += [createDate(line)]
  92
  93     #Computation time
  94     computationTime = [a - b for a, b in zip(endedTime, startedTime)]
  95     computationTimeSecond = numpy.array([time.total_seconds() for time in computationTime])
  96     waitingTime = [a - b for a, b in zip(endedTime, submittedTime)]
  97     waitingTimeSecond = numpy.array([time.total_seconds() for time in waitingTime])
  98     computePart =computationTimeSecond / waitingTimeSecond * 100.0
  99
 100     #Write the output
 101     with open('statJobs.txt', 'w') as output:
 102         output.write(str(len(filePaths)) + " run jobs\n\n")
 103         output.write("Starting submission at " + str(min(submittedTime)) + "\n")
 104         output.write("Finishing submission at " + str(max(submittedTime)) + "\n\n")
 105         output.write("First job started computation at " + str(min(startedTime)) + "\n")
 106         output.write("Last job started computation at " + str(max(startedTime)) + "\n\n")
 107         output.write("First job ended computation at " + str(min(endedTime)) + "\n")
 108         output.write("Last job ended computation at " + str(max(endedTime)) + "\n\n")
 109         output.write("Min computation time " + str(min(computationTime)) + "\n")
 110         output.write("Max computation time " + str(max(computationTime)) + "\n")
 111         output.write("Mean computation time " + str(datetime.timedelta(seconds=numpy.mean(computationTimeSecond))) + "\n")
 112         output.write("Std computation time " + str(datetime.timedelta(seconds=numpy.std(computationTimeSecond))) + "\n\n")
 113         output.write("Min waiting time " + str(min(waitingTime)) + "\n")
 114         output.write("Max waiting time " + str(max(waitingTime)) + "\n")
 115         output.write("Mean waiting time " + str(datetime.timedelta(seconds=numpy.mean(waitingTimeSecond))) + "\n")
 116         output.write("Std waiting " + str(datetime.timedelta(seconds=numpy.std(waitingTimeSecond))) + "\n\n")
 117         output.write("Min efficiency "  + str(min(computePart)) + " %\n")
 118         output.write("Max efficiency " + str(max(computePart)) + " %\n")
 119         output.write("Mean efficiency " + str(numpy.mean(computePart)) + " %\n")
 120         output.write("Std efficiency " + str(numpy.std(computePart)) + " %\n\n")
 121         output.write("Speed up " + str(numpy.mean(computationTimeSecond)*len(filePaths)/max(waitingTimeSecond)) + "\n\n")
 122
 123     return
 124
 125 if __name__ == "__main__":
 126     if len(sys.argv) != 3:
 127         usage()
 128     else:
 129         computeEnlapsedTime()