bin/parse-perfcheck.py

   1 #!/usr/bin/python
   2
   3 # This file is part of the LibreOffice project.
   4 # This Source Code Form is subject to the terms of the Mozilla Public
   5 # License, v. 2.0. If a copy of the MPL was not distributed with this
   6 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   7
   8 import sys
   9 import os
  10 import getopt
  11 import csv
  12
  13
  14 colsResult = {}
  15 allTests = []
  16
  17 def parseFile(dirname, filename, lastCommit):
  18
  19   curTestComment, total = None, None
  20
  21   path = os.path.join(dirname, filename)
  22
  23   trigger = "desc: Trigger: Client Request: "
  24   trigger_len = len(trigger)
  25   totals = "totals: "
  26   totals_len = len(totals)
  27
  28   with open(path,'r') as callgrindFile:
  29      lines = callgrindFile.readlines()
  30
  31      for line in lines:
  32          if line.startswith(trigger):
  33              curTestComment = line[trigger_len:].replace("\n","")
  34          elif line.startswith(totals):
  35              total = line[totals_len:].replace("\n","")
  36
  37   if curTestComment is None or total is None:
  38     return None
  39
  40   testName = os.path.basename(dirname).replace(".test.core","")
  41
  42   lastCommitId, lastCommitDate = lastCommit
  43   if lastCommitId not in colsResult:
  44     colsResult[lastCommitId] = {}
  45     colsResult[lastCommitId]['date'] = lastCommitDate
  46     colsResult[lastCommitId]['values'] = {}
  47
  48   colsResult[lastCommitId]['values'][curTestComment] = total
  49
  50   return [lastCommitId, lastCommitDate, testName, curTestComment, total, filename]
  51
  52 def processDirectory(rootDir, needsCsvHeader, lastCommit):
  53
  54     results = []
  55
  56     if needsCsvHeader:
  57         results.append(["lastCommit", "lastCommitDate", "test filename", "dump comment", "count", "filename"])
  58
  59     for dirName, subdirList, fileList in os.walk(rootDir):
  60         files = [f for f in fileList if f.startswith("callgrind.out.")]
  61         for fname in files:
  62             found = parseFile(dirName, fname, lastCommit)
  63             if found is not None:
  64                 results.append(found)
  65     return results
  66
  67 def getLastCommitInfo():
  68
  69   stream = os.popen("git log --date=iso")
  70   line = stream.readline()
  71   commitId = line.replace("commit ","").replace("\n","")
  72   line = stream.readline()
  73   line = stream.readline()
  74   commitDate = line.replace("Date: ","").replace("\n","").strip()
  75
  76   return commitId, commitDate
  77
  78 def displayUsage():
  79
  80   usage = """
  81
  82 Parses the callgrind results of make perfcheck
  83
  84 Arguments :
  85
  86   --csv-file\t\t the target CSV file - new or containing previous tests - default : perfcheckResult.csv
  87   --source-directory\t directory containing make perfcheck output - default : ./workdir/CppunitTest
  88   --alert-type\t\t mode for calculating alerts - valid values : previous first
  89   --alert-value\t\t alert threshold in % - default = 10
  90
  91   --help\t\t this message
  92
  93 Columned output is dumped into csv-file + ".col"
  94
  95 Alerts, if any, are displayed in standard output
  96
  97 """
  98   print(usage)
  99
 100 class WrongArguments(Exception):
 101     pass
 102
 103 def analyzeArgs(args):
 104
 105     try:
 106         opts, args = getopt.getopt(args, 'x', [
 107             'csv-file=', 'source-directory=', 'alert-type=', 'alert-value=', 'help'])
 108     except getopt.GetoptError:
 109         raise WrongArguments
 110
 111     targetFileName = "perfcheckResult.csv"
 112     sourceDirectory = "./workdir/CppunitTest"
 113     alertType = ""
 114     alertValue = 10
 115
 116     for o, a in opts:
 117         if o == '--help':
 118             displayUsage()
 119             sys.exit()
 120         elif o == "--csv-file":
 121             targetFileName = a
 122         elif o == "--source-directory":
 123             sourceDirectory = a
 124         elif o == "--alert-type":
 125             alertType = a
 126         elif o == "--alert-value":
 127             alertValue = float(a)
 128         else:
 129             raise WrongArguments
 130
 131     return targetFileName, sourceDirectory, alertType, alertValue
 132
 133 def readCsvFile(targetFilename):
 134
 135     with open(targetFilename, 'r') as csvfile:
 136         reader = csv.reader(csvfile, delimiter="\t")
 137         # skip header
 138         next(reader)
 139         for line in reader:
 140
 141             # do not process empty lines
 142             if not line:
 143                 continue
 144
 145             curId, curDate, curTestName, curTestComment, curValue = line
 146
 147             if curTestComment not in allTests:
 148                 allTests.append(curTestComment)
 149
 150             if curId not in colsResult:
 151                 colsResult[curId] = {}
 152                 colsResult[curId]['date'] = curDate
 153                 colsResult[curId]['values'] = {}
 154
 155             colsResult[curId]['values'][curTestComment] = curValue
 156
 157 if __name__ == '__main__':
 158
 159   #check args
 160   try:
 161       targetFileName, sourceDirectory, alertType, alertValue = analyzeArgs(sys.argv[1:])
 162   except WrongArguments:
 163       displayUsage()
 164       sys.exit(1)
 165
 166   # check if sourceDirectory exists
 167   if not os.path.isdir(sourceDirectory):
 168     print("sourceDirectory %s not found - Aborting" % (sourceDirectory))
 169     sys.exit(1)
 170
 171   # read the complete CSV file
 172   if os.path.isfile(targetFileName):
 173     readCsvFile(targetFileName)
 174     needsCsvHeader = False
 175   else:
 176     needsCsvHeader = True
 177
 178   # last commit Id
 179   lastCommitId, lastCommitDate = getLastCommitInfo()
 180
 181   # walker through directory
 182   if lastCommitId not in colsResult:
 183
 184     lastCommit = (lastCommitId, lastCommitDate)
 185     results = processDirectory(sourceDirectory, needsCsvHeader, lastCommit)
 186     ppResults = "\n".join(["\t".join(row) for row in results])
 187
 188     print('\nNew results\n' + ppResults)
 189
 190     # append raw result
 191     with open(targetFileName,'a') as csvfile:
 192         writer = csv.writer(csvfile, delimiter='\t')
 193         writer.writerows(results)
 194         print("\nCSV file written at " + targetFileName + '\n')
 195
 196   else:
 197       print("\nCSV file up to date " + targetFileName + '\n')
 198
 199
 200   # build columned output
 201
 202   # header
 203   mLine = '\t'.join(["commit", "date"] + allTests) + '\n'
 204
 205   alertTest = {}
 206
 207   with open(targetFileName + '.col','w') as fileResult:
 208       for k in colsResult:
 209           mLine += k + "\t" + colsResult[k]['date'] + "\t"
 210           for t in allTests:
 211               if t in colsResult[k]['values']:
 212                   mValue= colsResult[k]['values'][t]
 213                   if t not in alertTest:
 214                       alertTest[t] = {}
 215                   alertTest[t][colsResult[k]['date']] = mValue
 216               else:
 217                   mValue = ""
 218               mLine += mValue + "\t"
 219           mLine += "\n"
 220
 221       # write columned result
 222       fileResult.write(mLine)
 223
 224   print("Columned file written at " + targetFileName + '.col\n')
 225
 226   # check for Alerts
 227
 228   if alertType == "":
 229     sys.exit(1)
 230
 231   alertResult = ""
 232
 233   for t in alertTest:
 234
 235     testDict = alertTest[t]
 236
 237     # sort
 238     keylist = sorted(testDict.keys())
 239     maxVal = float(testDict[keylist[-1]])
 240     minVal = 0
 241
 242     if alertType == "previous":
 243       if len(keylist) > 1:
 244         minVal = float(testDict[keylist[-2]])
 245     else:
 246       minVal = float(testDict[keylist[0]])
 247
 248     if minVal != 0:
 249       delta = 100 * ((maxVal-minVal)/minVal)
 250     else:
 251       delta = 0
 252
 253     if delta > float(alertValue):
 254       alertResult +=  t + "\t" + "{:.2f}".format(delta) + " %\n"
 255
 256   if alertResult != "":
 257     print("!!!!!!!! ALERT !!!!!!!\n")
 258     print(alertResult)