lld/utils/benchmark.py

   1 #!/usr/bin/env python
   2 #
   3 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 # See https://llvm.org/LICENSE.txt for license information.
   5 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 #
   7 # ==------------------------------------------------------------------------==#
   8
   9 import os
  10 import glob
  11 import re
  12 import subprocess
  13 import json
  14 import datetime
  15 import argparse
  16 try:
  17     from urllib.parse import urlencode
  18     from urllib.request import urlopen, Request
  19 except ImportError:
  20     from urllib import urlencode
  21     from urllib2 import urlopen, Request
  22
  23
  24 parser = argparse.ArgumentParser()
  25 parser.add_argument('benchmark_directory')
  26 parser.add_argument('--runs', type=int, default=10)
  27 parser.add_argument('--wrapper', default='')
  28 parser.add_argument('--machine', required=True)
  29 parser.add_argument('--revision', required=True)
  30 parser.add_argument('--threads', action='store_true')
  31 parser.add_argument('--url', help='The lnt server url to send the results to',
  32                     default='http://localhost:8000/db_default/v4/link/submitRun')
  33 args = parser.parse_args()
  34
  35 class Bench:
  36     def __init__(self, directory, variant):
  37         self.directory = directory
  38         self.variant = variant
  39     def __str__(self):
  40         if not self.variant:
  41             return self.directory
  42         return '%s-%s' % (self.directory, self.variant)
  43
  44 def getBenchmarks():
  45     ret = []
  46     for i in glob.glob('*/response*.txt'):
  47         m = re.match('response-(.*)\.txt', os.path.basename(i))
  48         variant = m.groups()[0] if m else None
  49         ret.append(Bench(os.path.dirname(i), variant))
  50     return ret
  51
  52 def parsePerfNum(num):
  53     num = num.replace(b',',b'')
  54     try:
  55         return int(num)
  56     except ValueError:
  57         return float(num)
  58
  59 def parsePerfLine(line):
  60     ret = {}
  61     line = line.split(b'#')[0].strip()
  62     if len(line) != 0:
  63         p = line.split()
  64         ret[p[1].strip().decode('ascii')] = parsePerfNum(p[0])
  65     return ret
  66
  67 def parsePerf(output):
  68     ret = {}
  69     lines = [x.strip() for x in output.split(b'\n')]
  70
  71     seconds = [x for x in lines if b'seconds time elapsed' in x][0]
  72     seconds = seconds.strip().split()[0].strip()
  73     ret['seconds-elapsed'] = parsePerfNum(seconds)
  74
  75     measurement_lines = [x for x in lines if b'#' in x]
  76     for l in measurement_lines:
  77         ret.update(parsePerfLine(l))
  78     return ret
  79
  80 def run(cmd):
  81     try:
  82         return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
  83     except subprocess.CalledProcessError as e:
  84         print(e.output)
  85         raise e
  86
  87 def combinePerfRun(acc, d):
  88     for k,v in d.items():
  89         a = acc.get(k, [])
  90         a.append(v)
  91         acc[k] = a
  92
  93 def perf(cmd):
  94     # Discard the first run to warm up any system cache.
  95     run(cmd)
  96
  97     ret = {}
  98     wrapper_args = [x for x in args.wrapper.split(',') if x]
  99     for i in range(args.runs):
 100         os.unlink('t')
 101         out = run(wrapper_args + ['perf', 'stat'] + cmd)
 102         r = parsePerf(out)
 103         combinePerfRun(ret, r)
 104     os.unlink('t')
 105     return ret
 106
 107 def runBench(bench):
 108     thread_arg = [] if args.threads else ['--no-threads']
 109     os.chdir(bench.directory)
 110     suffix = '-%s' % bench.variant if bench.variant else ''
 111     response = 'response' + suffix + '.txt'
 112     ret = perf(['../ld.lld', '@' + response, '-o', 't'] + thread_arg)
 113     ret['name'] = str(bench)
 114     os.chdir('..')
 115     return ret
 116
 117 def buildLntJson(benchmarks):
 118     start = datetime.datetime.utcnow().isoformat()
 119     tests = [runBench(b) for b in benchmarks]
 120     end = datetime.datetime.utcnow().isoformat()
 121     ret = {
 122         'format_version' : 2,
 123         'machine' : { 'name' : args.machine },
 124         'run' : {
 125             'end_time' : start,
 126             'start_time' : end,
 127             'llvm_project_revision': args.revision
 128         },
 129         'tests' : tests
 130     }
 131     return json.dumps(ret, sort_keys=True, indent=4)
 132
 133 def submitToServer(data):
 134     data2 = urlencode({ 'input_data' : data }).encode('ascii')
 135     urlopen(Request(args.url, data2))
 136
 137 os.chdir(args.benchmark_directory)
 138 data = buildLntJson(getBenchmarks())
 139 submitToServer(data)