framework/core.py

   1 #!/usr/bin/env python
   2 #
   3 # Permission is hereby granted, free of charge, to any person
   4 # obtaining a copy of this software and associated documentation
   5 # files (the "Software"), to deal in the Software without
   6 # restriction, including without limitation the rights to use,
   7 # copy, modify, merge, publish, distribute, sublicense, and/or
   8 # sell copies of the Software, and to permit persons to whom the
   9 # Software is furnished to do so, subject to the following
  10 # conditions:
  11 #
  12 # This permission notice shall be included in all copies or
  13 # substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  16 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  17 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  18 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  19 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  20 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  21 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22 # DEALINGS IN THE SOFTWARE.
  23
  24 # Piglit core
  25
  26 import errno
  27 import os
  28 import platform
  29 import stat
  30 import subprocess
  31 import sys
  32 import time
  33 import traceback
  34
  35 __all__ = [
  36         'Environment',
  37         'checkDir',
  38         'loadTestProfile',
  39         'TestrunResult',
  40         'GroupResult',
  41         'TestResult',
  42         'TestProfile',
  43         'Group',
  44         'Test',
  45         'testBinDir'
  46 ]
  47
  48
  49 #############################################################################
  50 ##### Helper functions
  51 #############################################################################
  52
  53 # Ensure the given directory exists
  54 def checkDir(dirname, failifexists):
  55         exists = True
  56         try:
  57                 os.stat(dirname)
  58         except OSError, e:
  59                 if e.errno == errno.ENOENT or e.errno == errno.ENOTDIR:
  60                         exists = False
  61
  62         if exists and failifexists:
  63                 print >>sys.stderr, "%(dirname)s exists already.\nUse --overwrite if you want to overwrite it.\n" % locals()
  64                 exit(1)
  65
  66         try:
  67                 os.makedirs(dirname)
  68         except OSError, e:
  69                 if e.errno != errno.EEXIST:
  70                         raise
  71
  72 # Encode a string
  73 def encode(text):
  74         return text.encode("string_escape")
  75
  76 def decode(text):
  77         return text.decode("string_escape")
  78
  79 testBinDir = os.path.dirname(__file__) + '/../bin/'
  80
  81
  82 #############################################################################
  83 ##### Result classes
  84 #############################################################################
  85
  86 class TestResult(dict):
  87         def __init__(self, *args):
  88                 dict.__init__(self)
  89
  90                 assert(len(args) == 0 or len(args) == 2)
  91
  92                 if len(args) == 2:
  93                         for k in args[0]:
  94                                 self.__setattr__(k, args[0][k])
  95
  96                         self.update(args[1])
  97
  98         def __repr__(self):
  99                 attrnames = set(dir(self)) - set(dir(self.__class__()))
 100                 return '%(class)s(%(dir)s,%(dict)s)' % {
 101                         'class': self.__class__.__name__,
 102                         'dir': dict([(k, self.__getattribute__(k)) for k in attrnames]),
 103                         'dict': dict.__repr__(self)
 104                 }
 105
 106         def allTestResults(self, name):
 107                 return {name: self}
 108
 109         def write(self, file, path):
 110                 print >>file, "@test: " + encode(path)
 111                 for k in self:
 112                         v = self[k]
 113                         if type(v) == list:
 114                                 print >>file, k + "!"
 115                                 for s in v:
 116                                         print >>file, " " + encode(str(s))
 117                                 print >>file, "!"
 118                         else:
 119                                 print >>file, k + ": " + encode(str(v))
 120                 print >>file, "!"
 121
 122
 123 class GroupResult(dict):
 124         def __init__(self, *args):
 125                 dict.__init__(self)
 126
 127                 assert(len(args) == 0 or len(args) == 2)
 128
 129                 if len(args) == 2:
 130                         for k in args[0]:
 131                                 self.__setattr__(k, args[0][k])
 132
 133                         self.update(args[1])
 134
 135         def __repr__(self):
 136                 attrnames = set(dir(self)) - set(dir(self.__class__()))
 137                 return '%(class)s(%(dir)s,%(dict)s)' % {
 138                         'class': self.__class__.__name__,
 139                         'dir': dict([(k, self.__getattribute__(k)) for k in attrnames]),
 140                         'dict': dict.__repr__(self)
 141                 }
 142
 143         def allTestResults(self, groupName):
 144                 collection = {}
 145                 for name, sub in self.items():
 146                         subfullname = name
 147                         if len(groupName) > 0:
 148                                 subfullname = groupName + '/' + subfullname
 149                         collection.update(sub.allTestResults(subfullname))
 150                 return collection
 151
 152         def write(self, file, groupName):
 153                 for name, sub in self.items():
 154                         subfullname = name
 155                         if len(groupName) > 0:
 156                                 subfullname = groupName + '/' + subfullname
 157                         sub.write(file, subfullname)
 158
 159
 160 class TestrunResult:
 161         def __init__(self, *args):
 162                 self.name = ''
 163                 self.globalkeys = ['name', 'href', 'glxinfo', 'lspci', 'time']
 164                 self.results = GroupResult()
 165
 166         def allTestResults(self):
 167                 '''Return a dictionary containing (name: TestResult) mappings.
 168                 Note that writing to this dictionary has no effect.'''
 169                 return self.results.allTestResults('')
 170
 171         def write(self, file):
 172                 for key in self.globalkeys:
 173                         if key in self.__dict__:
 174                                 print >>file, "%s: %s" % (key, encode(self.__dict__[key]))
 175
 176                 self.results.write(file,'')
 177
 178         def parseFile(self, file):
 179                 def arrayparser(a):
 180                         def cb(line):
 181                                 if line == '!':
 182                                         del stack[-1]
 183                                 else:
 184                                         a.append(line[1:])
 185                         return cb
 186
 187                 def dictparser(d):
 188                         def cb(line):
 189                                 if line == '!':
 190                                         del stack[-1]
 191                                         return
 192
 193                                 colon = line.find(':')
 194                                 if colon < 0:
 195                                         excl = line.find('!')
 196                                         if excl < 0:
 197                                                 raise Exception("Line %(linenr)d: Bad format" % locals())
 198
 199                                         key = line[:excl]
 200                                         d[key] = []
 201                                         stack.append(arrayparser(d[key]))
 202                                         return
 203
 204                                 key = line[:colon]
 205                                 value = decode(line[colon+2:])
 206                                 d[key] = value
 207                         return cb
 208
 209                 def toplevel(line):
 210                         colon = line.find(':')
 211                         if colon < 0:
 212                                 raise Exception("Line %(linenr)d: Bad format" % locals())
 213
 214                         key = line[:colon]
 215                         value = decode(line[colon+2:])
 216                         if key in self.globalkeys:
 217                                 self.__dict__[key] = value
 218                         elif key == '@test':
 219                                 comp = value.split('/')
 220                                 group = self.results
 221                                 for name in comp[:-1]:
 222                                         if name not in group:
 223                                                 group[name] = GroupResult()
 224                                         group = group[name]
 225
 226                                 result = TestResult()
 227                                 group[comp[-1]] = result
 228
 229                                 stack.append(dictparser(result))
 230                         else:
 231                                 raise Exception("Line %d: Unknown key %s" % (linenr, key))
 232
 233                 stack = [toplevel]
 234                 linenr = 1
 235                 for line in file:
 236                         if line[-1] == '\n':
 237                                 stack[-1](line[0:-1])
 238                         linenr = linenr + 1
 239
 240         def parseDir(self, path, PreferSummary):
 241                 main = None
 242                 filelist = [path + '/main', path + '/summary']
 243                 if PreferSummary:
 244                         filelist[:0] = [path + '/summary']
 245                 for filename in filelist:
 246                         try:
 247                                 main = open(filename, 'U')
 248                                 break
 249                         except:
 250                                 pass
 251                 if not main:
 252                         raise Exception("Failed to open %(path)s" % locals())
 253                 self.parseFile(main)
 254                 main.close()
 255
 256
 257 #############################################################################
 258 ##### Generic Test classes
 259 #############################################################################
 260
 261 class Environment:
 262         def __init__(self):
 263                 self.file = sys.stdout
 264                 self.execute = True
 265                 self.filter = []
 266                 self.exclude_filter = []
 267
 268         def run(self, command):
 269                 try:
 270                         p = subprocess.Popen(
 271                                 command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 272                         (stdout, stderr) = p.communicate()
 273                 except:
 274                         return "Failed to run " + command
 275                 return stderr+stdout
 276
 277         def collectData(self):
 278                 if platform.system() != 'Windows':
 279                         print >>self.file, "glxinfo:", '@@@' + encode(self.run('glxinfo'))
 280                 if platform.system() == 'Linux':
 281                         print >>self.file, "lspci:", '@@@' + encode(self.run('lspci'))
 282
 283
 284 class Test:
 285         ignoreErrors = []
 286         sleep = 0
 287
 288         def __init__(self):
 289                 pass
 290
 291         def run(self):
 292                 raise NotImplementedError
 293
 294         def doRun(self, env, path):
 295                 # Exclude tests that don't match the filter regexp
 296                 if len(env.filter) > 0:
 297                         if not True in map(lambda f: f.search(path) != None, env.filter):
 298                                 return None
 299
 300                 # And exclude tests that do match the exclude_filter regexp
 301                 if len(env.exclude_filter) > 0:
 302                         if True in map(lambda f: f.search(path) != None, env.exclude_filter):
 303                                 return None
 304                 # Run the test
 305                 if env.execute:
 306                         try:
 307                                 print "Test: %(path)s" % locals()
 308                                 time_start = time.time()
 309                                 result = self.run()
 310                                 time_end = time.time()
 311                                 if 'time' not in result:
 312                                         result['time'] = time_end - time_start
 313                                 if 'result' not in result:
 314                                         result['result'] = 'fail'
 315                                 if not isinstance(result, TestResult):
 316                                         result = TestResult({}, result)
 317                                         result['result'] = 'warn'
 318                                         result['note'] = 'Result not returned as an instance of TestResult'
 319                         except:
 320                                 result = TestResult()
 321                                 result['result'] = 'fail'
 322                                 result['exception'] = str(sys.exc_info()[0]) + str(sys.exc_info()[1])
 323                                 result['traceback'] = '@@@' + "".join(traceback.format_tb(sys.exc_info()[2]))
 324
 325                         if result['result'] != 'pass':
 326                                 print "    result: %(result)s" % { 'result': result['result'] }
 327
 328                         result.write(env.file, path)
 329                         if Test.sleep:
 330                                 time.sleep(Test.sleep)
 331                 else:
 332                         print "Dry-run: %(path)s" % locals()
 333
 334         # Returns True iff the given error message should be ignored
 335         def isIgnored(self, error):
 336                 for pattern in Test.ignoreErrors:
 337                         if pattern.search(error):
 338                                 return True
 339
 340                 return False
 341
 342         # Default handling for stderr messages
 343         def handleErr(self, results, err):
 344                 errors = filter(lambda s: len(s) > 0, map(lambda s: s.strip(), err.split('\n')))
 345
 346                 ignored = [s for s in errors if self.isIgnored(s)]
 347                 errors = [s for s in errors if s not in ignored]
 348
 349                 if len(errors) > 0:
 350                         results['errors'] = errors
 351
 352                         if results['result'] == 'pass':
 353                                 results['result'] = 'warn'
 354
 355                 if len(ignored) > 0:
 356                         results['errors_ignored'] = ignored
 357
 358
 359 class Group(dict):
 360         def doRun(self, env, path):
 361                 for sub in sorted(self):
 362                         spath = sub
 363                         if len(path) > 0:
 364                                 spath = path + '/' + spath
 365                         self[sub].doRun(env, spath)
 366
 367
 368 class TestProfile:
 369         def __init__(self):
 370                 self.tests = Group()
 371                 self.sleep = 0
 372
 373         def run(self, env):
 374                 time_start = time.time()
 375                 self.tests.doRun(env, '')
 376                 time_end = time.time()
 377                 print >>env.file, "time:",(time_end-time_start)
 378
 379 #############################################################################
 380 ##### Loaders
 381 #############################################################################
 382
 383 def loadTestProfile(filename):
 384         try:
 385                 ns = {
 386                         '__file__': filename
 387                 }
 388                 execfile(filename, ns)
 389                 return ns['profile']
 390         except:
 391                 traceback.print_exc()
 392                 raise Exception('Could not read tests profile')
 393
 394 def loadTestResults(path, PreferSummary=False):
 395         try:
 396                 mode = os.stat(path)[stat.ST_MODE]
 397                 testrun = TestrunResult()
 398                 if stat.S_ISDIR(mode):
 399                         testrun.parseDir(path, PreferSummary)
 400                 else:
 401                         file = open(path, 'r')
 402                         testrun.parseFile(file)
 403                         file.close()
 404
 405                 if len(testrun.name) == 0:
 406                         if path[-1] == '/':
 407                                 testrun.name = os.path.basename(path[0:-1])
 408                         else:
 409                                 testrun.name = os.path.basename(path)
 410
 411                 return testrun
 412         except:
 413                 traceback.print_exc()
 414                 raise Exception('Could not read tests results')