framework/profile.py

   1 # coding=utf-8
   2 # Permission is hereby granted, free of charge, to any person
   3 # obtaining a copy of this software and associated documentation
   4 # files (the "Software"), to deal in the Software without
   5 # restriction, including without limitation the rights to use,
   6 # copy, modify, merge, publish, distribute, sublicense, and/or
   7 # sell copies of the Software, and to permit persons to whom the
   8 # Software is furnished to do so, subject to the following
   9 # conditions:
  10 #
  11 # This permission notice shall be included in all copies or
  12 # substantial portions of the Software.
  13 #
  14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  15 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  16 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  17 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  18 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  19 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  20 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  21 # DEALINGS IN THE SOFTWARE.
  22
  23 """Classes dealing with groups of Tests.
  24
  25 In piglit tests are grouped into "profiles", which are equivalent to "suites"
  26 in some other testing nomenclature. A profile is a way to tell the framework
  27 that you have a group of tests you want to run, here are the names of those
  28 tests, and the Test instance.
  29 """
  30
  31 import ast
  32 import collections
  33 import contextlib
  34 import copy
  35 import gzip
  36 import importlib
  37 import itertools
  38 import multiprocessing
  39 import multiprocessing.dummy
  40 import os
  41 import re
  42 import xml.etree.ElementTree as et
  43
  44 from framework import grouptools, exceptions, status
  45 from framework.dmesg import get_dmesg
  46 from framework.log import LogManager
  47 from framework.monitoring import Monitoring
  48 from framework.test.base import Test, DummyTest
  49 from framework.test.piglit_test import (
  50     PiglitCLTest, PiglitGLTest, ASMParserTest, BuiltInConstantsTest,
  51     CLProgramTester, VkRunnerTest, ROOT_DIR,
  52 )
  53 from framework.test.shader_test import ShaderTest, MultiShaderTest
  54 from framework.test.glsl_parser_test import GLSLParserTest
  55 from framework.test.xorg import XTSTest, RendercheckTest
  56 from framework.options import OPTIONS
  57
  58 __all__ = [
  59     'RegexFilter',
  60     'TestDict',
  61     'TestProfile',
  62     'load_test_profile',
  63     'run',
  64 ]
  65
  66
  67 class RegexFilter(object):
  68     """An object to be passed to TestProfile.filter.
  69
  70     This object takes a list (or list-like object) of strings which it converts
  71     to re.compiled objects (so use raw strings for escape sequences), and acts
  72     as a callable for filtering tests. If a test matches any of the regex then
  73     it will be scheduled to run. When the inverse keyword argument is True then
  74     a test that matches any regex will not be scheduled. Regardless of the
  75     value of the inverse flag if filters is empty then the test will be run.
  76
  77     Arguments:
  78     filters -- a list of regex compiled objects.
  79
  80     Keyword Arguments:
  81     inverse -- Inverse the sense of the match.
  82     """
  83
  84     def __init__(self, filters, inverse=False):
  85         self.filters = [re.compile(f, flags=re.IGNORECASE) for f in filters]
  86         self.inverse = inverse
  87
  88     def __call__(self, name, _):  # pylint: disable=invalid-name
  89         # This needs to match the signature (name, test), since it doesn't need
  90         # the test instance use _.
  91
  92         # If self.filters is empty then return True, we don't want to remove
  93         # any tests from the run.
  94         if not self.filters:
  95             return True
  96
  97         if not self.inverse:
  98             return any(r.search(name) for r in self.filters)
  99         else:
 100             return not any(r.search(name) for r in self.filters)
 101
 102
 103 class TestDict(collections.abc.MutableMapping):
 104     """A special kind of dict for tests.
 105
 106     This mapping lowers the names of keys by default, and enforces that keys be
 107     strings (not bytes) and that values are Test derived objects. It is also a
 108     wrapper around collections.OrderedDict.
 109
 110     This class doesn't accept keyword arguments, this is intentional. This is
 111     because the TestDict class is ordered, and keyword arguments are unordered,
 112     which is a design mismatch.
 113     """
 114     def __init__(self):
 115         # This counter is incremented once when the allow_reassignment context
 116         # manager is opened, and decremented each time it is closed. This
 117         # allows stacking of the context manager
 118         self.__allow_reassignment = 0
 119         self.__container = collections.OrderedDict()
 120
 121     def __setitem__(self, key, value):
 122         """Enforce types on set operations.
 123
 124         Keys should only be strings, and values should only be Tests.
 125
 126         This method makes one additional requirement, it lowers the key before
 127         adding it. This solves a couple of problems, namely that we want to be
 128         able to use file-system hierarchies as groups in some cases, and those
 129         are assumed to be all lowercase to avoid problems on case insensitive
 130         file-systems.
 131         """
 132         # keys should be strings
 133         if not isinstance(key, str):
 134             raise exceptions.PiglitFatalError(
 135                 "TestDict keys must be strings, but was {}".format(type(key)))
 136
 137         # Values should either be more Tests
 138         if not isinstance(value, Test):
 139             raise exceptions.PiglitFatalError(
 140                 "TestDict values must be a Test, but was a {}".format(
 141                     type(value)))
 142
 143         # This must be lowered before the following test, or the test can pass
 144         # in error if the key has capitals in it.
 145         key = key.lower()
 146
 147         # If there is already a test of that value in the tree it is an error
 148         if not self.__allow_reassignment and key in self.__container:
 149             if self.__container[key] != value:
 150                 error = (
 151                     'Further, the two tests are not the same,\n'
 152                     'The original test has this command:   "{0}"\n'
 153                     'The new test has this command:        "{1}"'.format(
 154                         ' '.join(self.__container[key].command),
 155                         ' '.join(value.command))
 156                 )
 157             else:
 158                 error = "and both tests are the same."
 159
 160             raise exceptions.PiglitFatalError(
 161                 "A test has already been assigned the name: {}\n{}".format(
 162                     key, error))
 163
 164         self.__container[key] = value
 165
 166     def __getitem__(self, key):
 167         """Lower the value before returning."""
 168         return self.__container[key.lower()]
 169
 170     def __delitem__(self, key):
 171         """Lower the value before returning."""
 172         del self.__container[key.lower()]
 173
 174     def __len__(self):
 175         return len(self.__container)
 176
 177     def __iter__(self):
 178         return iter(self.__container)
 179
 180     @contextlib.contextmanager
 181     def group_manager(self, test_class, group, **default_args):
 182         """A context manager to make working with flat groups simple.
 183
 184         This provides a simple way to replace add_plain_test,
 185         add_concurrent_test, etc. Basic usage would be to use the with
 186         statement to yield and adder instance, and then add tests.
 187
 188         This does not provide for a couple of cases.
 189         1) When you need to alter the test after initialization. If you need to
 190            set instance.env, for example, you will need to do so manually. It
 191            is recommended to not use this function for that case, but to
 192            manually assign the test and set env together, for code clearness.
 193         2) When you need to use a function that modifies the TestProfile.
 194
 195         Arguments:
 196         test_class -- a Test derived class that. Instances of this class will
 197                       be added to the profile.
 198         group      -- a string or unicode that will be used as the key for the
 199                       test in profile.
 200
 201         Keyword Arguments:
 202         **         -- any additional keyword arguments will be considered
 203                       default arguments to all tests added by the adder. They
 204                       will always be overwritten by **kwargs passed to the
 205                       adder function
 206
 207         >>> from framework.test import PiglitGLTest
 208         >>> p = TestProfile()
 209         >>> with p.group_manager(PiglitGLTest, 'a') as g:
 210         ...     g(['test'])
 211         ...     g(['power', 'test'], 'powertest')
 212         """
 213         assert isinstance(group, str), type(group)
 214
 215         def adder(args, name=None, override_class=None, **kwargs):
 216             """Helper function that actually adds the tests.
 217
 218             Arguments:
 219             args   -- arguments to be passed to the test_class constructor.
 220                       This must be appropriate for the underlying class
 221
 222             Keyword Arguments:
 223             name   -- If this is a a truthy value that value will be used as
 224                       the key for the test. If name is falsy then args will be
 225                       ' '.join'd and used as name. Default: None
 226             kwargs -- Any additional args will be passed directly to the test
 227                       constructor as keyword args.
 228             """
 229             # If there is no name, join the arguments list together to make
 230             # the name
 231             if not name:
 232                 assert isinstance(args, list) # //
 233                 name = ' '.join(args)
 234
 235             assert isinstance(name, str)
 236             lgroup = grouptools.join(group, name)
 237
 238             class_ = override_class or test_class
 239
 240             self[lgroup] = class_(
 241                 args,
 242                 **dict(itertools.chain(default_args.items(), kwargs.items())))
 243
 244         yield adder
 245
 246     @property
 247     @contextlib.contextmanager
 248     def allow_reassignment(self):
 249         """Context manager that allows keys to be reassigned.
 250
 251         Normally reassignment happens in error, but sometimes one actually
 252         wants to do reassignment, say to add extra options in a reduced
 253         profile. This method allows reassignment, but only within its context,
 254         making it an explicit choice to do so.
 255
 256         It is safe to nest this contextmanager.
 257
 258         This is not thread safe, or even co-routine safe.
 259         """
 260         self.__allow_reassignment += 1
 261         yield
 262         self.__allow_reassignment -= 1
 263
 264
 265 class Filters(collections.abc.MutableSequence):
 266
 267     def __init__(self, iterable=None):
 268         if iterable:
 269             self.__container = list(iterable)
 270         else:
 271             self.__container = []
 272
 273     def __getitem__(self, index):
 274         return self.__container[index]
 275
 276     def __setitem__(self, index, value):
 277         self.__container[index] = value
 278
 279     def __delitem__(self, index):
 280         del self.__container[index]
 281
 282     def __len__(self):
 283         return len(self.__container)
 284
 285     def __add__(self, other):
 286         return type(self)(itertools.chain(iter(self), iter(other)))
 287
 288     def insert(self, index, value):
 289         self.__container.insert(index, value)
 290
 291     def run(self, iterable):
 292         for f in self.__container:
 293             if hasattr(f, 'reset'):
 294                 f.reset()
 295
 296         for k, v in iterable:
 297             if all(f(k, v) for f in self.__container):
 298                 yield k, v
 299
 300
 301 def make_test(element):
 302     """Rebuild a test instance from xml."""
 303     def process(elem, opt):
 304         k = elem.attrib['name']
 305         v = elem.attrib['value']
 306         try:
 307             opt[k] = ast.literal_eval(v)
 308         except ValueError:
 309             opt[k] = v
 310
 311     type_ = element.attrib['type']
 312     options = {}
 313     for e in element.findall('./option'):
 314         process(e, options)
 315     options['env'] = {e.attrib['name']: e.attrib['value']
 316                       for e in element.findall('./environment/env')}
 317
 318     if type_ == 'gl':
 319         return PiglitGLTest(**options)
 320     if type_ == 'gl_builtin':
 321         return BuiltInConstantsTest(**options)
 322     if type_ == 'cl':
 323         return PiglitCLTest(**options)
 324     if type_ == 'cl_prog':
 325         return CLProgramTester(**options)
 326     if type_ == 'shader':
 327         return ShaderTest(**options)
 328     if type_ == 'glsl_parser':
 329         return GLSLParserTest(**options)
 330     if type_ == 'asm_parser':
 331         return ASMParserTest(**options)
 332     if type_ == 'vkrunner':
 333         return VkRunnerTest(**options)
 334     if type_ == 'multi_shader':
 335         options['skips'] = []
 336         for e in element.findall('./Skips/Skip/option'):
 337             skips = {}
 338             process(e, skips)
 339             options['skips'].append(skips)
 340         return MultiShaderTest(**options)
 341     if type_ == 'xts':
 342         return XTSTest(**options)
 343     if type_ == 'rendercheck':
 344         return RendercheckTest(**options)
 345     raise Exception('Unreachable')
 346
 347
 348 class XMLProfile(object):
 349
 350     def __init__(self, filename):
 351         self.filename = filename
 352         self.forced_test_list = []
 353         self.filters = Filters()
 354         self.options = {
 355             'dmesg': get_dmesg(False),
 356             'monitor': Monitoring(False),
 357             'ignore_missing': False,
 358         }
 359
 360     def __len__(self):
 361         if not (self.filters or self.forced_test_list):
 362             with gzip.open(self.filename, 'rt') as f:
 363                 iter_ = et.iterparse(f, events=(b'start', ))
 364                 for _, elem in iter_:
 365                     if elem.tag == 'PiglitTestList':
 366                         return int(elem.attrib['count'])
 367         return sum(1 for _ in self.itertests())
 368
 369     def setup(self):
 370         pass
 371
 372     def teardown(self):
 373         pass
 374
 375     def _itertests(self):
 376         """Always iterates tests instead of using the forced test_list."""
 377         def _iter():
 378             with gzip.open(self.filename, 'rt') as f:
 379                 doc = et.iterparse(f, events=(b'end', ))
 380                 _, root = next(doc)  # get the root so we can keep clearing it
 381                 for _, e in doc:
 382                     if e.tag != 'Test':
 383                         continue
 384                     k = e.attrib['name']
 385                     v = make_test(e)
 386                     yield k, v
 387                     root.clear()
 388
 389         for k, v in self.filters.run(_iter()):
 390             yield k, v
 391
 392     def itertests(self):
 393         if self.forced_test_list:
 394             alltests = dict(self._itertests())
 395             opts = collections.OrderedDict()
 396             for n in self.forced_test_list:
 397                 if self.options['ignore_missing'] and n not in alltests:
 398                     opts[n] = DummyTest(n, status.NOTRUN)
 399                 else:
 400                     opts[n] = alltests[n]
 401             return opts.items()
 402         else:
 403             return iter(self._itertests())
 404
 405
 406 class MetaProfile(object):
 407
 408     """Holds multiple profiles but acts like one.
 409
 410     This is meant to allow classic profiles like all to exist after being
 411     split.
 412     """
 413
 414     def __init__(self, filename):
 415         self.forced_test_list = []
 416         self.filters = Filters()
 417         self.options = {
 418             'dmesg': get_dmesg(False),
 419             'monitor': Monitoring(False),
 420             'ignore_missing': False,
 421         }
 422
 423         tree = et.parse(filename)
 424         root = tree.getroot()
 425         self._profiles = [load_test_profile(p.text)
 426                           for p in root.findall('.//Profile')]
 427
 428         for p in self._profiles:
 429             p.options = self.options
 430
 431     def __len__(self):
 432         if self.forced_test_list or self.filters:
 433             return sum(1 for _ in self.itertests())
 434         return sum(len(p) for p in self._profiles)
 435
 436     def setup(self):
 437         pass
 438
 439     def teardown(self):
 440         pass
 441
 442     def _itertests(self):
 443         def _iter():
 444             for p in self._profiles:
 445                 for k, v in p.itertests():
 446                     yield k, v
 447
 448         for k, v in self.filters.run(_iter()):
 449             yield k, v
 450
 451     def itertests(self):
 452         if self.forced_test_list:
 453             alltests = dict(self._itertests())
 454             opts = collections.OrderedDict()
 455             for n in self.forced_test_list:
 456                 if self.options['ignore_missing'] and n not in alltests:
 457                     opts[n] = DummyTest(n, status.NOTRUN)
 458                 else:
 459                     opts[n] = alltests[n]
 460             return opts.items()
 461         else:
 462             return iter(self._itertests())
 463
 464
 465 class TestProfile(object):
 466     """Class that holds a list of tests for execution.
 467
 468     This class represents a single testsuite, it has a mapping (dictionary-like
 469     object) of tests attached (TestDict). This is a mapping of <str>:<Test>
 470     (python 3 str, python 2 unicode), and the key is delimited by
 471     grouptools.SEPARATOR.
 472
 473     The group_manager method provides a context_manager to make adding test to
 474     the test_list easier, by doing more validation and enforcement.
 475     >>> t = TestProfile()
 476     >>> with t.group_manager(Test, 'foo@bar') as g:
 477     ...     g(['foo'])
 478
 479     This class does not provide a way to execute itself, instead that is
 480     handled by the run function in this module, which is able to process and
 481     run multiple TestProfile objects at once.
 482     """
 483     def __init__(self):
 484         self.test_list = TestDict()
 485         self.forced_test_list = []
 486         self.filters = Filters()
 487         self.options = {
 488             'dmesg': get_dmesg(False),
 489             'monitor': Monitoring(False),
 490             'ignore_missing': False,
 491         }
 492
 493     def __len__(self):
 494         return sum(1 for _ in self.itertests())
 495
 496     def setup(self):
 497         """Method to do pre-run setup."""
 498
 499     def teardown(self):
 500         """Method to do post-run teardown."""
 501
 502     def copy(self):
 503         """Create a copy of the TestProfile.
 504
 505         This method creates a copy with references to the original instance
 506         using copy.copy. This allows profiles to be "subclassed" by other
 507         profiles, without modifying the original.
 508
 509         copy.deepcopy is used for the filters so the original is
 510         actually not modified in this case.
 511         """
 512         new = copy.copy(self)
 513         new.test_list = copy.copy(self.test_list)
 514         new.forced_test_list = copy.copy(self.forced_test_list)
 515         new.filters = copy.deepcopy(self.filters)
 516         return new
 517
 518     def itertests(self):
 519         """Iterate over tests while filtering.
 520
 521         This iterator is non-destructive.
 522         """
 523         if self.forced_test_list:
 524             opts = collections.OrderedDict()
 525             for n in self.forced_test_list:
 526                 if self.options['ignore_missing'] and n not in self.test_list:
 527                     opts[n] = DummyTest(n, status.NOTRUN)
 528                 else:
 529                     opts[n] = self.test_list[n]
 530         else:
 531             opts = collections.OrderedDict()
 532             test_keys = list(self.test_list.keys())
 533             test_keys.sort()
 534             for k in test_keys:
 535                 opts[k] = self.test_list[k]
 536
 537         for k, v in self.filters.run(opts.items()):
 538             yield k, v
 539
 540
 541 def load_test_profile(filename, python=None):
 542     """Load a python module and return it's profile attribute.
 543
 544     All of the python test files provide a profile attribute which is a
 545     TestProfile instance. This loads that module and returns it or raises an
 546     error.
 547
 548     This method doesn't care about file extensions as a way to be backwards
 549     compatible with script wrapping piglit. 'tests/quick', 'tests/quick.tests',
 550     'tests/quick.py', and 'quick' are all equally valid for filename.
 551
 552     This will raise a FatalError if the module doesn't exist, or if the module
 553     doesn't have a profile attribute.
 554
 555     Raises:
 556     PiglitFatalError -- if the module cannot be imported for any reason, or if
 557                         the module lacks a "profile" attribute.
 558
 559     Arguments:
 560     filename -- the name of a python module to get a 'profile' from
 561
 562     Keyword Arguments:
 563     python -- If this is None (the default) XML is tried, and then a python
 564               module. If True, then only python is tried, if False then only
 565               XML is tried.
 566     """
 567     name, ext = os.path.splitext(os.path.basename(filename))
 568     if ext == '.no_isolation':
 569         name = filename
 570
 571     if not python:
 572         # If process-isolation is false then try to load a profile named
 573         # {name}.no_isolation instead. This is only valid for xml based
 574         # profiles.
 575         if ext != '.no_isolation' and not OPTIONS.process_isolation:
 576             try:
 577                 return load_test_profile(name + '.no_isolation' + ext, python)
 578             except exceptions.PiglitFatalError:
 579                 # There might not be a .no_isolation version, try to load the
 580                 # regular version in that case.
 581                 pass
 582
 583         if os.path.isabs(filename):
 584             if '.meta' in filename:
 585                 return MetaProfile(filename)
 586             if '.xml' in filename:
 587                 return XMLProfile(filename)
 588
 589         meta = os.path.join(ROOT_DIR, 'tests', name + '.meta.xml')
 590         if os.path.exists(meta):
 591             return MetaProfile(meta)
 592
 593
 594         xml = os.path.join(ROOT_DIR, 'tests', name + '.xml.gz')
 595         if os.path.exists(xml):
 596             return XMLProfile(xml)
 597
 598     if python is False:
 599         raise exceptions.PiglitFatalError(
 600             'Cannot open "tests/{0}.xml.gz" or "tests/{0}.meta.xml"'.format(name))
 601
 602     try:
 603         mod = importlib.import_module('tests.{0}'.format(name))
 604     except ImportError:
 605         raise exceptions.PiglitFatalError(
 606             'Failed to import "{}", there is either something wrong with the '
 607             'module or it doesn\'t exist. Check your spelling?'.format(
 608                 filename))
 609
 610     try:
 611         return mod.profile
 612     except AttributeError:
 613         raise exceptions.PiglitFatalError(
 614             'There is no "profile" attribute in module {}.\n'
 615             'Did you specify the right file?'.format(filename))
 616
 617
 618 def run(profiles, logger, backend, concurrency, jobs):
 619     """Runs all tests using Thread pool.
 620
 621     When called this method will flatten out self.tests into self.test_list,
 622     then will prepare a logger, and begin executing tests through it's Thread
 623     pools.
 624
 625     Based on the value of concurrency it will either run all the tests
 626     concurrently, all serially, or first the thread safe tests then the
 627     serial tests.
 628
 629     Finally it will print a final summary of the tests.
 630
 631     Arguments:
 632     profiles -- a list of Profile instances.
 633     logger   -- a log.LogManager instance.
 634     backend  -- a results.Backend derived instance.
 635     jobs     -- maximum number of concurrent jobs. Use os.cpu_count() by default
 636     """
 637     chunksize = 1
 638
 639     profiles = [(p, p.itertests()) for p in profiles]
 640     log = LogManager(logger, sum(len(p) for p, _ in profiles))
 641
 642     def test(name, test, profile, this_pool=None):
 643         """Function to call test.execute from map"""
 644         with backend.write_test(name) as w:
 645             test.execute(name, log.get(), profile.options)
 646             w(test.result)
 647         if profile.options['monitor'].abort_needed:
 648             this_pool.terminate()
 649
 650     def run_threads(pool, profile, test_list, filterby=None):
 651         """ Open a pool, close it, and join it """
 652         if filterby:
 653             # Although filterby could be attached to TestProfile as a filter,
 654             # it would have to be removed when run_threads exits, requiring
 655             # more code, and adding side-effects
 656             test_list = (x for x in test_list if filterby(x))
 657
 658         for n, t in test_list:
 659             pool.apply_async(test, [n, t, profile, pool])
 660
 661     def run_profile(profile, test_list):
 662         """Run an individual profile."""
 663         profile.setup()
 664         if concurrency == "all":
 665             run_threads(multi, profile, test_list)
 666         elif concurrency == "none":
 667             run_threads(single, profile, test_list)
 668         else:
 669             assert concurrency == "some"
 670             # test_list is an iterator, we need to copy it to run it twice.
 671             test_list = itertools.tee(test_list, 2)
 672
 673             # Filter and return only thread safe tests to the threaded pool
 674             run_threads(multi, profile, test_list[0],
 675                         lambda x: x[1].run_concurrent)
 676
 677             # Filter and return the non thread safe tests to the single
 678             # pool
 679             run_threads(single, profile, test_list[1],
 680                         lambda x: not x[1].run_concurrent)
 681         profile.teardown()
 682
 683     # Multiprocessing.dummy is a wrapper around Threading that provides a
 684     # multiprocessing compatible API
 685     #
 686     # The default value of pool is the number of virtual processor cores
 687     single = multiprocessing.dummy.Pool(1)
 688     multi = multiprocessing.dummy.Pool(jobs)
 689
 690     try:
 691         for p in profiles:
 692             run_profile(*p)
 693
 694         for pool in [single, multi]:
 695             pool.close()
 696             pool.join()
 697     finally:
 698         log.get().summary()
 699
 700     for p, _ in profiles:
 701         if p.options['monitor'].abort_needed:
 702             raise exceptions.PiglitAbort(p.options['monitor'].error_message)