framework/profile.py

   1 # Permission is hereby granted, free of charge, to any person
   2 # obtaining a copy of this software and associated documentation
   3 # files (the "Software"), to deal in the Software without
   4 # restriction, including without limitation the rights to use,
   5 # copy, modify, merge, publish, distribute, sublicense, and/or
   6 # sell copies of the Software, and to permit persons to whom the
   7 # Software is furnished to do so, subject to the following
   8 # conditions:
   9 #
  10 # This permission notice shall be included in all copies or
  11 # substantial portions of the Software.
  12 #
  13 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  14 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  15 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  16 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  17 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  18 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  19 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20 # DEALINGS IN THE SOFTWARE.
  21
  22 """Classes dealing with groups of Tests.
  23
  24 In piglit tests are grouped into "profiles", which are equivalent to "suites"
  25 in some other testing nomenclature. A profile is a way to tell the framework
  26 that you have a group of tests you want to run, here are the names of those
  27 tests, and the Test instance.
  28 """
  29
  30 from __future__ import (
  31     absolute_import, division, print_function, unicode_literals
  32 )
  33 import collections
  34 import contextlib
  35 import copy
  36 import importlib
  37 import itertools
  38 import multiprocessing
  39 import multiprocessing.dummy
  40 import os
  41 import re
  42
  43 import six
  44
  45 from framework import grouptools, exceptions, status
  46 from framework.dmesg import get_dmesg
  47 from framework.log import LogManager
  48 from framework.monitoring import Monitoring
  49 from framework.test.base import Test, DummyTest
  50
  51 __all__ = [
  52     'RegexFilter',
  53     'TestDict',
  54     'TestProfile',
  55     'load_test_profile',
  56     'run',
  57 ]
  58
  59
  60 class RegexFilter(object):
  61     """An object to be passed to TestProfile.filter.
  62
  63     This object takes a list (or list-like object) of strings which it converts
  64     to re.compiled objects (so use raw strings for escape sequences), and acts
  65     as a callable for filtering tests. If a test matches any of the regex then
  66     it will be scheduled to run. When the inverse keyword argument is True then
  67     a test that matches any regex will not be scheduled. Regardless of the
  68     value of the inverse flag if filters is empty then the test will be run.
  69
  70     Arguments:
  71     filters -- a list of regex compiled objects.
  72
  73     Keyword Arguments:
  74     inverse -- Inverse the sense of the match.
  75     """
  76
  77     def __init__(self, filters, inverse=False):
  78         self.filters = [re.compile(f, flags=re.IGNORECASE) for f in filters]
  79         self.inverse = inverse
  80
  81     def __call__(self, name, _):  # pylint: disable=invalid-name
  82         # This needs to match the signature (name, test), since it doesn't need
  83         # the test instance use _.
  84
  85         # If self.filters is empty then return True, we don't want to remove
  86         # any tests from the run.
  87         if not self.filters:
  88             return True
  89
  90         if not self.inverse:
  91             return any(r.search(name) for r in self.filters)
  92         else:
  93             return not any(r.search(name) for r in self.filters)
  94
  95
  96 class TestDict(collections.MutableMapping):
  97     """A special kind of dict for tests.
  98
  99     This mapping lowers the names of keys by default, and enforces that keys be
 100     strings (not bytes) and that values are Test derived objects. It is also a
 101     wrapper around collections.OrderedDict.
 102
 103     This class doesn't accept keyword arguments, this is intentional. This is
 104     because the TestDict class is ordered, and keyword arguments are unordered,
 105     which is a design mismatch.
 106     """
 107     def __init__(self):
 108         # This counter is incremented once when the allow_reassignment context
 109         # manager is opened, and decremented each time it is closed. This
 110         # allows stacking of the context manager
 111         self.__allow_reassignment = 0
 112         self.__container = collections.OrderedDict()
 113
 114     def __setitem__(self, key, value):
 115         """Enforce types on set operations.
 116
 117         Keys should only be strings, and values should only be Tests.
 118
 119         This method makes one additional requirement, it lowers the key before
 120         adding it. This solves a couple of problems, namely that we want to be
 121         able to use file-system hierarchies as groups in some cases, and those
 122         are assumed to be all lowercase to avoid problems on case insensitive
 123         file-systems.
 124         """
 125         # keys should be strings
 126         if not isinstance(key, six.text_type):
 127             raise exceptions.PiglitFatalError(
 128                 "TestDict keys must be strings, but was {}".format(type(key)))
 129
 130         # Values should either be more Tests
 131         if not isinstance(value, Test):
 132             raise exceptions.PiglitFatalError(
 133                 "TestDict values must be a Test, but was a {}".format(
 134                     type(value)))
 135
 136         # This must be lowered before the following test, or the test can pass
 137         # in error if the key has capitals in it.
 138         key = key.lower()
 139
 140         # If there is already a test of that value in the tree it is an error
 141         if not self.__allow_reassignment and key in self.__container:
 142             if self.__container[key] != value:
 143                 error = (
 144                     'Further, the two tests are not the same,\n'
 145                     'The original test has this command:   "{0}"\n'
 146                     'The new test has this command:        "{1}"'.format(
 147                         ' '.join(self.__container[key].command),
 148                         ' '.join(value.command))
 149                 )
 150             else:
 151                 error = "and both tests are the same."
 152
 153             raise exceptions.PiglitFatalError(
 154                 "A test has already been assigned the name: {}\n{}".format(
 155                     key, error))
 156
 157         self.__container[key] = value
 158
 159     def __getitem__(self, key):
 160         """Lower the value before returning."""
 161         return self.__container[key.lower()]
 162
 163     def __delitem__(self, key):
 164         """Lower the value before returning."""
 165         del self.__container[key.lower()]
 166
 167     def __len__(self):
 168         return len(self.__container)
 169
 170     def __iter__(self):
 171         return iter(self.__container)
 172
 173     @contextlib.contextmanager
 174     def group_manager(self, test_class, group, **default_args):
 175         """A context manager to make working with flat groups simple.
 176
 177         This provides a simple way to replace add_plain_test,
 178         add_concurrent_test, etc. Basic usage would be to use the with
 179         statement to yield and adder instance, and then add tests.
 180
 181         This does not provide for a couple of cases.
 182         1) When you need to alter the test after initialization. If you need to
 183            set instance.env, for example, you will need to do so manually. It
 184            is recommended to not use this function for that case, but to
 185            manually assign the test and set env together, for code clearness.
 186         2) When you need to use a function that modifies the TestProfile.
 187
 188         Arguments:
 189         test_class -- a Test derived class that. Instances of this class will
 190                       be added to the profile.
 191         group      -- a string or unicode that will be used as the key for the
 192                       test in profile.
 193
 194         Keyword Arguments:
 195         **         -- any additional keyword arguments will be considered
 196                       default arguments to all tests added by the adder. They
 197                       will always be overwritten by **kwargs passed to the
 198                       adder function
 199
 200         >>> from framework.test import PiglitGLTest
 201         >>> p = TestProfile()
 202         >>> with p.group_manager(PiglitGLTest, 'a') as g:
 203         ...     g(['test'])
 204         ...     g(['power', 'test'], 'powertest')
 205         """
 206         assert isinstance(group, six.string_types), type(group)
 207
 208         def adder(args, name=None, **kwargs):
 209             """Helper function that actually adds the tests.
 210
 211             Arguments:
 212             args   -- arguments to be passed to the test_class constructor.
 213                       This must be appropriate for the underlying class
 214
 215             Keyword Arguments:
 216             name   -- If this is a a truthy value that value will be used as
 217                       the key for the test. If name is falsy then args will be
 218                       ' '.join'd and used as name. Default: None
 219             kwargs -- Any additional args will be passed directly to the test
 220                       constructor as keyword args.
 221             """
 222             # If there is no name, join the arguments list together to make
 223             # the name
 224             if not name:
 225                 assert isinstance(args, list) # //
 226                 name = ' '.join(args)
 227
 228             assert isinstance(name, six.string_types)
 229             lgroup = grouptools.join(group, name)
 230
 231             self[lgroup] = test_class(
 232                 args,
 233                 **dict(itertools.chain(six.iteritems(default_args),
 234                                        six.iteritems(kwargs))))
 235
 236         yield adder
 237
 238     @property
 239     @contextlib.contextmanager
 240     def allow_reassignment(self):
 241         """Context manager that allows keys to be reassigned.
 242
 243         Normally reassignment happens in error, but sometimes one actually
 244         wants to do reassignment, say to add extra options in a reduced
 245         profile. This method allows reassignment, but only within its context,
 246         making it an explicit choice to do so.
 247
 248         It is safe to nest this contextmanager.
 249
 250         This is not thread safe, or even co-routine safe.
 251         """
 252         self.__allow_reassignment += 1
 253         yield
 254         self.__allow_reassignment -= 1
 255
 256
 257 class TestProfile(object):
 258     """Class that holds a list of tests for execution.
 259
 260     This class represents a single testsuite, it has a mapping (dictionary-like
 261     object) of tests attached (TestDict). This is a mapping of <str>:<Test>
 262     (python 3 str, python 2 unicode), and the key is delimited by
 263     grouptools.SEPARATOR.
 264
 265     The group_manager method provides a context_manager to make adding test to
 266     the test_list easier, by doing more validation and enforcement.
 267     >>> t = TestProfile()
 268     >>> with t.group_manager(Test, 'foo@bar') as g:
 269     ...     g(['foo'])
 270
 271     This class does not provide a way to execute itself, instead that is
 272     handled by the run function in this module, which is able to process and
 273     run multiple TestProfile objects at once.
 274     """
 275     def __init__(self):
 276         self.test_list = TestDict()
 277         self.forced_test_list = []
 278         self.filters = []
 279         self.options = {
 280             'dmesg': get_dmesg(False),
 281             'monitor': Monitoring(False),
 282             'ignore_missing': False,
 283         }
 284
 285     def setup(self):
 286         """Method to do pre-run setup."""
 287
 288     def teardown(self):
 289         """Method to do post-run teardown."""
 290
 291     def copy(self):
 292         """Create a copy of the TestProfile.
 293
 294         This method creates a copy with references to the original instance
 295         using copy.copy. This allows profiles to be "subclassed" by other
 296         profiles, without modifying the original.
 297         """
 298         new = copy.copy(self)
 299         new.test_list = copy.copy(self.test_list)
 300         new.forced_test_list = copy.copy(self.forced_test_list)
 301         new.filters = copy.copy(self.filters)
 302         return new
 303
 304     def itertests(self):
 305         """Iterate over tests while filtering.
 306
 307         This iterator is non-destructive.
 308         """
 309         if self.forced_test_list:
 310             opts = collections.OrderedDict()
 311             for n in self.forced_test_list:
 312                 if self.options['ignore_missing'] and n not in self.test_list:
 313                     opts[n] = DummyTest(n, status.NOTRUN)
 314                 else:
 315                     opts[n] = self.test_list[n]
 316         else:
 317             opts = self.test_list  # pylint: disable=redefined-variable-type
 318
 319         for k, v in six.iteritems(opts):
 320             if all(f(k, v) for f in self.filters):
 321                 yield k, v
 322
 323
 324 def load_test_profile(filename):
 325     """Load a python module and return it's profile attribute.
 326
 327     All of the python test files provide a profile attribute which is a
 328     TestProfile instance. This loads that module and returns it or raises an
 329     error.
 330
 331     This method doesn't care about file extensions as a way to be backwards
 332     compatible with script wrapping piglit. 'tests/quick', 'tests/quick.tests',
 333     'tests/quick.py', and 'quick' are all equally valid for filename.
 334
 335     This will raise a FatalError if the module doesn't exist, or if the module
 336     doesn't have a profile attribute.
 337
 338     Raises:
 339     PiglitFatalError -- if the module cannot be imported for any reason, or if
 340                         the module lacks a "profile" attribute.
 341
 342     Arguments:
 343     filename -- the name of a python module to get a 'profile' from
 344     """
 345     try:
 346         mod = importlib.import_module('tests.{0}'.format(
 347             os.path.splitext(os.path.basename(filename))[0]))
 348     except ImportError:
 349         raise exceptions.PiglitFatalError(
 350             'Failed to import "{}", there is either something wrong with the '
 351             'module or it doesn\'t exist. Check your spelling?'.format(
 352                 filename))
 353
 354     try:
 355         return mod.profile
 356     except AttributeError:
 357         raise exceptions.PiglitFatalError(
 358             'There is no "profile" attribute in module {}.\n'
 359             'Did you specify the right file?'.format(filename))
 360
 361
 362 def run(profiles, logger, backend, concurrency):
 363     """Runs all tests using Thread pool.
 364
 365     When called this method will flatten out self.tests into self.test_list,
 366     then will prepare a logger, and begin executing tests through it's Thread
 367     pools.
 368
 369     Based on the value of concurrency it will either run all the tests
 370     concurrently, all serially, or first the thread safe tests then the
 371     serial tests.
 372
 373     Finally it will print a final summary of the tests.
 374
 375     Arguments:
 376     profiles -- a list of Profile instances.
 377     logger   -- a log.LogManager instance.
 378     backend  -- a results.Backend derived instance.
 379     """
 380     chunksize = 1
 381
 382     # The logger needs to know how many tests are running. Because of filters
 383     # there's no way to do that without making a concrete list out of the
 384     # filters profiles.
 385     profiles = [(p, list(p.itertests())) for p in profiles]
 386     log = LogManager(logger, sum(len(l) for _, l in profiles))
 387
 388     # check that after the filters are run there are actually tests to run.
 389     if not any(l for _, l in profiles):
 390         raise exceptions.PiglitUserError('no matching tests')
 391
 392     def test(name, test, profile, this_pool=None):
 393         """Function to call test.execute from map"""
 394         with backend.write_test(name) as w:
 395             test.execute(name, log.get(), profile.options)
 396             w(test.result)
 397         if profile.options['monitor'].abort_needed:
 398             this_pool.terminate()
 399
 400     def run_threads(pool, profile, test_list, filterby=None):
 401         """ Open a pool, close it, and join it """
 402         if filterby:
 403             # Although filterby could be attached to TestProfile as a filter,
 404             # it would have to be removed when run_threads exits, requiring
 405             # more code, and adding side-effects
 406             test_list = (x for x in test_list if filterby(x))
 407
 408         pool.imap(lambda pair: test(pair[0], pair[1], profile, pool),
 409                   test_list, chunksize)
 410
 411     def run_profile(profile, test_list):
 412         """Run an individual profile."""
 413         profile.setup()
 414         if concurrency == "all":
 415             run_threads(multi, profile, test_list)
 416         elif concurrency == "none":
 417             run_threads(single, profile, test_list)
 418         else:
 419             assert concurrency == "some"
 420             # Filter and return only thread safe tests to the threaded pool
 421             run_threads(multi, profile, test_list,
 422                         lambda x: x[1].run_concurrent)
 423
 424             # Filter and return the non thread safe tests to the single
 425             # pool
 426             run_threads(single, profile, test_list,
 427                         lambda x: not x[1].run_concurrent)
 428         profile.teardown()
 429
 430     # Multiprocessing.dummy is a wrapper around Threading that provides a
 431     # multiprocessing compatible API
 432     #
 433     # The default value of pool is the number of virtual processor cores
 434     single = multiprocessing.dummy.Pool(1)
 435     multi = multiprocessing.dummy.Pool()
 436
 437     try:
 438         for p in profiles:
 439             run_profile(*p)
 440
 441         for pool in [single, multi]:
 442             pool.close()
 443             pool.join()
 444     finally:
 445         log.get().summary()
 446
 447     for p, _ in profiles:
 448         if p.options['monitor'].abort_needed:
 449             raise exceptions.PiglitAbort(p.options['monitor'].error_message)