framework/test/base.py

   1 # coding=utf-8
   2 #
   3 # Permission is hereby granted, free of charge, to any person
   4 # obtaining a copy of this software and associated documentation
   5 # files (the "Software"), to deal in the Software without
   6 # restriction, including without limitation the rights to use,
   7 # copy, modify, merge, publish, distribute, sublicense, and/or
   8 # sell copies of the Software, and to permit persons to whom the
   9 # Software is furnished to do so, subject to the following
  10 # conditions:
  11 #
  12 # This permission notice shall be included in all copies or
  13 # substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  16 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  17 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  18 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  19 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  20 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  21 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22 # DEALINGS IN THE SOFTWARE.
  23
  24 """ Module provides a base class for Tests """
  25
  26 import abc
  27 import copy
  28 import errno
  29 import itertools
  30 import os
  31 import signal
  32 import subprocess
  33 import sys
  34 import time
  35 import traceback
  36 import warnings
  37
  38 from framework import exceptions
  39 from framework import status
  40 from framework.options import OPTIONS
  41 from framework.results import TestResult
  42
  43 _EXTRA_POPEN_ARGS = {}
  44
  45 if os.name == 'posix':
  46     # This should work for all *nix systems, Linux, the BSDs, and OSX.
  47     # This speicifically creates a session group for each test, so that
  48     # it's children can be killed if it times out.
  49     _EXTRA_POPEN_ARGS = {'start_new_session': True}
  50
  51 # pylint: enable=wrong-import-position,wrong-import-order
  52
  53
  54 __all__ = [
  55     'Test',
  56     'TestIsSkip',
  57     'TestRunError',
  58     'ValgrindMixin',
  59     'WindowResizeMixin',
  60     'is_crash_returncode',
  61 ]
  62
  63 # Allows timeouts to be suppressed by setting the environment variable
  64 # PIGLIT_NO_TIMEOUT to anything that bool() will resolve as True
  65 _SUPPRESS_TIMEOUT = bool(os.environ.get('PIGLIT_NO_TIMEOUT', False))
  66
  67
  68 class TestIsSkip(exceptions.PiglitException):
  69     """Exception raised in is_skip() if the test is a skip."""
  70     def __init__(self, reason):
  71         super(TestIsSkip, self).__init__()
  72         self.reason = reason
  73
  74
  75 class TestRunError(exceptions.PiglitException):
  76     """Exception raised if the test fails to run."""
  77     def __init__(self, message, status):
  78         super(TestRunError, self).__init__(message)
  79         self.status = status
  80
  81
  82 def is_crash_returncode(returncode):
  83     """Determine whether the given process return code correspond to a
  84     crash.
  85     """
  86     # In python 2 NoneType and other types can be compaired, in python3 this
  87     # isn't allowed.
  88     if returncode is None:
  89         return False
  90
  91     if sys.platform == 'win32':
  92         # On Windows:
  93         # - For uncaught exceptions the process terminates with the exception
  94         # code, which is usually negative
  95         # - MSVCRT's abort() terminates process with exit code 3
  96         return returncode < 0 or returncode == 3
  97     else:
  98         return returncode < 0
  99
 100
 101 class Test(metaclass=abc.ABCMeta):
 102     """ Abstract base class for Test classes
 103
 104     This class provides the framework for running tests, with several methods
 105     and properties that can be overwritten to produce a specialized class for
 106     running test suites other than piglit.
 107
 108     It provides two methods for running tests, execute and run.
 109     execute() provides lots of features, and is invoked when running piglit
 110     from the command line, run() is a more basic method for running the test,
 111     and is called internally by execute(), but is can be useful outside of it.
 112
 113     Arguments:
 114     command -- a value to be passed to subprocess.Popen
 115
 116     Keyword Arguments:
 117     run_concurrent -- If True the test is thread safe. Default: False
 118
 119     """
 120     __slots__ = ['run_concurrent', 'env', 'result', 'cwd', '_command']
 121     timeout = None
 122
 123     def __init__(self, command, run_concurrent=False, env=None, cwd=None):
 124         assert isinstance(command, list), command
 125
 126         self.run_concurrent = run_concurrent
 127         self._command = copy.copy(command)
 128         self.env = env or {}
 129         self.result = TestResult()
 130         self.cwd = cwd
 131
 132     def execute(self, path, log, options):
 133         """ Run a test
 134
 135         Run a test, but with features. This times the test, uses dmesg checking
 136         (if requested), and runs the logger.
 137
 138         Arguments:
 139         path    -- the name of the test
 140         log     -- a log.Log instance
 141         options -- a dictionary containing dmesg and monitoring objects
 142         """
 143         log.start(path)
 144         # Run the test
 145         if OPTIONS.execute:
 146             try:
 147                 self.result.time.start = time.time()
 148                 options['dmesg'].update_dmesg()
 149                 options['monitor'].update_monitoring()
 150                 self.run()
 151                 self.result.time.end = time.time()
 152                 self.result = options['dmesg'].update_result(self.result)
 153                 options['monitor'].check_monitoring()
 154             # This is a rare case where a bare exception is okay, since we're
 155             # using it to log exceptions
 156             except:
 157                 exc_type, exc_value, exc_traceback = sys.exc_info()
 158                 traceback.print_exc(file=sys.stderr)
 159                 self.result.result = 'fail'
 160                 self.result.exception = "{}{}".format(exc_type, exc_value)
 161                 self.result.traceback = "".join(
 162                     traceback.format_tb(exc_traceback))
 163
 164             log.log(self.result.result)
 165         else:
 166             log.log('dry-run')
 167
 168     @property
 169     def command(self):
 170         assert self._command
 171         return self._command
 172
 173     @command.setter
 174     def command(self, new):
 175         assert isinstance(new, list), 'Test.command must be a list'
 176         self._command = new
 177
 178     @abc.abstractmethod
 179     def interpret_result(self):
 180         """Convert the raw output of the test into a form piglit understands.
 181         """
 182         if is_crash_returncode(self.result.returncode):
 183             self.result.result = status.CRASH
 184             if self.result.subtests:
 185                 # We know because subtests are ordered that the first test with
 186                 # a status of NOTRUN is the subtest that crashed, mark that
 187                 # test and move on.
 188                 for k, v in self.result.subtests.items():
 189                     if v == status.NOTRUN:
 190                         self.result.subtests[k] = status.CRASH
 191                         break
 192         elif self.result.returncode != 0:
 193             if self.result.result == status.PASS:
 194                 self.result.result = status.WARN
 195             else:
 196                 self.result.result = status.FAIL
 197
 198     def run(self):
 199         """
 200         Run a test.  The return value will be a dictionary with keys
 201         including 'result', 'info', 'returncode' and 'command'.
 202         * For 'result', the value may be one of 'pass', 'fail', 'skip',
 203           'crash', or 'warn'.
 204         * For 'info', the value will include stderr/out text.
 205         * For 'returncode', the value will be the numeric exit code/value.
 206         * For 'command', the value will be command line program and arguments.
 207         """
 208         self.result.command = ' '.join(map(str, self.command))
 209         self.result.environment = " ".join(
 210             '{0}="{1}"'.format(k, v) for k, v in itertools.chain(
 211                 OPTIONS.env.items(), self.env.items()))
 212
 213         try:
 214             self.is_skip()
 215         except TestIsSkip as e:
 216             self.result.result = status.SKIP
 217             for each in self.result.subtests.keys():
 218                 self.result.subtests[each] = status.SKIP
 219             self.result.out = e.reason
 220             self.result.returncode = None
 221             return
 222
 223         try:
 224             self._run_command()
 225         except TestRunError as e:
 226             self.result.result = str(e.status)
 227             for each in self.result.subtests.keys():
 228                 self.result.subtests[each] = str(e.status)
 229             self.result.out = str(e)
 230             self.result.returncode = None
 231             return
 232
 233         self.interpret_result()
 234
 235     def is_skip(self):
 236         """ Application specific check for skip
 237
 238         If this function returns a truthy value then the current test will be
 239         skipped. The base version will always return False
 240
 241         """
 242         pass
 243
 244     def _run_command(self, **kwargs):
 245         """ Run the test command and get the result
 246
 247         This method sets environment options, then runs the executable. If the
 248         executable isn't found it sets the result to skip.
 249
 250         """
 251         # This allows the ReducedProcessMixin to work without having to whack
 252         # self.command (which should be treated as immutable), but is
 253         # considered private.
 254         command = kwargs.pop('_command', self.command)
 255
 256         # Setup the environment for the test. Environment variables are taken
 257         # from the following sources, listed in order of increasing precedence:
 258         #
 259         #   1. This process's current environment.
 260         #   2. Global test options. (Some of these are command line options to
 261         #      Piglit's runner script).
 262         #   3. Per-test environment variables set in all.py.
 263         #
 264         # Piglit chooses this order because Unix tradition dictates that
 265         # command line options (2) override environment variables (1); and
 266         # Piglit considers environment variables set in all.py (3) to be test
 267         # requirements.
 268         _base = itertools.chain(os.environ.items(),
 269                                 OPTIONS.env.items(),
 270                                 self.env.items())
 271         fullenv = {str(k): str(v) for k, v in _base}
 272
 273         try:
 274             proc = subprocess.Popen(map(str, command),
 275                                     stdout=subprocess.PIPE,
 276                                     stderr=subprocess.PIPE,
 277                                     cwd=self.cwd,
 278                                     env=fullenv,
 279                                     universal_newlines=True,
 280                                     **_EXTRA_POPEN_ARGS)
 281
 282             self.result.pid.append(proc.pid)
 283             if not _SUPPRESS_TIMEOUT:
 284                 out, err = proc.communicate(timeout=self.timeout)
 285             else:
 286                 out, err = proc.communicate()
 287             returncode = proc.returncode
 288         except OSError as e:
 289             # Different sets of tests get built under different build
 290             # configurations.  If a developer chooses to not build a test,
 291             # Piglit should not report that test as having failed.
 292             if e.errno == errno.ENOENT:
 293                 raise TestRunError("Test executable not found.\n", 'skip')
 294             else:
 295                 raise e
 296         except subprocess.TimeoutExpired:
 297             # This can only be reached if subprocess32 is present or on python
 298             # 3.x, since # TimeoutExpired is never raised by the python 2.7
 299             # fallback code.
 300
 301             proc.terminate()
 302
 303             # XXX: This is probably broken on windows, since os.getpgid doesn't
 304             # exist on windows. What is the right way to handle this?
 305             if proc.poll() is None:
 306                 time.sleep(3)
 307                 os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
 308
 309             # Since the process isn't running it's safe to get any remaining
 310             # stdout/stderr values out and store them.
 311             self.result.out, self.result.err = proc.communicate()
 312
 313             raise TestRunError(
 314                 'Test run time exceeded timeout value ({} seconds)\n'.format(
 315                     self.timeout),
 316                 'timeout')
 317         # LLVM prints colored text into stdout/stderr on error, which raises:
 318         except UnicodeDecodeError as e:
 319             raise TestRunError("UnicodeDecodeError.\n", 'crash')
 320
 321         # The setter handles the bytes/unicode conversion
 322         self.result.out = out
 323         self.result.err = err
 324         self.result.returncode = returncode
 325
 326     def __eq__(self, other):
 327         return self.command == other.command
 328
 329     def __ne__(self, other):
 330         return not self == other
 331
 332
 333 class DummyTest(Test):
 334     def __init__(self, name, result):
 335         super(DummyTest, self).__init__([name])
 336         self.result.result = result
 337
 338     def execute(self, path, log, options):
 339         pass
 340
 341     def interpret_result(self):
 342         pass
 343
 344
 345 class WindowResizeMixin(object):
 346     """ Mixin class that deals with spurious window resizes
 347
 348     On gnome (and possible other DE's) the window manager may decide to resize
 349     a window. This causes the test to fail even though otherwise would not.
 350     This Mixin overides the _run_command method to run the test 5 times, each
 351     time searching for the string 'Got suprious window resize' in the output,
 352     if it fails to find it it will break the loop and continue.
 353
 354     see: https://bugzilla.gnome.org/show_bug.cgi?id=680214
 355
 356     """
 357     def _run_command(self, *args, **kwargs):
 358         """Run a test up 5 times when window resize is detected.
 359
 360         Rerun the command up to 5 times if the window size changes, if it
 361         changes 6 times mark the test as fail and return True, which will cause
 362         Test.run() to return early.
 363
 364         """
 365         for _ in range(5):
 366             super(WindowResizeMixin, self)._run_command(*args, **kwargs)
 367             if "Got spurious window resize" not in self.result.out:
 368                 return
 369
 370         # If we reach this point then there has been no error, but spurious
 371         # resize was detected more than 5 times. Set the result to fail
 372         raise TestRunError('Got spurious resize more than 5 times', 'fail')
 373
 374
 375 class ValgrindMixin(object):
 376     """Mixin class that adds support for running tests through valgrind.
 377
 378     This mixin allows a class to run with the --valgrind option.
 379
 380     """
 381     @Test.command.getter
 382     def command(self):
 383         return super(ValgrindMixin, self).command
 384
 385     def keys(self):
 386         if OPTIONS.valgrind:
 387             return ['valgrind', '--quiet', '--error-exitcode=1',
 388                     '--tool=memcheck']
 389         else:
 390             return []
 391
 392     def interpret_result(self):
 393         """Set the status to the valgrind status.
 394
 395         It is important that the valgrind interpret_results code is run last,
 396         since it depends on the statuses already set and passed to it,
 397         including the Test.interpret_result() method. To this end it executes
 398         super().interpret_result(), then calls it's own result.
 399
 400         """
 401         super(ValgrindMixin, self).interpret_result()
 402
 403         if OPTIONS.valgrind:
 404             # If the underlying test failed, simply report
 405             # 'skip' for this valgrind test.
 406             if self.result.result != 'pass' and (
 407                     self.result.result != 'warn' and
 408                     self.result.returncode != 0):
 409                 self.result.result = 'skip'
 410             elif self.result.returncode == 0:
 411                 # Test passes and is valgrind clean.
 412                 self.result.result = 'pass'
 413             else:
 414                 # Test passed but has valgrind errors.
 415                 self.result.result = 'fail'
 416
 417
 418 class ReducedProcessMixin(metaclass=abc.ABCMeta):
 419     """This Mixin simplifies writing Test classes that run more than one test
 420     in a single process.
 421
 422     Although one of the benefits of piglit is it's process isolation, there are
 423     times that process isolation is too expensive for day to day runs, and
 424     running more than one test in a single process is a valid trade-off for
 425     decreased run times. This class helps to ease writing a Test class for such
 426     a purpose, while not suffering all of the drawback of the approach.
 427
 428     The first way that this helps is that it provides crash detection and
 429     recovery, allowing a single subtest to crash
 430     """
 431
 432     def __init__(self, command, subtests=None, **kwargs):
 433         assert subtests is not None
 434         super(ReducedProcessMixin, self).__init__(command, **kwargs)
 435         self._expected = subtests
 436         self._populate_subtests()
 437
 438     def is_skip(self):
 439         """Skip if the length of expected is 0."""
 440         if not self._expected:
 441             raise TestIsSkip('All subtests skipped')
 442         super(ReducedProcessMixin, self).is_skip()
 443
 444     def __find_sub(self):
 445         """Helper for getting the next index."""
 446         return len([l for l in self.result.out.split('\n')
 447                     if self._is_subtest(l)])
 448
 449     @staticmethod
 450     def _subtest_name(test):
 451         """If the name provided isn't the subtest name, this method does."""
 452         return test
 453
 454     def _stop_status(self):
 455         """This method returns the status of the test that stopped the run.
 456
 457         By default this will return status.CRASH, but this may not be suitable
 458         for some suites, which may require special considerations and need to
 459         require a different status in some cases, like SKIP.
 460         """
 461         return status.CRASH
 462
 463     def _run_command(self, *args, **kwargs):
 464         """Run the command until all of the subtests have completed or crashed.
 465
 466         This method will try to run all of the subtests, resuming the run if
 467         it's interrupted, and combining the stdout and stderr attributes
 468         together for parsing later. I will separate those values with
 469         "\n\n====RESUME====\n\n".
 470         """
 471         super(ReducedProcessMixin, self)._run_command(*args, **kwargs)
 472
 473         if not self._is_cherry():
 474             returncode = self.result.returncode
 475             out = [self.result.out]
 476             err = [self.result.err]
 477             cur_sub = self.__find_sub() or 1
 478             last = len(self._expected)
 479
 480             while cur_sub < last:
 481                 self.result.subtests[
 482                     self._subtest_name(self._expected[cur_sub - 1])] = \
 483                         self._stop_status()
 484
 485                 super(ReducedProcessMixin, self)._run_command(
 486                     _command=self._resume(cur_sub) + list(args), **kwargs)
 487
 488                 out.append(self.result.out)
 489                 err.append(self.result.err)
 490
 491                 # If the index is 0 the next test failed without printing a
 492                 # name, increase by 1 so that test will be marked crash and we
 493                 # don't get stuck in an infinite loop, otherwise return the
 494                 # number of tests that did complete.
 495                 cur_sub += self.__find_sub() or 1
 496
 497             if not self._is_cherry():
 498                 self.result.subtests[
 499                     self._subtest_name(self._expected[cur_sub - 1])] = \
 500                         self._stop_status()
 501
 502             # Restore and keep the original returncode (so that it remains a
 503             # non-pass, since only one test might fail and the resumed part
 504             # might return 0)
 505             self.result.returncode = returncode
 506             self.result.out = '\n\n====RESUME====\n\n'.join(out)
 507             self.result.err = '\n\n====RESUME====\n\n'.join(err)
 508
 509     def _is_cherry(self):
 510         """Method used to determine if rerunning is required.
 511
 512         If this returns False then the rerun path will be entered, otherwise
 513         _run_command is effectively a bare call to super().
 514
 515         Classes using this mixin may need to overwrite this if the binary
 516         they're calling can stop prematurely but return 0.
 517         """
 518         return self.result.returncode == 0
 519
 520     def _populate_subtests(self):
 521         """Default implementation of subtest prepopulation.
 522
 523         It may be necissary to override this depending on the subtest format.
 524         """
 525         self.result.subtests.update({x: status.NOTRUN for x in self._expected})
 526
 527     @abc.abstractmethod
 528     def _resume(self, current):
 529         """Method that defines how to resume the case if it crashes.
 530
 531         This method will be provided with a completed count, which is the index
 532         into self._expected of the first subtest that hasn't been run. This
 533         method should return the command to restart, and the ReduceProcessMixin
 534         will handle actually restarting the the process with the new command.
 535         """
 536
 537     @abc.abstractmethod
 538     def _is_subtest(self, line):
 539         """Determines if a line in stdout contains a subtest name.
 540
 541         This method is used during the resume detection phase of the
 542         _run_command method to determine how many subtests have successfully
 543         been run.
 544
 545         Should simply return True if the line reprents a test starting, or
 546         False if it does not.
 547         """