framework/test/base.py

   1 # coding=utf-8
   2 #
   3 # Permission is hereby granted, free of charge, to any person
   4 # obtaining a copy of this software and associated documentation
   5 # files (the "Software"), to deal in the Software without
   6 # restriction, including without limitation the rights to use,
   7 # copy, modify, merge, publish, distribute, sublicense, and/or
   8 # sell copies of the Software, and to permit persons to whom the
   9 # Software is furnished to do so, subject to the following
  10 # conditions:
  11 #
  12 # This permission notice shall be included in all copies or
  13 # substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
  16 # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
  17 # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  18 # PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
  19 # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
  20 # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
  21 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22 # DEALINGS IN THE SOFTWARE.
  23
  24 """ Module provides a base class for Tests """
  25
  26 import abc
  27 import copy
  28 import errno
  29 import itertools
  30 import os
  31 import signal
  32 import subprocess
  33 import sys
  34 import time
  35 import traceback
  36 import warnings
  37
  38 from framework import exceptions
  39 from framework import status
  40 from framework.options import OPTIONS
  41 from framework.results import TestResult
  42
  43 _EXTRA_POPEN_ARGS = {}
  44
  45 if os.name == 'posix':
  46     # This should work for all *nix systems, Linux, the BSDs, and OSX.
  47     # This speicifically creates a session group for each test, so that
  48     # it's children can be killed if it times out.
  49     _EXTRA_POPEN_ARGS = {'start_new_session': True}
  50
  51 # pylint: enable=wrong-import-position,wrong-import-order
  52
  53
  54 __all__ = [
  55     'Test',
  56     'TestIsSkip',
  57     'TestRunError',
  58     'ValgrindMixin',
  59     'WindowResizeMixin',
  60     'is_crash_returncode',
  61 ]
  62
  63 # Allows timeouts to be suppressed by setting the environment variable
  64 # PIGLIT_NO_TIMEOUT to anything that bool() will resolve as True
  65 _SUPPRESS_TIMEOUT = bool(os.environ.get('PIGLIT_NO_TIMEOUT', False))
  66
  67
  68 class TestIsSkip(exceptions.PiglitException):
  69     """Exception raised in is_skip() if the test is a skip."""
  70     def __init__(self, reason):
  71         super(TestIsSkip, self).__init__()
  72         self.reason = reason
  73
  74
  75 class TestRunError(exceptions.PiglitException):
  76     """Exception raised if the test fails to run."""
  77     def __init__(self, message, status):
  78         super(TestRunError, self).__init__(message)
  79         self.status = status
  80
  81
  82 def is_crash_returncode(returncode):
  83     """Determine whether the given process return code correspond to a
  84     crash.
  85     """
  86     # In Python 3 isn't allowed to compare NoneType and other types.
  87     if returncode is None:
  88         return False
  89
  90     if sys.platform == 'win32':
  91         # On Windows:
  92         # - For uncaught exceptions the process terminates with the exception
  93         # code, which is usually negative
  94         # - MSVCRT's abort() terminates process with exit code 3
  95         return returncode < 0 or returncode == 3
  96     else:
  97         return returncode < 0
  98
  99
 100 class Test(metaclass=abc.ABCMeta):
 101     """ Abstract base class for Test classes
 102
 103     This class provides the framework for running tests, with several methods
 104     and properties that can be overwritten to produce a specialized class for
 105     running test suites other than piglit.
 106
 107     It provides two methods for running tests, execute and run.
 108     execute() provides lots of features, and is invoked when running piglit
 109     from the command line, run() is a more basic method for running the test,
 110     and is called internally by execute(), but is can be useful outside of it.
 111
 112     Arguments:
 113     command -- a value to be passed to subprocess.Popen
 114
 115     Keyword Arguments:
 116     run_concurrent -- If True the test is thread safe. Default: False
 117
 118     """
 119     __slots__ = ['run_concurrent', 'env', 'result', 'cwd', '_command']
 120     timeout = None
 121
 122     def __init__(self, command, run_concurrent=False, env=None, cwd=None):
 123         assert isinstance(command, list), command
 124
 125         self.run_concurrent = run_concurrent
 126         self._command = copy.copy(command)
 127         self.env = env or {}
 128         self.result = TestResult()
 129         self.cwd = cwd
 130
 131     def execute(self, path, log, options):
 132         """ Run a test
 133
 134         Run a test, but with features. This times the test, uses dmesg checking
 135         (if requested), and runs the logger.
 136
 137         Arguments:
 138         path    -- the name of the test
 139         log     -- a log.Log instance
 140         options -- a dictionary containing dmesg and monitoring objects
 141         """
 142         log.start(path)
 143         # Run the test
 144         if OPTIONS.execute:
 145             try:
 146                 self.result.time.start = time.time()
 147                 options['dmesg'].update_dmesg()
 148                 options['monitor'].update_monitoring()
 149                 self.run()
 150                 self.result.time.end = time.time()
 151                 self.result = options['dmesg'].update_result(self.result)
 152                 options['monitor'].check_monitoring()
 153             # This is a rare case where a bare exception is okay, since we're
 154             # using it to log exceptions
 155             except:
 156                 exc_type, exc_value, exc_traceback = sys.exc_info()
 157                 traceback.print_exc(file=sys.stderr)
 158                 self.result.result = 'fail'
 159                 self.result.exception = "{}{}".format(exc_type, exc_value)
 160                 self.result.traceback = "".join(
 161                     traceback.format_tb(exc_traceback))
 162
 163             log.log(self.result.result)
 164         else:
 165             log.log('dry-run')
 166
 167     @property
 168     def command(self):
 169         assert self._command
 170         return self._command
 171
 172     @command.setter
 173     def command(self, new):
 174         assert isinstance(new, list), 'Test.command must be a list'
 175         self._command = new
 176
 177     @abc.abstractmethod
 178     def interpret_result(self):
 179         """Convert the raw output of the test into a form piglit understands.
 180         """
 181         if is_crash_returncode(self.result.returncode):
 182             self.result.result = status.CRASH
 183             if self.result.subtests:
 184                 # We know because subtests are ordered that the first test with
 185                 # a status of NOTRUN is the subtest that crashed, mark that
 186                 # test and move on.
 187                 for k, v in self.result.subtests.items():
 188                     if v == status.NOTRUN:
 189                         self.result.subtests[k] = status.CRASH
 190                         break
 191         elif self.result.returncode != 0:
 192             if self.result.result == status.PASS:
 193                 self.result.result = status.WARN
 194             else:
 195                 self.result.result = status.FAIL
 196
 197     def run(self):
 198         """
 199         Run a test.  The return value will be a dictionary with keys
 200         including 'result', 'info', 'returncode' and 'command'.
 201         * For 'result', the value may be one of 'pass', 'fail', 'skip',
 202           'crash', or 'warn'.
 203         * For 'info', the value will include stderr/out text.
 204         * For 'returncode', the value will be the numeric exit code/value.
 205         * For 'command', the value will be command line program and arguments.
 206         """
 207         self.result.command = ' '.join(map(str, self.command))
 208         self.result.environment = " ".join(
 209             '{0}="{1}"'.format(k, v) for k, v in itertools.chain(
 210                 OPTIONS.env.items(), self.env.items()))
 211
 212         try:
 213             self.is_skip()
 214         except TestIsSkip as e:
 215             self.result.result = status.SKIP
 216             for each in self.result.subtests.keys():
 217                 self.result.subtests[each] = status.SKIP
 218             self.result.out = e.reason
 219             self.result.returncode = None
 220             return
 221
 222         try:
 223             self._run_command()
 224         except TestRunError as e:
 225             self.result.result = str(e.status)
 226             for each in self.result.subtests.keys():
 227                 self.result.subtests[each] = str(e.status)
 228             self.result.out = str(e)
 229             self.result.returncode = None
 230             return
 231
 232         self.interpret_result()
 233
 234     def is_skip(self):
 235         """ Application specific check for skip
 236
 237         If this function returns a truthy value then the current test will be
 238         skipped. The base version will always return False
 239
 240         """
 241         pass
 242
 243     def _run_command(self, **kwargs):
 244         """ Run the test command and get the result
 245
 246         This method sets environment options, then runs the executable. If the
 247         executable isn't found it sets the result to skip.
 248
 249         """
 250         # This allows the ReducedProcessMixin to work without having to whack
 251         # self.command (which should be treated as immutable), but is
 252         # considered private.
 253         command = kwargs.pop('_command', self.command)
 254
 255         # Setup the environment for the test. Environment variables are taken
 256         # from the following sources, listed in order of increasing precedence:
 257         #
 258         #   1. This process's current environment.
 259         #   2. Global test options. (Some of these are command line options to
 260         #      Piglit's runner script).
 261         #   3. Per-test environment variables set in all.py.
 262         #
 263         # Piglit chooses this order because Unix tradition dictates that
 264         # command line options (2) override environment variables (1); and
 265         # Piglit considers environment variables set in all.py (3) to be test
 266         # requirements.
 267         _base = itertools.chain(os.environ.items(),
 268                                 OPTIONS.env.items(),
 269                                 self.env.items())
 270         fullenv = {str(k): str(v) for k, v in _base}
 271
 272         try:
 273             proc = subprocess.Popen(map(str, command),
 274                                     stdout=subprocess.PIPE,
 275                                     stderr=subprocess.PIPE,
 276                                     cwd=self.cwd,
 277                                     env=fullenv,
 278                                     universal_newlines=True,
 279                                     **_EXTRA_POPEN_ARGS)
 280
 281             self.result.pid.append(proc.pid)
 282             if not _SUPPRESS_TIMEOUT:
 283                 out, err = proc.communicate(timeout=self.timeout)
 284             else:
 285                 out, err = proc.communicate()
 286             returncode = proc.returncode
 287         except OSError as e:
 288             # Different sets of tests get built under different build
 289             # configurations.  If a developer chooses to not build a test,
 290             # Piglit should not report that test as having failed.
 291             if e.errno == errno.ENOENT:
 292                 raise TestRunError("Test executable not found.\n", 'skip')
 293             else:
 294                 raise e
 295         except subprocess.TimeoutExpired:
 296             # This can only be reached if subprocess32 is present or on python
 297             # 3.x, since # TimeoutExpired is never raised by the python 2.7
 298             # fallback code.
 299
 300             proc.terminate()
 301
 302             if sys.platform != 'win32':
 303                 if proc.poll() is None:
 304                     time.sleep(3)
 305                     os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
 306
 307             # Since the process isn't running it's safe to get any remaining
 308             # stdout/stderr values out and store them.
 309             self.result.out, self.result.err = proc.communicate()
 310
 311             raise TestRunError(
 312                 'Test run time exceeded timeout value ({} seconds)\n'.format(
 313                     self.timeout),
 314                 'timeout')
 315         # LLVM prints colored text into stdout/stderr on error, which raises:
 316         except UnicodeDecodeError as e:
 317             raise TestRunError("UnicodeDecodeError.\n", 'crash')
 318
 319         # The setter handles the bytes/unicode conversion
 320         self.result.out = out
 321         self.result.err = err
 322         self.result.returncode = returncode
 323
 324     def __eq__(self, other):
 325         return self.command == other.command
 326
 327     def __ne__(self, other):
 328         return not self == other
 329
 330
 331 class DummyTest(Test):
 332     def __init__(self, name, result):
 333         super(DummyTest, self).__init__([name])
 334         self.result.result = result
 335
 336     def execute(self, path, log, options):
 337         pass
 338
 339     def interpret_result(self):
 340         pass
 341
 342
 343 class WindowResizeMixin(object):
 344     """ Mixin class that deals with spurious window resizes
 345
 346     On gnome (and possible other DE's) the window manager may decide to resize
 347     a window. This causes the test to fail even though otherwise would not.
 348     This Mixin overrides the _run_command method to run the test 5 times, each
 349     time searching for the string 'Got spurious window resize' in the output,
 350     if it fails to find it it will break the loop and continue.
 351
 352     see: https://bugzilla.gnome.org/show_bug.cgi?id=680214
 353
 354     """
 355     def _run_command(self, *args, **kwargs):
 356         """Run a test up 5 times when window resize is detected.
 357
 358         Rerun the command up to 5 times if the window size changes, if it
 359         changes 6 times mark the test as fail and return True, which will cause
 360         Test.run() to return early.
 361
 362         """
 363         for _ in range(5):
 364             super(WindowResizeMixin, self)._run_command(*args, **kwargs)
 365             if "Got spurious window resize" not in self.result.out:
 366                 return
 367
 368         # If we reach this point then there has been no error, but spurious
 369         # resize was detected more than 5 times. Set the result to fail
 370         raise TestRunError('Got spurious resize more than 5 times', 'fail')
 371
 372
 373 class ValgrindMixin(object):
 374     """Mixin class that adds support for running tests through valgrind.
 375
 376     This mixin allows a class to run with the --valgrind option.
 377
 378     """
 379     @Test.command.getter
 380     def command(self):
 381         return super(ValgrindMixin, self).command
 382
 383     def keys(self):
 384         if OPTIONS.valgrind:
 385             return ['valgrind', '--quiet', '--error-exitcode=1',
 386                     '--tool=memcheck']
 387         else:
 388             return []
 389
 390     def interpret_result(self):
 391         """Set the status to the valgrind status.
 392
 393         It is important that the valgrind interpret_results code is run last,
 394         since it depends on the statuses already set and passed to it,
 395         including the Test.interpret_result() method. To this end it executes
 396         super().interpret_result(), then calls it's own result.
 397
 398         """
 399         super(ValgrindMixin, self).interpret_result()
 400
 401         if OPTIONS.valgrind:
 402             # If the underlying test failed, simply report
 403             # 'skip' for this valgrind test.
 404             if self.result.result != 'pass' and (
 405                     self.result.result != 'warn' and
 406                     self.result.returncode != 0):
 407                 self.result.result = 'skip'
 408             elif self.result.returncode == 0:
 409                 # Test passes and is valgrind clean.
 410                 self.result.result = 'pass'
 411             else:
 412                 # Test passed but has valgrind errors.
 413                 self.result.result = 'fail'
 414
 415
 416 class ReducedProcessMixin(metaclass=abc.ABCMeta):
 417     """This Mixin simplifies writing Test classes that run more than one test
 418     in a single process.
 419
 420     Although one of the benefits of piglit is it's process isolation, there are
 421     times that process isolation is too expensive for day to day runs, and
 422     running more than one test in a single process is a valid trade-off for
 423     decreased run times. This class helps to ease writing a Test class for such
 424     a purpose, while not suffering all of the drawback of the approach.
 425
 426     The first way that this helps is that it provides crash detection and
 427     recovery, allowing a single subtest to crash
 428     """
 429
 430     def __init__(self, command, subtests=None, **kwargs):
 431         assert subtests is not None
 432         super(ReducedProcessMixin, self).__init__(command, **kwargs)
 433         self._expected = subtests
 434         self._populate_subtests()
 435
 436     def is_skip(self):
 437         """Skip if the length of expected is 0."""
 438         if not self._expected:
 439             raise TestIsSkip('All subtests skipped')
 440         super(ReducedProcessMixin, self).is_skip()
 441
 442     def __find_sub(self):
 443         """Helper for getting the next index."""
 444         return len([l for l in self.result.out.split('\n')
 445                     if self._is_subtest(l)])
 446
 447     @staticmethod
 448     def _subtest_name(test):
 449         """If the name provided isn't the subtest name, this method does."""
 450         return test
 451
 452     def _stop_status(self):
 453         """This method returns the status of the test that stopped the run.
 454
 455         By default this will return status.CRASH, but this may not be suitable
 456         for some suites, which may require special considerations and need to
 457         require a different status in some cases, like SKIP.
 458         """
 459         return status.CRASH
 460
 461     def _run_command(self, *args, **kwargs):
 462         """Run the command until all of the subtests have completed or crashed.
 463
 464         This method will try to run all of the subtests, resuming the run if
 465         it's interrupted, and combining the stdout and stderr attributes
 466         together for parsing later. I will separate those values with
 467         "\n\n====RESUME====\n\n".
 468         """
 469         super(ReducedProcessMixin, self)._run_command(*args, **kwargs)
 470
 471         if not self._is_cherry():
 472             returncode = self.result.returncode
 473             out = [self.result.out]
 474             err = [self.result.err]
 475             cur_sub = self.__find_sub() or 1
 476             last = len(self._expected)
 477
 478             while cur_sub < last:
 479                 self.result.subtests[
 480                     self._subtest_name(self._expected[cur_sub - 1])] = \
 481                         self._stop_status()
 482
 483                 super(ReducedProcessMixin, self)._run_command(
 484                     _command=self._resume(cur_sub) + list(args), **kwargs)
 485
 486                 out.append(self.result.out)
 487                 err.append(self.result.err)
 488
 489                 # If the index is 0 the next test failed without printing a
 490                 # name, increase by 1 so that test will be marked crash and we
 491                 # don't get stuck in an infinite loop, otherwise return the
 492                 # number of tests that did complete.
 493                 cur_sub += self.__find_sub() or 1
 494
 495             if not self._is_cherry():
 496                 self.result.subtests[
 497                     self._subtest_name(self._expected[cur_sub - 1])] = \
 498                         self._stop_status()
 499
 500             # Restore and keep the original returncode (so that it remains a
 501             # non-pass, since only one test might fail and the resumed part
 502             # might return 0)
 503             self.result.returncode = returncode
 504             self.result.out = '\n\n====RESUME====\n\n'.join(out)
 505             self.result.err = '\n\n====RESUME====\n\n'.join(err)
 506
 507     def _is_cherry(self):
 508         """Method used to determine if rerunning is required.
 509
 510         If this returns False then the rerun path will be entered, otherwise
 511         _run_command is effectively a bare call to super().
 512
 513         Classes using this mixin may need to overwrite this if the binary
 514         they're calling can stop prematurely but return 0.
 515         """
 516         return self.result.returncode == 0
 517
 518     def _populate_subtests(self):
 519         """Default implementation of subtest prepopulation.
 520
 521         It may be necissary to override this depending on the subtest format.
 522         """
 523         self.result.subtests.update({x: status.NOTRUN for x in self._expected})
 524
 525     @abc.abstractmethod
 526     def _resume(self, current):
 527         """Method that defines how to resume the case if it crashes.
 528
 529         This method will be provided with a completed count, which is the index
 530         into self._expected of the first subtest that hasn't been run. This
 531         method should return the command to restart, and the ReduceProcessMixin
 532         will handle actually restarting the the process with the new command.
 533         """
 534
 535     @abc.abstractmethod
 536     def _is_subtest(self, line):
 537         """Determines if a line in stdout contains a subtest name.
 538
 539         This method is used during the resume detection phase of the
 540         _run_command method to determine how many subtests have successfully
 541         been run.
 542
 543         Should simply return True if the line reprents a test starting, or
 544         False if it does not.
 545         """