unittests/framework/test_dmesg.py

   1 # coding=utf-8
   2 # Copyright (c) 2015-2016, 2019 Intel Corporation
   3
   4 # Permission is hereby granted, free of charge, to any person obtaining a copy
   5 # of this software and associated documentation files (the "Software"), to deal
   6 # in the Software without restriction, including without limitation the rights
   7 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 # copies of the Software, and to permit persons to whom the Software is
   9 # furnished to do so, subject to the following conditions:
  10
  11 # The above copyright notice and this permission notice shall be included in
  12 # all copies or substantial portions of the Software.
  13
  14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20 # SOFTWARE.
  21
  22 """Tests for the dmesg module.
  23
  24 This module makes extensive use of mock to avoid actually calling into dmesg,
  25 which allows us to test all classes on all platforms, including windows.
  26
  27 """
  28
  29 import collections
  30 import re
  31 try:
  32     import mock
  33 except ImportError:
  34     from unittest import mock
  35
  36 import pytest
  37
  38 from framework import dmesg
  39 from framework import status
  40 from framework import results
  41 from framework import exceptions
  42
  43 from . import skip
  44
  45 # pylint: disable=invalid-name,no-self-use
  46
  47
  48 class _DmesgTester(dmesg.BaseDmesg):
  49     """Test Dmesg class. stubs update_dmesg and __init__"""
  50
  51     def __init__(self):
  52         super(_DmesgTester, self).__init__()
  53         self._new_messages = ['some', 'new', 'messages']
  54
  55     def update_dmesg(self, *args, **kwargs):
  56         pass
  57
  58
  59 class TestBaseDmesg(object):
  60     """Tests for the BaseDmesg class."""
  61
  62     result = None
  63
  64     @classmethod
  65     def setup_class(cls):
  66         cls.dmesg = _DmesgTester()
  67
  68     def setup(self):
  69         self.result = results.TestResult()
  70         self.result.dmesg = mock.sentinel.dmesg
  71
  72     def test_update_result_dmesg(self):
  73         """dmesg.BaseDmesg.update_result: records new dmesg content in result"""
  74         self.dmesg.update_result(self.result)
  75         assert self.result.dmesg is not mock.sentinel.dmesg
  76
  77     @pytest.mark.parametrize(
  78         "initial,expected",
  79         [
  80             (status.PASS, status.DMESG_WARN),
  81             (status.WARN, status.DMESG_FAIL),
  82             (status.FAIL, status.DMESG_FAIL),
  83             (status.CRASH, status.CRASH),
  84             (status.SKIP, status.SKIP),
  85             (status.NOTRUN, status.NOTRUN),
  86             (status.TIMEOUT, status.TIMEOUT),
  87         ],
  88         ids=str)
  89     def test_update_result_status(self, initial, expected):
  90         """Test that when update_result is called status change when they
  91         should, and don't when they shouldn't.
  92         """
  93         self.result.result = initial
  94         self.dmesg.update_result(self.result)
  95         assert self.result.result is expected
  96
  97     @pytest.mark.parametrize(
  98         "initial,expected",
  99         [
 100             (status.PASS, status.DMESG_WARN),
 101             (status.WARN, status.DMESG_FAIL),
 102             (status.FAIL, status.DMESG_FAIL),
 103             (status.CRASH, status.CRASH),
 104             (status.SKIP, status.SKIP),
 105             (status.NOTRUN, status.NOTRUN),
 106             (status.TIMEOUT, status.TIMEOUT),
 107         ],
 108         ids=str)
 109     def test_update_result_subtests(self, initial, expected):
 110         """Test that when update_result is called subtest statuses change when
 111         they should, and don't when they shouldn't.
 112         """
 113         self.result.subtests['foo'] = initial
 114         self.result.subtests['bar'] = initial
 115
 116         self.dmesg.update_result(self.result)
 117
 118         assert self.result.subtests['foo'] is expected
 119         assert self.result.subtests['bar'] is expected
 120
 121     def test_update_result_regex_no_match(self):
 122         """dmesg.BaseDmesg.update_result: if no regex matches don't change
 123         status.
 124         """
 125         self.dmesg.regex = re.compile(r'nomatchforthisreally')
 126         self.result.result = status.PASS
 127
 128         self.dmesg.update_result(self.result)
 129
 130         assert self.result.result is status.PASS
 131
 132     def test_update_result_regex_match(self):
 133         """dmesg.BaseDmesg.update_result: if regex matches change status."""
 134         self.dmesg.regex = re.compile(r'.*')
 135         self.result.result = status.PASS
 136
 137         self.dmesg.update_result(self.result)
 138
 139         assert self.result.result is status.DMESG_WARN
 140
 141
 142 class TestLinuxDmesgTimestamps(object):
 143     """Tests for the LinuxDmesg.__init__ timestampe detection.
 144
 145     The linux path will try to open /proc/config.gz and look for
 146     CONFIG_PRINTK_TIME=y, there are a couple of things that can go wrong it
 147     should recover from, in both of the following cases it's expected to go
 148     on and fall back to less exact checking.
 149
 150     The first is that there is no /proc/config.gz in that case we'll get a
 151     OSError in python2 or it's descendent FileNotFoundError in python 3.
 152
 153     The second is that it could get an IOError in python 2.x or
 154     PermissionError in python 3.x, if the file cannot be read.
 155
 156     The fallback path will try read dmesg and see if there's evidence of a
 157     timestamp, or warn that it can't find anything.
 158
 159     """
 160     def test_warn(self, mocker):
 161         """Test that if /proc/config is not available, and there are no values
 162         to check in _last_message a RuntimeWarning should be issued.
 163         """
 164         # Mocking this will prevent LinuxDmesg._last_message from being
 165         # updated, which will force us down the warn path
 166         mocker.patch('framework.dmesg.LinuxDmesg.update_dmesg')
 167
 168         # OSError was picked because it will work for both python2 and python3
 169         # (FileNotFoundError is a descendent of OSError)
 170         mocker.patch('framework.dmesg.gzip.open', side_effect=OSError)
 171
 172         with pytest.warns(RuntimeWarning):
 173             dmesg.LinuxDmesg()
 174
 175     # Implementation notes:
 176     #
 177     # It would seem like a parametrized test would be what we want here, since
 178     # these tests share so much state, but FileNotFoundError and
 179     # PermissionError don't exist in python 2.x, and the parametrizer will
 180     # cause an error. Using a class with a shared method is the next best
 181     # solution.
 182     @staticmethod
 183     def _do_test(error, mocker):
 184         # Mocking this will prevent LinuxDmesg._last_message from being
 185         # updated, which will force us down the warn path, which is convenient
 186         # for assertion puproses.
 187         mocker.patch('framework.dmesg.LinuxDmesg.update_dmesg')
 188         mocker.patch('framework.dmesg.gzip.open', side_effect=error)
 189
 190         with pytest.warns(RuntimeWarning):
 191             dmesg.LinuxDmesg()
 192
 193     def test_config_filenotfounderror(self, mocker):
 194         """Test that on python 3.x if an FileNotFound is raised by gzip.open
 195         operation doesn't stop.
 196         """
 197         self._do_test(FileNotFoundError, mocker)
 198
 199     def test_config_permissionerror(self, mocker):
 200         """Test that on python 3.x if an PermissionError is raised by gzip.open
 201         operation doesn't stop.
 202         """
 203         self._do_test(PermissionError, mocker)
 204
 205     def test_not_timestamps(self, mocker):
 206         """If _last_message is populated but doesn't have a valid timestamp
 207         then an PiglitFatalException shoudl be raised.
 208         """
 209         mocker.patch('framework.dmesg.subprocess.check_output',
 210                      mocker.Mock(return_value=b'foo\nbar\n'))
 211         # This error will work for python 2.x and 3.x
 212         mocker.patch('framework.dmesg.gzip.open', side_effect=OSError)
 213
 214         with pytest.raises(exceptions.PiglitFatalError):
 215             dmesg.LinuxDmesg()
 216
 217     @skip.linux
 218     def test_partial_wrap(self):
 219         """dmesg.LinuxDmesg.update_dmesg: correctly handles partial wrap.
 220
 221         Since dmesg is a ringbuffer it can roll over, and we need to ensure
 222         that we're handling that correctly.
 223         """
 224         result = results.TestResult()
 225
 226         mock_out = mock.Mock(return_value=b'[1.0]This\n[2.0]is\n[3.0]dmesg')
 227         with mock.patch('framework.dmesg.subprocess.check_output', mock_out):
 228             test = dmesg.LinuxDmesg()
 229
 230         mock_out.return_value = b'[3.0]dmesg\n[4.0]whoo!'
 231         with mock.patch('framework.dmesg.subprocess.check_output', mock_out):
 232             test.update_result(result)
 233
 234         assert result.dmesg == '[4.0]whoo!'
 235
 236     @skip.linux
 237     def test_complete_wrap(self):
 238         """dmesg.LinuxDmesg.update_dmesg: correctly handles complete wrap.
 239
 240         Since dmesg is a ringbuffer (at least on Linux) it can roll over, and we
 241         need to ensure that we're handling that correctly.
 242         """
 243         result = results.TestResult()
 244
 245         mock_out = mock.Mock(return_value=b'[1.0]This\n[2.0]is\n[3.0]dmesg')
 246         with mock.patch('framework.dmesg.subprocess.check_output', mock_out):
 247             test = dmesg.LinuxDmesg()
 248
 249         mock_out.return_value = b'[4.0]whoo!\n[5.0]doggy'
 250         with mock.patch('framework.dmesg.subprocess.check_output', mock_out):
 251             test.update_result(result)
 252
 253         assert result.dmesg == '[4.0]whoo!\n[5.0]doggy'
 254
 255
 256 class TestLinuxDmesg(object):
 257     """Tests for LinuxDmesg methods."""
 258
 259     def test_update_result_sets_result_attr(self):
 260         """When update_dmesg is called on a value it should set the dmesg
 261         attribute of the results.
 262         """
 263         result = results.TestResult(status.PASS)
 264
 265         with mock.patch('framework.dmesg.subprocess.check_output',
 266                         mock.Mock(return_value=b'[1.0]this')):
 267             test = dmesg.LinuxDmesg()
 268         with mock.patch('framework.dmesg.subprocess.check_output',
 269                         mock.Mock(
 270                             return_value=b'[1.0]this\n[2.0]is\n[2.5]dmesg!\n')):
 271             test.update_result(result)
 272
 273         assert result.dmesg == '[2.0]is\n[2.5]dmesg!'
 274
 275     def test_update_result_no_change(self):
 276         """When update_result is called but no changes to dmesg have occured it
 277         should not set the dmesg attribute.
 278         """
 279         result = results.TestResult('pass')
 280         result.dmesg = mock.sentinel.dmesg
 281
 282         with mock.patch('framework.dmesg.subprocess.check_output',
 283                         mock.Mock(return_value=b'[1.0]this')):
 284             test = dmesg.LinuxDmesg()
 285             test.update_result(result)
 286
 287         assert result.dmesg is mock.sentinel.dmesg
 288
 289     def test_repr(self):
 290         with mock.patch('framework.dmesg.subprocess.check_output',
 291                         mock.Mock(return_value=b'[1.0]this')):
 292             assert repr(dmesg.LinuxDmesg()) == 'LinuxDmesg()'
 293
 294
 295 class TestDummyDmesg(object):
 296     """Tests for the DummyDmesg class."""
 297     _Namespace = collections.namedtuple('_Namespace', ['dmesg', 'result'])
 298
 299     @pytest.fixture
 300     def testers(self):
 301         # The setter for result.TestResult checks types, rather than trying to
 302         # make the sentinel pass that test, just make a mock that mostly acts
 303         # like TestResult
 304         result = mock.Mock(spec=results.TestResult)
 305         result.dmesg = mock.sentinel.dmesg
 306         result.result = mock.sentinel.result
 307         return self._Namespace(dmesg.DummyDmesg(), result)
 308
 309     def test_update_result(self, testers):
 310         """DummyDmesg.update_results shouldn't do anything."""
 311         testers.dmesg.update_result(testers.result)
 312         assert testers.result.dmesg is mock.sentinel.dmesg
 313         assert testers.result.result is mock.sentinel.result
 314
 315     def test_repr(self):
 316         assert repr(dmesg.DummyDmesg()) == 'DummyDmesg()'
 317
 318
 319 def _name_get_dmesg(value):
 320     """Function that names TestGetDmesg.test_get_dmesg."""
 321     if isinstance(value, bool):
 322         return 'real' if not value else 'dummy'
 323     elif isinstance(value, str):
 324         return value
 325     elif isinstance(value, dmesg.BaseDmesg):
 326         return repr(value)
 327     else:
 328         raise Exception('unreachable')
 329
 330
 331 class TestGetDmesg(object):
 332     """Tests for get_dmesg factory."""
 333
 334     @pytest.mark.parametrize(
 335         'platform,dummy,expected',
 336         [
 337             ('win32', False, dmesg.DummyDmesg),
 338             ('win32', True, dmesg.DummyDmesg),
 339             skip.linux(('linux', False, dmesg.DummyDmesg)),
 340             skip.linux(('linux', True, dmesg.LinuxDmesg)),
 341         ],
 342         ids=_name_get_dmesg)
 343     def test_get_dmesg(self, platform, dummy, expected, mocker):
 344         """Test that get_dmesg returns the expected dmesg type on variuos
 345         platforms with various configurations.
 346         """
 347         mocker.patch('framework.dmesg.sys.platform', platform)
 348
 349         with mock.patch('framework.dmesg.subprocess.check_output',
 350                         mock.Mock(return_value=b'[1.0]foo')):
 351             actual = dmesg.get_dmesg(not_dummy=dummy)
 352
 353         # We don't want a subclass, we want the *exact* class. This is a
 354         # unittest after all
 355         assert type(actual) == expected  # pylint: disable=unidiomatic-typecheck