unittests/framework/test_profile.py

   1 # coding=utf-8
   2 # Copyright (c) 2014, 2016, 2019 Intel Corporation
   3
   4 # Permission is hereby granted, free of charge, to any person obtaining a copy
   5 # of this software and associated documentation files (the "Software"), to deal
   6 # in the Software without restriction, including without limitation the rights
   7 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   8 # copies of the Software, and to permit persons to whom the Software is
   9 # furnished to do so, subject to the following conditions:
  10
  11 # The above copyright notice and this permission notice shall be included in
  12 # all copies or substantial portions of the Software.
  13
  14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  17 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  19 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  20 # SOFTWARE.
  21
  22 """ Provides test for the framework.profile modules """
  23
  24 import pytest
  25
  26 from framework import exceptions
  27 from framework import grouptools
  28 from framework import profile
  29 from . import utils
  30
  31 # pylint: disable=invalid-name,no-self-use,protected-access
  32
  33
  34 class TestLoadTestProfile(object):
  35     """Tests for profile.load_test_profile."""
  36
  37     def test_no_profile_attribute(self, tmpdir):
  38         """profile.load_test_profile: Loading a module with no profile name
  39         exits.
  40
  41         Because load_test_profile uses test.{} to load a module we need a
  42         module in tests that doesn't have a profile attribute. The only module
  43         that currently meets that requirement is __init__.py
  44         """
  45         p = tmpdir.join('foo.profile')
  46
  47         with pytest.raises(exceptions.PiglitFatalError):
  48             profile.load_test_profile(str(p))
  49
  50     def test_no_such_module(self, tmpdir):
  51         """profile.load_test_profile: Trying to load a non-existent module
  52         exits.
  53         """
  54         # Ensure that the module will not exist by moving to an empty temporary
  55         # directory
  56         tmpdir.chdir()
  57         with pytest.raises(exceptions.PiglitFatalError):
  58             profile.load_test_profile('this_module_will_never_ever_exist')
  59
  60
  61 class TestTestProfile(object):
  62     """Tests for profile.TestProfile."""
  63
  64     class TestCopy(object):
  65         """Tests for the copy method."""
  66
  67         @pytest.fixture
  68         def fixture(self):
  69             orig = profile.TestProfile()
  70             orig.test_list['foo'] = utils.Test(['foo'])
  71             orig.test_list['bar'] = utils.Test(['bar'])
  72             orig.filters = [lambda name, _: name != 'bar']
  73             orig.forced_test_list = ['foo']
  74             return orig
  75
  76         def test_filters(self, fixture):
  77             """The filters attribute is copied correctly."""
  78             new = fixture.copy()
  79
  80             # Assert that the fixtures are equivalent, but not the same
  81             assert fixture.filters == new.filters
  82             assert fixture.filters is not new.filters
  83
  84             # And double check by modifying one of them and asserting that the
  85             # other has not changed.
  86             new.filters.append(lambda name, _: name != 'oink')
  87             assert len(fixture.filters) == 1
  88
  89         def test_forced_test_list(self, fixture):
  90             """The forced_test_list attribute is copied correctly."""
  91             new = fixture.copy()
  92
  93             # Assert that the fixtures are equivalent, but not the same
  94             assert fixture.forced_test_list == new.forced_test_list
  95             assert fixture.forced_test_list is not new.forced_test_list
  96
  97             # And double check by modifying one of them and asserting that the
  98             # other has not changed.
  99             del new.forced_test_list[0]
 100             assert fixture.forced_test_list[0] == 'foo'
 101
 102         def test_test_list(self, fixture):
 103             """The test_list attribute is copied correctly."""
 104             new = fixture.copy()
 105
 106             # Assert that the fixtures are equivalent, but not the same
 107             assert fixture.test_list == new.test_list
 108             assert fixture.test_list is not new.test_list
 109
 110
 111 class TestTestDict(object):
 112     """Tests for the TestDict object."""
 113
 114     test = None
 115
 116     def setup(self):
 117         self.test = profile.TestDict()
 118
 119     @pytest.mark.parametrize(
 120         "arg",
 121         [None, utils.Test(['foo']), ['a'], {'a': 1}],
 122         ids=str)
 123     def test_key_not_string(self, arg):
 124         """profile.TestDict: If key value isn't a string an exception is raised.
 125
 126         This throws a few different things at the key value and expects an
 127         error to be raised. It isn't a perfect test, but it was the best I
 128         could come up with.
 129         """
 130         with pytest.raises(exceptions.PiglitFatalError):
 131             self.test[arg] = utils.Test(['foo'])
 132
 133     def test_reassignment(self):
 134         """reassigning a key raises an exception."""
 135         self.test['foo'] = utils.Test(['foo'])
 136         with pytest.raises(exceptions.PiglitFatalError):
 137             self.test['foo'] = utils.Test(['foo', 'bar'])
 138
 139     class TestAllowReassignment(object):
 140         """Tests for TestDict.allow_reassignment."""
 141
 142         @pytest.fixture
 143         def test(self):
 144             return profile.TestDict()
 145
 146         def test_case_insensitve(self, test):
 147             """reassigning a key raises an exception (capitalization is
 148             ignored)."""
 149             test['foo'] = utils.Test(['foo'])
 150             with pytest.raises(exceptions.PiglitFatalError):
 151                 test['Foo'] = utils.Test(['foo', 'bar'])
 152
 153         def test_simple(self, test):
 154             """profile.TestDict.allow_reassignment works."""
 155             test['a'] = utils.Test(['foo'])
 156             with test.allow_reassignment:
 157                 test['a'] = utils.Test(['bar'])
 158
 159             assert test['a'].command == ['bar']
 160
 161         def test_with_groupmanager(self, test):
 162             """profile.TestProfile: allow_reassignment wrapper works with
 163             groupmanager.
 164             """
 165             testname = grouptools.join('a', 'b')
 166             test[testname] = utils.Test(['foo'])
 167             with test.allow_reassignment:
 168                 with test.group_manager(utils.Test, 'a') as g:
 169                     g(['bar'], 'b')
 170
 171             assert test[testname].command == ['bar']
 172
 173         def test_stacked(self, test):
 174             """profile.profile.TestDict.allow_reassignment: check stacking
 175             cornercase.
 176
 177             There is an odd corner case in the original (obvious)
 178             implementation of this function, If one opens two context managers
 179             and then returns from the inner one assignment will not be allowed,
 180             even though one is still inside the first context manager.
 181             """
 182             test['a'] = utils.Test(['foo'])
 183             with test.allow_reassignment:
 184                 with test.allow_reassignment:
 185                     pass
 186                 test['a'] = utils.Test(['bar'])
 187
 188             assert test['a'].command == ['bar']
 189
 190     class TestGroupManager(object):
 191         """Test the group_manager method."""
 192
 193         @pytest.fixture
 194         def inst(self):
 195             return profile.TestDict()
 196
 197         def test_no_name_args_eq_one(self, inst):
 198             """no name and len(args) == 1 is valid."""
 199             with inst.group_manager(utils.Test, 'foo') as g:
 200                 g(['a'])
 201
 202             assert grouptools.join('foo', 'a') in inst
 203
 204         def test_no_name_args_gt_one(self, inst):
 205             """no name and len(args) > 1 is valid."""
 206             with inst.group_manager(utils.Test, 'foo') as g:
 207                 g(['a', 'b'])
 208
 209             assert grouptools.join('foo', 'a b') in inst
 210
 211         def test_name(self, inst):
 212             """name plus len(args) > 1 is valid."""
 213             with inst.group_manager(utils.Test, 'foo') as g:
 214                 g(['a', 'b'], 'a')
 215
 216             assert grouptools.join('foo', 'a') in inst
 217
 218         def test_adder_kwargs_passed(self, inst):
 219             """Extra kwargs passed to the adder function are passed to the
 220             Test.
 221             """
 222             with inst.group_manager(utils.Test, 'foo') as g:
 223                 g(['a'], run_concurrent=True)
 224             test = inst[grouptools.join('foo', 'a')]
 225
 226             assert test.run_concurrent is True
 227
 228         def test_context_manager_keyword_args_passed(self, inst):
 229             """kwargs passed to the context_manager are passed to the Test."""
 230             with inst.group_manager(
 231                     utils.Test, 'foo', run_concurrent=True) as g:  # pylint: disable=bad-continuation
 232                     # This is a pylint bug, there's an upstream report
 233                 g(['a'])
 234             test = inst[grouptools.join('foo', 'a')]
 235
 236             assert test.run_concurrent is True
 237
 238         def test_adder_kwargs_overwrite_context_manager_kwargs(self, inst):
 239             """default_args are overwritten by kwargs."""
 240             with inst.group_manager(
 241                     utils.Test, 'foo', run_concurrent=True) as g:  # pylint: disable=bad-continuation
 242                     # This is a pylint bug, there's an upstream report
 243                 g(['a'], run_concurrent=False)
 244
 245             test = inst[grouptools.join('foo', 'a')]
 246             assert test.run_concurrent is False
 247
 248
 249 class TestRegexFilter(object):
 250     """Tests for the RegexFilter class."""
 251
 252     class TestNormal(object):
 253         """Tests for inverse set to False (default)."""
 254
 255         def test_empty(self):
 256             """Returns True when no filters are provided."""
 257             test = profile.RegexFilter([])
 258             assert test('foobob', None)
 259
 260         def test_matches(self):
 261             """Returns True when the test matches any regex."""
 262             test = profile.RegexFilter([r'foo', r'bar'])
 263             assert test('foobob', None)
 264
 265         def test_not_matches(self):
 266             """Returns True when the test matches any regex."""
 267             test = profile.RegexFilter([r'fob', r'bar'])
 268             assert not test('foobob', None)
 269
 270     class TestInverse(object):
 271         """Tests for inverse set to True."""
 272
 273         def test_empty(self):
 274             """Returns True when no filters are provided."""
 275             test = profile.RegexFilter([], inverse=True)
 276             assert test('foobob', None)
 277
 278         def test_matches(self):
 279             """Returns False when the test matches any regex."""
 280             test = profile.RegexFilter([r'foo', r'bar'], inverse=True)
 281             assert not test('foobob', None)
 282
 283         def test_not_matches(self):
 284             """Returns False when the test matches any regex."""
 285             test = profile.RegexFilter([r'fob', r'bar'], inverse=True)
 286             assert test('foobob', None)