unittests/framework/replay/test_compare_replay.py

   1 # coding=utf-8
   2 #
   3 # Copyright © 2020 Valve Corporation.
   4 # Copyright © 2022 Collabora Ltd.
   5 #
   6 # Permission is hereby granted, free of charge, to any person obtaining a
   7 # copy of this software and associated documentation files (the "Software"),
   8 # to deal in the Software without restriction, including without limitation
   9 # the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10 # and/or sell copies of the Software, and to permit persons to whom the
  11 # Software is furnished to do so, subject to the following conditions:
  12 #
  13 # The above copyright notice and this permission notice shall be included
  14 # in all copies or substantial portions of the Software.
  15 #
  16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19 # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20 # OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21 # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22 # OTHER DEALINGS IN THE SOFTWARE.
  23 #
  24 # SPDX-License-Identifier: MIT
  25
  26
  27 """Tests for replayer's compare_replay module."""
  28
  29 import pytest
  30
  31 import contextlib
  32 import io
  33 import os
  34
  35 from os import path
  36
  37 from framework import exceptions, status
  38 from framework.replay import backends
  39 from framework.replay import compare_replay
  40 from framework.replay.options import OPTIONS
  41
  42
  43 class TestCompareReplay(object):
  44     """Tests for compare_replay methods."""
  45
  46     @staticmethod
  47     def _create_dump(trace_path, results_path, calls):
  48         p = path.join(results_path,
  49                       path.basename(trace_path) + '-' + str(calls) + '.png')
  50         os.makedirs(path.dirname(p), exist_ok=True)
  51         with open(p, 'w') as f:
  52             f.write('content')
  53
  54     @staticmethod
  55     def mock_backends_dump(trace_path, results_path, calls):
  56         if trace_path.endswith('KhronosGroup-Vulkan-Tools/amd/polaris10/vkcube.gfxr'):
  57             TestCompareReplay._create_dump(trace_path, results_path, 99)
  58
  59             return True
  60         elif trace_path.endswith('pathfinder/demo.trace'):
  61             TestCompareReplay._create_dump(trace_path, results_path, 78)
  62
  63             return True
  64         elif trace_path.endswith('glmark2/jellyfish.rdc'):
  65             return True
  66         elif trace_path.endswith('Wicked-Engine/Tests:Cloth_Physics_Test.trace-dxgi'):
  67             return False
  68         elif trace_path.endswith('unimplemented/backend.vktrace'):
  69             raise backends.DumpBackendNotImplementedError(
  70                 'DumpBackend for "vktrace" is not implemented')
  71         elif trace_path.endswith('unexisting/back.end'):
  72             raise backends.DumpBackendError(
  73                 'No module supports file extensions "end"')
  74         else:
  75             raise exceptions.PiglitFatalError(
  76                 'Non treated trace path: {}'.format(trace_path))
  77
  78     @staticmethod
  79     def mock_hexdigest_from_image(image_file):
  80         if image_file.endswith(
  81                 'KhronosGroup-Vulkan-Tools/amd/polaris10/vkcube.gfxr-99.png'):
  82             return '917cbbf4f09dd62ea26d247a1c70c16e'
  83         elif image_file.endswith(
  84                 'pathfinder/demo.trace-78.png'):
  85             return 'e624d76c70cc3c532f4f54439e13659a'
  86         else:
  87             raise exceptions.PiglitFatalError(
  88                 'Non treated image file: {}'.format(image_file))
  89
  90     @staticmethod
  91     def mock_qty_load_yaml(yaml_file):
  92         if yaml_file == 'empty.yml':
  93             return {}
  94         elif yaml_file == 'no-device.yml':
  95             return {"traces": {
  96                 "glmark2/desktop-blur-radius=5:effect=blur:passes=1:separable=true:windows=4.rdc": {
  97                     "gl-vmware-llvmpipe": {
  98                         "checksum": "8867f3a41f180626d0d4b7661ff5c0f4"
  99                     }
 100                 }
 101             }}
 102         elif yaml_file == 'one-trace.yml':
 103             return {"traces": {
 104                 "KhronosGroup-Vulkan-Tools/amd/polaris10/vkcube.gfxr": {
 105                     OPTIONS.device_name: {
 106                         "checksum": "917cbbf4f09dd62ea26d247a1c70c16e"
 107                     }
 108                 }
 109             }}
 110         elif yaml_file == 'two-traces.yml':
 111             return {"traces": {
 112                 "pathfinder/demo.trace": {
 113                     OPTIONS.device_name: {
 114                         "checksum": "e624d76c70cc3c532f4f54439e13659a"
 115                     }
 116                 },
 117                 "KhronosGroup-Vulkan-Tools/amd/polaris10/vkcube.gfxr": {
 118                     OPTIONS.device_name: {
 119                         "checksum": "917cbbf4f09dd62ea26d247a1c70c16e"
 120                     }
 121                 }
 122             }}
 123         else:
 124             raise exceptions.PiglitFatalError(
 125                 'Non treated YAML file: {}'.format(yaml_file))
 126
 127     @pytest.fixture(autouse=True)
 128     def setup(self, mocker, tmpdir):
 129         """Setup for TestCompareReplay.
 130
 131         This create the basic environment for testing.
 132         """
 133
 134         OPTIONS.device_name = 'test-device'
 135         OPTIONS.db_path = tmpdir.mkdir('db-path').strpath
 136         OPTIONS.results_path = tmpdir.mkdir('results').strpath
 137         self.trace_path = 'KhronosGroup-Vulkan-Tools/amd/polaris10/vkcube.gfxr'
 138         self.exp_checksum = '917cbbf4f09dd62ea26d247a1c70c16e'
 139         self.results_partial_path = path.join('results/trace',
 140                                               OPTIONS.device_name)
 141         self.m_qty_load_yaml = mocker.patch(
 142             'framework.replay.compare_replay.qty.load_yaml',
 143             side_effect=TestCompareReplay.mock_qty_load_yaml)
 144         self.m_ensure_file = mocker.patch(
 145             'framework.replay.compare_replay.ensure_file',
 146             return_value=None)
 147         self.m_backends_dump = mocker.patch(
 148             'framework.replay.compare_replay.backends.dump',
 149             side_effect=TestCompareReplay.mock_backends_dump)
 150         self.m_hexdigest_from_image = mocker.patch(
 151             'framework.replay.compare_replay.hexdigest_from_image',
 152             side_effect=TestCompareReplay.mock_hexdigest_from_image)
 153         self.tmpdir = tmpdir
 154
 155     def test_from_yaml_empty(self):
 156         """compare_replay.from_yaml: compare using an empty YAML file"""
 157
 158         f = io.StringIO()
 159         with contextlib.redirect_stdout(f):
 160             assert (compare_replay.from_yaml('empty.yml')
 161                     is status.PASS)
 162         self.m_qty_load_yaml.assert_called_once()
 163         s = f.getvalue()
 164         assert s == ''
 165
 166     def test_from_yaml_no_device(self):
 167         """compare_replay.from_yaml: compare using a YAML without expectations for the used device"""
 168
 169         f = io.StringIO()
 170         with contextlib.redirect_stdout(f):
 171             assert (compare_replay.from_yaml('no-device.yml')
 172                     is status.PASS)
 173         self.m_qty_load_yaml.assert_called_once()
 174         s = f.getvalue()
 175         assert s == ''
 176
 177     def test_from_yaml_one_trace(self):
 178         """compare_replay.from_yaml: compare using a YAML with just one expectation for the used device"""
 179
 180         f = io.StringIO()
 181         with contextlib.redirect_stdout(f):
 182             assert (compare_replay.from_yaml('one-trace.yml')
 183                     is status.PASS)
 184         self.m_qty_load_yaml.assert_called_once()
 185         self.m_ensure_file.assert_called_once()
 186         self.m_backends_dump.assert_called_once()
 187         self.m_hexdigest_from_image.assert_called_once()
 188         dumped_image_path = '{}-99.png'.format(self.trace_path)
 189         root, ext = path.splitext(dumped_image_path)
 190         assert not self.tmpdir.join(self.results_partial_path,
 191                                     dumped_image_path).check()
 192         assert not self.tmpdir.join(self.results_partial_path,
 193                                     '{}-{}{}'.format(
 194                                         root, self.exp_checksum, ext)).check()
 195         s = f.getvalue()
 196         assert s == ('[check_image]\n'
 197                      '    actual: ' + self.exp_checksum + '\n'
 198                      '  expected: ' + self.exp_checksum + '\n'
 199                      '[check_image] Images match for:\n'
 200                      '  ' + self.trace_path + '\n'
 201                      '\n')
 202
 203     def test_from_yaml_two_traces(self):
 204         """compare_replay.from_yaml: compare using a YAML with more than one expectation for the used device"""
 205
 206         second_trace_path = 'pathfinder/demo.trace'
 207         second_exp_checksum = 'e624d76c70cc3c532f4f54439e13659a'
 208         f = io.StringIO()
 209         with contextlib.redirect_stdout(f):
 210             assert (compare_replay.from_yaml('two-traces.yml')
 211                     is status.PASS)
 212         self.m_qty_load_yaml.assert_called_once()
 213         assert self.m_ensure_file.call_count == 2
 214         assert self.m_backends_dump.call_count == 2
 215         assert self.m_hexdigest_from_image.call_count == 2
 216         dumped_image_path = '{}-78.png'.format(second_trace_path)
 217         root, ext = path.splitext(dumped_image_path)
 218         assert not self.tmpdir.join(self.results_partial_path,
 219                                     dumped_image_path).check()
 220         assert not self.tmpdir.join(
 221             self.results_partial_path,
 222             '{}-{}{}'.format(root, second_exp_checksum, ext)).check()
 223         dumped_image_path = '{}-99.png'.format(self.trace_path)
 224         root, ext = path.splitext(dumped_image_path)
 225         assert not self.tmpdir.join(self.results_partial_path,
 226                                     dumped_image_path).check()
 227         assert not self.tmpdir.join(self.results_partial_path,
 228                                     '{}-{}{}'.format(
 229                                         root, self.exp_checksum, ext)).check()
 230         s = f.getvalue()
 231         assert s == ('[check_image]\n'
 232                      '    actual: ' + second_exp_checksum + '\n'
 233                      '  expected: ' + second_exp_checksum + '\n'
 234                      '[check_image] Images match for:\n'
 235                      '  ' + second_trace_path + '\n'
 236                      '\n'
 237                      '[check_image]\n'
 238                      '    actual: ' + self.exp_checksum + '\n'
 239                      '  expected: ' + self.exp_checksum + '\n'
 240                      '[check_image] Images match for:\n'
 241                      '  ' + self.trace_path + '\n'
 242                      '\n')
 243
 244     def test_trace_success(self):
 245         """compare_replay.trace: compare a trace successfully"""
 246
 247         f = io.StringIO()
 248         with contextlib.redirect_stdout(f):
 249             assert (compare_replay.trace(self.trace_path, self.exp_checksum)
 250                     is status.PASS)
 251         self.m_qty_load_yaml.assert_not_called()
 252         self.m_ensure_file.assert_called_once()
 253         self.m_backends_dump.assert_called_once()
 254         self.m_hexdigest_from_image.assert_called_once()
 255         dumped_image_path = '{}-99.png'.format(self.trace_path)
 256         root, ext = path.splitext(dumped_image_path)
 257         assert not self.tmpdir.join(self.results_partial_path,
 258                                     dumped_image_path).check()
 259         final_image_pathlib = self.tmpdir.join(
 260             self.results_partial_path, '{}-{}{}'.format(
 261                 root, self.exp_checksum, ext))
 262         assert not final_image_pathlib.check()
 263         s = f.getvalue()
 264         assert s.endswith('PIGLIT: '
 265                           '{"images": [{'
 266                           '"image_desc": "' + self.trace_path + '", '
 267                           '"checksum_ref": "' + self.exp_checksum + '", '
 268                           '"checksum_render": "' + self.exp_checksum + '", '
 269                           '"image_ref": "' + self.exp_checksum + '.png", '
 270                           '"image_render": "' + self.exp_checksum + '.png"'
 271                           '}], "result": "pass"}\n')
 272
 273     def test_trace_success_keep_image(self):
 274         """compare_replay.trace: compare a trace successfully and set the option to keep the dumped image"""
 275
 276         OPTIONS.keep_image = True
 277         f = io.StringIO()
 278         with contextlib.redirect_stdout(f):
 279             assert (compare_replay.trace(self.trace_path, self.exp_checksum)
 280                     is status.PASS)
 281         self.m_qty_load_yaml.assert_not_called()
 282         self.m_ensure_file.assert_called_once()
 283         self.m_backends_dump.assert_called_once()
 284         self.m_hexdigest_from_image.assert_called_once()
 285         dumped_image_path = '{}-99.png'.format(self.trace_path)
 286         root, ext = path.splitext(dumped_image_path)
 287         assert not self.tmpdir.join(self.results_partial_path,
 288                                     dumped_image_path).check()
 289         final_image_pathlib = self.tmpdir.join(
 290             self.results_partial_path, '{}-{}{}'.format(
 291                 root, self.exp_checksum, ext))
 292         assert final_image_pathlib.check()
 293         s = f.getvalue()
 294         assert s.endswith('PIGLIT: '
 295                           '{"images": [{'
 296                           '"image_desc": "' + self.trace_path + '", '
 297                           '"checksum_ref": "' + self.exp_checksum + '", '
 298                           '"checksum_render": "' + self.exp_checksum + '", '
 299                           '"image_ref": "' + self.exp_checksum + '.png", '
 300                           '"image_render": "' + final_image_pathlib.strpath +
 301                           '"}], "result": "pass"}\n')
 302
 303     def test_trace_fail(self):
 304         """compare_replay.trace: fail comparing a trace"""
 305
 306         wrong_checksum = '917cbbf4f09dd62ea26d247a1c70c16f'
 307         f = io.StringIO()
 308         with contextlib.redirect_stdout(f):
 309             assert (compare_replay.trace(self.trace_path, wrong_checksum)
 310                     is status.FAIL)
 311         self.m_qty_load_yaml.assert_not_called()
 312         self.m_ensure_file.assert_called_once()
 313         self.m_backends_dump.assert_called_once()
 314         self.m_hexdigest_from_image.assert_called_once()
 315         dumped_image_path = '{}-99.png'.format(self.trace_path)
 316         root, ext = path.splitext(dumped_image_path)
 317         assert not self.tmpdir.join(self.results_partial_path,
 318                                     dumped_image_path).check()
 319         final_image_pathlib = self.tmpdir.join(
 320             self.results_partial_path, '{}-{}{}'.format(
 321                 root, self.exp_checksum, ext))
 322         assert final_image_pathlib.check()
 323         s = f.getvalue()
 324         assert s.endswith('PIGLIT: '
 325                           '{"images": [{'
 326                           '"image_desc": "' + self.trace_path + '", '
 327                           '"checksum_ref": "' + wrong_checksum + '", '
 328                           '"checksum_render": "' + self.exp_checksum + '", '
 329                           '"image_ref": "' + wrong_checksum + '.png", '
 330                           '"image_render": "' + final_image_pathlib.strpath +
 331                           '"}], "result": "fail"}\n')
 332
 333     @pytest.mark.parametrize('trace_path', [
 334         ('glmark2/jellyfish.rdc'),
 335         ('unimplemented/backend.vktrace'),
 336         ('unexisting/back.end'),
 337     ])
 338     def test_trace_no_dumped_images(self, trace_path):
 339         """compare_replay.trace: dump succeeds but no images are generated"""
 340
 341         third_exp_checksum = 'ebaa1e2d04d7dfe5a91499510722c46e'
 342         f = io.StringIO()
 343         with contextlib.redirect_stdout(f):
 344             assert (compare_replay.trace(trace_path, third_exp_checksum)
 345                     is status.CRASH)
 346         self.m_qty_load_yaml.assert_not_called()
 347         self.m_ensure_file.assert_called_once()
 348         self.m_backends_dump.assert_called_once()
 349         self.m_hexdigest_from_image.assert_not_called()
 350         root, ext = path.splitext('{}-99.png'.format(self.trace_path))
 351         final_image_pathlib = self.tmpdir.join(
 352             self.results_partial_path, '{}-{}{}'.format(
 353                 root, third_exp_checksum, ext))
 354         assert not final_image_pathlib.check()
 355         s = f.getvalue()
 356         assert s.endswith('PIGLIT: '
 357                           '{"images": [{'
 358                           '"image_desc": "' + trace_path + '", '
 359                           '"checksum_ref": "' + third_exp_checksum + '", '
 360                           '"checksum_render": null, '
 361                           '"image_ref": "' + third_exp_checksum + '.png", '
 362                           '"image_render": null}], "result": "crash"}\n')
 363
 364     @pytest.mark.parametrize('trace_path', [
 365         ('Wicked-Engine/Tests:Cloth_Physics_Test.trace-dxgi'),
 366         ('unimplemented/backend.vktrace'),
 367         ('unexisting/back.end'),
 368     ])
 369     def test_trace_dump_crash(self, trace_path):
 370         """compare_replay.trace: dump crashes"""
 371
 372         third_exp_checksum = '6b6d27df609b8d086cc3335e6d103581'
 373         f = io.StringIO()
 374         with contextlib.redirect_stdout(f):
 375             assert (compare_replay.trace(trace_path, third_exp_checksum)
 376                     is status.CRASH)
 377         self.m_qty_load_yaml.assert_not_called()
 378         self.m_ensure_file.assert_called_once()
 379         self.m_backends_dump.assert_called_once()
 380         self.m_hexdigest_from_image.assert_not_called()
 381         root, ext = path.splitext('{}-99.png'.format(self.trace_path))
 382         final_image_pathlib = self.tmpdir.join(
 383             self.results_partial_path, '{}-{}{}'.format(
 384                 root, third_exp_checksum, ext))
 385         assert not final_image_pathlib.check()
 386         s = f.getvalue()
 387         assert s.endswith('PIGLIT: '
 388                           '{"images": [{'
 389                           '"image_desc": "' + trace_path + '", '
 390                           '"checksum_ref": "' + third_exp_checksum + '", '
 391                           '"checksum_render": null, '
 392                           '"image_ref": "' + third_exp_checksum + '.png", '
 393                           '"image_render": null}], "result": "crash"}\n')