tools/flakiness/is_flaky_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Unit tests for is_flaky."""
   6
   7 import is_flaky
   8 import subprocess
   9 import sys
  10 import threading
  11 import unittest
  12
  13
  14 class IsFlakyTest(unittest.TestCase):
  15
  16   def setUp(self):
  17     self.original_subprocess_check_call = subprocess.check_call
  18     subprocess.check_call = self.mock_check_call
  19     self.check_call_calls = []
  20     self.check_call_results = []
  21     is_flaky.load_options = self.mock_load_options
  22
  23   def tearDown(self):
  24     subprocess.check_call = self.original_subprocess_check_call
  25
  26   def mock_check_call(self, command, stdout, stderr):
  27     self.check_call_calls.append(command)
  28     if self.check_call_results:
  29       return self.check_call_results.pop(0)
  30     else:
  31       return 0
  32
  33   def mock_load_options(self):
  34     class MockOptions():
  35       jobs = 2
  36       retries = 10
  37       threshold = 0.3
  38       command = ['command', 'param1', 'param2']
  39     return MockOptions()
  40
  41   def testExecutesTestCorrectNumberOfTimes(self):
  42     is_flaky.main()
  43     self.assertEqual(len(self.check_call_calls), 10)
  44
  45   def testExecutesTestWithCorrectArguments(self):
  46     is_flaky.main()
  47     for call in self.check_call_calls:
  48       self.assertEqual(call, ['command', 'param1', 'param2'])
  49
  50   def testReturnsNonFlakyForAllSuccesses(self):
  51     self.check_call_results = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
  52     ret_code = is_flaky.main()
  53     self.assertEqual(ret_code, 0)
  54
  55   def testReturnsNonFlakyForAllFailures(self):
  56     self.check_call_results = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
  57     ret_code = is_flaky.main()
  58     self.assertEqual(ret_code, 0)
  59
  60   def testReturnsNonFlakyForSmallNumberOfFailures(self):
  61     self.check_call_results = [1, 0, 1, 0, 0, 0, 0, 0, 0, 0]
  62     ret_code = is_flaky.main()
  63     self.assertEqual(ret_code, 0)
  64
  65   def testReturnsFlakyForLargeNumberOfFailures(self):
  66     self.check_call_results = [1, 1, 1, 0, 1, 0, 0, 0, 0, 0]
  67     ret_code = is_flaky.main()
  68     self.assertEqual(ret_code, 1)
  69
  70
  71 if __name__ == '__main__':
  72   unittest.main()