tools/auto_bisect/ttest_test.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Unit tests for ttest module."""
   6
   7 import unittest
   8
   9 import ttest
  10
  11
  12 # This test case accesses private functions of the ttest module.
  13 # pylint: disable=W0212
  14 class TTestTest(unittest.TestCase):
  15   """Tests for the t-test functions."""
  16
  17   def testWelchsFormula(self):
  18     """Tests calculation of the t value."""
  19     # Results can be verified by directly plugging variables into Welch's
  20     # equation (e.g. using a calculator or the Python interpreter).
  21     self.assertEqual(
  22         -0.2796823595120407,
  23         ttest._TValue(0.299, 0.307, 0.05, 0.08, 150, 165))
  24
  25     # Note that a negative t value is obtained when the first sample has a
  26     # smaller mean than the second, otherwise a positive value is returned.
  27     self.assertEqual(
  28         0.2796823595120407,
  29         ttest._TValue(0.307, 0.299, 0.08, 0.05, 165, 150))
  30
  31   def testWelchSatterthwaiteFormula(self):
  32     """Tests calculation of estimated degrees of freedom."""
  33     # Note that since the Welch-Satterthwaite equation gives an estimate of
  34     # degrees of freedom, the result may not be an integer.
  35     self.assertEqual(
  36         307.1987997516727,
  37         ttest._DegreesOfFreedom(0.05, 0.08, 150, 165))
  38
  39   def testWelchsTTest(self):
  40     """Tests the t value and degrees of freedom output of Welch's t-test."""
  41     # The t-value can be checked with scipy.stats.ttest_ind(equal_var=False).
  42     t, df, _ = ttest.WelchsTTest([2, 3, 2, 3, 2, 3], [4, 5, 4, 5, 4, 5])
  43     self.assertAlmostEqual(10.0, df)
  44
  45     # The t-value produced by scipy.stats.ttest_ind is -6.32455532034.
  46     # Our function produces slightly different results.
  47     # Possibly due to differences in rounding error?
  48     self.assertAlmostEqual(-6.325, t, delta=1.0)
  49
  50   def testTTestEqualSamples(self):
  51     """Checks that t = 0 and p = 1 when the samples are the same."""
  52     t, _, p = ttest.WelchsTTest([1, 2, 3], [1, 2, 3])
  53     self.assertEqual(0, t)
  54     self.assertEqual(1, p)
  55
  56     t, _, p = ttest.WelchsTTest([1, 2], [1, 2])
  57     self.assertEqual(0, t)
  58     self.assertEqual(1, p)
  59
  60   def testTTestVeryDifferentSamples(self):
  61     """Checks that p is very low when the samples are clearly different."""
  62     t, _, p = ttest.WelchsTTest(
  63         [100, 101, 100, 101, 100], [1, 2, 1, 2, 1, 2, 1, 2])
  64     self.assertGreaterEqual(t, 250)
  65     self.assertLessEqual(p, 0.01)
  66
  67   def testTTestVariance(self):
  68     """Verifies that higher variance -> higher p value."""
  69     _, _, p_low_var = ttest.WelchsTTest([2, 3, 2, 3], [4, 5, 4, 5])
  70     _, _, p_high_var = ttest.WelchsTTest([1, 4, 1, 4], [3, 6, 3, 6])
  71     self.assertLess(p_low_var, p_high_var)
  72
  73   def testTTestSampleSize(self):
  74     """Verifies that smaller sample size -> higher p value."""
  75     _, _, p_larger_sample = ttest.WelchsTTest([2, 3, 2, 3], [4, 5, 4, 5])
  76     _, _, p_smaller_sample = ttest.WelchsTTest([2, 3, 2, 3], [4, 5])
  77     self.assertLess(p_larger_sample, p_smaller_sample)
  78
  79   def testTTestMeanDifference(self):
  80     """Verifies that smaller difference between means -> higher p value."""
  81     _, _, p_far_means = ttest.WelchsTTest([2, 3, 2, 3], [5, 6, 5, 6])
  82     _, _, p_near_means = ttest.WelchsTTest([2, 3, 2, 3], [3, 4, 3, 4])
  83     self.assertLess(p_far_means, p_near_means)
  84
  85
  86 class LookupTableTest(unittest.TestCase):
  87   """Tests for functionality related to lookup of p-values in a table."""
  88
  89   def setUp(self):
  90     self.original_TWO_TAIL = ttest.TWO_TAIL
  91     self.original_TABLE = ttest.TABLE
  92     ttest.TWO_TAIL = [1, 0.2, 0.1, 0.05, 0.02, 0.01]
  93     ttest.TABLE = {
  94         1: [0, 6.314, 12.71, 31.82, 63.66, 318.31],
  95         2: [0, 2.920, 4.303, 6.965, 9.925, 22.327],
  96         3: [0, 2.353, 3.182, 4.541, 5.841, 10.215],
  97         4: [0, 2.132, 2.776, 3.747, 4.604, 7.173],
  98     }
  99
 100   def tearDown(self):
 101     ttest.TWO_TAIL = self.original_TWO_TAIL
 102     ttest.TABLE = self.original_TABLE
 103
 104   def testLookupExactMatch(self):
 105     """Tests a lookup when there is an exact match."""
 106     self.assertEqual(0.1, ttest._LookupPValue(3.182, 3))
 107     self.assertEqual(0.1, ttest._LookupPValue(-3.182, 3))
 108
 109   def testLookupAbove(self):
 110     """Tests a lookup when the given value is above an entry in the table."""
 111     self.assertEqual(0.2, ttest._LookupPValue(3.1, 2))
 112     self.assertEqual(0.2, ttest._LookupPValue(-3.1, 2))
 113
 114   def testLookupLargeTValue(self):
 115     """Tests a lookup when the given t-value is very large."""
 116     self.assertEqual(0.01, ttest._LookupPValue(500.0, 1))
 117     self.assertEqual(0.01, ttest._LookupPValue(-500.0, 1))
 118
 119   def testLookupZeroTValue(self):
 120     """Tests a lookup when the given t-value is zero."""
 121     self.assertEqual(1, ttest._LookupPValue(0.0, 1))
 122     self.assertEqual(1, ttest._LookupPValue(0.0, 2))
 123
 124   def testLookupLargeDF(self):
 125     """Tests a lookup when the given degrees of freedom is large."""
 126     self.assertEqual(0.02, ttest._LookupPValue(5.0, 50))
 127
 128
 129 if __name__ == '__main__':
 130   unittest.main()