solver-benchmarks/tests/HackageBenchmarkTest.hs

   1 import HackageBenchmark
   2 import Statistics.Types (mkPValue)
   3 import Test.Tasty (TestTree, defaultMain, testGroup)
   4 import Test.Tasty.HUnit (assertBool, testCase, (@?=))
   5
   6 main :: IO ()
   7 main = defaultMain tests
   8
   9 tests :: TestTree
  10 tests = testGroup "unit tests" [
  11
  12     testGroup "isSignificantTimeDifference" [
  13
  14         testCase "detect increase in distribution" $ assertBool "" $
  15             isSignificantTimeDifference (mkPValue 0.05) [1,2..7] [4,5..10]
  16
  17       , testCase "detect decrease in distribution" $ assertBool "" $
  18             isSignificantTimeDifference (mkPValue 0.05) [1,2..7] [-2,-1..4]
  19
  20       , testCase "ignore same data" $ assertBool "" $
  21             not $ isSignificantTimeDifference (mkPValue 0.05) [1,2..10] [1,2..10]
  22
  23       , testCase "same data with high p-value is significant" $ assertBool "" $
  24             isSignificantTimeDifference (mkPValue 0.9) [1,2..10] [1,2..10]
  25
  26       , testCase "ignore outlier" $ assertBool "" $
  27             not $ isSignificantTimeDifference (mkPValue 0.05) [1, 2, 1, 1, 1] [2, 1, 50, 1, 1]
  28       ]
  29
  30   , testGroup "combineTrialResults" [
  31
  32         testCase "convert unexpected difference to Unknown" $
  33             combineTrialResults [NoInstallPlan, BackjumpLimit] @?= Unknown
  34
  35       , testCase "return one of identical errors" $
  36             combineTrialResults [NoInstallPlan, NoInstallPlan] @?= NoInstallPlan
  37
  38       , testCase "return one of identical successes" $
  39             combineTrialResults [Solution, Solution] @?= Solution
  40
  41       , testCase "timeout overrides other results" $
  42             combineTrialResults [Solution, Timeout, Solution] @?= Timeout
  43
  44       , testCase "convert unexpected difference to Unknown, even with timeout" $
  45             combineTrialResults [Solution, Timeout, NoInstallPlan] @?= Unknown
  46     ]
  47
  48   , testGroup "isSignificantResult" [
  49
  50         testCase "different results are significant" $ assertBool "" $
  51             isSignificantResult NoInstallPlan BackjumpLimit
  52
  53       , testCase "unknown result is significant" $ assertBool "" $
  54             isSignificantResult Unknown Unknown
  55
  56       , testCase "PkgNotFound is significant" $ assertBool "" $
  57             isSignificantResult PkgNotFound PkgNotFound
  58
  59       , testCase "same expected error is not significant" $ assertBool "" $
  60             not $ isSignificantResult NoInstallPlan NoInstallPlan
  61
  62       , testCase "success is not significant" $ assertBool "" $
  63             not $ isSignificantResult Solution Solution
  64     ]
  65
  66   , testGroup "shouldContinueAfterFirstTrial" [
  67
  68         testCase "rerun when min difference is zero" $ assertBool "" $
  69                   shouldContinueAfterFirstTrial 0 1.0 1.0 Solution Solution
  70
  71       , testCase "rerun when min difference is zero, even with timeout" $
  72                   assertBool "" $
  73                   shouldContinueAfterFirstTrial 0 1.0 1.0 Timeout Timeout
  74
  75       , testCase "treat timeouts as the same time" $ assertBool "" $
  76             not $ shouldContinueAfterFirstTrial 0.000001 89.9 92.0 Timeout Timeout
  77
  78       , testCase "skip when times are too close - 1" $ assertBool "" $
  79             not $ shouldContinueAfterFirstTrial 10 1.0 0.91  Solution Solution
  80
  81       , testCase "skip when times are too close - 2" $ assertBool "" $
  82             not $ shouldContinueAfterFirstTrial 10 1.0 1.09  Solution Solution
  83
  84       , testCase "rerun when times aren't too close - 1" $ assertBool "" $
  85                   shouldContinueAfterFirstTrial 10 1.0 0.905 Solution Solution
  86
  87       , testCase "rerun when times aren't too close - 2" $ assertBool "" $
  88                   shouldContinueAfterFirstTrial 10 1.0 1.1   Solution Solution
  89     ]
  90   ]