tools/gn/bin/compare_test_lists.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 # This script compares the gtest test list for two different builds.
   6 #
   7 # Usage:
   8 #   compare_test_lists.py <build_dir_1> <build_dir_2> <binary_name>
   9 #
  10 # For example, from the "src" directory:
  11 #   python tools/gn/bin/compare_test_lists.py out/Debug out/gnbuild ipc_tests
  12 #
  13 # This will compile the given binary in both output directories, then extracts
  14 # the test lists and prints missing or extra tests between the first and the
  15 # second build.
  16
  17 import os
  18 import subprocess
  19 import sys
  20
  21 def BuildBinary(build_dir, binary_name):
  22   """Builds the given binary in the given directory with Ninja.
  23
  24   Returns True on success."""
  25   return subprocess.call(["ninja", "-C", build_dir, binary_name]) == 0
  26
  27
  28 def GetTestList(path_to_binary):
  29   """Returns a set of full test names.
  30
  31   Each test will be of the form "Case.Test". There will be a separate line
  32   for each combination of Case/Test (there are often multiple tests in each
  33   case).
  34
  35   Throws an exception on failure."""
  36   raw_output = subprocess.check_output([path_to_binary, "--gtest_list_tests"])
  37   input_lines = raw_output.split('\n')
  38
  39   # The format of the gtest_list_tests output is:
  40   # "Case1."
  41   # "  Test1  # <Optional extra stuff>"
  42   # "  Test2"
  43   # "Case2."
  44   # "  Test1"
  45   case_name = ''  # Includes trailing dot.
  46   test_set = set()
  47   for line in input_lines:
  48     if len(line) > 1:
  49       if line[0] == ' ':
  50         # Indented means a test in previous case.
  51         test_set.add(case_name + line[:line.find('#')].strip())
  52       else:
  53         # New test case.
  54         case_name = line.strip()
  55
  56   return test_set
  57
  58
  59 def PrintSetDiff(a_name, a, b_name, b, binary_name):
  60   """Prints the test list difference between the given sets a and b.
  61
  62   a_name and b_name will be used to refer to the directories of the two sets,
  63   and the binary name will be shown as the source of the output."""
  64
  65   a_not_b = list(a - b)
  66   if len(a_not_b):
  67     print "\n", binary_name, "tests in", a_name, "but not", b_name
  68     a_not_b.sort()
  69     for cur in a_not_b:
  70       print "  ", cur
  71
  72   b_not_a = list(b - a)
  73   if len(b_not_a):
  74     print "\n", binary_name, "tests in", b_name, "but not", a_name
  75     b_not_a.sort()
  76     for cur in b_not_a:
  77       print "  ", cur
  78
  79   if len(a_not_b) == 0 and len(b_not_a) == 0:
  80     print "\nTests match!"
  81
  82
  83 def Run(a_dir, b_dir, binary_name):
  84   if not BuildBinary(a_dir, binary_name):
  85     print "Building", binary_name, "in", a_dir, "failed"
  86     return 1
  87   if not BuildBinary(b_dir, binary_name):
  88     print "Building", binary_name, "in", b_dir, "failed"
  89     return 1
  90
  91   a_tests = GetTestList(os.path.join(a_dir, binary_name))
  92   b_tests = GetTestList(os.path.join(b_dir, binary_name))
  93
  94   PrintSetDiff(a_dir, a_tests, b_dir, b_tests, binary_name)
  95
  96
  97 if len(sys.argv) != 4:
  98   print "Usage: compare_test_lists.py <build_dir_1> <build_dir_2> " \
  99       "<test_binary_name>"
 100   sys.exit(1)
 101 sys.exit(Run(sys.argv[1], sys.argv[2], sys.argv[3]))