media/tools/layout_tests/layouttests.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Layout tests module that is necessary for the layout analyzer.
   6
   7 Layout tests are stored in an SVN repository and LayoutTestCaseManager collects
   8 these layout test cases (including description).
   9 """
  10
  11 import copy
  12 import csv
  13 import locale
  14 import re
  15 import sys
  16 import urllib2
  17
  18 import pysvn
  19
  20 # LayoutTests SVN root location.
  21 DEFAULT_LAYOUTTEST_LOCATION = (
  22     'http://src.chromium.org/blink/trunk/LayoutTests/')
  23 # LayoutTests SVN view link
  24 DEFAULT_LAYOUTTEST_SVN_VIEW_LOCATION = (
  25     'http://src.chromium.org/viewvc/blink/trunk/LayoutTests/')
  26
  27
  28 # When parsing the test HTML file and finding the test description,
  29 # this script tries to find the test description using sentences
  30 # starting with these keywords. This is adhoc but it is the only way
  31 # since there is no standard for writing test description.
  32 KEYWORDS_FOR_TEST_DESCRIPTION = ['This test', 'Tests that', 'Test ']
  33
  34 # If cannot find the keywords, this script tries to find test case
  35 # description by the following tags.
  36 TAGS_FOR_TEST_DESCRIPTION = ['title', 'p', 'div']
  37
  38 # If cannot find the tags, this script tries to find the test case
  39 # description in the sentence containing following words.
  40 KEYWORD_FOR_TEST_DESCRIPTION_FAIL_SAFE = ['PASSED ', 'PASS:']
  41
  42
  43 class LayoutTests(object):
  44   """A class to store test names in layout tests.
  45
  46   The test names (including regular expression patterns) are read from a CSV
  47   file and used for getting layout test names from repository.
  48   """
  49
  50   def __init__(self, layouttest_root_path=DEFAULT_LAYOUTTEST_LOCATION,
  51                parent_location_list=None, filter_names=None,
  52                recursion=False):
  53     """Initialize LayoutTests using root and CSV file.
  54
  55     Args:
  56       layouttest_root_path: A location string where layout tests are stored.
  57       parent_location_list: A list of parent directories that are needed for
  58           getting layout tests.
  59       filter_names: A list of test name patterns that are used for filtering
  60           test names (e.g., media/*.html).
  61       recursion: a boolean indicating whether the test names are sought
  62           recursively.
  63     """
  64
  65     if layouttest_root_path.startswith('http://'):
  66       name_map = self.GetLayoutTestNamesFromSVN(parent_location_list,
  67                                                 layouttest_root_path,
  68                                                 recursion)
  69     else:
  70       # TODO(imasaki): support other forms such as CSV for reading test names.
  71       pass
  72     self.name_map = copy.copy(name_map)
  73     if filter_names:
  74       # Filter names.
  75       for lt_name in name_map.iterkeys():
  76         match = False
  77         for filter_name in filter_names:
  78           if re.search(filter_name, lt_name):
  79             match = True
  80             break
  81         if not match:
  82           del self.name_map[lt_name]
  83     # We get description only for the filtered names.
  84     for lt_name in self.name_map.iterkeys():
  85       self.name_map[lt_name] = 'No description available'
  86
  87   @staticmethod
  88   def ExtractTestDescription(txt):
  89     """Extract the description description from test code in HTML.
  90
  91     Currently, we have 4 rules described in the code below.
  92     (This example falls into rule 1):
  93       <p>
  94       This tests the intrinsic size of a video element is the default
  95       300,150 before metadata is loaded, and 0,0 after
  96       metadata is loaded for an audio-only file.
  97       </p>
  98     The strategy is very adhoc since the original test case files
  99     (in HTML format) do not have standard way to store test description.
 100
 101     Args:
 102       txt: A HTML text which may or may not contain test description.
 103
 104     Returns:
 105       A string that contains test description. Returns 'UNKNOWN' if the
 106           test description is not found.
 107     """
 108     # (1) Try to find test description that contains keywords such as
 109     #     'test that' and surrounded by p tag.
 110     #     This is the most common case.
 111     for keyword in KEYWORDS_FOR_TEST_DESCRIPTION:
 112       # Try to find <p> and </p>.
 113       pattern = r'<p>(.*' + keyword + '.*)</p>'
 114       matches = re.search(pattern, txt)
 115       if matches is not None:
 116         return matches.group(1).strip()
 117
 118     # (2) Try to find it by using more generic keywords such as 'PASS' etc.
 119     for keyword in KEYWORD_FOR_TEST_DESCRIPTION_FAIL_SAFE:
 120       # Try to find new lines.
 121       pattern = r'\n(.*' + keyword + '.*)\n'
 122       matches = re.search(pattern, txt)
 123       if matches is not None:
 124         # Remove 'p' tag.
 125         text = matches.group(1).strip()
 126         return text.replace('<p>', '').replace('</p>', '')
 127
 128     # (3) Try to find it by using HTML tag such as title.
 129     for tag in TAGS_FOR_TEST_DESCRIPTION:
 130       pattern = r'<' + tag + '>(.*)</' + tag + '>'
 131       matches = re.search(pattern, txt)
 132       if matches is not None:
 133         return matches.group(1).strip()
 134
 135     # (4) Try to find it by using test description and remove 'p' tag.
 136     for keyword in KEYWORDS_FOR_TEST_DESCRIPTION:
 137       # Try to find <p> and </p>.
 138       pattern = r'\n(.*' + keyword + '.*)\n'
 139       matches = re.search(pattern, txt)
 140       if matches is not None:
 141         # Remove 'p' tag.
 142         text = matches.group(1).strip()
 143         return text.replace('<p>', '').replace('</p>', '')
 144
 145     # (5) cannot find test description using existing rules.
 146     return 'UNKNOWN'
 147
 148   @staticmethod
 149   def GetLayoutTestNamesFromSVN(parent_location_list,
 150                                 layouttest_root_path, recursion):
 151     """Get LayoutTest names from SVN.
 152
 153     Args:
 154       parent_location_list: a list of locations of parent directories. This is
 155           used when getting layout tests using PySVN.list().
 156       layouttest_root_path: the root path of layout tests directory.
 157       recursion: a boolean indicating whether the test names are sought
 158           recursively.
 159
 160     Returns:
 161       a map containing test names as keys for de-dupe.
 162     """
 163     client = pysvn.Client()
 164     # Get directory structure in the repository SVN.
 165     name_map = {}
 166     for parent_location in parent_location_list:
 167       if parent_location.endswith('/'):
 168         full_path = layouttest_root_path + parent_location
 169         try:
 170           file_list = client.list(full_path, recurse=recursion)
 171           for file_name in file_list:
 172             if sys.stdout.isatty():
 173               default_encoding = sys.stdout.encoding
 174             else:
 175               default_encoding = locale.getpreferredencoding()
 176             file_name = file_name[0].repos_path.encode(default_encoding)
 177             # Remove the word '/truck/LayoutTests'.
 178             file_name = file_name.replace('/trunk/LayoutTests/', '')
 179             if file_name.endswith('.html'):
 180               name_map[file_name] = True
 181         except:
 182           print 'Unable to list tests in %s.' % full_path
 183     return name_map
 184
 185   @staticmethod
 186   def GetLayoutTestNamesFromCSV(csv_file_path):
 187     """Get layout test names from CSV file.
 188
 189     Args:
 190       csv_file_path: the path for the CSV file containing test names (including
 191           regular expression patterns). The CSV file content has one column and
 192           each row contains a test name.
 193
 194     Returns:
 195        a list of test names in string.
 196     """
 197     file_object = file(csv_file_path, 'r')
 198     reader = csv.reader(file_object)
 199     names = [row[0] for row in reader]
 200     file_object.close()
 201     return names
 202
 203   @staticmethod
 204   def GetParentDirectoryList(names):
 205     """Get parent directory list from test names.
 206
 207     Args:
 208       names: a list of test names. The test names also have path information as
 209           well (e.g., media/video-zoom.html).
 210
 211     Returns:
 212       a list of parent directories for the given test names.
 213     """
 214     pd_map = {}
 215     for name in names:
 216       p_dir = name[0:name.rfind('/') + 1]
 217       pd_map[p_dir] = True
 218     return list(pd_map.iterkeys())
 219
 220   def JoinWithTestExpectation(self, test_expectations):
 221     """Join layout tests with the test expectation file using test name as key.
 222
 223     Args:
 224       test_expectations: a test expectations object.
 225
 226     Returns:
 227       test_info_map contains test name as key and another map as value. The
 228           other map contains test description and the test expectation
 229           information which contains keyword (e.g., 'GPU') as key (we do
 230           not care about values). The map data structure is used since we
 231           have to look up these keywords several times.
 232     """
 233     test_info_map = {}
 234     for (lt_name, desc) in self.name_map.items():
 235       test_info_map[lt_name] = {}
 236       test_info_map[lt_name]['desc'] = desc
 237       for (te_name, te_info) in (
 238           test_expectations.all_test_expectation_info.items()):
 239         if te_name == lt_name or (
 240             te_name in lt_name and te_name.endswith('/')):
 241           # Only keep the first match when found.
 242           test_info_map[lt_name]['te_info'] = te_info
 243           break
 244     return test_info_map