media/tools/layout_tests/layouttest_analyzer.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Main functions for the Layout Test Analyzer module."""
   7
   8 from datetime import datetime
   9 import optparse
  10 import os
  11 import sys
  12 import time
  13
  14 import layouttest_analyzer_helpers
  15 from layouttest_analyzer_helpers import DEFAULT_REVISION_VIEW_URL
  16 import layouttests
  17 from layouttests import DEFAULT_LAYOUTTEST_SVN_VIEW_LOCATION
  18
  19 from test_expectations import TestExpectations
  20 from trend_graph import TrendGraph
  21
  22 # Predefined result directory.
  23 DEFAULT_RESULT_DIR = 'result'
  24 # TODO(shadi): Remove graph functions as they are not used any more.
  25 DEFAULT_GRAPH_FILE = os.path.join('graph', 'graph.html')
  26 # TODO(shadi): Check if these files are needed any more.
  27 DEFAULT_STATS_CSV_FILENAME = 'stats.csv'
  28 DEFAULT_ISSUES_CSV_FILENAME = 'issues.csv'
  29 # TODO(shadi): These are used only for |debug| mode. What is debug mode for?
  30 #              AFAIK, we don't run debug mode, should be safe to remove.
  31 # Predefined result files for debug.
  32 CUR_TIME_FOR_DEBUG = '2011-09-11-19'
  33 CURRENT_RESULT_FILE_FOR_DEBUG = os.path.join(DEFAULT_RESULT_DIR,
  34                                              CUR_TIME_FOR_DEBUG)
  35 PREV_TIME_FOR_DEBUG = '2011-09-11-18'
  36
  37 # Text to append at the end of every analyzer result email.
  38 DEFAULT_EMAIL_APPEND_TEXT = (
  39     '<b><a href="https://groups.google.com/a/google.com/group/'
  40     'layout-test-analyzer-result/topics">Email History</a></b><br>'
  41   )
  42
  43
  44 def ParseOption():
  45   """Parse command-line options using OptionParser.
  46
  47   Returns:
  48       an object containing all command-line option information.
  49   """
  50   option_parser = optparse.OptionParser()
  51
  52   option_parser.add_option('-r', '--receiver-email-address',
  53                            dest='receiver_email_address',
  54                            help=('receiver\'s email address. '
  55                                  'Result email is not sent if this is not '
  56                                  'specified.'))
  57   option_parser.add_option('-g', '--debug-mode', dest='debug',
  58                            help=('Debug mode is used when you want to debug '
  59                                  'the analyzer by using local file rather '
  60                                  'than getting data from SVN. This shortens '
  61                                  'the debugging time (off by default).'),
  62                            action='store_true', default=False)
  63   option_parser.add_option('-t', '--trend-graph-location',
  64                            dest='trend_graph_location',
  65                            help=('Location of the bug trend file; '
  66                                  'file is expected to be in Google '
  67                                  'Visualization API trend-line format '
  68                                  '(defaults to %default).'),
  69                            default=DEFAULT_GRAPH_FILE)
  70   option_parser.add_option('-n', '--test-group-file-location',
  71                            dest='test_group_file_location',
  72                            help=('Location of the test group file; '
  73                                  'file is expected to be in CSV format '
  74                                  'and lists all test name patterns. '
  75                                  'When this option is not specified, '
  76                                  'the value of --test-group-name is used '
  77                                  'for a test name pattern.'),
  78                            default=None)
  79   option_parser.add_option('-x', '--test-group-name',
  80                            dest='test_group_name',
  81                            help=('A name of test group. Either '
  82                                  '--test_group_file_location or this option '
  83                                  'needs to be specified.'))
  84   option_parser.add_option('-d', '--result-directory-location',
  85                            dest='result_directory_location',
  86                            help=('Name of result directory location '
  87                                  '(default to %default).'),
  88                            default=DEFAULT_RESULT_DIR)
  89   option_parser.add_option('-b', '--email-appended-text-file-location',
  90                            dest='email_appended_text_file_location',
  91                            help=('File location of the email appended text. '
  92                                  'The text is appended in the status email. '
  93                                  '(default to %default and no text is '
  94                                  'appended in that case).'),
  95                            default=None)
  96   option_parser.add_option('-c', '--email-only-change-mode',
  97                            dest='email_only_change_mode',
  98                            help=('With this mode, email is sent out '
  99                                  'only when there is a change in the '
 100                                  'analyzer result compared to the previous '
 101                                  'result (off by default)'),
 102                            action='store_true', default=False)
 103   option_parser.add_option('-q', '--dashboard-file-location',
 104                            dest='dashboard_file_location',
 105                            help=('Location of dashboard file. The results are '
 106                                  'not reported to the dashboard if this '
 107                                  'option is not specified.'))
 108   option_parser.add_option('-z', '--issue-detail-mode',
 109                            dest='issue_detail_mode',
 110                            help=('With this mode, email includes issue details '
 111                                  '(links to the flakiness dashboard)'
 112                                  ' (off by default)'),
 113                            action='store_true', default=False)
 114   return option_parser.parse_args()[0]
 115
 116
 117 def GetCurrentAndPreviousResults(debug, test_group_file_location,
 118                                  test_group_name, result_directory_location):
 119   """Get current and the latest previous analyzer results.
 120
 121   In debug mode, they are read from predefined files. In non-debug mode,
 122   current analyzer results are dynamically obtained from Blink SVN and
 123   the latest previous result is read from the corresponding file.
 124
 125   Args:
 126     debug: please refer to |options|.
 127     test_group_file_location: please refer to |options|.
 128     test_group_name: please refer to |options|.
 129     result_directory_location: please refer to |options|.
 130
 131   Returns:
 132     a tuple of the following:
 133        prev_time: the previous time string that is compared against.
 134        prev_analyzer_result_map: previous analyzer result map. Please refer to
 135           layouttest_analyzer_helpers.AnalyzerResultMap.
 136        analyzer_result_map: current analyzer result map. Please refer to
 137           layouttest_analyzer_helpers.AnalyzerResultMap.
 138   """
 139   if not debug:
 140     if not test_group_file_location and not test_group_name:
 141       print ('Either --test-group-name or --test_group_file_location must be '
 142              'specified. Exiting this program.')
 143       sys.exit()
 144     filter_names = []
 145     if test_group_file_location and os.path.exists(test_group_file_location):
 146       filter_names = layouttests.LayoutTests.GetLayoutTestNamesFromCSV(
 147           test_group_file_location)
 148       parent_location_list = (
 149           layouttests.LayoutTests.GetParentDirectoryList(filter_names))
 150       recursion = True
 151     else:
 152       # When test group CSV file is not specified, test group name
 153       # (e.g., 'media') is used for getting layout tests.
 154       # The tests are in
 155       #     http://src.chromium.org/blink/trunk/LayoutTests/media
 156       # Filtering is not set so all HTML files are considered as valid tests.
 157       # Also, we look for the tests recursively.
 158       if not test_group_file_location or (
 159           not os.path.exists(test_group_file_location)):
 160         print ('Warning: CSV file (%s) does not exist. So it is ignored and '
 161                '%s is used for obtaining test names') % (
 162                    test_group_file_location, test_group_name)
 163       if not test_group_name.endswith('/'):
 164         test_group_name += '/'
 165       parent_location_list = [test_group_name]
 166       filter_names = None
 167       recursion = True
 168     layouttests_object = layouttests.LayoutTests(
 169         parent_location_list=parent_location_list, recursion=recursion,
 170         filter_names=filter_names)
 171     analyzer_result_map = layouttest_analyzer_helpers.AnalyzerResultMap(
 172         layouttests_object.JoinWithTestExpectation(TestExpectations()))
 173     result = layouttest_analyzer_helpers.FindLatestResult(
 174         result_directory_location)
 175     if result:
 176       (prev_time, prev_analyzer_result_map) = result
 177     else:
 178       prev_time = None
 179       prev_analyzer_result_map = None
 180   else:
 181     analyzer_result_map = layouttest_analyzer_helpers.AnalyzerResultMap.Load(
 182         CURRENT_RESULT_FILE_FOR_DEBUG)
 183     prev_time = PREV_TIME_FOR_DEBUG
 184     prev_analyzer_result_map = (
 185         layouttest_analyzer_helpers.AnalyzerResultMap.Load(
 186             os.path.join(DEFAULT_RESULT_DIR, prev_time)))
 187   return (prev_time, prev_analyzer_result_map, analyzer_result_map)
 188
 189
 190 def SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map,
 191               appended_text_to_email, email_only_change_mode, debug,
 192               receiver_email_address, test_group_name, issue_detail_mode):
 193   """Send result status email.
 194
 195   Args:
 196     prev_time: the previous time string that is compared against.
 197     prev_analyzer_result_map: previous analyzer result map. Please refer to
 198         layouttest_analyzer_helpers.AnalyzerResultMap.
 199     analyzer_result_map: current analyzer result map. Please refer to
 200         layouttest_analyzer_helpers.AnalyzerResultMap.
 201     appended_text_to_email: the text string to append to the status email.
 202     email_only_change_mode: please refer to |options|.
 203     debug: please refer to |options|.
 204     receiver_email_address: please refer to |options|.
 205     test_group_name: please refer to |options|.
 206     issue_detail_mode: please refer to |options|.
 207
 208   Returns:
 209     a tuple of the following:
 210         result_change: a boolean indicating whether there is a change in the
 211             result compared with the latest past result.
 212         diff_map: please refer to
 213             layouttest_analyzer_helpers.SendStatusEmail().
 214         simple_rev_str: a simple version of revision string that is sent in
 215             the email.
 216         rev: the latest revision number for the given test group.
 217         rev_date: the latest revision date for the given test group.
 218         email_content:  email content string (without
 219             |appended_text_to_email|) that will be shown on the dashboard.
 220   """
 221   rev = ''
 222   rev_date = ''
 223   email_content = ''
 224   if prev_analyzer_result_map:
 225     diff_map = analyzer_result_map.CompareToOtherResultMap(
 226         prev_analyzer_result_map)
 227     result_change = (any(diff_map['whole']) or any(diff_map['skip']) or
 228                      any(diff_map['nonskip']))
 229     # Email only when |email_only_change_mode| is False or there
 230     # is a change in the result compared to the last result.
 231     simple_rev_str = ''
 232     if not email_only_change_mode or result_change:
 233       prev_time_in_float = datetime.strptime(prev_time, '%Y-%m-%d-%H')
 234       prev_time_in_float = time.mktime(prev_time_in_float.timetuple())
 235       if debug:
 236         cur_time_in_float = datetime.strptime(CUR_TIME_FOR_DEBUG,
 237                                               '%Y-%m-%d-%H')
 238         cur_time_in_float = time.mktime(cur_time_in_float.timetuple())
 239       else:
 240         cur_time_in_float = time.time()
 241       (rev_str, simple_rev_str, rev, rev_date) = (
 242           layouttest_analyzer_helpers.GetRevisionString(prev_time_in_float,
 243                                                         cur_time_in_float,
 244                                                         diff_map))
 245       email_content = analyzer_result_map.ConvertToString(prev_time,
 246                                                           diff_map,
 247                                                           issue_detail_mode)
 248       if receiver_email_address:
 249         layouttest_analyzer_helpers.SendStatusEmail(
 250             prev_time, analyzer_result_map, diff_map,
 251             receiver_email_address, test_group_name,
 252             appended_text_to_email, email_content, rev_str,
 253             email_only_change_mode)
 254     if simple_rev_str:
 255       simple_rev_str = '\'' + simple_rev_str + '\''
 256     else:
 257       simple_rev_str = 'undefined'  # GViz uses undefined for NONE.
 258   else:
 259     # Initial result should be written to tread-graph if there are no previous
 260     # results.
 261     result_change = True
 262     diff_map = None
 263     simple_rev_str = 'undefined'
 264     email_content = analyzer_result_map.ConvertToString(None, diff_map,
 265                                                         issue_detail_mode)
 266   return (result_change, diff_map, simple_rev_str, rev, rev_date,
 267           email_content)
 268
 269
 270 def UpdateTrendGraph(start_time, analyzer_result_map, diff_map, simple_rev_str,
 271                      trend_graph_location):
 272   """Update trend graph in GViz.
 273
 274   Annotate the graph with revision information.
 275
 276   Args:
 277     start_time: the script starting time as a float value.
 278     analyzer_result_map: current analyzer result map. Please refer to
 279         layouttest_analyzer_helpers.AnalyzerResultMap.
 280     diff_map: a map that has 'whole', 'skip' and 'nonskip' as keys.
 281         Please refer to |diff_map| in
 282         |layouttest_analyzer_helpers.SendStatusEmail()|.
 283     simple_rev_str: a simple version of revision string that is sent in
 284         the email.
 285     trend_graph_location: the location of the trend graph that needs to be
 286         updated.
 287
 288   Returns:
 289      a dictionary that maps result data category ('whole', 'skip', 'nonskip',
 290          'passingrate') to information tuple (a dictionary that maps test name
 291          to its description, annotation, simple_rev_string) of the given result
 292          data category. These tuples are used for trend graph update.
 293   """
 294   # Trend graph update (if specified in the command-line argument) when
 295   # there is change from the last result.
 296   # Currently, there are two graphs (graph1 is for 'whole', 'skip',
 297   # 'nonskip' and the graph2 is for 'passingrate'). Please refer to
 298   # graph/graph.html.
 299   # Sample JS annotation for graph1:
 300   #   [new Date(2011,8,12,10,41,32),224,undefined,'',52,undefined,
 301   #    undefined, 12, 'test1,','<a href="http://t</a>,',],
 302   # This example lists 'whole' triple and 'skip' triple and
 303   # 'nonskip' triple. Each triple is (the number of tests that belong to
 304   # the test group, linked text, a link). The following code generates this
 305   # automatically based on rev_string etc.
 306   trend_graph = TrendGraph(trend_graph_location)
 307   datetime_string = start_time.strftime('%Y,%m,%d,%H,%M,%S')
 308   data_map = {}
 309   passingrate_anno = ''
 310   for test_group in ['whole', 'skip', 'nonskip']:
 311     anno = 'undefined'
 312     # Extract test description.
 313     test_map = {}
 314     for (test_name, value) in (
 315         analyzer_result_map.result_map[test_group].iteritems()):
 316       test_map[test_name] = value['desc']
 317     test_str = ''
 318     links = ''
 319     if diff_map and diff_map[test_group]:
 320       for i in [0, 1]:
 321         for (name, _) in diff_map[test_group][i]:
 322           test_str += name + ','
 323           # This is link to test HTML in SVN.
 324           links += ('<a href="%s%s">%s</a>' %
 325                     (DEFAULT_LAYOUTTEST_SVN_VIEW_LOCATION, name, name))
 326       if test_str:
 327         anno = '\'' + test_str + '\''
 328         # The annotation of passing rate is a union of all annotations.
 329         passingrate_anno += anno
 330     if links:
 331       links = '\'' + links + '\''
 332     else:
 333       links = 'undefined'
 334     if test_group is 'whole':
 335       data_map[test_group] = (test_map, anno, links)
 336     else:
 337       data_map[test_group] = (test_map, anno, simple_rev_str)
 338   if not passingrate_anno:
 339     passingrate_anno = 'undefined'
 340   data_map['passingrate'] = (
 341       str(analyzer_result_map.GetPassingRate()), passingrate_anno,
 342       simple_rev_str)
 343   trend_graph.Update(datetime_string, data_map)
 344   return data_map
 345
 346
 347 def UpdateDashboard(dashboard_file_location, test_group_name, data_map,
 348                     layouttest_root_path, rev, rev_date, email,
 349                     email_content):
 350   """Update dashboard HTML file.
 351
 352   Args:
 353     dashboard_file_location: the file location for the dashboard file.
 354     test_group_name: please refer to |options|.
 355     data_map: a dictionary that maps result data category ('whole', 'skip',
 356         'nonskip', 'passingrate') to information tuple (a dictionary that maps
 357         test name to its description, annotation, simple_rev_string) of the
 358         given result data category. These tuples are used for trend graph
 359         update.
 360     layouttest_root_path: A location string where layout tests are stored.
 361     rev: the latest revision number for the given test group.
 362     rev_date: the latest revision date for the given test group.
 363     email: email address of the owner for the given test group.
 364     email_content:  email content string (without |appended_text_to_email|)
 365         that will be shown on the dashboard.
 366   """
 367   # Generate a HTML file that contains all test names for each test group.
 368   escaped_tg_name = test_group_name.replace('/', '_')
 369   for tg in ['whole', 'skip', 'nonskip']:
 370     file_name = os.path.join(
 371         os.path.dirname(dashboard_file_location),
 372         escaped_tg_name + '_' + tg + '.html')
 373     file_object = open(file_name, 'wb')
 374     file_object.write('<table border="1">')
 375     sorted_testnames = data_map[tg][0].keys()
 376     sorted_testnames.sort()
 377     for testname in sorted_testnames:
 378       file_object.write((
 379           '<tr><td><a href="%s">%s</a></td><td><a href="%s">dashboard</a>'
 380           '</td><td>%s</td></tr>') % (
 381               layouttest_root_path + testname, testname,
 382               ('http://test-results.appspot.com/dashboards/'
 383                'flakiness_dashboard.html#tests=%s') % testname,
 384               data_map[tg][0][testname]))
 385     file_object.write('</table>')
 386     file_object.close()
 387   email_content_with_link = ''
 388   if email_content:
 389     file_name = os.path.join(os.path.dirname(dashboard_file_location),
 390                              escaped_tg_name + '_email.html')
 391     file_object = open(file_name, 'wb')
 392     file_object.write(email_content)
 393     file_object.close()
 394     email_content_with_link = '<a href="%s_email.html">info</a>' % (
 395         escaped_tg_name)
 396   test_group_str = (
 397       '<td><a href="%(test_group_path)s">%(test_group_name)s</a></td>'
 398       '<td><a href="%(graph_path)s">graph</a></td>'
 399       '<td><a href="%(all_tests_path)s">%(all_tests_count)d</a></td>'
 400       '<td><a href="%(skip_tests_path)s">%(skip_tests_count)d</a></td>'
 401       '<td><a href="%(nonskip_tests_path)s">%(nonskip_tests_count)d</a></td>'
 402       '<td>%(fail_rate)d%%</td>'
 403       '<td>%(passing_rate)d%%</td>'
 404       '<td><a href="%(rev_url)s">%(rev)s</a></td>'
 405       '<td>%(rev_date)s</td>'
 406       '<td><a href="mailto:%(email)s">%(email)s</a></td>'
 407       '<td>%(email_content)s</td>\n') % {
 408           # Dashboard file and graph must be in the same directory
 409           # to make the following link work.
 410           'test_group_path': layouttest_root_path + '/' + test_group_name,
 411           'test_group_name': test_group_name,
 412           'graph_path': escaped_tg_name + '.html',
 413           'all_tests_path': escaped_tg_name + '_whole.html',
 414           'all_tests_count': len(data_map['whole'][0]),
 415           'skip_tests_path': escaped_tg_name + '_skip.html',
 416           'skip_tests_count': len(data_map['skip'][0]),
 417           'nonskip_tests_path': escaped_tg_name + '_nonskip.html',
 418           'nonskip_tests_count': len(data_map['nonskip'][0]),
 419           'fail_rate': 100 - float(data_map['passingrate'][0]),
 420           'passing_rate': float(data_map['passingrate'][0]),
 421           'rev_url': DEFAULT_REVISION_VIEW_URL % rev,
 422           'rev': rev,
 423           'rev_date': rev_date,
 424           'email': email,
 425           'email_content': email_content_with_link
 426       }
 427   layouttest_analyzer_helpers.ReplaceLineInFile(
 428       dashboard_file_location, '<td>' + test_group_name + '</td>',
 429       test_group_str)
 430
 431
 432 def main():
 433   """A main function for the analyzer."""
 434   options = ParseOption()
 435   start_time = datetime.now()
 436
 437   (prev_time, prev_analyzer_result_map, analyzer_result_map) = (
 438       GetCurrentAndPreviousResults(options.debug,
 439                                    options.test_group_file_location,
 440                                    options.test_group_name,
 441                                    options.result_directory_location))
 442   (result_change, diff_map, simple_rev_str, rev, rev_date, email_content) = (
 443       SendEmail(prev_time, prev_analyzer_result_map, analyzer_result_map,
 444                 DEFAULT_EMAIL_APPEND_TEXT,
 445                 options.email_only_change_mode, options.debug,
 446                 options.receiver_email_address, options.test_group_name,
 447                 options.issue_detail_mode))
 448
 449   # Create CSV texts and save them for bug spreadsheet.
 450   (stats, issues_txt) = analyzer_result_map.ConvertToCSVText(
 451       start_time.strftime('%Y-%m-%d-%H'))
 452   file_object = open(os.path.join(options.result_directory_location,
 453                                   DEFAULT_STATS_CSV_FILENAME), 'wb')
 454   file_object.write(stats)
 455   file_object.close()
 456   file_object = open(os.path.join(options.result_directory_location,
 457                                   DEFAULT_ISSUES_CSV_FILENAME), 'wb')
 458   file_object.write(issues_txt)
 459   file_object.close()
 460
 461   if not options.debug and (result_change or not prev_analyzer_result_map):
 462     # Save the current result when result is changed or the script is
 463     # executed for the first time.
 464     date = start_time.strftime('%Y-%m-%d-%H')
 465     file_path = os.path.join(options.result_directory_location, date)
 466     analyzer_result_map.Save(file_path)
 467   if result_change or not prev_analyzer_result_map:
 468     data_map = UpdateTrendGraph(start_time, analyzer_result_map, diff_map,
 469                                 simple_rev_str, options.trend_graph_location)
 470     # Report the result to dashboard.
 471     if options.dashboard_file_location:
 472       UpdateDashboard(options.dashboard_file_location, options.test_group_name,
 473                       data_map, layouttests.DEFAULT_LAYOUTTEST_LOCATION, rev,
 474                       rev_date, options.receiver_email_address,
 475                       email_content)
 476
 477
 478 if '__main__' == __name__:
 479   main()