tools/auto_bisect/request_build.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This module contains functionality for starting build try jobs via HTTP.
   6
   7 This includes both sending a request to start a job, and also related code
   8 for querying the status of the job.
   9
  10 This module can be either run as a stand-alone script to send a request to a
  11 builder, or imported and used by calling the public functions below.
  12 """
  13
  14 import getpass
  15 import json
  16 import optparse
  17 import os
  18 import sys
  19 import urllib
  20 import urllib2
  21
  22 import fetch_build
  23
  24 # URL template for fetching JSON data about builds.
  25 BUILDER_JSON_URL = ('%(server_url)s/json/builders/%(bot_name)s/builds/'
  26                     '%(build_num)s?as_text=1&filter=0')
  27
  28 # URL template for displaying build steps.
  29 BUILDER_HTML_URL = '%(server_url)s/builders/%(bot_name)s/builds/%(build_num)s'
  30
  31 # Try server status page URLs, used to get build status.
  32 PERF_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.perf'
  33 LINUX_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.linux'
  34
  35 # Status codes that can be returned by the GetBuildStatus method
  36 # From buildbot.status.builder.
  37 # See: http://docs.buildbot.net/current/developer/results.html
  38 SUCCESS, WARNINGS, FAILURE, SKIPPED, EXCEPTION, RETRY, TRYPENDING = range(7)
  39 OK = (SUCCESS, WARNINGS)  # These indicate build is complete.
  40 FAILED = (FAILURE, EXCEPTION, SKIPPED)  # These indicate build failure.
  41 PENDING = (RETRY, TRYPENDING)  # These indicate in progress or in pending queue.
  42
  43
  44 class ServerAccessError(Exception):
  45
  46   def __str__(self):
  47     return '%s\nSorry, cannot connect to server.' % self.args[0]
  48
  49
  50 def _IsBuildRunning(build_data):
  51   """Checks whether the build is in progress on buildbot.
  52
  53   Presence of currentStep element in build JSON indicates build is in progress.
  54
  55   Args:
  56     build_data: A dictionary with build data, loaded from buildbot JSON API.
  57
  58   Returns:
  59     True if build is in progress, otherwise False.
  60   """
  61   current_step = build_data.get('currentStep')
  62   if (current_step and current_step.get('isStarted') and
  63       current_step.get('results') is None):
  64     return True
  65   return False
  66
  67
  68 def _IsBuildFailed(build_data):
  69   """Checks whether the build failed on buildbot.
  70
  71   Sometime build status is marked as failed even though compile and packaging
  72   steps are successful. This may happen due to some intermediate steps of less
  73   importance such as gclient revert, generate_telemetry_profile are failed.
  74   Therefore we do an addition check to confirm if build was successful by
  75   calling _IsBuildSuccessful.
  76
  77   Args:
  78     build_data: A dictionary with build data, loaded from buildbot JSON API.
  79
  80   Returns:
  81     True if revision is failed build, otherwise False.
  82   """
  83   if (build_data.get('results') in FAILED and
  84       not _IsBuildSuccessful(build_data)):
  85     return True
  86   return False
  87
  88
  89 def _IsBuildSuccessful(build_data):
  90   """Checks whether the build succeeded on buildbot.
  91
  92   We treat build as successful if the package_build step is completed without
  93   any error i.e., when results attribute of the this step has value 0 or 1
  94   in its first element.
  95
  96   Args:
  97     build_data: A dictionary with build data, loaded from buildbot JSON API.
  98
  99   Returns:
 100     True if revision is successfully build, otherwise False.
 101   """
 102   if build_data.get('steps'):
 103     for item in build_data.get('steps'):
 104       # The 'results' attribute of each step consists of two elements,
 105       # results[0]: This represents the status of build step.
 106       # See: http://docs.buildbot.net/current/developer/results.html
 107       # results[1]: List of items, contains text if step fails, otherwise empty.
 108       if (item.get('name') == 'package_build' and
 109           item.get('isFinished') and
 110           item.get('results')[0] in OK):
 111         return True
 112   return False
 113
 114
 115 def _FetchBuilderData(builder_url):
 116   """Fetches JSON data for the all the builds from the tryserver.
 117
 118   Args:
 119     builder_url: A tryserver URL to fetch builds information.
 120
 121   Returns:
 122     A dictionary with information of all build on the tryserver.
 123   """
 124   data = None
 125   try:
 126     url = urllib2.urlopen(builder_url)
 127   except urllib2.URLError as e:
 128     print ('urllib2.urlopen error %s, waterfall status page down.[%s]' % (
 129         builder_url, str(e)))
 130     return None
 131   if url is not None:
 132     try:
 133       data = url.read()
 134     except IOError as e:
 135       print 'urllib2 file object read error %s, [%s].' % (builder_url, str(e))
 136   return data
 137
 138
 139 def _GetBuildData(buildbot_url):
 140   """Gets build information for the given build id from the tryserver.
 141
 142   Args:
 143     buildbot_url: A tryserver URL to fetch build information.
 144
 145   Returns:
 146     A dictionary with build information if build exists, otherwise None.
 147   """
 148   builds_json = _FetchBuilderData(buildbot_url)
 149   if builds_json:
 150     return json.loads(builds_json)
 151   return None
 152
 153
 154 def _GetBuildBotUrl(builder_type):
 155   """Gets build bot URL for fetching build info.
 156
 157   Bisect builder bots are hosted on tryserver.chromium.perf, though we cannot
 158   access this tryserver using host and port number directly, so we use another
 159   tryserver URL for the perf tryserver.
 160
 161   Args:
 162     builder_type: Determines what type of builder is used, e.g. "perf".
 163
 164   Returns:
 165     URL of the buildbot as a string.
 166   """
 167   if builder_type == fetch_build.PERF_BUILDER:
 168     return PERF_TRY_SERVER_URL
 169   if builder_type == fetch_build.FULL_BUILDER:
 170     return LINUX_TRY_SERVER_URL
 171   raise NotImplementedError('Unsupported builder type "%s".' % builder_type)
 172
 173
 174 def GetBuildStatus(build_num, bot_name, builder_type):
 175   """Gets build status from the buildbot status page for a given build number.
 176
 177   Args:
 178     build_num: A build number on tryserver to determine its status.
 179     bot_name: Name of the bot where the build information is scanned.
 180     builder_type: Type of builder, e.g. "perf".
 181
 182   Returns:
 183     A pair which consists of build status (SUCCESS, FAILED or PENDING) and a
 184     link to build status page on the waterfall.
 185   """
 186   # TODO(prasadv, qyearsley): Make this a method of BuildArchive
 187   # (which may be renamed to BuilderTryBot or Builder).
 188   results_url = None
 189   if build_num:
 190     # Get the URL for requesting JSON data with status information.
 191     server_url = _GetBuildBotUrl(builder_type)
 192     buildbot_url = BUILDER_JSON_URL % {
 193         'server_url': server_url,
 194         'bot_name': bot_name,
 195         'build_num': build_num,
 196     }
 197     build_data = _GetBuildData(buildbot_url)
 198     if build_data:
 199       # Link to build on the buildbot showing status of build steps.
 200       results_url = BUILDER_HTML_URL % {
 201           'server_url': server_url,
 202           'bot_name': bot_name,
 203           'build_num': build_num,
 204       }
 205       if _IsBuildFailed(build_data):
 206         return (FAILED, results_url)
 207
 208       elif _IsBuildSuccessful(build_data):
 209         return (OK, results_url)
 210   return (PENDING, results_url)
 211
 212
 213 def GetBuildNumFromBuilder(build_reason, bot_name, builder_type):
 214   """Gets build number on build status page for a given 'build reason'.
 215
 216   This function parses the JSON data from buildbot page and collects basic
 217   information about the all the builds, and then uniquely identifies the build
 218   based on the 'reason' attribute in builds's JSON data.
 219
 220   The 'reason' attribute set is when a build request is posted, and it is used
 221   to identify the build on status page.
 222
 223   Args:
 224     build_reason: A unique build name set to build on tryserver.
 225     bot_name: Name of the bot where the build information is scanned.
 226     builder_type: Type of builder, e.g. "perf".
 227
 228   Returns:
 229     A build number as a string if found, otherwise None.
 230   """
 231   # TODO(prasadv, qyearsley): Make this a method of BuildArchive
 232   # (which may be renamed to BuilderTryBot or Builder).
 233   # Gets the buildbot url for the given host and port.
 234   server_url = _GetBuildBotUrl(builder_type)
 235   buildbot_url = BUILDER_JSON_URL % {
 236       'server_url': server_url,
 237       'bot_name': bot_name,
 238       'build_num': '_all',
 239   }
 240   builds_json = _FetchBuilderData(buildbot_url)
 241   if builds_json:
 242     builds_data = json.loads(builds_json)
 243     for current_build in builds_data:
 244       if builds_data[current_build].get('reason') == build_reason:
 245         return builds_data[current_build].get('number')
 246   return None