build/pgo/automation.py.in

   1 #
   2 # ***** BEGIN LICENSE BLOCK *****
   3 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   4 #
   5 # The contents of this file are subject to the Mozilla Public License Version
   6 # 1.1 (the "License"); you may not use this file except in compliance with
   7 # the License. You may obtain a copy of the License at
   8 # http://www.mozilla.org/MPL/
   9 #
  10 # Software distributed under the License is distributed on an "AS IS" basis,
  11 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  12 # for the specific language governing rights and limitations under the
  13 # License.
  14 #
  15 # The Original Code is mozilla.org code.
  16 #
  17 # The Initial Developer of the Original Code is
  18 # Mozilla Foundation.
  19 # Portions created by the Initial Developer are Copyright (C) 2008
  20 # the Initial Developer. All Rights Reserved.
  21 #
  22 # Contributor(s):
  23 #   Robert Sayre <sayrer@gmail.com>
  24 #   Jeff Walden <jwalden+bmo@mit.edu>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39
  40 import codecs
  41 from datetime import datetime
  42 import itertools
  43 import logging
  44 import shutil
  45 import os
  46 import re
  47 import signal
  48 import sys
  49 import threading
  50
  51 """
  52 Runs the browser from a script, and provides useful utilities
  53 for setting up the browser environment.
  54 """
  55
  56 __all__ = [
  57            "UNIXISH",
  58            "IS_WIN32",
  59            "IS_MAC",
  60            "runApp",
  61            "Process",
  62            "initializeProfile",
  63            "DIST_BIN",
  64            "DEFAULT_APP",
  65           ]
  66
  67 # These are generated in mozilla/build/Makefile.in
  68 #expand DIST_BIN = "./" + __XPC_BIN_PATH__
  69 #expand IS_WIN32 = len("__WIN32__") != 0
  70 #expand IS_MAC = __IS_MAC__ != 0
  71 #ifdef IS_CYGWIN
  72 #expand IS_CYGWIN = __IS_CYGWIN__ == 1
  73 #else
  74 IS_CYGWIN = False
  75 #endif
  76 #expand IS_CAMINO = __IS_CAMINO__ != 0
  77
  78 UNIXISH = not IS_WIN32 and not IS_MAC
  79
  80 #expand DEFAULT_APP = "./" + __BROWSER_PATH__
  81
  82 ###########
  83 # LOGGING #
  84 ###########
  85
  86 # We use the logging system here primarily because it'll handle multiple
  87 # threads, which is needed to process the output of the server and application
  88 # processes simultaneously.
  89 log = logging.getLogger()
  90 handler = logging.StreamHandler(sys.stdout)
  91 log.setLevel(logging.INFO)
  92 log.addHandler(handler)
  93
  94
  95 #################
  96 # SUBPROCESSING #
  97 #################
  98
  99 class Process:
 100   """
 101   Represents a subprocess of this process.  We don't just directly use the
 102   subprocess module here because we want compatibility with Python 2.3 on
 103   non-Windows platforms.  :-(
 104   """
 105
 106   def __init__(self, command, args, env):
 107     """
 108     Creates a process representing the execution of the given command, which
 109     must be an absolute path, with the given arguments in the given environment.
 110     The process is then started.
 111     """
 112     command = os.path.abspath(command)
 113     if IS_WIN32:
 114       import subprocess
 115       cmd = [command]
 116       cmd.extend(args)
 117       p = subprocess.Popen(cmd, env = env,
 118                            stdout = subprocess.PIPE,
 119                            stderr = subprocess.STDOUT)
 120       self._out = p.stdout
 121     else:
 122       import popen2
 123       cmd = []
 124       for (k, v) in env.iteritems():
 125         cmd.append(k + "='" + v + "' ")
 126       cmd.append("'" + command + "'")
 127       cmd.extend(map(lambda x: "'" + x + "'", args))
 128       cmd = " ".join(cmd)
 129       p = popen2.Popen4(cmd)
 130       self._out = p.fromchild
 131
 132     self._process = p
 133     self.pid = p.pid
 134
 135     self._thread = threading.Thread(target = lambda: self._run())
 136     self._thread.start()
 137
 138   def _run(self):
 139     "Continues execution of this process on a separate thread."
 140     p = self._process
 141     out = self._out
 142
 143     if IS_WIN32:
 144       running = lambda: p.poll() is None
 145     else:
 146       running = lambda: p.poll() == -1
 147
 148     # read in lines until the process finishes, then read in any last remaining
 149     # buffered lines
 150     while running():
 151       line = out.readline().rstrip()
 152       if len(line) > 0:
 153         log.info(line)
 154     for line in out:
 155       line = line.rstrip()
 156       if len(line) > 0:
 157         log.info(line)
 158     self._status = p.poll()
 159
 160   def wait(self):
 161     "Waits for this process to finish, then returns the process's status."
 162     self._thread.join()
 163     return self._status
 164
 165   def kill(self):
 166     "Kills this process."
 167     try:
 168       if not IS_WIN32: # XXX
 169         os.kill(self._process.pid, signal.SIGKILL)
 170     except:
 171       pass
 172
 173
 174 #################
 175 # PROFILE SETUP #
 176 #################
 177
 178 class SyntaxError(Exception):
 179   "Signifies a syntax error on a particular line in server-locations.txt."
 180
 181   def __init__(self, lineno, msg = None):
 182     self.lineno = lineno
 183     self.msg = msg
 184
 185   def __str__(self):
 186     s = "Syntax error on line " + str(self.lineno)
 187     if self.msg:
 188       s += ": %s." % self.msg
 189     else:
 190       s += "."
 191     return s
 192
 193
 194 class Location:
 195   "Represents a location line in server-locations.txt."
 196
 197   def __init__(self, scheme, host, port, options):
 198     self.scheme = scheme
 199     self.host = host
 200     self.port = port
 201     self.options = options
 202
 203
 204 def readLocations():
 205   """
 206   Reads the locations at which the Mochitest HTTP server is available from
 207   server-locations.txt.
 208   """
 209
 210   locationFile = codecs.open("server-locations.txt", "r", "UTF-8")
 211
 212   # Perhaps more detail than necessary, but it's the easiest way to make sure
 213   # we get exactly the format we want.  See server-locations.txt for the exact
 214   # format guaranteed here.
 215   lineRe = re.compile(r"^(?P<scheme>[a-z][-a-z0-9+.]*)"
 216                       r"://"
 217                       r"(?P<host>"
 218                         r"\d+\.\d+\.\d+\.\d+"
 219                         r"|"
 220                         r"(?:[a-z0-9](?:[-a-z0-9]*[a-z0-9])?\.)*"
 221                         r"[a-z](?:[-a-z0-9]*[a-z0-9])?"
 222                       r")"
 223                       r":"
 224                       r"(?P<port>\d+)"
 225                       r"(?:"
 226                       r"\s+"
 227                       r"(?P<options>\w+(?:,\w+)*)"
 228                       r")?$")
 229   locations = []
 230   lineno = 0
 231   seenPrimary = False
 232   for line in locationFile:
 233     lineno += 1
 234     if line.startswith("#") or line == "\n":
 235       continue
 236
 237     match = lineRe.match(line)
 238     if not match:
 239       raise SyntaxError(lineno)
 240
 241     options = match.group("options")
 242     if options:
 243       options = options.split(",")
 244       if "primary" in options:
 245         if seenPrimary:
 246           raise SyntaxError(lineno, "multiple primary locations")
 247         seenPrimary = True
 248     else:
 249       options = []
 250
 251     locations.append(Location(match.group("scheme"), match.group("host"),
 252                               match.group("port"), options))
 253
 254   if not seenPrimary:
 255     raise SyntaxError(lineno + 1, "missing primary location")
 256
 257   return locations
 258
 259
 260 def initializeProfile(profileDir):
 261   "Sets up the standard testing profile."
 262
 263   # Start with a clean slate.
 264   shutil.rmtree(profileDir, True)
 265   os.mkdir(profileDir)
 266
 267   prefs = []
 268
 269   part = """\
 270 user_pref("browser.dom.window.dump.enabled", true);
 271 user_pref("dom.allow_scripts_to_close_windows", true);
 272 user_pref("dom.disable_open_during_load", false);
 273 user_pref("dom.max_script_run_time", 0); // no slow script dialogs
 274 user_pref("signed.applets.codebase_principal_support", true);
 275 user_pref("security.warn_submit_insecure", false);
 276 user_pref("browser.shell.checkDefaultBrowser", false);
 277 user_pref("browser.warnOnQuit", false);
 278 user_pref("accessibility.typeaheadfind.autostart", false);
 279 user_pref("javascript.options.showInConsole", true);
 280 user_pref("layout.debug.enable_data_xbl", true);
 281 user_pref("browser.EULA.override", true);
 282
 283 user_pref("camino.warn_when_closing", false); // Camino-only, harmless to others
 284 """
 285   prefs.append(part)
 286
 287   locations = readLocations()
 288
 289   # Grant God-power to all the privileged servers on which tests run.
 290   privileged = filter(lambda loc: "privileged" in loc.options, locations)
 291   for (i, l) in itertools.izip(itertools.count(1), privileged):
 292     part = """
 293 user_pref("capability.principal.codebase.p%(i)d.granted",
 294           "UniversalXPConnect UniversalBrowserRead UniversalBrowserWrite \
 295            UniversalPreferencesRead UniversalPreferencesWrite \
 296            UniversalFileRead");
 297 user_pref("capability.principal.codebase.p%(i)d.id", "%(origin)s");
 298 user_pref("capability.principal.codebase.p%(i)d.subjectName", "");
 299 """  % { "i": i,
 300          "origin": (l.scheme + "://" + l.host + ":" + l.port) }
 301     prefs.append(part)
 302
 303   # We need to proxy every server but the primary one.
 304   origins = ["'%s://%s:%s'" % (l.scheme, l.host, l.port)
 305              for l in filter(lambda l: "primary" not in l.options, locations)]
 306   origins = ", ".join(origins)
 307
 308   pacURL = """data:text/plain,
 309 function FindProxyForURL(url, host)
 310 {
 311   var origins = [%(origins)s];
 312   var regex = new RegExp('^([a-z][-a-z0-9+.]*)' +
 313                          '://' +
 314                          '(?:[^/@]*@)?' +
 315                          '(.*?)' +
 316                          '(?::(\\\\\\\\d+))?/');
 317   var matches = regex.exec(url);
 318   if (!matches)
 319     return 'DIRECT';
 320   var isHttp = matches[1] == 'http';
 321   if (!matches[3])
 322     matches[3] = isHttp ? '80' : '443';
 323   var origin = matches[1] + '://' + matches[2] + ':' + matches[3];
 324   if (origins.indexOf(origin) < 0)
 325     return 'DIRECT';
 326   if (isHttp)
 327     return 'PROXY localhost:8888';
 328   return 'DIRECT';
 329 }""" % { "origins": origins }
 330   pacURL = "".join(pacURL.splitlines())
 331
 332   part = """
 333 user_pref("network.proxy.type", 2);
 334 user_pref("network.proxy.autoconfig_url", "%(pacURL)s");
 335
 336 user_pref("camino.use_system_proxy_settings", false); // Camino-only, harmless to others
 337 """ % {"pacURL": pacURL}
 338   prefs.append(part)
 339
 340   # write the preferences
 341   prefsFile = open(profileDir + "/" + "user.js", "a")
 342   prefsFile.write("".join(prefs))
 343   prefsFile.close()
 344
 345
 346 ###############
 347 # RUN THE APP #
 348 ###############
 349
 350 def runApp(testURL, env, app, profileDir, extraArgs):
 351   "Run the app, returning the time at which it was started."
 352   # mark the start
 353   start = datetime.now()
 354
 355   # now run with the profile we created
 356   cmd = app
 357   if IS_MAC and not IS_CAMINO and not cmd.endswith("-bin"):
 358     cmd += "-bin"
 359   cmd = os.path.abspath(cmd)
 360
 361   args = []
 362   if IS_MAC:
 363     args.append("-foreground")
 364
 365   if IS_CYGWIN:
 366     profileDirectory = commands.getoutput("cygpath -w \"" + profileDir + "/\"")
 367   else:
 368     profileDirectory = profileDir + "/"
 369
 370   args.extend(("-no-remote", "-profile", profileDirectory))
 371   if IS_CAMINO:
 372     args.extend(("-url", testURL))
 373   else:
 374     args.append((testURL))
 375   args.extend(extraArgs)
 376   proc = Process(cmd, args, env = env)
 377   log.info("Application pid: %d", proc.pid)
 378   status = proc.wait()
 379   if status != 0:
 380     log.info("ERROR FAIL Exited with code %d during test run", status)
 381
 382   return start