tools/site_compare/scrapers/firefox/firefox2.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Does scraping for Firefox 2.0."""
   7
   8 import pywintypes
   9 import time
  10 import types
  11
  12 from drivers import keyboard
  13 from drivers import mouse
  14 from drivers import windowing
  15
  16 # Default version
  17 version = "2.0.0.6"
  18
  19 DEFAULT_PATH = r"c:\program files\mozilla firefox\firefox.exe"
  20
  21 # TODO(jhaas): the Firefox scraper is a bit rickety at the moment. Known
  22 # issues: 1) won't work if the default profile puts toolbars in different
  23 # locations, 2) uses sleep() statements rather than more robust checks,
  24 # 3) fails badly if an existing Firefox window is open when the scrape
  25 # is invoked. This needs to be fortified at some point.
  26
  27 def GetBrowser(path):
  28   """Invoke the Firefox browser and return the process and window.
  29
  30   Args:
  31     path: full path to browser
  32
  33   Returns:
  34     A tuple of (process handle, render pane)
  35   """
  36   if not path: path = DEFAULT_PATH
  37
  38   # Invoke Firefox
  39   (proc, wnd) = windowing.InvokeAndWait(path)
  40
  41   # Get the content pane
  42   render_pane = windowing.FindChildWindow(
  43     wnd,
  44     "MozillaWindowClass/MozillaWindowClass/MozillaWindowClass")
  45
  46   return (proc, wnd, render_pane)
  47
  48
  49 def InvokeBrowser(path):
  50   """Invoke the Firefox browser.
  51
  52   Args:
  53     path: full path to browser
  54
  55   Returns:
  56     A tuple of (main window, process handle, render pane)
  57   """
  58   # Reuse an existing instance of the browser if we can find one. This
  59   # may not work correctly, especially if the window is behind other windows.
  60   wnds = windowing.FindChildWindows(0, "MozillaUIWindowClass")
  61   if len(wnds):
  62     wnd = wnds[0]
  63     proc = None
  64   else:
  65     # Invoke Firefox
  66     (proc, wnd) = windowing.InvokeAndWait(path)
  67
  68   # Get the content pane
  69   render_pane = windowing.FindChildWindow(
  70     wnd,
  71     "MozillaWindowClass/MozillaWindowClass/MozillaWindowClass")
  72
  73   return (wnd, proc, render_pane)
  74
  75
  76 def Scrape(urls, outdir, size, pos, timeout=20, **kwargs):
  77   """Invoke a browser, send it to a series of URLs, and save its output.
  78
  79   Args:
  80     urls: list of URLs to scrape
  81     outdir: directory to place output
  82     size: size of browser window to use
  83     pos: position of browser window
  84     timeout: amount of time to wait for page to load
  85     kwargs: miscellaneous keyword args
  86
  87   Returns:
  88     None if success, else an error string
  89   """
  90   if "path" in kwargs and kwargs["path"]: path = kwargs["path"]
  91   else: path = DEFAULT_PATH
  92
  93   (wnd, proc, render_pane) = InvokeBrowser(path)
  94
  95   # Resize and reposition the frame
  96   windowing.MoveAndSizeWindow(wnd, pos, size, render_pane)
  97
  98   time.sleep(3)
  99
 100   # Firefox is a bit of a pain: it doesn't use standard edit controls,
 101   # and it doesn't display a throbber when there's no tab. Let's make
 102   # sure there's at least one tab, then select the first one
 103
 104   mouse.ClickInWindow(wnd)
 105   keyboard.TypeString("[t]", True)
 106   mouse.ClickInWindow(wnd, (30, 115))
 107   time.sleep(2)
 108
 109   timedout = False
 110
 111   # Visit each URL we're given
 112   if type(urls) in types.StringTypes: urls = [urls]
 113
 114   for url in urls:
 115
 116     # Use keyboard shortcuts
 117     keyboard.TypeString("{d}", True)
 118     keyboard.TypeString(url)
 119     keyboard.TypeString("\n")
 120
 121     # Wait for the page to finish loading
 122     load_time = windowing.WaitForThrobber(wnd, (10, 96, 26, 112), timeout)
 123     timedout = load_time < 0
 124
 125     if timedout:
 126       break
 127
 128     # Scrape the page
 129     image = windowing.ScrapeWindow(render_pane)
 130
 131     # Save to disk
 132     if "filename" in kwargs:
 133       if callable(kwargs["filename"]):
 134         filename = kwargs["filename"](url)
 135       else:
 136         filename = kwargs["filename"]
 137     else:
 138       filename = windowing.URLtoFilename(url, outdir, ".bmp")
 139     image.save(filename)
 140
 141   # Close all the tabs, cheesily
 142   mouse.ClickInWindow(wnd)
 143
 144   while len(windowing.FindChildWindows(0, "MozillaUIWindowClass")):
 145     keyboard.TypeString("[w]", True)
 146     time.sleep(1)
 147
 148   if timedout:
 149     return "timeout"
 150
 151
 152 def Time(urls, size, timeout, **kwargs):
 153   """Measure how long it takes to load each of a series of URLs
 154
 155   Args:
 156     urls: list of URLs to time
 157     size: size of browser window to use
 158     timeout: amount of time to wait for page to load
 159     kwargs: miscellaneous keyword args
 160
 161   Returns:
 162     A list of tuples (url, time). "time" can be "crashed" or "timeout"
 163   """
 164   if "path" in kwargs and kwargs["path"]: path = kwargs["path"]
 165   else: path = DEFAULT_PATH
 166   proc = None
 167
 168   # Visit each URL we're given
 169   if type(urls) in types.StringTypes: urls = [urls]
 170
 171   ret = []
 172   for url in urls:
 173     try:
 174       # Invoke the browser if necessary
 175       if not proc:
 176         (wnd, proc, render_pane) = InvokeBrowser(path)
 177
 178         # Resize and reposition the frame
 179         windowing.MoveAndSizeWindow(wnd, (0,0), size, render_pane)
 180
 181         time.sleep(3)
 182
 183         # Firefox is a bit of a pain: it doesn't use standard edit controls,
 184         # and it doesn't display a throbber when there's no tab. Let's make
 185         # sure there's at least one tab, then select the first one
 186
 187         mouse.ClickInWindow(wnd)
 188         keyboard.TypeString("[t]", True)
 189         mouse.ClickInWindow(wnd, (30, 115))
 190         time.sleep(2)
 191
 192       # Use keyboard shortcuts
 193       keyboard.TypeString("{d}", True)
 194       keyboard.TypeString(url)
 195       keyboard.TypeString("\n")
 196
 197       # Wait for the page to finish loading
 198       load_time = windowing.WaitForThrobber(wnd, (10, 96, 26, 112), timeout)
 199       timedout = load_time < 0
 200
 201       if timedout:
 202         load_time = "timeout"
 203
 204         # Try to close the browser; if this fails it's probably a crash
 205         mouse.ClickInWindow(wnd)
 206
 207         count = 0
 208         while (len(windowing.FindChildWindows(0, "MozillaUIWindowClass"))
 209           and count < 5):
 210           keyboard.TypeString("[w]", True)
 211           time.sleep(1)
 212           count = count + 1
 213
 214         if len(windowing.FindChildWindows(0, "MozillaUIWindowClass")):
 215           windowing.EndProcess(proc)
 216           load_time = "crashed"
 217
 218         proc = None
 219     except pywintypes.error:
 220       proc = None
 221       load_time = "crashed"
 222
 223     ret.append( (url, load_time) )
 224
 225   if proc:
 226     count = 0
 227     while (len(windowing.FindChildWindows(0, "MozillaUIWindowClass"))
 228       and count < 5):
 229       keyboard.TypeString("[w]", True)
 230       time.sleep(1)
 231       count = count + 1
 232   return ret
 233
 234
 235 def main():
 236   # We're being invoked rather than imported, so run some tests
 237   path = r"c:\sitecompare\scrapes\Firefox\2.0.0.6"
 238   windowing.PreparePath(path)
 239
 240   # Scrape three sites and save the results
 241   Scrape(
 242     ["http://www.microsoft.com", "http://www.google.com",
 243      "http://www.sun.com"],
 244     path, (1024, 768), (0, 0))
 245   return 0
 246
 247
 248 if __name__ == "__main__":
 249   sys.exit(main())