tools/telemetry_tools/telemetry_bootstrap.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Bootstrap Chrome Telemetry by downloading all its files from SVN servers.
   6
   7 Requires a DEPS file to specify which directories on which SVN servers
   8 are required to run Telemetry. Format of that DEPS file is a subset of the
   9 normal DEPS file format[1]; currently only only the "deps" dictionary is
  10 supported and nothing else.
  11
  12 Fetches all files in the specified directories using WebDAV (SVN is WebDAV under
  13 the hood).
  14
  15 [1] http://dev.chromium.org/developers/how-tos/depottools#TOC-DEPS-file
  16 """
  17
  18 import imp
  19 import logging
  20 import os
  21 import urllib
  22 import urlparse
  23
  24 # Link to file containing the 'davclient' WebDAV client library.
  25 _DAVCLIENT_URL = ('https://src.chromium.org/viewvc/chrome/trunk/src/tools/' +
  26                   'telemetry/third_party/davclient/davclient.py')
  27
  28 # Dummy module for Davclient.
  29 _davclient = None
  30
  31 def _download_and_import_davclient_module():
  32   """Dynamically import davclient helper library."""
  33   global _davclient
  34   davclient_src = urllib.urlopen(_DAVCLIENT_URL).read()
  35   _davclient = imp.new_module('davclient')
  36   exec davclient_src in _davclient.__dict__
  37
  38
  39 class DAVClientWrapper():
  40   """Knows how to retrieve subdirectories and files from WebDAV/SVN servers."""
  41
  42   def __init__(self, root_url):
  43     """Initialize SVN server root_url, save files to local dest_dir.
  44
  45     Args:
  46       root_url: string url of SVN/WebDAV server
  47     """
  48     self.root_url = root_url
  49     self.client = _davclient.DAVClient(root_url)
  50
  51   @staticmethod
  52   def __norm_path_keys(dict_with_path_keys):
  53     """Returns a dictionary with os.path.normpath called on every key."""
  54     return dict((os.path.normpath(k), v) for (k, v) in
  55                 dict_with_path_keys.items())
  56
  57   def GetDirList(self, path):
  58     """Returns string names of all files and subdirs of path on the server."""
  59     props = self.__norm_path_keys(self.client.propfind(path, depth=1))
  60     # remove this path
  61     del props[os.path.normpath(path)]
  62     return [os.path.basename(p) for p in props.keys()]
  63
  64   def IsFile(self, path):
  65     """Returns True if the path is a file on the server, False if directory."""
  66     props = self.__norm_path_keys(self.client.propfind(path, depth=1))
  67     return props[os.path.normpath(path)]['resourcetype'] is None
  68
  69   def Traverse(self, src_path, dst_path):
  70     """Walks the directory hierarchy pointed to by src_path download all files.
  71
  72     Recursively walks src_path and saves all files and subfolders into
  73     dst_path.
  74
  75     Args:
  76       src_path: string path on SVN server to save (absolute path on server).
  77       dest_path: string local path (relative or absolute) to save to.
  78     """
  79     if self.IsFile(src_path):
  80       if not os.path.exists(os.path.dirname(dst_path)):
  81         logging.info("creating %s", os.path.dirname(dst_path))
  82         os.makedirs(os.path.dirname(dst_path))
  83       logging.info("Saving %s to %s", self.root_url + src_path, dst_path)
  84       urllib.urlretrieve(self.root_url + src_path, dst_path)
  85       return
  86     else:
  87       for subdir in self.GetDirList(src_path):
  88         self.Traverse(os.path.join(src_path, subdir),
  89                       os.path.join(dst_path, subdir))
  90
  91
  92 def ListAllDepsPaths(deps_content):
  93   """Recursively returns a list of all paths indicated in this deps file.
  94
  95   Note that this discards information about where path dependencies come from,
  96   so this is only useful in the context of a Chromium source checkout that has
  97   already fetched all dependencies.
  98
  99   Args:
 100     deps_content: String containing deps information to be evaluated, in the
 101                   format given in the header of this file.
 102   Returns: A list of string paths starting under src that are required by the
 103            given deps file, and all of its sub-dependencies. This amounts to
 104            the keys of the 'deps' dictionary.
 105   """
 106   chrome_root = os.path.dirname(__file__)
 107   while os.path.basename(chrome_root) != 'src':
 108     chrome_root = os.path.abspath(os.path.join(chrome_root, '..'))
 109   deps = imp.new_module('deps')
 110   exec deps_content in deps.__dict__
 111
 112   deps_paths = deps.deps.keys()
 113
 114   if hasattr(deps, 'deps_includes'):
 115     for path in deps.deps_includes.keys():
 116       # Need to localize the paths.
 117       path = os.path.join(chrome_root, '..', path)
 118       deps_paths = deps_paths + ListAllDepsPaths(open(path).read())
 119
 120   return deps_paths
 121
 122
 123 def DownloadDepsURL(destination_dir, url):
 124   """Wrapper around DownloadDeps that takes a string URL to the deps file.
 125
 126   Args:
 127     destination_dir: String path to local directory to download files into.
 128     url: URL of deps file (see DownloadDeps for format).
 129   """
 130   logging.warning('Downloading deps from %s...', url)
 131   DownloadDeps(destination_dir, urllib.urlopen(url).read())
 132
 133
 134 def DownloadDeps(destination_dir, deps_content):
 135   """Saves all the dependencies in deps_path.
 136
 137   Reads deps_content, assuming the contents are in the simple DEPS-like file
 138   format specified in the header of this file, then download all
 139   files/directories listed to the destination_dir.
 140
 141   Args:
 142     destination_dir: String path to directory to download files into.
 143     deps_content: String containing deps information to be evaluated.
 144   """
 145   # TODO(wiltzius): Add a parameter for which revision to pull.
 146   _download_and_import_davclient_module()
 147
 148   deps = imp.new_module('deps')
 149   exec deps_content in deps.__dict__
 150
 151   for dst_path, src_path in deps.deps.iteritems():
 152     full_dst_path = os.path.join(destination_dir, dst_path)
 153     parsed_url = urlparse.urlparse(src_path)
 154     root_url = parsed_url.scheme + '://' + parsed_url.netloc
 155
 156     dav_client = DAVClientWrapper(root_url)
 157     dav_client.Traverse(parsed_url.path, full_dst_path)
 158
 159   if hasattr(deps, 'deps_includes'):
 160     for url in deps.deps_includes.values():
 161       DownloadDepsURL(destination_dir, url)
 162