tools/telemetry/catapult_base/dependency_manager/dependency_manager.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import stat
   8
   9 from catapult_base import cloud_storage
  10 from catapult_base import support_binaries
  11 from catapult_base.dependency_manager import base_config
  12 from catapult_base.dependency_manager import exceptions
  13
  14
  15 DEFAULT_TYPE = 'default'
  16
  17
  18 class DependencyManager(object):
  19   def __init__(self, configs, supported_config_types=None):
  20     """Manages file dependencies found locally or in cloud_storage.
  21
  22     Args:
  23         configs: A list of instances of BaseConfig or it's subclasses, passed
  24             in decreasing order of precedence.
  25         supported_config_types: A list of whitelisted config_types.
  26             No restrictions if None is specified.
  27
  28     Raises:
  29         ValueError: If |configs| is not a list of instances of BaseConfig or
  30             its subclasses.
  31         UnsupportedConfigFormatError: If supported_config_types is specified and
  32             configs contains a config not in the supported config_types.
  33
  34     Example: DependencyManager([config1, config2, config3])
  35         No requirements on the type of Config, and any dependencies that have
  36         local files for the same platform will first look in those from
  37         config1, then those from config2, and finally those from config3.
  38     """
  39     if configs is None or type(configs) != list:
  40       raise ValueError(
  41           'Must supply a list of config files to DependencyManager')
  42     # self._lookup_dict is a dictionary with the following format:
  43     # { dependency1: {platform1: dependency_info1,
  44     #                 platform2: dependency_info2}
  45     #   dependency2: {platform1: dependency_info3,
  46     #                  ...}
  47     #   ...}
  48     #
  49     # Where the dependencies and platforms are strings, and the
  50     # dependency_info's are DependencyInfo instances.
  51     self._lookup_dict = {}
  52     self.supported_configs = supported_config_types or []
  53     for config in configs:
  54       self._UpdateDependencies(config)
  55
  56   def FetchPath(self, dependency, platform):
  57     """Get a path to an executable for |dependency|, downloading as needed.
  58
  59     A path to a default executable may be returned if a platform specific
  60     version is not specified in the config(s).
  61
  62     Args:
  63         dependency: Name of the desired dependency, as given in the config(s)
  64             used in this DependencyManager.
  65         platform: Name of the platform the dependency will run on. Often of the
  66             form 'os_architecture'. Must match those specified in the config(s)
  67             used in this DependencyManager.
  68
  69     Returns:
  70         A path to an executable of |dependency| that will run on |platform|,
  71         downloading from cloud storage if needed.
  72
  73     Raises:
  74         NoPathFoundError: If a local copy of the executable cannot be found and
  75             a remote path could not be downloaded from cloud_storage.
  76         CredentialsError: If cloud_storage credentials aren't configured.
  77         PermissionError: If cloud_storage credentials are configured, but not
  78             with an account that has permission to download the remote file.
  79         NotFoundError: If the remote file does not exist where expected in
  80             cloud_storage.
  81         ServerError: If an internal server error is hit while downloading the
  82             remote file.
  83         CloudStorageError: If another error occured while downloading the remote
  84             path.
  85         FileNotFoundError: If an attempted download was otherwise unsuccessful.
  86
  87     """
  88     dependency_info = self._GetDependencyInfo(dependency, platform)
  89     if not dependency_info:
  90       # TODO(aiolos): Replace the support_binaries call with an error once all
  91       # binary dependencies are moved over to the new system.
  92
  93       # platform should be of the form '%s_%s' % (os_name, arch_name) when
  94       # called from the binary_manager.
  95       platform_parts = platform.split('_', 1)
  96       assert len(platform_parts) == 2
  97       platform_os, platform_arch = platform_parts
  98       logging.info('Calling into support_binaries with dependency %s, platform '
  99                    '%s and arch %s.' % (dependency, platform_os,
 100                                         platform_arch))
 101       return support_binaries.FindPath(dependency, platform_arch,
 102                                        platform_os)
 103     logging.info('Looking for dependency %s, on platform %s in the dependency '
 104                  'manager.' % (dependency, platform))
 105     path = self._LocalPath(dependency_info)
 106     if not path or not os.path.exists(path):
 107       path = self._CloudStoragePath(dependency_info)
 108       if not path or not os.path.exists(path):
 109         raise exceptions.NoPathFoundError(dependency, platform)
 110     return path
 111
 112   def LocalPath(self, dependency, platform):
 113     """Get a path to a locally stored executable for |dependency|.
 114
 115     A path to a default executable may be returned if a platform specific
 116     version is not specified in the config(s).
 117     Will not download the executable.
 118
 119     Args:
 120         dependency: Name of the desired dependency, as given in the config(s)
 121             used in this DependencyManager.
 122         platform: Name of the platform the dependency will run on. Often of the
 123             form 'os_architecture'. Must match those specified in the config(s)
 124             used in this DependencyManager.
 125
 126     Returns:
 127         A path to an executable for |dependency| that will run on |platform|.
 128
 129     Raises:
 130         NoPathFoundError: If a local copy of the executable cannot be found.
 131     """
 132     # TODO(aiolos): Replace the support_binaries call with an error once all
 133     # binary dependencies are moved over to the new system.
 134     dependency_info = self._GetDependencyInfo(dependency, platform)
 135     if not dependency_info:
 136       return support_binaries.FindLocallyBuiltPath(dependency)
 137     local_path = self._LocalPath(dependency_info)
 138     if not local_path or not os.path.exists(local_path):
 139       raise exceptions.NoPathFoundError(dependency, platform)
 140     return local_path
 141
 142   def _UpdateDependencies(self, config):
 143     """Add the dependency information stored in |config| to this instance.
 144
 145     Args:
 146         config: An instances of BaseConfig or a subclasses.
 147
 148     Raises:
 149         UnsupportedConfigFormatError: If supported_config_types was specified
 150         and config is not in the supported config_types.
 151     """
 152     if not isinstance(config, base_config.BaseConfig):
 153       raise ValueError('Must use a BaseConfig or subclass instance with the '
 154                        'DependencyManager.')
 155     if (self.supported_configs and
 156         config.GetConfigType() not in self.supported_configs):
 157       raise exceptions.UnsupportedConfigFormatError(config.GetConfigType(),
 158                                                     config.config_path)
 159     for dep_info in config.IterDependencyInfo():
 160       dependency = dep_info.dependency
 161       platform = dep_info.platform
 162       if dependency not in self._lookup_dict:
 163         self._lookup_dict[dependency] = {}
 164       if platform not in self._lookup_dict[dependency]:
 165         self._lookup_dict[dependency][platform] = dep_info
 166       else:
 167         self._lookup_dict[dependency][platform].Update(dep_info)
 168
 169
 170   def _GetDependencyInfo(self, dependency, platform):
 171     """Get information for |dependency| on |platform|, or a default if needed.
 172
 173     Args:
 174         dependency: Name of the desired dependency, as given in the config(s)
 175             used in this DependencyManager.
 176         platform: Name of the platform the dependency will run on. Often of the
 177             form 'os_architecture'. Must match those specified in the config(s)
 178             used in this DependencyManager.
 179
 180     Returns: The dependency_info for |dependency| on |platform| if it exists.
 181         Or the default version of |dependency| if it exists, or None if neither
 182         exist.
 183     """
 184     if not self._lookup_dict or dependency not in self._lookup_dict:
 185       return None
 186     dependency_dict = self._lookup_dict[dependency]
 187     device_type = platform
 188     if not device_type in dependency_dict:
 189       device_type = DEFAULT_TYPE
 190     return dependency_dict.get(device_type)
 191
 192   @staticmethod
 193   def _LocalPath(dependency_info):
 194     """Return a path to a locally stored file for |dependency_info|.
 195
 196     Will not download the file.
 197
 198     Args:
 199         dependency_info: A DependencyInfo instance for the dependency to be
 200             found and the platform it should run on.
 201
 202     Returns: A path to a local file, or None if not found.
 203     """
 204     if dependency_info:
 205       paths = dependency_info.local_paths
 206       for local_path in paths:
 207         if os.path.exists(local_path):
 208           return local_path
 209     return None
 210
 211   @staticmethod
 212   def _CloudStoragePath(dependency_info):
 213     """Return a path to a downloaded file for |dependency_info|.
 214
 215     May not download the file if it has already been downloaded.
 216
 217     Args:
 218         dependency_info: A DependencyInfo instance for the dependency to be
 219             found and the platform it should run on.
 220
 221     Returns: A path to an executable that was stored in cloud_storage, or None
 222        if not found.
 223
 224     Raises:
 225         CredentialsError: If cloud_storage credentials aren't configured.
 226         PermissionError: If cloud_storage credentials are configured, but not
 227             with an account that has permission to download the needed file.
 228         NotFoundError: If the needed file does not exist where expected in
 229             cloud_storage.
 230         ServerError: If an internal server error is hit while downloading the
 231             needed file.
 232         CloudStorageError: If another error occured while downloading the remote
 233             path.
 234         FileNotFoundError: If the download was otherwise unsuccessful.
 235     """
 236     if not dependency_info:
 237       return None
 238     cs_path = dependency_info.cs_remote_path
 239     cs_hash = dependency_info.cs_hash
 240     cs_bucket = dependency_info.cs_bucket
 241     download_path = dependency_info.download_path
 242     if not cs_path or not cs_bucket or not cs_hash or not download_path:
 243       return None
 244
 245     download_dir = os.path.dirname(download_path)
 246     if not os.path.exists(download_dir):
 247       os.makedirs(download_dir)
 248
 249     cloud_storage.GetIfHashChanged(cs_path, download_path, cs_bucket, cs_hash)
 250     if not os.path.exists(download_path):
 251       raise exceptions.FileNotFoundError(download_path)
 252     #TODO(aiolos): Add support for unzipping files.
 253     os.chmod(download_path,
 254              stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP)
 255     return os.path.abspath(download_path)
 256