tools/telemetry/catapult_base/dependency_manager/dependency_manager.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import logging
   6 import os
   7 import stat
   8
   9 from catapult_base import cloud_storage
  10 from catapult_base import support_binaries
  11 from catapult_base.dependency_manager import base_config
  12 from catapult_base.dependency_manager import exceptions
  13
  14
  15 DEFAULT_TYPE = 'default'
  16
  17
  18 class DependencyManager(object):
  19   def __init__(self, configs, supported_config_types=None):
  20     """Manages file dependencies found locally or in cloud_storage.
  21
  22     Args:
  23         configs: A list of instances of BaseConfig or it's subclasses, passed
  24             in decreasing order of precedence.
  25         supported_config_types: A list of whitelisted config_types.
  26             No restrictions if None is specified.
  27
  28     Raises:
  29         ValueError: If |configs| is not a list of instances of BaseConfig or
  30             its subclasses.
  31         UnsupportedConfigFormatError: If supported_config_types is specified and
  32             configs contains a config not in the supported config_types.
  33
  34     Example: DependencyManager([config1, config2, config3])
  35         No requirements on the type of Config, and any dependencies that have
  36         local files for the same platform will first look in those from
  37         config1, then those from config2, and finally those from config3.
  38     """
  39     if configs is None or type(configs) != list:
  40       raise ValueError(
  41           'Must supply a list of config files to DependencyManager')
  42     # self._lookup_dict is a dictionary with the following format:
  43     # { dependency1: {platform1: dependency_info1,
  44     #                 platform2: dependency_info2}
  45     #   dependency2: {platform1: dependency_info3,
  46     #                  ...}
  47     #   ...}
  48     #
  49     # Where the dependencies and platforms are strings, and the
  50     # dependency_info's are DependencyInfo instances.
  51     self._lookup_dict = {}
  52     self.supported_configs = supported_config_types or []
  53     for config in configs:
  54       self._UpdateDependencies(config)
  55
  56   def FetchPath(self, dependency, platform):
  57     """Get a path to an executable for |dependency|, downloading as needed.
  58
  59     A path to a default executable may be returned if a platform specific
  60     version is not specified in the config(s).
  61
  62     Args:
  63         dependency: Name of the desired dependency, as given in the config(s)
  64             used in this DependencyManager.
  65         platform: Name of the platform the dependency will run on. Often of the
  66             form 'os_architecture'. Must match those specified in the config(s)
  67             used in this DependencyManager.
  68
  69     Returns:
  70         A path to an executable of |dependency| that will run on |platform|,
  71         downloading from cloud storage if needed.
  72
  73     Raises:
  74         NoPathFoundError: If a local copy of the executable cannot be found and
  75             a remote path could not be downloaded from cloud_storage.
  76         CredentialsError: If cloud_storage credentials aren't configured.
  77         PermissionError: If cloud_storage credentials are configured, but not
  78             with an account that has permission to download the remote file.
  79         NotFoundError: If the remote file does not exist where expected in
  80             cloud_storage.
  81         ServerError: If an internal server error is hit while downloading the
  82             remote file.
  83         CloudStorageError: If another error occured while downloading the remote
  84             path.
  85         FileNotFoundError: If an attempted download was otherwise unsuccessful.
  86
  87     """
  88     if not self._lookup_dict or not dependency in self._lookup_dict:
  89       # TODO(aiolos): Replace the support_binaries call with an error once all
  90       # binary dependencies are moved over to the new system.
  91
  92       # platform should be of the form '%s_%s' % (os_name, arch_name) when
  93       # called from the binary_manager.
  94       platform_parts = platform.split('_', 1)
  95       assert len(platform_parts) == 2
  96       platform_os, platform_arch = platform_parts
  97       logging.info('Calling into support_binaries with dependency %s, platform '
  98                    '%s and arch %s.' % (dependency, platform_os,
  99                                         platform_arch))
 100       return support_binaries.FindPath(dependency, platform_arch,
 101                                        platform_os)
 102     logging.info('Looking for dependency %s, on platform %s in the dependency '
 103                  'manager.' % (dependency, platform))
 104     dependency_info = self._GetDependencyInfo(self._lookup_dict, dependency,
 105                                               platform)
 106     path = self._LocalPath(dependency_info)
 107     if not path or not os.path.exists(path):
 108       path = self._CloudStoragePath(dependency_info)
 109       if not path or not os.path.exists(path):
 110         raise exceptions.NoPathFoundError(dependency, platform)
 111     return path
 112
 113   def LocalPath(self, dependency, platform):
 114     """Get a path to a locally stored executable for |dependency|.
 115
 116     A path to a default executable may be returned if a platform specific
 117     version is not specified in the config(s).
 118     Will not download the executable.
 119
 120     Args:
 121         dependency: Name of the desired dependency, as given in the config(s)
 122             used in this DependencyManager.
 123         platform: Name of the platform the dependency will run on. Often of the
 124             form 'os_architecture'. Must match those specified in the config(s)
 125             used in this DependencyManager.
 126
 127     Returns:
 128         A path to an executable for |dependency| that will run on |platform|.
 129
 130     Raises:
 131         NoPathFoundError: If a local copy of the executable cannot be found.
 132     """
 133     # TODO(aiolos): Replace the support_binaries call with an error once all
 134     # binary dependencies are moved over to the new system.
 135     if not self._lookup_dict or not dependency in self._lookup_dict:
 136       return support_binaries.FindLocallyBuiltPath(dependency)
 137     dependency_info = self._GetDependencyInfo(
 138         self._lookup_dict, dependency, platform)
 139     local_path = self._LocalPath(dependency_info)
 140     if not local_path or not os.path.exists(local_path):
 141       raise exceptions.NoPathFoundError(dependency, platform)
 142     return local_path
 143
 144   def _UpdateDependencies(self, config):
 145     """Add the dependency information stored in |config| to this instance.
 146
 147     Args:
 148         config: An instances of BaseConfig or a subclasses.
 149
 150     Raises:
 151         UnsupportedConfigFormatError: If supported_config_types was specified
 152         and config is not in the supported config_types.
 153     """
 154     if not isinstance(config, base_config.BaseConfig):
 155       raise ValueError('Must use a BaseConfig or subclass instance with the '
 156                        'DependencyManager.')
 157     if (self.supported_configs and
 158         config.GetConfigType() not in self.supported_configs):
 159       raise exceptions.UnsupportedConfigFormatError(config.GetConfigType(),
 160                                                     config.config_path)
 161     for dep_info in config.IterDependencyInfo():
 162       dependency = dep_info.dependency
 163       platform = dep_info.platform
 164       if dependency not in self._lookup_dict:
 165         self._lookup_dict[dependency] = {}
 166       if platform not in self._lookup_dict[dependency]:
 167         self._lookup_dict[dependency][platform] = dep_info
 168       else:
 169         self._lookup_dict[dependency][platform].Update(dep_info)
 170
 171
 172   def _GetDependencyInfo(self, dependency, platform):
 173     """Get information for |dependency| on |platform|, or a default if needed.
 174
 175     Args:
 176         dependency: Name of the desired dependency, as given in the config(s)
 177             used in this DependencyManager.
 178         platform: Name of the platform the dependency will run on. Often of the
 179             form 'os_architecture'. Must match those specified in the config(s)
 180             used in this DependencyManager.
 181
 182     Returns: The dependency_info for |dependency| on |platform| if it exists.
 183         Or the default version of |dependency| if it exists, or None if neither
 184         exist.
 185     """
 186     if not self._lookup_dict or dependency not in self._lookup_dict:
 187       return None
 188     dependency_dict = self._lookup_dict[dependency]
 189     device_type = platform
 190     if not device_type in dependency_dict:
 191       device_type = DEFAULT_TYPE
 192     return dependency_dict.get(device_type)
 193
 194   @staticmethod
 195   def _LocalPath(dependency_info):
 196     """Return a path to a locally stored file for |dependency_info|.
 197
 198     Will not download the file.
 199
 200     Args:
 201         dependency_info: A DependencyInfo instance for the dependency to be
 202             found and the platform it should run on.
 203
 204     Returns: A path to a local file, or None if not found.
 205     """
 206     if dependency_info:
 207       paths = dependency_info.local_paths
 208       for local_path in paths:
 209         if os.path.exists(local_path):
 210           return local_path
 211     return None
 212
 213   @staticmethod
 214   def _CloudStoragePath(dependency_info):
 215     """Return a path to a downloaded file for |dependency_info|.
 216
 217     May not download the file if it has already been downloaded.
 218
 219     Args:
 220         dependency_info: A DependencyInfo instance for the dependency to be
 221             found and the platform it should run on.
 222
 223     Returns: A path to an executable that was stored in cloud_storage, or None
 224        if not found.
 225
 226     Raises:
 227         CredentialsError: If cloud_storage credentials aren't configured.
 228         PermissionError: If cloud_storage credentials are configured, but not
 229             with an account that has permission to download the needed file.
 230         NotFoundError: If the needed file does not exist where expected in
 231             cloud_storage.
 232         ServerError: If an internal server error is hit while downloading the
 233             needed file.
 234         CloudStorageError: If another error occured while downloading the remote
 235             path.
 236         FileNotFoundError: If the download was otherwise unsuccessful.
 237     """
 238     if not dependency_info:
 239       return None
 240     cs_path = dependency_info.cs_remote_path
 241     cs_hash = dependency_info.cs_hash
 242     cs_bucket = dependency_info.cs_bucket
 243     download_path = dependency_info.download_path
 244     if not cs_path or not cs_bucket or not cs_hash or not download_path:
 245       return None
 246
 247     download_dir = os.path.dirname(download_path)
 248     if not os.path.exists(download_dir):
 249       os.makedirs(download_dir)
 250
 251     cloud_storage.GetIfHashChanged(cs_path, download_path, cs_bucket, cs_hash)
 252     if not os.path.exists(download_path):
 253       raise exceptions.FileNotFoundError(download_path)
 254     #TODO(aiolos): Add support for unzipping files.
 255     os.chmod(download_path,
 256              stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP)
 257     return os.path.abspath(download_path)
 258