tools/telemetry/catapult_base/dependency_manager/base_config.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import json
   6 import os
   7
   8 from catapult_base.dependency_manager import dependency_info
   9 from catapult_base.dependency_manager import exceptions
  10
  11
  12 class BaseConfig(object):
  13   """A basic config class for use with the DependencyManager.
  14
  15   Initiated with a json file in the following format:
  16
  17             {  "config_type": "BaseConfig",
  18                "dependencies": {
  19                  "dep_name1": {
  20                    "cloud_storage_base_folder": "base_folder1",
  21                    "cloud_storage_bucket": "bucket1",
  22                    "file_info": {
  23                      "platform1": {
  24                         "cloud_storage_hash": "hash_for_platform1",
  25                         "download_path": "download_path111",
  26                         "version_in_cs": "1.11.1.11."
  27                         "local_paths": ["local_path1110", "local_path1111"]
  28                       },
  29                       "platform2": {
  30                         "cloud_storage_hash": "hash_for_platform2",
  31                         "download_path": "download_path2",
  32                         "local_paths": ["local_path20", "local_path21"]
  33                       },
  34                       ...
  35                    }
  36                  },
  37                  "dependency_name_2": {
  38                     ...
  39                  },
  40                   ...
  41               }
  42             }
  43
  44     Required fields: "dependencies" and "config_type".
  45                      Note that config_type must be "BaseConfig"
  46
  47     Assumptions:
  48         "cloud_storage_base_folder" is a top level folder in the given
  49           "cloud_storage_bucket" where all of the dependency files are stored
  50           at "dependency_name"_"cloud_storage_hash".
  51
  52         "download_path" and all paths in "local_paths" are relative to the
  53           config file's location.
  54
  55         All or none of the following cloud storage related fields must be
  56           included in each platform dictionary:
  57           "cloud_storage_hash", "download_path", "cs_remote_path"
  58
  59         "version_in_cs" is an optional cloud storage field, but is dependent
  60           on the above cloud storage related fields.
  61
  62
  63     Also note that platform names are often of the form os_architechture.
  64     Ex: "win_AMD64"
  65
  66     More information on the fields can be found in dependencies_info.py
  67   """
  68   def __init__(self, file_path, writable=False):
  69     """ Initialize a BaseConfig for the DependencyManager.
  70
  71     Args:
  72         writable: False: This config will be used to lookup information.
  73                   True: This config will be used to update information.
  74
  75         file_path: Path to a file containing a json dictionary in the expected
  76                    json format for this config class. Base format expected:
  77
  78                    { "config_type": config_type,
  79                      "dependencies": dependencies_dict }
  80
  81                    config_type: must match the return value of GetConfigType.
  82                    dependencies: A dictionary with the information needed to
  83                        create dependency_info instances for the given
  84                        dependencies.
  85
  86                    See dependency_info.py for more information.
  87     """
  88     self._config_path = file_path
  89     self._writable = writable
  90     if not file_path:
  91       raise ValueError('Must supply config file path.')
  92     if not os.path.exists(file_path):
  93       if not writable:
  94         raise exceptions.EmptyConfigError(file_path)
  95       self._config_data = {}
  96       self.CreateEmptyConfig(file_path)
  97     else:
  98       with open(file_path, 'r') as f:
  99         config_data = json.load(f)
 100       if not config_data:
 101         raise exceptions.EmptyConfigError(file_path)
 102       config_type = config_data.pop('config_type', None)
 103       if config_type != self.GetConfigType():
 104         raise ValueError(
 105             'Supplied config_type (%s) is not the expected type (%s) in file '
 106             '%s' % (config_type, self.GetConfigType(), file_path))
 107       self._config_data = config_data.get('dependencies', {})
 108
 109   def IterDependencyInfo(self):
 110     """ Yields a DependencyInfo for each dependency/platform pair.
 111
 112     Raises:
 113         ReadWriteError: If called when the config is writable.
 114         ValueError: If any of the dependencies contain partial information for
 115             downloading from cloud_storage. (See dependency_info.py)
 116     """
 117     if self._writable:
 118       raise exceptions.ReadWriteError(
 119           'Trying to read dependency info from a  writable config. File for '
 120           'config: %s' % self._config_path)
 121     for dep in self._config_data:
 122
 123       base_path = os.path.dirname(self._config_path)
 124       dependency_dict = self._config_data.get(dep, {})
 125       platforms_dict = dependency_dict.get('file_info')
 126       cs_bucket = dependency_dict.get('cloud_storage_bucket', None)
 127       cs_base_folder = dependency_dict.get('cloud_storage_base_folder', '')
 128       for platform in platforms_dict:
 129         platform_info = platforms_dict.get(platform)
 130         local_paths = platform_info.get('local_paths', [])
 131         if local_paths:
 132           paths = []
 133           for path in local_paths:
 134             path = self._FormatPath(path)
 135             paths.append(os.path.abspath(os.path.join(base_path, path)))
 136           local_paths = paths
 137
 138         download_path = platform_info.get('download_path', None)
 139         if download_path:
 140           download_path = self._FormatPath(download_path)
 141           download_path = os.path.abspath(
 142               os.path.join(base_path, download_path))
 143
 144         cs_remote_path = None
 145         cs_hash = platform_info.get('cloud_storage_hash', None)
 146         if cs_hash:
 147           cs_remote_file = '%s_%s' % (dep, cs_hash)
 148           cs_remote_path = cs_remote_file if not cs_base_folder else (
 149               '%s/%s' % (cs_base_folder, cs_remote_file))
 150
 151         version_in_cs = platform_info.get('version_in_cs', None)
 152
 153         if download_path or cs_remote_path or cs_hash or version_in_cs:
 154           dep_info = dependency_info.DependencyInfo(
 155               dep, platform, self._config_path, cs_bucket=cs_bucket,
 156               cs_remote_path=cs_remote_path, download_path=download_path,
 157               cs_hash=cs_hash, version_in_cs=version_in_cs,
 158               local_paths=local_paths)
 159         else:
 160           dep_info = dependency_info.DependencyInfo(
 161               dep, platform, self._config_path, local_paths=local_paths)
 162         yield dep_info
 163
 164   @classmethod
 165   def CreateEmptyConfig(cls, file_path):
 166     """Create an empty BaseConfig json dict and write it out to |file_path|.
 167
 168     Raises:
 169         ValueError: If the path already exists.
 170     """
 171     if os.path.exists(file_path):
 172       raise ValueError('File already exists, and would be overwritten.')
 173     json_dict = {'config_type': cls.GetConfigType(),
 174                  'dependencies': {}}
 175     with open(file_path, 'w') as outfile:
 176       json.dump(json_dict, outfile, indent=2, sort_keys=True)
 177     return json_dict
 178
 179   @classmethod
 180   def GetConfigType(cls):
 181     return 'BaseConfig'
 182
 183   @property
 184   def config_path(self):
 185     return self._config_path
 186
 187   def UpdateCloudStorageDependency(
 188       self, dependency, platform, dependency_path, version=None):
 189     """Update the cloud storage hash and the version for the given dependency.
 190     """
 191     # TODO(aiolos): Only allow the config to be updated if writable is True to
 192     # avoid data changing underneath the dependency manager.
 193     raise NotImplementedError
 194
 195   def GetVersion(self, dependency, platform):
 196     """Return the Version information for the given dependency."""
 197     if not self._config_data(dependency):
 198       raise ValueError('Dependency %s is not in config.' % dependency)
 199     if not self.config_data[dependency].get(platform):
 200       raise ValueError('Dependency %s has no information for platform %s in '
 201                        'this config.' % (dependency, platform))
 202     return self._config_data[dependency][platform].get('version_in_cs')
 203
 204   @classmethod
 205   def _FormatPath(cls, file_path):
 206     """Format |file_path| for the current file system.
 207
 208     We may be downloading files for another platform, so paths must be
 209     downloadable on the current system.
 210     """
 211     if not file_path:
 212       return file_path
 213     if os.path.sep != '\\':
 214       return file_path.replace('\\', os.path.sep)
 215     elif os.path.sep != '/':
 216       return file_path.replace('/', os.path.sep)
 217     return file_path
 218