tools/telemetry/catapult_base/dependency_manager/base_config.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import json
   6 import os
   7
   8 from catapult_base.dependency_manager import dependency_info
   9 from catapult_base.dependency_manager import exceptions
  10
  11
  12 class BaseConfig(object):
  13   """A basic config class for use with the DependencyManager.
  14
  15   Initiated with a json file in the following format:
  16
  17             {  "config_type": "BaseConfig",
  18                "dependencies": {
  19                  "dep_name1": {
  20                    "cloud_storage_base_folder": "base_folder1",
  21                    "cloud_storage_bucket": "bucket1",
  22                    "file_info": {
  23                      "platform1": {
  24                         "cloud_storage_hash": "hash_for_platform1",
  25                         "download_path": "download_path111",
  26                         "cs_remote_path": "cs_path111",
  27                         "local_paths": ["local_path1110", "local_path1111"]
  28                       },
  29                       "platform2": {
  30                         "cloud_storage_hash": "hash_for_platform2",
  31                         "download_path": "download_path2",
  32                         "cs_remote_path": "cs_path2",
  33                         "local_paths": ["local_path20", "local_path21"]
  34                       },
  35                       ...
  36                    }
  37                  },
  38                  "dependency_name_2": {
  39                     ...
  40                  },
  41                   ...
  42               }
  43             }
  44
  45     Required feilds: "dependencies" and "config_type".
  46                      Note that config_type must be "BaseConfig"
  47     Assumptions:
  48         |cloud_storage_base_folder| is a top level folder in the given
  49           |cloud_storage_bucket| where all of the dependency files are stored
  50           at |dependency_name|_|cloud_storage_hash|.
  51
  52         |download_path| and all paths in |local_paths| are relative to the
  53           config file's location.
  54
  55     Also note that platform names are often of the form os_architechture.
  56     Ex: "win_AMD64"
  57
  58     More information on the fields can be found in dependencies_info.py
  59   """
  60   def __init__(self, file_path, writable=False):
  61     """ Initialize a BaseConfig for the DependencyManager.
  62
  63     Args:
  64         writable: False: This config will be used to lookup information.
  65                   True: This config will be used to update information.
  66
  67         file_path: Path to a file containing a json dictionary in the expected
  68                    json format for this config class. Base format expected:
  69
  70                    { "config_type": config_type,
  71                      "dependencies": dependencies_dict }
  72
  73                    config_type: must match the return value of GetConfigType.
  74                    dependencies: A dictionary with the information needed to
  75                        create dependency_info instances for the given
  76                        dependencies.
  77
  78                    See dependency_info.py for more information.
  79     """
  80     self._config_path = file_path
  81     self._writable = writable
  82     if not file_path:
  83       raise ValueError('Must supply config file path.')
  84     if not os.path.exists(file_path):
  85       if not writable:
  86         raise exceptions.EmptyConfigError(file_path)
  87       self._config_data = {}
  88       self.CreateEmptyConfig(file_path)
  89     else:
  90       with open(file_path, 'r') as f:
  91         config_data = json.load(f)
  92       if not config_data:
  93         raise exceptions.EmptyConfigError(file_path)
  94       config_type = config_data.pop('config_type', None)
  95       if config_type != self.GetConfigType():
  96         raise ValueError(
  97             'Supplied config_type (%s) is not the expected type (%s) in file '
  98             '%s' % (config_type, self.GetConfigType(), file_path))
  99       self._config_data = config_data.get('dependencies', {})
 100
 101   def IterDependencyInfo(self):
 102     """ Yields a DependencyInfo for each dependency/platform pair.
 103
 104     Raises:
 105         ReadWriteError: If called when the config is writable.
 106         ValueError: If any of the dependencies contain partial information for
 107             downloading from cloud_storage. (See dependency_info.py)
 108     """
 109     if self._writable:
 110       raise exceptions.ReadWriteError(
 111           'Trying to read dependency info from a  writable config. File for '
 112           'config: %s' % self._config_path)
 113     for dep in self._config_data:
 114
 115       base_path = os.path.dirname(self._config_path)
 116       dependency_dict = self._config_data.get(dep, {})
 117       platforms_dict = dependency_dict.get('file_info')
 118       cs_bucket = dependency_dict.get('cloud_storage_bucket', None)
 119       cs_base_folder = dependency_dict.get('cloud_storage_base_folder', '')
 120       for platform in platforms_dict:
 121         platform_info = platforms_dict.get(platform)
 122         local_paths = platform_info.get('local_paths', [])
 123         if local_paths:
 124           paths = []
 125           for path in local_paths:
 126             path = self._FormatPath(path)
 127             paths.append(os.path.abspath(os.path.join(base_path, path)))
 128           local_paths = paths
 129
 130         download_path = platform_info.get('download_path', None)
 131         if download_path:
 132           download_path = self._FormatPath(download_path)
 133           download_path = os.path.abspath(
 134               os.path.join(base_path, download_path))
 135
 136         cs_remote_path = None
 137         cs_hash = platform_info.get('cloud_storage_hash', None)
 138         if cs_hash:
 139           cs_remote_file = '%s_%s' % (dep, cs_hash)
 140           cs_remote_path = cs_remote_file if not cs_base_folder else (
 141               '%s/%s' % (cs_base_folder, cs_remote_file))
 142
 143         if download_path or cs_remote_path or cs_hash:
 144           dep_info = dependency_info.DependencyInfo(
 145               dep, platform, self._config_path, cs_bucket=cs_bucket,
 146               cs_remote_path=cs_remote_path, download_path=download_path,
 147               cs_hash=cs_hash, local_paths=local_paths)
 148         else:
 149           dep_info = dependency_info.DependencyInfo(
 150               dep, platform, self._config_path, local_paths=local_paths)
 151         yield dep_info
 152
 153   @classmethod
 154   def CreateEmptyConfig(cls, file_path):
 155     """Create an empty BaseConfig json dict and write it out to |file_path|.
 156
 157     Raises:
 158         ValueError: If the path already exists.
 159     """
 160     if os.path.exists(file_path):
 161       raise ValueError('File already exists, and would be overwritten.')
 162     json_dict = {'config_type': cls.GetConfigType(),
 163                  'dependencies': {}}
 164     with open(file_path, 'w') as outfile:
 165       json.dump(json_dict, outfile, indent=2, sort_keys=True)
 166     return json_dict
 167
 168   @classmethod
 169   def GetConfigType(cls):
 170     return 'BaseConfig'
 171
 172   @property
 173   def config_path(self):
 174     return self._config_path
 175
 176   def UpdateCloudStorageDependency(
 177       self, dependency, platform, dependency_path, version=None):
 178     """Update the cloud storage hash and the version for the given dependency.
 179     """
 180     # TODO(aiolos): Only allow the config to be updated if writable is True to
 181     # avoid data changing underneath the dependency manager.
 182     raise NotImplementedError
 183
 184   def GetVersion(self, dependency, platform):
 185     """Return the Version information for the given dependency."""
 186     raise NotImplementedError
 187
 188   @classmethod
 189   def _FormatPath(cls, file_path):
 190     """Format |file_path| for the current file system.
 191
 192     We may be downloading files for another platform, so paths must be
 193     downloadable on the current system.
 194     """
 195     if not file_path:
 196       return file_path
 197     if os.path.sep != '\\':
 198       return file_path.replace('\\', os.path.sep)
 199     elif os.path.sep != '/':
 200       return file_path.replace('/', os.path.sep)
 201     return file_path
 202