tools/telemetry/third_party/gsutilz/gslib/cloud_api_delegator.py

   1 # -*- coding: utf-8 -*-
   2 # Copyright 2013 Google Inc. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """Gsutil API delegator for interacting with cloud storage providers."""
  16
  17 from __future__ import absolute_import
  18
  19 import boto
  20 from boto import config
  21 from gslib.cloud_api import ArgumentException
  22 from gslib.cloud_api import CloudApi
  23 from gslib.cs_api_map import ApiMapConstants
  24 from gslib.cs_api_map import ApiSelector
  25
  26
  27 class CloudApiDelegator(CloudApi):
  28   """Class that handles delegating requests to gsutil Cloud API implementations.
  29
  30   This class is responsible for determining at runtime which gsutil Cloud API
  31   implementation should service the request based on the Cloud storage provider,
  32   command-level API support, and configuration file override.
  33
  34   During initialization it takes as an argument a gsutil_api_map which maps
  35   providers to their default and supported gsutil Cloud API implementations
  36   (see comments in cs_api_map for details).
  37
  38   Instantiation of multiple delegators per-thread is required for multiprocess
  39   and/or multithreaded operations. Calling methods on the same delegator in
  40   multiple threads is unsafe.
  41   """
  42
  43   def __init__(self, bucket_storage_uri_class, gsutil_api_map, logger,
  44                provider=None, debug=0):
  45     """Performs necessary setup for delegating cloud storage requests.
  46
  47     This function has different arguments than the gsutil Cloud API __init__
  48     function because of the delegation responsibilties of this class.
  49
  50     Args:
  51       bucket_storage_uri_class: boto storage_uri class, used by APIs that
  52                                 provide boto translation or mocking.
  53       gsutil_api_map: Map of providers and API selector tuples to api classes
  54                       which can be used to communicate with those providers.
  55       logger: logging.logger for outputting log messages.
  56       provider: Default provider prefix describing cloud storage provider to
  57                 connect to.
  58       debug: Debug level for the API implementation (0..3).
  59     """
  60     super(CloudApiDelegator, self).__init__(bucket_storage_uri_class, logger,
  61                                             provider=provider, debug=debug)
  62     self.api_map = gsutil_api_map
  63     self.prefer_api = boto.config.get('GSUtil', 'prefer_api', '').upper()
  64     self.loaded_apis = {}
  65
  66     if not self.api_map[ApiMapConstants.API_MAP]:
  67       raise ArgumentException('No apiclass supplied for gsutil Cloud API map.')
  68
  69   def _GetApi(self, provider):
  70     """Returns a valid CloudApi for use by the caller.
  71
  72     This function lazy-loads connection and credentials using the API map
  73     and credential store provided during class initialization.
  74
  75     Args:
  76       provider: Provider to load API for. If None, class-wide default is used.
  77
  78     Raises:
  79       ArgumentException if there is no matching API available in the API map.
  80
  81     Returns:
  82       Valid API instance that can be used to communicate with the Cloud
  83       Storage provider.
  84     """
  85     provider = provider or self.provider
  86     if not provider:
  87       raise ArgumentException('No provider selected for _GetApi')
  88
  89     provider = str(provider)
  90     if provider not in self.loaded_apis:
  91       self.loaded_apis[provider] = {}
  92
  93     api_selector = self.GetApiSelector(provider)
  94     if api_selector not in self.loaded_apis[provider]:
  95       # Need to load the API.
  96       self._LoadApi(provider, api_selector)
  97
  98     return self.loaded_apis[provider][api_selector]
  99
 100   def _LoadApi(self, provider, api_selector):
 101     """Loads a CloudApi into the loaded_apis map for this class.
 102
 103     Args:
 104       provider: Provider to load the API for.
 105       api_selector: cs_api_map.ApiSelector defining the API type.
 106     """
 107     if provider not in self.api_map[ApiMapConstants.API_MAP]:
 108       raise ArgumentException(
 109           'gsutil Cloud API map contains no entry for provider %s.' % provider)
 110     if api_selector not in self.api_map[ApiMapConstants.API_MAP][provider]:
 111       raise ArgumentException(
 112           'gsutil Cloud API map does not support API %s for provider %s.' %
 113           (api_selector, provider))
 114     self.loaded_apis[provider][api_selector] = (
 115         self.api_map[ApiMapConstants.API_MAP][provider][api_selector](
 116             self.bucket_storage_uri_class,
 117             self.logger,
 118             provider=provider,
 119             debug=self.debug))
 120
 121   def GetApiSelector(self, provider=None):
 122     """Returns a cs_api_map.ApiSelector based on input and configuration.
 123
 124     Args:
 125       provider: Provider to return the ApiSelector for.  If None, class-wide
 126                 default is used.
 127
 128     Returns:
 129       cs_api_map.ApiSelector that will be used for calls to the delegator
 130       for this provider.
 131     """
 132     selected_provider = provider or self.provider
 133     if not selected_provider:
 134       raise ArgumentException('No provider selected for CloudApi')
 135
 136     if (selected_provider not in self.api_map[ApiMapConstants.DEFAULT_MAP] or
 137         self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider] not in
 138         self.api_map[ApiMapConstants.API_MAP][selected_provider]):
 139       raise ArgumentException('No default api available for provider %s' %
 140                               selected_provider)
 141
 142     if selected_provider not in self.api_map[ApiMapConstants.SUPPORT_MAP]:
 143       raise ArgumentException('No supported apis available for provider %s' %
 144                               selected_provider)
 145
 146     api = self.api_map[ApiMapConstants.DEFAULT_MAP][selected_provider]
 147
 148     # If we have only HMAC credentials for Google Cloud Storage, we must use
 149     # the XML API as the JSON API does not support HMAC.
 150     #
 151     # Technically if we have only HMAC credentials, we should still be able to
 152     # access public read resources via the JSON API, but the XML API can do
 153     # that just as well. It is better to use it than inspect the credentials on
 154     # every HTTP call.
 155     if (provider == 'gs' and
 156         not config.has_option('Credentials', 'gs_oauth2_refresh_token') and
 157         not (config.has_option('Credentials', 'gs_service_client_id')
 158              and config.has_option('Credentials', 'gs_service_key_file')) and
 159         (config.has_option('Credentials', 'gs_access_key_id')
 160          and config.has_option('Credentials', 'gs_secret_access_key'))):
 161       api = ApiSelector.XML
 162     # Try to force the user's preference to a supported API.
 163     elif self.prefer_api in (self.api_map[ApiMapConstants.SUPPORT_MAP]
 164                              [selected_provider]):
 165       api = self.prefer_api
 166     return api
 167
 168   # For function docstrings, see CloudApi class.
 169   def GetBucket(self, bucket_name, provider=None, fields=None):
 170     return self._GetApi(provider).GetBucket(bucket_name, fields=fields)
 171
 172   def ListBuckets(self, project_id=None, provider=None, fields=None):
 173     return self._GetApi(provider).ListBuckets(project_id=project_id,
 174                                               fields=fields)
 175
 176   def PatchBucket(self, bucket_name, metadata, canned_acl=None,
 177                   canned_def_acl=None, preconditions=None, provider=None,
 178                   fields=None):
 179     return self._GetApi(provider).PatchBucket(
 180         bucket_name, metadata, canned_acl=canned_acl,
 181         canned_def_acl=canned_def_acl, preconditions=preconditions,
 182         fields=fields)
 183
 184   def CreateBucket(self, bucket_name, project_id=None, metadata=None,
 185                    provider=None, fields=None):
 186     return self._GetApi(provider).CreateBucket(
 187         bucket_name, project_id=project_id, metadata=metadata, fields=fields)
 188
 189   def DeleteBucket(self, bucket_name, preconditions=None, provider=None):
 190     return self._GetApi(provider).DeleteBucket(bucket_name,
 191                                                preconditions=preconditions)
 192
 193   def ListObjects(self, bucket_name, prefix=None, delimiter=None,
 194                   all_versions=None, provider=None, fields=None):
 195     return self._GetApi(provider).ListObjects(
 196         bucket_name, prefix=prefix, delimiter=delimiter,
 197         all_versions=all_versions, fields=fields)
 198
 199   def GetObjectMetadata(self, bucket_name, object_name, generation=None,
 200                         provider=None, fields=None):
 201     return self._GetApi(provider).GetObjectMetadata(
 202         bucket_name, object_name, generation=generation, fields=fields)
 203
 204   def PatchObjectMetadata(self, bucket_name, object_name, metadata,
 205                           canned_acl=None, generation=None, preconditions=None,
 206                           provider=None, fields=None):
 207     return self._GetApi(provider).PatchObjectMetadata(
 208         bucket_name, object_name, metadata, canned_acl=canned_acl,
 209         generation=generation, preconditions=preconditions, fields=fields)
 210
 211   def GetObjectMedia(
 212       self, bucket_name, object_name, download_stream, provider=None,
 213       generation=None, object_size=None,
 214       download_strategy=CloudApi.DownloadStrategy.ONE_SHOT,
 215       start_byte=0, end_byte=None, progress_callback=None,
 216       serialization_data=None, digesters=None):
 217     return self._GetApi(provider).GetObjectMedia(
 218         bucket_name, object_name, download_stream,
 219         download_strategy=download_strategy, start_byte=start_byte,
 220         end_byte=end_byte, generation=generation, object_size=object_size,
 221         progress_callback=progress_callback,
 222         serialization_data=serialization_data, digesters=digesters)
 223
 224   def UploadObject(self, upload_stream, object_metadata, size=None,
 225                    canned_acl=None, preconditions=None, progress_callback=None,
 226                    provider=None, fields=None):
 227     return self._GetApi(provider).UploadObject(
 228         upload_stream, object_metadata, size=size, canned_acl=canned_acl,
 229         preconditions=preconditions, progress_callback=progress_callback,
 230         fields=fields)
 231
 232   def UploadObjectStreaming(self, upload_stream, object_metadata,
 233                             canned_acl=None, preconditions=None,
 234                             progress_callback=None, provider=None, fields=None):
 235     return self._GetApi(provider).UploadObjectStreaming(
 236         upload_stream, object_metadata, canned_acl=canned_acl,
 237         preconditions=preconditions, progress_callback=progress_callback,
 238         fields=fields)
 239
 240   def UploadObjectResumable(
 241       self, upload_stream, object_metadata, canned_acl=None, preconditions=None,
 242       provider=None, fields=None, size=None, serialization_data=None,
 243       tracker_callback=None, progress_callback=None):
 244     return self._GetApi(provider).UploadObjectResumable(
 245         upload_stream, object_metadata, canned_acl=canned_acl,
 246         preconditions=preconditions, size=size, fields=fields,
 247         serialization_data=serialization_data,
 248         tracker_callback=tracker_callback, progress_callback=progress_callback)
 249
 250   def CopyObject(self, src_obj_metadata, dst_obj_metadata, src_generation=None,
 251                  canned_acl=None, preconditions=None, progress_callback=None,
 252                  max_bytes_per_call=None, provider=None, fields=None):
 253     return self._GetApi(provider).CopyObject(
 254         src_obj_metadata, dst_obj_metadata, src_generation=src_generation,
 255         canned_acl=canned_acl, preconditions=preconditions,
 256         progress_callback=progress_callback,
 257         max_bytes_per_call=max_bytes_per_call, fields=fields)
 258
 259   def ComposeObject(self, src_objs_metadata, dst_obj_metadata,
 260                     preconditions=None, provider=None, fields=None):
 261     return self._GetApi(provider).ComposeObject(
 262         src_objs_metadata, dst_obj_metadata, preconditions=preconditions,
 263         fields=fields)
 264
 265   def DeleteObject(self, bucket_name, object_name, preconditions=None,
 266                    generation=None, provider=None):
 267     return self._GetApi(provider).DeleteObject(
 268         bucket_name, object_name, preconditions=preconditions,
 269         generation=generation)
 270
 271   def WatchBucket(self, bucket_name, address, channel_id, token=None,
 272                   provider=None, fields=None):
 273     return self._GetApi(provider).WatchBucket(
 274         bucket_name, address, channel_id, token=token, fields=fields)
 275
 276   def StopChannel(self, channel_id, resource_id, provider=None):
 277     return self._GetApi(provider).StopChannel(channel_id, resource_id)
 278
 279   def XmlPassThroughGetAcl(self, storage_url, def_obj_acl=False, provider=None):
 280     """XML compatibility function for getting ACLs.
 281
 282     Args:
 283       storage_url: StorageUrl object.
 284       def_obj_acl: If true, get the default object ACL on a bucket.
 285       provider: Cloud storage provider to connect to.  If not present,
 286                 class-wide default is used.
 287
 288     Raises:
 289       ArgumentException for errors during input validation.
 290       ServiceException for errors interacting with cloud storage providers.
 291
 292     Returns:
 293       ACL XML for the resource specified by storage_url.
 294     """
 295     return self._GetApi(provider).XmlPassThroughGetAcl(storage_url,
 296                                                        def_obj_acl=def_obj_acl)
 297
 298   def XmlPassThroughSetAcl(self, acl_text, storage_url, canned=True,
 299                            def_obj_acl=False, provider=None):
 300     """XML compatibility function for setting ACLs.
 301
 302     Args:
 303       acl_text: XML ACL or canned ACL string.
 304       storage_url: StorageUrl object.
 305       canned: If true, acl_text is treated as a canned ACL string.
 306       def_obj_acl: If true, set the default object ACL on a bucket.
 307       provider: Cloud storage provider to connect to.  If not present,
 308                 class-wide default is used.
 309
 310     Raises:
 311       ArgumentException for errors during input validation.
 312       ServiceException for errors interacting with cloud storage providers.
 313
 314     Returns:
 315       None.
 316     """
 317     self._GetApi(provider).XmlPassThroughSetAcl(
 318         acl_text, storage_url, canned=canned, def_obj_acl=def_obj_acl)
 319
 320   def XmlPassThroughGetCors(self, storage_url, provider=None):
 321     """XML compatibility function for getting CORS configuration on a bucket.
 322
 323     Args:
 324       storage_url: StorageUrl object.
 325       provider: Cloud storage provider to connect to.  If not present,
 326                 class-wide default is used.
 327
 328     Raises:
 329       ArgumentException for errors during input validation.
 330       ServiceException for errors interacting with cloud storage providers.
 331
 332     Returns:
 333       CORS configuration XML for the bucket specified by storage_url.
 334     """
 335     return self._GetApi(provider).XmlPassThroughGetCors(storage_url)
 336
 337   def XmlPassThroughSetCors(self, cors_text, storage_url, provider=None):
 338     """XML compatibility function for setting CORS configuration on a bucket.
 339
 340     Args:
 341       cors_text: Raw CORS XML string.
 342       storage_url: StorageUrl object.
 343       provider: Cloud storage provider to connect to.  If not present,
 344                 class-wide default is used.
 345
 346     Raises:
 347       ArgumentException for errors during input validation.
 348       ServiceException for errors interacting with cloud storage providers.
 349
 350     Returns:
 351       None.
 352     """
 353     self._GetApi(provider).XmlPassThroughSetCors(cors_text, storage_url)
 354
 355   def XmlPassThroughGetLifecycle(self, storage_url, provider=None):
 356     """XML compatibility function for getting lifecycle config on a bucket.
 357
 358     Args:
 359       storage_url: StorageUrl object.
 360       provider: Cloud storage provider to connect to.  If not present,
 361                 class-wide default is used.
 362
 363     Raises:
 364       ArgumentException for errors during input validation.
 365       ServiceException for errors interacting with cloud storage providers.
 366
 367     Returns:
 368       Lifecycle configuration XML for the bucket specified by storage_url.
 369     """
 370     return self._GetApi(provider).XmlPassThroughGetLifecycle(storage_url)
 371
 372   def XmlPassThroughSetLifecycle(self, lifecycle_text, storage_url,
 373                                  provider=None):
 374     """XML compatibility function for setting CORS configuration on a bucket.
 375
 376     Args:
 377       lifecycle_text: Raw lifecycle configuration XML string.
 378       storage_url: StorageUrl object.
 379       provider: Cloud storage provider to connect to.  If not present,
 380                 class-wide default is used.
 381
 382     Raises:
 383       ArgumentException for errors during input validation.
 384       ServiceException for errors interacting with cloud storage providers.
 385
 386     Returns:
 387       None.
 388     """
 389     self._GetApi(provider).XmlPassThroughSetLifecycle(lifecycle_text,
 390                                                       storage_url)
 391
 392   def XmlPassThroughGetLogging(self, storage_url, provider=None):
 393     """XML compatibility function for getting logging configuration on a bucket.
 394
 395     Args:
 396       storage_url: StorageUrl object.
 397       provider: Cloud storage provider to connect to.  If not present,
 398                 class-wide default is used.
 399
 400     Raises:
 401       ArgumentException for errors during input validation.
 402       ServiceException for errors interacting with cloud storage providers.
 403
 404     Returns:
 405       Logging configuration XML for the bucket specified by storage_url.
 406     """
 407     return self._GetApi(provider).XmlPassThroughGetLogging(storage_url)
 408
 409   def XmlPassThroughGetWebsite(self, storage_url, provider=None):
 410     """XML compatibility function for getting website configuration on a bucket.
 411
 412     Args:
 413       storage_url: StorageUrl object.
 414       provider: Cloud storage provider to connect to.  If not present,
 415                 class-wide default is used.
 416
 417     Raises:
 418       ArgumentException for errors during input validation.
 419       ServiceException for errors interacting with cloud storage providers.
 420
 421     Returns:
 422       Website configuration XML for the bucket specified by storage_url.
 423     """
 424     return self._GetApi(provider).XmlPassThroughGetWebsite(storage_url)
 425