tools/telemetry/third_party/gsutilz/gslib/tests/mock_cloud_api.py

   1 # -*- coding: utf-8 -*-
   2 # Copyright 2014 Google Inc. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """Implements a simple mock gsutil Cloud API for unit testing.
  16
  17 gsutil 4 was primarily unit-tested using boto/gsutil 3's mock storage_uri class,
  18 since it was possible that changing out the underlying mocks would have had
  19 subtly different behavior and increased the risk of breaking back-compat.
  20
  21 Most unit and integration tests in gsutil 4 still set up the test objects with
  22 storage_uris and boto, and the unit tests interact with test objects via
  23 storage uris and boto.
  24
  25 This testing approach ties our tests heavily to boto; extending the
  26 boto mocks is difficult because it requires checking into boto. This also
  27 makes the unit test coverage boto-specific in several cases.
  28
  29 MockCloudApi was initially written to cover some parallel composite upload
  30 cases that the boto mocks couldn't handle. It is not yet a full implementation.
  31 Eventually, we can move to full a mock Cloud API implementation. However, we
  32 need to ensure we don't lose boto coverage from mock storage_uri.
  33 """
  34
  35
  36 from gslib.cloud_api import ServiceException
  37 from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
  38 from gslib.translation_helper import CreateBucketNotFoundException
  39 from gslib.translation_helper import CreateObjectNotFoundException
  40
  41
  42 class MockObject(object):
  43   """Defines a mock cloud storage provider object."""
  44
  45   def __init__(self, root_object, contents=''):
  46     self.root_object = root_object
  47     self.contents = contents
  48
  49   def __str__(self):
  50     return '%s/%s#%s' % (self.root_object.bucket,
  51                          self.root_object.name,
  52                          self.root_object.generation)
  53
  54   def __repr__(self):
  55     return str(self)
  56
  57
  58 class MockBucket(object):
  59   """Defines a mock cloud storage provider bucket."""
  60
  61   def __init__(self, bucket_name, versioned=False):
  62     self.root_object = apitools_messages.Bucket(
  63         name=bucket_name,
  64         versioning=apitools_messages.Bucket.VersioningValue(enabled=versioned))
  65     # Dict of object_name: (dict of 'live': MockObject
  66     #                               'versioned': ordered list of MockObject).
  67     self.objects = {}
  68
  69   def CreateObject(self, object_name, contents=''):
  70     return self.CreateObjectWithMetadata(MockObject(
  71         apitools_messages.Object(name=object_name, contents=contents)))
  72
  73   def CreateObjectWithMetadata(self, apitools_object, contents=''):
  74     """Creates an object in the bucket according to the input metadata.
  75
  76     This will create a new object version (ignoring the generation specified
  77     in the input object).
  78
  79     Args:
  80       apitools_object: apitools Object.
  81       contents: optional object contents.
  82
  83     Returns:
  84       apitools Object representing created object.
  85     """
  86     # This modifies the apitools_object with a generation number.
  87     object_name = apitools_object.name
  88     if (self.root_object.versioning and self.root_object.versioning.enabled and
  89         apitools_object.name in self.objects):
  90       if 'live' in self.objects[object_name]:
  91         # Versioning enabled and object exists, create an object with a
  92         # generation 1 higher.
  93         apitools_object.generation = (
  94             self.objects[object_name]['live'].root_object.generation + 1)
  95         # Move the live object to versioned.
  96         if 'versioned' not in self.objects[object_name]:
  97           self.objects[object_name]['versioned'] = []
  98         self.objects[object_name]['versioned'].append(
  99             self.objects[object_name]['live'])
 100       elif ('versioned' in self.objects[object_name] and
 101             self.objects[object_name]['versioned']):
 102         # Versioning enabled but only archived objects exist, pick a generation
 103         # higher than the highest versioned object (which will be at the end).
 104         apitools_object.generation = (
 105             self.objects[object_name]['versioned'][-1].root_object.generation
 106             + 1)
 107     else:
 108       # Versioning disabled or no objects exist yet with this name.
 109       apitools_object.generation = 1
 110       self.objects[object_name] = {}
 111     new_object = MockObject(apitools_object, contents=contents)
 112     self.objects[object_name]['live'] = new_object
 113     return new_object
 114
 115
 116 class MockCloudApi(object):
 117   """Simple mock service for buckets/objects that implements Cloud API.
 118
 119   Also includes some setup functions for tests.
 120   """
 121
 122   def __init__(self, provider='gs'):
 123     self.buckets = {}
 124     self.provider = provider
 125
 126   def MockCreateBucket(self, bucket_name):
 127     """Creates a simple bucket without exercising the API directly."""
 128     if bucket_name in self.buckets:
 129       raise ServiceException('Bucket %s already exists.' % bucket_name,
 130                              status=409)
 131     self.buckets[bucket_name] = MockBucket(bucket_name)
 132
 133   def MockCreateVersionedBucket(self, bucket_name):
 134     """Creates a simple bucket without exercising the API directly."""
 135     if bucket_name in self.buckets:
 136       raise ServiceException('Bucket %s already exists.' % bucket_name,
 137                              status=409)
 138     self.buckets[bucket_name] = MockBucket(bucket_name, versioned=True)
 139
 140   def MockCreateObject(self, bucket_name, object_name, contents=''):
 141     """Creates an object without exercising the API directly."""
 142     if bucket_name not in self.buckets:
 143       self.MockCreateBucket(bucket_name)
 144     self.buckets[bucket_name].CreateObject(object_name, contents=contents)
 145
 146   def MockCreateObjectWithMetadata(self, apitools_object, contents=''):
 147     """Creates an object without exercising the API directly."""
 148     assert apitools_object.bucket, 'No bucket specified for mock object'
 149     assert apitools_object.name, 'No object name specified for mock object'
 150     if apitools_object.bucket not in self.buckets:
 151       self.MockCreateBucket(apitools_object.bucket)
 152     return self.buckets[apitools_object.bucket].CreateObjectWithMetadata(
 153         apitools_object, contents=contents).root_object
 154
 155   # pylint: disable=unused-argument
 156   def GetObjectMetadata(self, bucket_name, object_name, generation=None,
 157                         provider=None, fields=None):
 158     """See CloudApi class for function doc strings."""
 159     if generation:
 160       generation = long(generation)
 161     if bucket_name in self.buckets:
 162       bucket = self.buckets[bucket_name]
 163       if object_name in bucket.objects and bucket.objects[object_name]:
 164         if generation:
 165           if 'versioned' in bucket.objects[object_name]:
 166             for obj in bucket.objects[object_name]['versioned']:
 167               if obj.root_object.generation == generation:
 168                 return obj.root_object
 169           if 'live' in bucket.objects[object_name]:
 170             if (bucket.objects[object_name]['live'].root_object.generation ==
 171                 generation):
 172               return bucket.objects[object_name]['live'].root_object
 173         else:
 174           # Return live object.
 175           if 'live' in bucket.objects[object_name]:
 176             return bucket.objects[object_name]['live'].root_object
 177       raise CreateObjectNotFoundException(404, self.provider, bucket_name,
 178                                           object_name)
 179     raise CreateBucketNotFoundException(404, self.provider, bucket_name)