tools/telemetry/third_party/gsutilz/gslib/tests/test_lifecycle.py

   1 # -*- coding: utf-8 -*-
   2 # Copyright 2013 Google Inc. All Rights Reserved.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #     http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 """Integration tests for lifecycle command."""
  16
  17 from __future__ import absolute_import
  18
  19 import json
  20 import posixpath
  21 from xml.dom.minidom import parseString
  22
  23 from gslib.cs_api_map import ApiSelector
  24 import gslib.tests.testcase as testcase
  25 from gslib.tests.testcase.integration_testcase import SkipForS3
  26 from gslib.tests.util import ObjectToURI as suri
  27 from gslib.tests.util import unittest
  28 from gslib.translation_helper import LifecycleTranslation
  29 from gslib.util import Retry
  30
  31
  32 @SkipForS3('Lifecycle command is only supported for gs:// URLs')
  33 class TestSetLifecycle(testcase.GsUtilIntegrationTestCase):
  34   """Integration tests for lifecycle command."""
  35
  36   empty_doc1 = '{}'
  37
  38   xml_doc = parseString(
  39       '<LifecycleConfiguration><Rule>'
  40       '<Action><Delete/></Action>'
  41       '<Condition><Age>365</Age></Condition>'
  42       '</Rule></LifecycleConfiguration>').toprettyxml(indent='    ')
  43
  44   bad_doc = (
  45       '{"rule": [{"action": {"type": "Add"}, "condition": {"age": 365}}]}\n')
  46
  47   lifecycle_doc = (
  48       '{"rule": [{"action": {"type": "Delete"}, "condition": {"age": 365}}]}\n')
  49   lifecycle_json_obj = json.loads(lifecycle_doc)
  50
  51   lifecycle_doc_bucket_style = (
  52       '{"lifecycle": {"rule": [{"action": {"type": "Delete"}, "condition": '
  53       '{"age": 365}}]}}\n')
  54
  55   lifecycle_created_before_doc = (
  56       '{"rule": [{"action": {"type": "Delete"}, "condition": '
  57       '{"createdBefore": "2014-10-01"}}]}\n')
  58   lifecycle_created_before_json_obj = json.loads(lifecycle_created_before_doc)
  59
  60   no_lifecycle_config = 'has no lifecycle configuration.'
  61
  62   def test_lifecycle_translation(self):
  63     """Tests lifecycle translation for various formats."""
  64     json_text = self.lifecycle_doc
  65     entries_list = LifecycleTranslation.JsonLifecycleToMessage(json_text)
  66     boto_lifecycle = LifecycleTranslation.BotoLifecycleFromMessage(entries_list)
  67     converted_entries_list = LifecycleTranslation.BotoLifecycleToMessage(
  68         boto_lifecycle)
  69     converted_json_text = LifecycleTranslation.JsonLifecycleFromMessage(
  70         converted_entries_list)
  71     self.assertEqual(json.loads(json_text), json.loads(converted_json_text))
  72
  73   def test_default_lifecycle(self):
  74     bucket_uri = self.CreateBucket()
  75     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket_uri)],
  76                             return_stdout=True)
  77     self.assertIn(self.no_lifecycle_config, stdout)
  78
  79   def test_set_empty_lifecycle1(self):
  80     bucket_uri = self.CreateBucket()
  81     fpath = self.CreateTempFile(contents=self.empty_doc1)
  82     self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)])
  83     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket_uri)],
  84                             return_stdout=True)
  85     self.assertIn(self.no_lifecycle_config, stdout)
  86
  87   def test_valid_lifecycle(self):
  88     bucket_uri = self.CreateBucket()
  89     fpath = self.CreateTempFile(contents=self.lifecycle_doc)
  90     self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)])
  91     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket_uri)],
  92                             return_stdout=True)
  93     self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
  94
  95   def test_valid_lifecycle_bucket_style(self):
  96     bucket_uri = self.CreateBucket()
  97     fpath = self.CreateTempFile(contents=self.lifecycle_doc_bucket_style)
  98     self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)])
  99     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket_uri)],
 100                             return_stdout=True)
 101     self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
 102
 103   def test_created_before_lifecycle(self):
 104     bucket_uri = self.CreateBucket()
 105     fpath = self.CreateTempFile(contents=self.lifecycle_created_before_doc)
 106     self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)])
 107     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket_uri)],
 108                             return_stdout=True)
 109     self.assertEqual(json.loads(stdout), self.lifecycle_created_before_json_obj)
 110
 111   def test_bad_lifecycle(self):
 112     bucket_uri = self.CreateBucket()
 113     fpath = self.CreateTempFile(contents=self.bad_doc)
 114     stderr = self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)],
 115                             expected_status=1, return_stderr=True)
 116     self.assertNotIn('XML lifecycle data provided', stderr)
 117
 118   def test_bad_xml_lifecycle(self):
 119     bucket_uri = self.CreateBucket()
 120     fpath = self.CreateTempFile(contents=self.xml_doc)
 121     stderr = self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)],
 122                             expected_status=1, return_stderr=True)
 123     self.assertIn('XML lifecycle data provided', stderr)
 124
 125   def test_set_lifecycle_and_reset(self):
 126     """Tests setting and turning off lifecycle configuration."""
 127     bucket_uri = self.CreateBucket()
 128     tmpdir = self.CreateTempDir()
 129     fpath = self.CreateTempFile(tmpdir=tmpdir, contents=self.lifecycle_doc)
 130     self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)])
 131     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket_uri)],
 132                             return_stdout=True)
 133     self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
 134
 135     fpath = self.CreateTempFile(tmpdir=tmpdir, contents=self.empty_doc1)
 136     self.RunGsUtil(['lifecycle', 'set', fpath, suri(bucket_uri)])
 137     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket_uri)],
 138                             return_stdout=True)
 139     self.assertIn(self.no_lifecycle_config, stdout)
 140
 141   def test_set_lifecycle_multi_buckets(self):
 142     """Tests setting lifecycle configuration on multiple buckets."""
 143     bucket1_uri = self.CreateBucket()
 144     bucket2_uri = self.CreateBucket()
 145     fpath = self.CreateTempFile(contents=self.lifecycle_doc)
 146     self.RunGsUtil(
 147         ['lifecycle', 'set', fpath, suri(bucket1_uri), suri(bucket2_uri)])
 148     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket1_uri)],
 149                             return_stdout=True)
 150     self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
 151     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket2_uri)],
 152                             return_stdout=True)
 153     self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
 154
 155   def test_set_lifecycle_wildcard(self):
 156     """Tests setting lifecycle with a wildcarded bucket URI."""
 157     if self.test_api == ApiSelector.XML:
 158       # This test lists buckets with wildcards, but it is possible that another
 159       # test being run in parallel (in the same project) deletes a bucket after
 160       # it is listed in this test. This causes the subsequent XML metadata get
 161       # for the lifecycle configuration to fail on that just-deleted bucket,
 162       # even though that bucket is not used directly in this test.
 163       return unittest.skip('XML wildcard behavior can cause test to flake '
 164                            'if a bucket in the same project is deleted '
 165                            'during execution.')
 166
 167     random_prefix = self.MakeRandomTestString()
 168     bucket1_name = self.MakeTempName('bucket', prefix=random_prefix)
 169     bucket2_name = self.MakeTempName('bucket', prefix=random_prefix)
 170     bucket1_uri = self.CreateBucket(bucket_name=bucket1_name)
 171     bucket2_uri = self.CreateBucket(bucket_name=bucket2_name)
 172     # This just double checks that the common prefix of the two buckets is what
 173     # we think it should be (based on implementation detail of CreateBucket).
 174     # We want to be careful when setting a wildcard on buckets to make sure we
 175     # don't step outside the test buckets to affect other buckets.
 176     common_prefix = posixpath.commonprefix([suri(bucket1_uri),
 177                                             suri(bucket2_uri)])
 178     self.assertTrue(common_prefix.startswith(
 179         'gs://%sgsutil-test-test_set_lifecycle_wildcard-' % random_prefix))
 180     wildcard = '%s*' % common_prefix
 181
 182     fpath = self.CreateTempFile(contents=self.lifecycle_doc)
 183
 184     # Use @Retry as hedge against bucket listing eventual consistency.
 185     expected = set([
 186         'Setting lifecycle configuration on %s/...' % suri(bucket1_uri),
 187         'Setting lifecycle configuration on %s/...' % suri(bucket2_uri)])
 188     actual = set()
 189     @Retry(AssertionError, tries=3, timeout_secs=1)
 190     def _Check1():
 191       stderr = self.RunGsUtil(['lifecycle', 'set', fpath, wildcard],
 192                               return_stderr=True)
 193       actual.update(stderr.splitlines())
 194       self.assertEqual(expected, actual)
 195       self.assertEqual(stderr.count('Setting lifecycle configuration'), 2)
 196     _Check1()
 197
 198     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket1_uri)],
 199                             return_stdout=True)
 200     self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)
 201     stdout = self.RunGsUtil(['lifecycle', 'get', suri(bucket2_uri)],
 202                             return_stdout=True)
 203     self.assertEqual(json.loads(stdout), self.lifecycle_json_obj)