tools/telemetry/catapult_base/cloud_storage_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import unittest
   7
   8 import mock
   9 from telemetry.testing import system_stub
  10
  11 from catapult_base import cloud_storage
  12
  13
  14 def _FakeReadHash(_):
  15   return 'hashthis!'
  16
  17 def _FakeCalulateHashMatchesRead(_):
  18   return 'hashthis!'
  19
  20 def _FakeCalulateHashNewHash(_):
  21   return 'omgnewhash'
  22
  23
  24 class CloudStorageUnitTest(unittest.TestCase):
  25
  26   def _FakeRunCommand(self, cmd):
  27     pass
  28
  29   def _FakeGet(self, bucket, remote_path, local_path):
  30     pass
  31
  32   def _assertRunCommandRaisesError(self, communicate_strs, error):
  33     stubs = system_stub.Override(cloud_storage, ['open', 'subprocess'])
  34     stubs.open.files = {'fake gsutil path':''}
  35     stubs.subprocess.Popen.returncode_result = 1
  36     try:
  37       for string in communicate_strs:
  38         stubs.subprocess.Popen.communicate_result = ('', string)
  39         self.assertRaises(error, cloud_storage._RunCommand, [])
  40     finally:
  41       stubs.Restore()
  42
  43   def testRunCommandCredentialsError(self):
  44     strs = ['You are attempting to access protected data with no configured',
  45              'Failure: No handler was ready to authenticate.']
  46     self._assertRunCommandRaisesError(strs, cloud_storage.CredentialsError)
  47
  48   def testRunCommandPermissionError(self):
  49     strs = ['status=403', 'status 403', '403 Forbidden']
  50     self._assertRunCommandRaisesError(strs, cloud_storage.PermissionError)
  51
  52   def testRunCommandNotFoundError(self):
  53     strs = ['InvalidUriError', 'No such object', 'No URLs matched',
  54             'One or more URLs matched no', 'InvalidUriError']
  55     self._assertRunCommandRaisesError(strs, cloud_storage.NotFoundError)
  56
  57   def testRunCommandServerError(self):
  58     strs = ['500 Internal Server Error']
  59     self._assertRunCommandRaisesError(strs, cloud_storage.ServerError)
  60
  61   def testRunCommandGenericError(self):
  62     strs = ['Random string']
  63     self._assertRunCommandRaisesError(strs, cloud_storage.CloudStorageError)
  64
  65   def testInsertCreatesValidCloudUrl(self):
  66     orig_run_command = cloud_storage._RunCommand
  67     try:
  68       cloud_storage._RunCommand = self._FakeRunCommand
  69       remote_path = 'test-remote-path.html'
  70       local_path = 'test-local-path.html'
  71       cloud_url = cloud_storage.Insert(cloud_storage.PUBLIC_BUCKET,
  72                                        remote_path, local_path)
  73       self.assertEqual('https://console.developers.google.com/m/cloudstorage'
  74                        '/b/chromium-telemetry/o/test-remote-path.html',
  75                        cloud_url)
  76     finally:
  77       cloud_storage._RunCommand = orig_run_command
  78
  79   def testExistsReturnsFalse(self):
  80     stubs = system_stub.Override(cloud_storage, ['subprocess'])
  81     try:
  82       stubs.subprocess.Popen.communicate_result = (
  83         '',
  84         'CommandException: One or more URLs matched no objects.\n')
  85       stubs.subprocess.Popen.returncode_result = 1
  86       self.assertFalse(cloud_storage.Exists('fake bucket',
  87                                             'fake remote path'))
  88     finally:
  89       stubs.Restore()
  90
  91   @mock.patch('catapult_base.cloud_storage.CalculateHash')
  92   @mock.patch('catapult_base.cloud_storage.Get')
  93   @mock.patch('catapult_base.cloud_storage.os.path')
  94   def testGetIfHashChanged(self, path_mock, get_mock, calc_hash_mock):
  95     path_mock.exists.side_effect = [False, True, True]
  96     calc_hash_mock.return_value = 'hash'
  97
  98     # The file at |local_path| doesn't exist. We should download file from cs.
  99     ret = cloud_storage.GetIfHashChanged(
 100         'remote_path', 'local_path', 'cs_bucket', 'hash')
 101     self.assertTrue(ret)
 102     get_mock.assert_called_once_with('cs_bucket', 'remote_path', 'local_path')
 103     get_mock.reset_mock()
 104     self.assertFalse(calc_hash_mock.call_args)
 105     calc_hash_mock.reset_mock()
 106
 107     # A local file exists at |local_path| but has the wrong hash.
 108     # We should download file from cs.
 109     ret = cloud_storage.GetIfHashChanged(
 110         'remote_path', 'local_path', 'cs_bucket', 'new_hash')
 111     self.assertTrue(ret)
 112     get_mock.assert_called_once_with('cs_bucket', 'remote_path', 'local_path')
 113     get_mock.reset_mock()
 114     calc_hash_mock.assert_called_once_with('local_path')
 115     calc_hash_mock.reset_mock()
 116
 117     # Downloaded file exists locally and has the right hash. Don't download.
 118     ret = cloud_storage.GetIfHashChanged(
 119         'remote_path', 'local_path', 'cs_bucket', 'hash')
 120     self.assertFalse(get_mock.call_args)
 121     self.assertFalse(ret)
 122     calc_hash_mock.reset_mock()
 123     get_mock.reset_mock()
 124
 125   def testGetIfChanged(self):
 126     stubs = system_stub.Override(cloud_storage, ['os', 'open'])
 127     orig_get = cloud_storage.Get
 128     orig_read_hash = cloud_storage.ReadHash
 129     orig_calculate_hash = cloud_storage.CalculateHash
 130     cloud_storage.ReadHash = _FakeReadHash
 131     cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
 132     file_path = 'test-file-path.wpr'
 133     hash_path = file_path + '.sha1'
 134     try:
 135       cloud_storage.Get = self._FakeGet
 136       # hash_path doesn't exist.
 137       self.assertFalse(cloud_storage.GetIfChanged(file_path,
 138                                                   cloud_storage.PUBLIC_BUCKET))
 139       # hash_path exists, but file_path doesn't.
 140       stubs.os.path.files.append(hash_path)
 141       self.assertTrue(cloud_storage.GetIfChanged(file_path,
 142                                                  cloud_storage.PUBLIC_BUCKET))
 143       # hash_path and file_path exist, and have same hash.
 144       stubs.os.path.files.append(file_path)
 145       self.assertFalse(cloud_storage.GetIfChanged(file_path,
 146                                                   cloud_storage.PUBLIC_BUCKET))
 147       # hash_path and file_path exist, and have different hashes.
 148       cloud_storage.CalculateHash = _FakeCalulateHashNewHash
 149       self.assertTrue(cloud_storage.GetIfChanged(file_path,
 150                                                  cloud_storage.PUBLIC_BUCKET))
 151     finally:
 152       stubs.Restore()
 153       cloud_storage.Get = orig_get
 154       cloud_storage.CalculateHash = orig_calculate_hash
 155       cloud_storage.ReadHash = orig_read_hash
 156
 157   def testGetFilesInDirectoryIfChanged(self):
 158     stubs = system_stub.Override(cloud_storage, ['os'])
 159     stubs.os._directory = {'dir1':['1file1.sha1', '1file2.txt', '1file3.sha1'],
 160                            'dir2':['2file.txt'], 'dir3':['3file1.sha1']}
 161     stubs.os.path.dirs = ['real_dir_path']
 162     def IncrementFilesUpdated(*_):
 163       IncrementFilesUpdated.files_updated += 1
 164     IncrementFilesUpdated.files_updated = 0
 165     orig_get_if_changed = cloud_storage.GetIfChanged
 166     cloud_storage.GetIfChanged = IncrementFilesUpdated
 167     try:
 168       self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged,
 169                         os.path.abspath(os.sep), cloud_storage.PUBLIC_BUCKET)
 170       self.assertEqual(0, IncrementFilesUpdated.files_updated)
 171       self.assertRaises(ValueError, cloud_storage.GetFilesInDirectoryIfChanged,
 172                         'fake_dir_path', cloud_storage.PUBLIC_BUCKET)
 173       self.assertEqual(0, IncrementFilesUpdated.files_updated)
 174       cloud_storage.GetFilesInDirectoryIfChanged('real_dir_path',
 175                                                  cloud_storage.PUBLIC_BUCKET)
 176       self.assertEqual(3, IncrementFilesUpdated.files_updated)
 177     finally:
 178       cloud_storage.GetIfChanged = orig_get_if_changed
 179       stubs.Restore()
 180
 181   def testCopy(self):
 182     orig_run_command = cloud_storage._RunCommand
 183     def AssertCorrectRunCommandArgs(args):
 184       self.assertEqual(expected_args, args)
 185     cloud_storage._RunCommand = AssertCorrectRunCommandArgs
 186     expected_args = ['cp', 'gs://bucket1/remote_path1',
 187                      'gs://bucket2/remote_path2']
 188     try:
 189       cloud_storage.Copy('bucket1', 'bucket2', 'remote_path1', 'remote_path2')
 190     finally:
 191       cloud_storage._RunCommand = orig_run_command