tools/telemetry/third_party/gsutilz/gslib/tests/test_parallel_cp.py

   1 # -*- coding: utf-8 -*-
   2 # Copyright 2010 Google Inc. All Rights Reserved.
   3 #
   4 # Permission is hereby granted, free of charge, to any person obtaining a
   5 # copy of this software and associated documentation files (the
   6 # "Software"), to deal in the Software without restriction, including
   7 # without limitation the rights to use, copy, modify, merge, publish, dis-
   8 # tribute, sublicense, and/or sell copies of the Software, and to permit
   9 # persons to whom the Software is furnished to do so, subject to the fol-
  10 # lowing conditions:
  11 #
  12 # The above copyright notice and this permission notice shall be included
  13 # in all copies or substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  16 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
  17 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
  18 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  19 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 # IN THE SOFTWARE.
  22 """Tests for parallel uploads ported from gsutil naming tests.
  23
  24 Currently, the mock storage service is not thread-safe and therefore not
  25 suitable for multiprocess/multithreaded testing. Since parallel composite
  26 uploads necessarily create at least one worker thread outside of main,
  27 these tests are present in this file as temporary (slower) integration tests
  28 to provide validation for parallel composite uploads until a thread-safe
  29 mock storage service rewrite.
  30
  31 Tests for relative paths are not included as integration_testcase does not
  32 support modifying the current working directory.
  33 """
  34
  35 import os
  36
  37 import gslib.tests.testcase as testcase
  38 from gslib.tests.util import ObjectToURI as suri
  39 from gslib.tests.util import PerformsFileToObjectUpload
  40 from gslib.util import Retry
  41
  42
  43 class TestParallelCp(testcase.GsUtilIntegrationTestCase):
  44   """Unit tests for gsutil naming logic."""
  45
  46   @PerformsFileToObjectUpload
  47   def testCopyingTopLevelFileToBucket(self):
  48     """Tests copying one top-level file to a bucket."""
  49     src_file = self.CreateTempFile(file_name='f0')
  50     dst_bucket_uri = self.CreateBucket()
  51     self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri)])
  52
  53     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1)
  54     self.assertEqual(suri(dst_bucket_uri, 'f0'), lines[0])
  55
  56   @PerformsFileToObjectUpload
  57   def testCopyingMultipleFilesToBucket(self):
  58     """Tests copying multiple files to a bucket."""
  59     src_file0 = self.CreateTempFile(file_name='f0')
  60     src_file1 = self.CreateTempFile(file_name='f1')
  61     dst_bucket_uri = self.CreateBucket()
  62     self.RunGsUtil(['cp', src_file0, src_file1, suri(dst_bucket_uri)])
  63
  64     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2)
  65     self.assertEqual(suri(dst_bucket_uri, 'f0'), lines[0])
  66     self.assertEqual(suri(dst_bucket_uri, 'f1'), lines[1])
  67
  68   @PerformsFileToObjectUpload
  69   def testCopyingNestedFileToBucketSubdir(self):
  70     """Tests copying a nested file to a bucket subdir.
  71
  72     Tests that we correctly translate local FS-specific delimiters ('\' on
  73     Windows) to bucket delimiter (/).
  74     """
  75     tmpdir = self.CreateTempDir()
  76     subdir = os.path.join(tmpdir, 'subdir')
  77     os.mkdir(subdir)
  78     src_file = self.CreateTempFile(tmpdir=tmpdir, file_name='obj', contents='')
  79     dst_bucket_uri = self.CreateBucket()
  80     # Make an object under subdir so next copy will treat subdir as a subdir.
  81     self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri, 'subdir/a')])
  82     self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri, 'subdir')])
  83
  84     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2)
  85     self.assertEqual(suri(dst_bucket_uri, 'subdir/a'), lines[0])
  86     self.assertEqual(suri(dst_bucket_uri, 'subdir/obj'), lines[1])
  87
  88   @PerformsFileToObjectUpload
  89   def testCopyingAbsolutePathDirToBucket(self):
  90     """Tests recursively copying absolute path directory to a bucket."""
  91     dst_bucket_uri = self.CreateBucket()
  92     src_dir_root = self.CreateTempDir(test_files=[
  93         'f0', 'f1', 'f2.txt', ('dir0', 'dir1', 'nested')])
  94     self.RunGsUtil(['cp', '-R', src_dir_root, suri(dst_bucket_uri)])
  95     src_tmpdir = os.path.split(src_dir_root)[1]
  96
  97     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 4)
  98     self.assertEqual(suri(dst_bucket_uri, src_tmpdir,
  99                           'dir0', 'dir1', 'nested'), lines[0])
 100     self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f0'), lines[1])
 101     self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f1'), lines[2])
 102     self.assertEqual(suri(dst_bucket_uri, src_tmpdir, 'f2.txt'), lines[3])
 103
 104   @PerformsFileToObjectUpload
 105   def testCopyingDirContainingOneFileToBucket(self):
 106     """Tests copying a directory containing 1 file to a bucket.
 107
 108     We test this case to ensure that correct bucket handling isn't dependent
 109     on the copy being treated as a multi-source copy.
 110     """
 111     dst_bucket_uri = self.CreateBucket()
 112     src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'foo')])
 113     self.RunGsUtil(['cp', '-R', os.path.join(src_dir, 'dir0', 'dir1'),
 114                     suri(dst_bucket_uri)])
 115
 116     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1)
 117     self.assertEqual(suri(dst_bucket_uri, 'dir1', 'foo'), lines[0])
 118
 119   @PerformsFileToObjectUpload
 120   def testCopyingFileToObjectWithConsecutiveSlashes(self):
 121     """Tests copying a file to an object containing consecutive slashes."""
 122     src_file = self.CreateTempFile(file_name='f0')
 123     dst_bucket_uri = self.CreateBucket()
 124     self.RunGsUtil(['cp', src_file, suri(dst_bucket_uri) + '//obj'])
 125
 126     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1)
 127     self.assertEqual(suri(dst_bucket_uri) + '//obj', lines[0])
 128
 129   @PerformsFileToObjectUpload
 130   def testCopyingObjsAndFilesToBucket(self):
 131     """Tests copying objects and files to a bucket."""
 132     src_bucket_uri = self.CreateBucket()
 133     self.CreateObject(src_bucket_uri, object_name='f1', contents='foo')
 134     src_dir = self.CreateTempDir(test_files=['f2'])
 135     dst_bucket_uri = self.CreateBucket()
 136     self.RunGsUtil(['cp', '-R', suri(src_bucket_uri, '**'),
 137                     '%s%s**' % (src_dir, os.sep), suri(dst_bucket_uri)])
 138
 139     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 2)
 140     self.assertEqual(suri(dst_bucket_uri, 'f1'), lines[0])
 141     self.assertEqual(suri(dst_bucket_uri, 'f2'), lines[1])
 142
 143   @PerformsFileToObjectUpload
 144   def testCopyingSubdirRecursiveToNonexistentSubdir(self):
 145     """Tests copying a directory with a single file recursively to a bucket.
 146
 147     The file should end up in a new bucket subdirectory with the file's
 148     directory structure starting below the recursive copy point, as in Unix cp.
 149
 150     Example:
 151       filepath: dir1/dir2/foo
 152       cp -r dir1 dir3
 153       Results in dir3/dir2/foo being created.
 154     """
 155     src_dir = self.CreateTempDir()
 156     self.CreateTempFile(tmpdir=src_dir + '/dir1/dir2', file_name='foo')
 157     dst_bucket_uri = self.CreateBucket()
 158     self.RunGsUtil(['cp', '-R', src_dir + '/dir1',
 159                     suri(dst_bucket_uri, 'dir3')])
 160
 161     lines = self.AssertNObjectsInBucket(dst_bucket_uri, 1)
 162     self.assertEqual(suri(dst_bucket_uri, 'dir3/dir2/foo'), lines[0])
 163
 164   @PerformsFileToObjectUpload
 165   def testCopyingWildcardedFilesToBucketSubDir(self):
 166     """Tests copying wildcarded files to a bucket subdir."""
 167     # Test with and without final slash on dest subdir.
 168     for final_dst_char in ('', '/'):
 169       dst_bucket_uri = self.CreateBucket()
 170       self.CreateObject(dst_bucket_uri, object_name='subdir0/existing',
 171                         contents='foo')
 172       self.CreateObject(dst_bucket_uri, object_name='subdir1/existing',
 173                         contents='foo')
 174       src_dir = self.CreateTempDir(test_files=['f0', 'f1', 'f2'])
 175
 176       for i in range(2):
 177         self.RunGsUtil(
 178             ['cp', os.path.join(src_dir, 'f?'),
 179              suri(dst_bucket_uri, 'subdir%d' % i) + final_dst_char])
 180
 181         @Retry(AssertionError, tries=3, timeout_secs=1)
 182         def _Check1():
 183           """Validate files were copied to the correct destinations."""
 184           stdout = self.RunGsUtil(['ls', suri(dst_bucket_uri, 'subdir%d' % i,
 185                                               '**')],
 186                                   return_stdout=True)
 187           lines = stdout.split('\n')
 188           self.assertEqual(5, len(lines))
 189           self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'existing'),
 190                            lines[0])
 191           self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f0'), lines[1])
 192           self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f1'), lines[2])
 193           self.assertEqual(suri(dst_bucket_uri, 'subdir%d' % i, 'f2'), lines[3])
 194         _Check1()
 195
 196   @PerformsFileToObjectUpload
 197   def testCopyingOneNestedFileToBucketSubDir(self):
 198     """Tests copying one nested file to a bucket subdir."""
 199     # Test with and without final slash on dest subdir.
 200     for final_dst_char in ('', '/'):
 201
 202       dst_bucket_uri = self.CreateBucket()
 203       self.CreateObject(dst_bucket_uri, object_name='d0/placeholder',
 204                         contents='foo')
 205       self.CreateObject(dst_bucket_uri, object_name='d1/placeholder',
 206                         contents='foo')
 207
 208       for i in range(2):
 209         src_dir = self.CreateTempDir(test_files=[('d3', 'd4', 'nested', 'f1')])
 210         self.RunGsUtil(['cp', '-r', suri(src_dir, 'd3'),
 211                         suri(dst_bucket_uri, 'd%d' % i) + final_dst_char])
 212
 213       lines = self.AssertNObjectsInBucket(dst_bucket_uri, 4)
 214       self.assertEqual(suri(dst_bucket_uri, 'd0', 'd3', 'd4', 'nested', 'f1'),
 215                        lines[0])
 216       self.assertEqual(suri(dst_bucket_uri, 'd0', 'placeholder'), lines[1])
 217       self.assertEqual(suri(dst_bucket_uri, 'd1', 'd3', 'd4', 'nested', 'f1'),
 218                        lines[2])
 219       self.assertEqual(suri(dst_bucket_uri, 'd1', 'placeholder'), lines[3])
 220