tools/telemetry/third_party/gsutilz/gslib/tests/test_naming.py

   1 # -*- coding: utf-8 -*-
   2 # Copyright 2010 Google Inc. All Rights Reserved.
   3 #
   4 # Permission is hereby granted, free of charge, to any person obtaining a
   5 # copy of this software and associated documentation files (the
   6 # "Software"), to deal in the Software without restriction, including
   7 # without limitation the rights to use, copy, modify, merge, publish, dis-
   8 # tribute, sublicense, and/or sell copies of the Software, and to permit
   9 # persons to whom the Software is furnished to do so, subject to the fol-
  10 # lowing conditions:
  11 #
  12 # The above copyright notice and this permission notice shall be included
  13 # in all copies or substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  16 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
  17 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
  18 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
  19 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 # IN THE SOFTWARE.
  22 """Tests for gsutil naming logic.
  23
  24 The test code in this file runs against an in-memory storage service mock,
  25 so runs very quickly. This is valuable for testing changes that impact the
  26 naming rules, since those rules are complex and it's useful to be able to
  27 make small incremental changes and rerun the tests frequently. Additional
  28 end-to-end tests (which send traffic to the production Google Cloud Storage
  29 service) are available via the gsutil test command.
  30 """
  31
  32 from __future__ import absolute_import
  33
  34 import gzip
  35 import os
  36 import StringIO
  37
  38 from gslib import copy_helper
  39 from gslib.cloud_api import NotFoundException
  40 from gslib.cloud_api import ServiceException
  41 from gslib.exception import CommandException
  42 from gslib.exception import HashMismatchException
  43 from gslib.storage_url import StorageUrlFromString
  44 import gslib.tests.testcase as testcase
  45 from gslib.tests.util import ObjectToURI as suri
  46 from gslib.tests.util import SetBotoConfigForTest
  47 from gslib.util import UTF8
  48
  49
  50 def _Overwrite(fp):
  51   """Overwrite first byte in an open file and flush contents."""
  52   fp.seek(0)
  53   fp.write('x')
  54   fp.flush()
  55
  56
  57 def _Append(fp):
  58   """Append a byte at end of an open file and flush contents."""
  59   fp.seek(0, 2)
  60   fp.write('x')
  61   fp.flush()
  62
  63
  64 # TODO: Re-enable PerformsFileToObjectUpload decorator on tests in this file
  65 # once we refactor to a thread-safe mock storage service implementation.
  66 class GsutilNamingTests(testcase.GsUtilUnitTestCase):
  67   """Unit tests for gsutil naming logic."""
  68
  69   def testGetPathBeforeFinalDir(self):
  70     """Tests GetPathBeforeFinalDir() (unit test)."""
  71     self.assertEqual(
  72         'gs://', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString(
  73             'gs://bucket/')))
  74     self.assertEqual(
  75         'gs://bucket', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString(
  76             'gs://bucket/dir/')))
  77     self.assertEqual(
  78         'gs://bucket', copy_helper.GetPathBeforeFinalDir(StorageUrlFromString(
  79             'gs://bucket/dir')))
  80     self.assertEqual(
  81         'gs://bucket/dir', copy_helper.GetPathBeforeFinalDir(
  82             StorageUrlFromString('gs://bucket/dir/obj')))
  83     src_dir = self.CreateTempDir()
  84     subdir = os.path.join(src_dir, 'subdir')
  85     os.mkdir(subdir)
  86     self.assertEqual(suri(src_dir),
  87                      copy_helper.GetPathBeforeFinalDir(
  88                          StorageUrlFromString(suri(subdir))))
  89
  90   # @PerformsFileToObjectUpload
  91   def testCopyingTopLevelFileToBucket(self):
  92     """Tests copying one top-level file to a bucket."""
  93     src_file = self.CreateTempFile(file_name='f0')
  94     dst_bucket_uri = self.CreateBucket()
  95     self.RunCommand('cp', [src_file, suri(dst_bucket_uri)])
  96     actual = list(self._test_wildcard_iterator(
  97         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
  98     self.assertEqual(1, len(actual))
  99     self.assertEqual('f0', actual[0].root_object.name)
 100
 101   # @PerformsFileToObjectUpload
 102   def testCopyingMultipleFilesToBucket(self):
 103     """Tests copying multiple files to a bucket."""
 104     src_file0 = self.CreateTempFile(file_name='f0')
 105     src_file1 = self.CreateTempFile(file_name='f1')
 106     dst_bucket_uri = self.CreateBucket()
 107     self.RunCommand('cp', [src_file0, src_file1, suri(dst_bucket_uri)])
 108     actual = set(str(u) for u in self._test_wildcard_iterator(
 109         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 110     expected = set([
 111         suri(dst_bucket_uri, 'f0'),
 112         suri(dst_bucket_uri, 'f1'),
 113     ])
 114     self.assertEqual(expected, actual)
 115
 116   # @PerformsFileToObjectUpload
 117   def testCopyingNestedFileToBucketSubdir(self):
 118     """Tests copying a nested file to a bucket subdir.
 119
 120     Tests that we correctly translate local FS-specific delimiters ('\' on
 121     Windows) to bucket delimiter (/).
 122     """
 123     tmpdir = self.CreateTempDir()
 124     subdir = os.path.join(tmpdir, 'subdir')
 125     os.mkdir(subdir)
 126     src_file = self.CreateTempFile(tmpdir=tmpdir, file_name='obj', contents='')
 127     dst_bucket_uri = self.CreateBucket()
 128     # Make an object under subdir so next copy will treat subdir as a subdir.
 129     self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'subdir/a')])
 130     self.RunCommand('cp', [src_file, suri(dst_bucket_uri, 'subdir')])
 131     actual = set(str(u) for u in self._test_wildcard_iterator(
 132         suri(dst_bucket_uri, '**')).IterObjects())
 133     expected = set([
 134         suri(dst_bucket_uri, 'subdir', 'a'),
 135         suri(dst_bucket_uri, 'subdir', 'obj'),
 136     ])
 137     self.assertEqual(expected, actual)
 138
 139   # @PerformsFileToObjectUpload
 140   def testCopyingAbsolutePathDirToBucket(self):
 141     """Tests recursively copying absolute path directory to a bucket."""
 142     dst_bucket_uri = self.CreateBucket()
 143     src_dir_root = self.CreateTempDir(test_files=[
 144         'f0', 'f1', 'f2.txt', ('dir0', 'dir1', 'nested')])
 145     self.RunCommand('cp', ['-R', src_dir_root, suri(dst_bucket_uri)])
 146     actual = set(str(u) for u in self._test_wildcard_iterator(
 147         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 148     src_tmpdir = os.path.split(src_dir_root)[1]
 149     expected = set([
 150         suri(dst_bucket_uri, src_tmpdir, 'f0'),
 151         suri(dst_bucket_uri, src_tmpdir, 'f1'),
 152         suri(dst_bucket_uri, src_tmpdir, 'f2.txt'),
 153         suri(dst_bucket_uri, src_tmpdir, 'dir0', 'dir1', 'nested')])
 154     self.assertEqual(expected, actual)
 155
 156   # @PerformsFileToObjectUpload
 157   def testCopyingRelativePathDirToBucket(self):
 158     """Tests recursively copying relative directory to a bucket."""
 159     dst_bucket_uri = self.CreateBucket()
 160     src_dir = self.CreateTempDir(test_files=[('dir0', 'f1')])
 161     self.RunCommand('cp', ['-R', 'dir0', suri(dst_bucket_uri)], cwd=src_dir)
 162     actual = set(str(u) for u in self._test_wildcard_iterator(
 163         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 164     expected = set([suri(dst_bucket_uri, 'dir0', 'f1')])
 165     self.assertEqual(expected, actual)
 166
 167   # @PerformsFileToObjectUpload
 168   def testCopyingRelPathSubDirToBucketSubdirWithDollarFolderObj(self):
 169     """Tests recursively copying relative sub-directory to bucket subdir.
 170
 171     Subdir is signified by a $folder$ object.
 172     """
 173     # Create a $folder$ object to simulate a folder created by GCS manager (or
 174     # various other tools), which gsutil understands to mean there is a folder
 175     # into which the object is being copied.
 176     dst_bucket_uri = self.CreateBucket()
 177     self.CreateObject(bucket_uri=dst_bucket_uri, object_name='abc_$folder$',
 178                       contents='')
 179     src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')])
 180     self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'),
 181                            suri(dst_bucket_uri, 'abc')], cwd=src_dir)
 182     actual = set(str(u) for u in self._test_wildcard_iterator(
 183         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 184     expected = set([suri(dst_bucket_uri, 'abc_$folder$'),
 185                     suri(dst_bucket_uri, 'abc', 'dir1', 'f1')])
 186     self.assertEqual(expected, actual)
 187
 188   # @PerformsFileToObjectUpload
 189   def testCopyingRelativePathSubDirToBucketSubdirSignifiedBySlash(self):
 190     """Tests recursively copying relative sub-directory to bucket subdir.
 191
 192     Subdir is signified by a / object.
 193     """
 194     dst_bucket_uri = self.CreateBucket()
 195     src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')])
 196     self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'),
 197                            suri(dst_bucket_uri, 'abc') + '/'], cwd=src_dir)
 198     actual = set(str(u) for u in self._test_wildcard_iterator(
 199         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 200     expected = set([suri(dst_bucket_uri, 'abc', 'dir1', 'f1')])
 201     self.assertEqual(expected, actual)
 202
 203   # @PerformsFileToObjectUpload
 204   def testCopyingRelativePathSubDirToBucket(self):
 205     """Tests recursively copying relative sub-directory to a bucket."""
 206     dst_bucket_uri = self.CreateBucket()
 207     src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'f1')])
 208     self.RunCommand('cp', ['-R', os.path.join('dir0', 'dir1'),
 209                            suri(dst_bucket_uri)], cwd=src_dir)
 210     actual = set(str(u) for u in self._test_wildcard_iterator(
 211         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 212     expected = set([suri(dst_bucket_uri, 'dir1', 'f1')])
 213     self.assertEqual(expected, actual)
 214
 215   # @PerformsFileToObjectUpload
 216   def testCopyingDotSlashToBucket(self):
 217     """Tests copying ./ to a bucket produces expected naming."""
 218     # When running a command like gsutil cp -r . gs://dest we expect the dest
 219     # obj names to be of the form gs://dest/abc, not gs://dest/./abc.
 220     dst_bucket_uri = self.CreateBucket()
 221     src_dir = self.CreateTempDir(test_files=['foo'])
 222     for rel_src_dir in ['.', '.%s' % os.sep]:
 223       self.RunCommand('cp', ['-R', rel_src_dir, suri(dst_bucket_uri)],
 224                       cwd=src_dir)
 225       actual = set(str(u) for u in self._test_wildcard_iterator(
 226           suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 227       expected = set([suri(dst_bucket_uri, 'foo')])
 228       self.assertEqual(expected, actual)
 229
 230   # @PerformsFileToObjectUpload
 231   def testCopyingDirContainingOneFileToBucket(self):
 232     """Tests copying a directory containing 1 file to a bucket.
 233
 234     We test this case to ensure that correct bucket handling isn't dependent
 235     on the copy being treated as a multi-source copy.
 236     """
 237     dst_bucket_uri = self.CreateBucket()
 238     src_dir = self.CreateTempDir(test_files=[('dir0', 'dir1', 'foo')])
 239     self.RunCommand('cp', ['-R', os.path.join(src_dir, 'dir0', 'dir1'),
 240                            suri(dst_bucket_uri)])
 241     actual = list((str(u) for u in self._test_wildcard_iterator(
 242         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True)))
 243     self.assertEqual(1, len(actual))
 244     self.assertEqual(suri(dst_bucket_uri, 'dir1', 'foo'), actual[0])
 245
 246   def testCopyingBucketToDir(self):
 247     """Tests copying from a bucket to a directory."""
 248     src_bucket_uri = self.CreateBucket(test_objects=['foo', 'dir/foo2'])
 249     dst_dir = self.CreateTempDir()
 250     # Mock objects don't support hash digestion.
 251     with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]):
 252       self.RunCommand('cp', ['-R', suri(src_bucket_uri), dst_dir])
 253     actual = set(str(u) for u in self._test_wildcard_iterator(
 254         '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True))
 255     expected = set([suri(dst_dir, src_bucket_uri.bucket_name, 'foo'),
 256                     suri(dst_dir, src_bucket_uri.bucket_name, 'dir', 'foo2')])
 257     self.assertEqual(expected, actual)
 258
 259   def testCopyingBucketToBucket(self):
 260     """Tests copying from a bucket-only URI to a bucket."""
 261     src_bucket_uri = self.CreateBucket(test_objects=['foo', 'dir/foo2'])
 262     dst_bucket_uri = self.CreateBucket()
 263     self.RunCommand('cp', ['-R', suri(src_bucket_uri), suri(dst_bucket_uri)])
 264     actual = set(str(u) for u in self._test_wildcard_iterator(
 265         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 266     expected = set([
 267         suri(dst_bucket_uri, src_bucket_uri.bucket_name, 'foo'),
 268         suri(dst_bucket_uri, src_bucket_uri.bucket_name, 'dir', 'foo2')])
 269     self.assertEqual(expected, actual)
 270
 271   def testCopyingDirectoryToDirectory(self):
 272     """Tests copying from a directory to a directory."""
 273     src_dir = self.CreateTempDir(test_files=['foo', ('dir', 'foo2')])
 274     dst_dir = self.CreateTempDir()
 275     self.RunCommand('cp', ['-R', src_dir, dst_dir])
 276     actual = set(str(u) for u in self._test_wildcard_iterator(
 277         '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True))
 278     src_dir_base = os.path.split(src_dir)[1]
 279     expected = set([suri(dst_dir, src_dir_base, 'foo'),
 280                     suri(dst_dir, src_dir_base, 'dir', 'foo2')])
 281     self.assertEqual(expected, actual)
 282
 283   def testCopyingFilesAndDirNonRecursive(self):
 284     """Tests copying containing files and a directory without -R."""
 285     src_dir = self.CreateTempDir(test_files=['foo', 'bar', ('d1', 'f2'),
 286                                              ('d2', 'f3'), ('d3', 'd4', 'f4')])
 287     dst_dir = self.CreateTempDir()
 288     self.RunCommand('cp', ['%s%s*' % (src_dir, os.sep), dst_dir])
 289     actual = set(str(u) for u in self._test_wildcard_iterator(
 290         '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True))
 291     expected = set([suri(dst_dir, 'foo'), suri(dst_dir, 'bar')])
 292     self.assertEqual(expected, actual)
 293
 294   def testCopyingFileToDir(self):
 295     """Tests copying one file to a directory."""
 296     src_file = self.CreateTempFile(file_name='foo')
 297     dst_dir = self.CreateTempDir()
 298     self.RunCommand('cp', [src_file, dst_dir])
 299     actual = list(self._test_wildcard_iterator(
 300         '%s%s*' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True))
 301     self.assertEqual(1, len(actual))
 302     self.assertEqual(suri(dst_dir, 'foo'), str(actual[0]))
 303
 304   # @PerformsFileToObjectUpload
 305   def testCopyingFileToObjectWithConsecutiveSlashes(self):
 306     """Tests copying a file to an object containing consecutive slashes."""
 307     src_file = self.CreateTempFile(file_name='f0')
 308     dst_bucket_uri = self.CreateBucket()
 309     self.RunCommand('cp', [src_file, suri(dst_bucket_uri) + '//obj'])
 310     actual = list(self._test_wildcard_iterator(
 311         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 312     self.assertEqual(1, len(actual))
 313     self.assertEqual('/obj', actual[0].root_object.name)
 314
 315   def testCopyingCompressedFileToBucket(self):
 316     """Tests copying one file with compression to a bucket."""
 317     src_file = self.CreateTempFile(contents='plaintext', file_name='f2.txt')
 318     dst_bucket_uri = self.CreateBucket()
 319     self.RunCommand('cp', ['-z', 'txt', src_file, suri(dst_bucket_uri)],)
 320     actual = list(self._test_wildcard_iterator(
 321         suri(dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True))
 322     self.assertEqual(1, len(actual))
 323     actual_obj = actual[0].root_object
 324     self.assertEqual('f2.txt', actual_obj.name)
 325     self.assertEqual('gzip', actual_obj.contentEncoding)
 326
 327     stdout = self.RunCommand('cat', [suri(dst_bucket_uri, 'f2.txt')],
 328                              return_stdout=True)
 329
 330     f = gzip.GzipFile(fileobj=StringIO.StringIO(stdout), mode='rb')
 331     try:
 332       self.assertEqual(f.read(), 'plaintext')
 333     finally:
 334       f.close()
 335
 336   def testCopyingObjectToObject(self):
 337     """Tests copying an object to an object."""
 338     src_bucket_uri = self.CreateBucket(test_objects=['obj'])
 339     dst_bucket_uri = self.CreateBucket()
 340     self.RunCommand('cp', [suri(src_bucket_uri, 'obj'), suri(dst_bucket_uri)])
 341     actual = list(self._test_wildcard_iterator(
 342         suri(dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True))
 343     self.assertEqual(1, len(actual))
 344     self.assertEqual('obj', actual[0].root_object.name)
 345
 346   def testCopyingObjectToObjectUsingDestWildcard(self):
 347     """Tests copying an object to an object using a dest wildcard."""
 348     src_bucket_uri = self.CreateBucket(test_objects=['obj'])
 349     dst_bucket_uri = self.CreateBucket(test_objects=['dstobj'])
 350     self.RunCommand('cp', [suri(src_bucket_uri, 'obj'),
 351                            '%s*' % dst_bucket_uri.uri])
 352     actual = set(str(u) for u in self._test_wildcard_iterator(
 353         suri(dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True))
 354     expected = set([suri(dst_bucket_uri, 'dstobj')])
 355     self.assertEqual(expected, actual)
 356
 357   def testCopyingObjsAndFilesToDir(self):
 358     """Tests copying objects and files to a directory."""
 359     src_bucket_uri = self.CreateBucket(test_objects=['f1'])
 360     src_dir = self.CreateTempDir(test_files=['f2'])
 361     dst_dir = self.CreateTempDir()
 362     # Mock objects don't support hash digestion.
 363     with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]):
 364       self.RunCommand('cp', ['-R', suri(src_bucket_uri, '**'),
 365                              os.path.join(src_dir, '**'), dst_dir])
 366     actual = set(str(u) for u in self._test_wildcard_iterator(
 367         os.path.join(dst_dir, '**')).IterAll(expand_top_level_buckets=True))
 368     expected = set([suri(dst_dir, 'f1'), suri(dst_dir, 'f2')])
 369     self.assertEqual(expected, actual)
 370
 371   def testCopyingObjToDot(self):
 372     """Tests that copying an object to . or ./ downloads to correct name."""
 373     src_bucket_uri = self.CreateBucket(test_objects=['f1'])
 374     dst_dir = self.CreateTempDir()
 375     for final_char in ('/', ''):
 376       # Mock objects don't support hash digestion.
 377       with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]):
 378         self.RunCommand('cp', [suri(src_bucket_uri, 'f1'), '.%s' % final_char],
 379                         cwd=dst_dir)
 380       actual = set()
 381       for dirname, dirnames, filenames in os.walk(dst_dir):
 382         for subdirname in dirnames:
 383           actual.add(os.path.join(dirname, subdirname))
 384         for filename in filenames:
 385           actual.add(os.path.join(dirname, filename))
 386       expected = set([os.path.join(dst_dir, 'f1')])
 387       self.assertEqual(expected, actual)
 388
 389   # @PerformsFileToObjectUpload
 390   def testCopyingObjsAndFilesToBucket(self):
 391     """Tests copying objects and files to a bucket."""
 392     src_bucket_uri = self.CreateBucket(test_objects=['f1'])
 393     src_dir = self.CreateTempDir(test_files=['f2'])
 394     dst_bucket_uri = self.CreateBucket()
 395     self.RunCommand('cp', ['-R', suri(src_bucket_uri, '**'),
 396                            '%s%s**' % (src_dir, os.sep), suri(dst_bucket_uri)])
 397     actual = set(str(u) for u in self._test_wildcard_iterator(
 398         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 399     expected = set([suri(dst_bucket_uri, 'f1'), suri(dst_bucket_uri, 'f2')])
 400     self.assertEqual(expected, actual)
 401
 402   # @PerformsFileToObjectUpload
 403   def testCopyingSubdirRecursiveToNonexistentSubdir(self):
 404     """Tests copying a directory with a single file recursively to a bucket.
 405
 406     The file should end up in a new bucket subdirectory with the file's
 407     directory structure starting below the recursive copy point, as in Unix cp.
 408
 409     Example:
 410       filepath: dir1/dir2/foo
 411       cp -r dir1 dir3
 412       Results in dir3/dir2/foo being created.
 413     """
 414     src_dir = self.CreateTempDir()
 415     self.CreateTempFile(tmpdir=src_dir + '/dir1/dir2', file_name='foo')
 416     dst_bucket_uri = self.CreateBucket()
 417     self.RunCommand('cp', ['-R', src_dir + '/dir1',
 418                            suri(dst_bucket_uri, 'dir3')])
 419     actual = set(str(u) for u in self._test_wildcard_iterator(
 420         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 421     expected = set([suri(dst_bucket_uri, 'dir3/dir2/foo')])
 422     self.assertEqual(expected, actual)
 423
 424   def testAttemptDirCopyWithoutRecursion(self):
 425     """Tests copying a directory without -R."""
 426     src_dir = self.CreateTempDir(test_files=1)
 427     dst_dir = self.CreateTempDir()
 428     try:
 429       self.RunCommand('cp', [src_dir, dst_dir])
 430       self.fail('Did not get expected CommandException')
 431     except CommandException, e:
 432       self.assertIn('No URLs matched', e.reason)
 433
 434   def testNonRecursiveFileAndSameNameSubdir(self):
 435     """Tests copying a file and subdirectory of the same name without -R."""
 436     src_bucket_uri = self.CreateBucket(test_objects=['f1', 'f1/f2'])
 437     dst_dir = self.CreateTempDir()
 438     # Mock objects don't support hash digestion.
 439     with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]):
 440       self.RunCommand('cp', [suri(src_bucket_uri, 'f1'), dst_dir])
 441     actual = list(self._test_wildcard_iterator(
 442         '%s%s*' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True))
 443     self.assertEqual(1, len(actual))
 444     self.assertEqual(suri(dst_dir, 'f1'), str(actual[0]))
 445     # TODO: Assert that we omit the prefix here when unit_testcase supports
 446     # returning stderr.
 447
 448   def testAttemptCopyingProviderOnlySrc(self):
 449     """Attempts to copy a src specified as a provider-only URI."""
 450     src_bucket_uri = self.CreateBucket()
 451     try:
 452       self.RunCommand('cp', ['gs://', suri(src_bucket_uri)])
 453       self.fail('Did not get expected CommandException')
 454     except CommandException, e:
 455       self.assertIn('provider-only', e.reason)
 456
 457   def testAttemptCopyingOverlappingSrcDstFile(self):
 458     """Attempts to an object atop itself."""
 459     src_file = self.CreateTempFile()
 460     try:
 461       self.RunCommand('cp', [src_file, src_file])
 462       self.fail('Did not get expected CommandException')
 463     except CommandException, e:
 464       self.assertIn('are the same file - abort', e.reason)
 465
 466   def testAttemptCopyingToMultiMatchWildcard(self):
 467     """Attempts to copy where dst wildcard matches >1 obj."""
 468     src_bucket_uri = self.CreateBucket(test_objects=2)
 469     try:
 470       self.RunCommand('cp', [suri(src_bucket_uri, 'obj0'),
 471                              suri(src_bucket_uri, '*')])
 472       self.fail('Did not get expected CommandException')
 473     except CommandException, e:
 474       self.assertNotEqual(e.reason.find('must match exactly 1 URL'), -1)
 475
 476   def testAttemptCopyingMultiObjsToFile(self):
 477     """Attempts to copy multiple objects to a file."""
 478     src_bucket_uri = self.CreateBucket(test_objects=2)
 479     dst_file = self.CreateTempFile()
 480     try:
 481       self.RunCommand('cp', ['-R', suri(src_bucket_uri, '*'), dst_file])
 482       self.fail('Did not get expected CommandException')
 483     except CommandException, e:
 484       self.assertIn('must name a directory, bucket, or', e.reason)
 485
 486   def testAttemptCopyingWithFileDirConflict(self):
 487     """Attempts to copy objects that cause a file/directory conflict."""
 488     # Create objects with name conflicts (a/b and a). Use 'dst' bucket because
 489     # it gets cleared after each test.
 490     bucket_uri = self.CreateBucket()
 491     self.CreateObject(bucket_uri=bucket_uri, object_name='a')
 492     self.CreateObject(bucket_uri=bucket_uri, object_name='b/a')
 493     dst_dir = self.CreateTempDir()
 494     try:
 495       self.RunCommand('cp', ['-R', suri(bucket_uri), dst_dir])
 496       self.fail('Did not get expected CommandException')
 497     except CommandException, e:
 498       self.assertNotEqual('exists where a directory needs to be created',
 499                           e.reason)
 500
 501   def testAttemptCopyingWithDirFileConflict(self):
 502     """Attempts to copy an object that causes a directory/file conflict."""
 503     # Create an object that conflicts with dest subdir.
 504     tmpdir = self.CreateTempDir()
 505     os.mkdir(os.path.join(tmpdir, 'abc'))
 506     src_uri = self.CreateObject(object_name='abc', contents='bar')
 507     try:
 508       self.RunCommand('cp', [suri(src_uri), tmpdir + '/'])
 509       self.fail('Did not get expected CommandException')
 510     except CommandException, e:
 511       self.assertNotEqual('where the file needs to be created', e.reason)
 512
 513   def testWildcardMoveWithinBucket(self):
 514     """Attempts to move using src wildcard that overlaps dest object.
 515
 516     We want to ensure that this doesn't stomp the result data.
 517     """
 518     dst_bucket_uri = self.CreateBucket(test_objects=['old'])
 519     self.RunCommand('mv', [suri(dst_bucket_uri, 'old*'),
 520                            suri(dst_bucket_uri, 'new')])
 521     actual = set(str(u) for u in self._test_wildcard_iterator(
 522         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 523     expected = set([suri(dst_bucket_uri, 'new')])
 524     self.assertEqual(expected, actual)
 525
 526   def testLsNonExistentObjectWithPrefixName(self):
 527     """Test ls of non-existent obj that matches prefix of existing objs."""
 528     # Use an object name that matches a prefix of other names at that level, to
 529     # ensure the ls subdir handling logic doesn't pick up anything extra.
 530     src_bucket_uri = self.CreateBucket(test_objects=['obj_with_suffix'])
 531     try:
 532       self.RunCommand('ls', [suri(src_bucket_uri, 'obj')])
 533     except CommandException, e:
 534       self.assertIn('matched no objects', e.reason)
 535
 536   def testLsBucketNonRecursive(self):
 537     """Test that ls of a bucket returns expected results."""
 538     src_bucket_uri = self.CreateBucket(test_objects=['foo1', 'd0/foo2',
 539                                                      'd1/d2/foo3'])
 540     output = self.RunCommand('ls', [suri(src_bucket_uri, '*')],
 541                              return_stdout=True)
 542     expected = set([suri(src_bucket_uri, 'foo1'),
 543                     suri(src_bucket_uri, 'd1', ':'),
 544                     suri(src_bucket_uri, 'd1', 'd2') + src_bucket_uri.delim,
 545                     suri(src_bucket_uri, 'd0', ':'),
 546                     suri(src_bucket_uri, 'd0', 'foo2')])
 547     expected.add('')  # Blank line between subdir listings.
 548     actual = set(output.split('\n'))
 549     self.assertEqual(expected, actual)
 550
 551   def testLsBucketRecursive(self):
 552     """Test that ls -R of a bucket returns expected results."""
 553     src_bucket_uri = self.CreateBucket(test_objects=['foo1', 'd0/foo2',
 554                                                      'd1/d2/foo3'])
 555     output = self.RunCommand('ls', ['-R', suri(src_bucket_uri, '*')],
 556                              return_stdout=True)
 557     expected = set([suri(src_bucket_uri, 'foo1'),
 558                     suri(src_bucket_uri, 'd1', ':'),
 559                     suri(src_bucket_uri, 'd1', 'd2', ':'),
 560                     suri(src_bucket_uri, 'd1', 'd2', 'foo3'),
 561                     suri(src_bucket_uri, 'd0', ':'),
 562                     suri(src_bucket_uri, 'd0', 'foo2')])
 563     expected.add('')  # Blank line between subdir listings.
 564     actual = set(output.split('\n'))
 565     self.assertEqual(expected, actual)
 566
 567   def testLsBucketRecursiveWithLeadingSlashObjectName(self):
 568     """Test that ls -R of a bucket with an object that has leading slash."""
 569     dst_bucket_uri = self.CreateBucket(test_objects=['f0'])
 570     output = self.RunCommand('ls', ['-R', suri(dst_bucket_uri) + '*'],
 571                              return_stdout=True)
 572     expected = set([suri(dst_bucket_uri, 'f0')])
 573     expected.add('')  # Blank line between subdir listings.
 574     actual = set(output.split('\n'))
 575     self.assertEqual(expected, actual)
 576
 577   def testLsBucketSubdirNonRecursive(self):
 578     """Test that ls of a bucket subdir returns expected results."""
 579     src_bucket_uri = self.CreateBucket(test_objects=['src_subdir/foo',
 580                                                      'src_subdir/nested/foo2'])
 581     output = self.RunCommand('ls', [suri(src_bucket_uri, 'src_subdir')],
 582                              return_stdout=True)
 583     expected = set([
 584         suri(src_bucket_uri, 'src_subdir', 'foo'),
 585         suri(src_bucket_uri, 'src_subdir', 'nested') + src_bucket_uri.delim])
 586     expected.add('')  # Blank line between subdir listings.
 587     actual = set(output.split('\n'))
 588     self.assertEqual(expected, actual)
 589
 590   def testLsBucketSubdirRecursive(self):
 591     """Test that ls -R of a bucket subdir returns expected results."""
 592     src_bucket_uri = self.CreateBucket(test_objects=['src_subdir/foo',
 593                                                      'src_subdir/nested/foo2'])
 594     for final_char in ('/', ''):
 595       output = self.RunCommand(
 596           'ls', ['-R', suri(src_bucket_uri, 'src_subdir') + final_char],
 597           return_stdout=True)
 598       expected = set([
 599           suri(src_bucket_uri, 'src_subdir', ':'),
 600           suri(src_bucket_uri, 'src_subdir', 'foo'),
 601           suri(src_bucket_uri, 'src_subdir', 'nested', ':'),
 602           suri(src_bucket_uri, 'src_subdir', 'nested', 'foo2')])
 603       expected.add('')  # Blank line between subdir listings.
 604       actual = set(output.split('\n'))
 605       self.assertEqual(expected, actual)
 606
 607   def testSetAclOnBucketRuns(self):
 608     """Test that the 'acl set' command basically runs."""
 609     # We don't test reading back the acl (via 'acl get' command) because at
 610     # present MockStorageService doesn't translate canned ACLs into actual ACL
 611     # XML.
 612     src_bucket_uri = self.CreateBucket()
 613     self.RunCommand('acl', ['set', 'private', suri(src_bucket_uri)])
 614
 615   def testSetAclOnWildcardNamedBucketRuns(self):
 616     """Test that 'acl set' basically runs against wildcard-named bucket."""
 617     # We don't test reading back the acl (via 'acl get' command) because at
 618     # present MockStorageService doesn't translate canned ACLs into actual ACL
 619     # XML.
 620     src_bucket_uri = self.CreateBucket(test_objects=['f0'])
 621     self.RunCommand('acl', ['set', 'private', suri(src_bucket_uri)[:-2] + '*'])
 622
 623   def testSetAclOnObjectRuns(self):
 624     """Test that the 'acl set' command basically runs."""
 625     src_bucket_uri = self.CreateBucket(test_objects=['f0'])
 626     self.RunCommand('acl', ['set', 'private', suri(src_bucket_uri, '*')])
 627
 628   def testSetDefAclOnBucketRuns(self):
 629     """Test that the 'defacl set' command basically runs."""
 630     src_bucket_uri = self.CreateBucket()
 631     self.RunCommand('defacl', ['set', 'private', suri(src_bucket_uri)])
 632
 633   def testSetDefAclOnObjectFails(self):
 634     """Test that the 'defacl set' command fails when run against an object."""
 635     src_bucket_uri = self.CreateBucket()
 636     try:
 637       self.RunCommand('defacl', ['set', 'private', suri(src_bucket_uri, '*')])
 638       self.fail('Did not get expected CommandException')
 639     except CommandException, e:
 640       self.assertIn('URL must name a bucket', e.reason)
 641
 642   # @PerformsFileToObjectUpload
 643   def testMinusDOptionWorks(self):
 644     """Tests using gsutil -D option."""
 645     src_file = self.CreateTempFile(file_name='f0')
 646     dst_bucket_uri = self.CreateBucket()
 647     self.RunCommand('cp', [src_file, suri(dst_bucket_uri)], debug=3)
 648     actual = list(self._test_wildcard_iterator(
 649         suri(dst_bucket_uri, '*')).IterAll(expand_top_level_buckets=True))
 650     self.assertEqual(1, len(actual))
 651     self.assertEqual('f0', actual[0].root_object.name)
 652
 653   def DownloadTestHelper(self, func):
 654     """Test resumable download with custom test function.
 655
 656     The custom function distorts downloaded data. We expect an exception to be
 657     raised and the dest file to be removed.
 658
 659     Args:
 660       func: Custom test function used to distort the downloaded data.
 661     """
 662     object_uri = self.CreateObject(contents='foo')
 663     # Need to explicitly tell the key to populate its etag so that hash
 664     # validation will be performed.
 665     object_uri.get_key().set_etag()
 666     dst_dir = self.CreateTempDir()
 667     got_expected_exception = False
 668     try:
 669       self.RunCommand('cp', [suri(object_uri), dst_dir], test_method=func)
 670       self.fail('Did not get expected CommandException')
 671     except HashMismatchException:
 672       self.assertFalse(os.listdir(dst_dir))
 673       got_expected_exception = True
 674     except Exception, e:
 675       self.fail('Unexpected exception raised: %s' % e)
 676     if not got_expected_exception:
 677       self.fail('Did not get expected CommandException')
 678
 679   def testDownloadWithObjectSizeChange(self):
 680     """Test resumable download on an object that changes size.
 681
 682     Size change occurs before the downloaded file's checksum is validated.
 683     """
 684     self.DownloadTestHelper(_Append)
 685
 686   def testDownloadWithFileContentChange(self):
 687     """Tests resumable download on an object that changes content.
 688
 689     Content change occurs before the downloaded file's checksum is validated.
 690     """
 691     self.DownloadTestHelper(_Overwrite)
 692
 693   # @PerformsFileToObjectUpload
 694   def testFlatCopyingObjsAndFilesToBucketSubDir(self):
 695     """Tests copying flatly listed objects and files to bucket subdir."""
 696     src_bucket_uri = self.CreateBucket(test_objects=['f0', 'd0/f1', 'd1/d2/f2'])
 697     src_dir = self.CreateTempDir(test_files=['f3', ('d3', 'f4'),
 698                                              ('d4', 'd5', 'f5')])
 699     dst_bucket_uri = self.CreateBucket(test_objects=['dst_subdir0/existing',
 700                                                      'dst_subdir1/existing'])
 701     # Test with and without final slash on dest subdir.
 702     for i, final_char in enumerate(('/', '')):
 703       self.RunCommand(
 704           'cp', ['-R', suri(src_bucket_uri, '**'), os.path.join(src_dir, '**'),
 705                  suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_char])
 706
 707     actual = set(str(u) for u in self._test_wildcard_iterator(
 708         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 709     expected = set()
 710     for i in range(2):
 711       expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing'))
 712       for j in range(6):
 713         expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'f%d' % j))
 714     self.assertEqual(expected, actual)
 715
 716   # @PerformsFileToObjectUpload
 717   def testRecursiveCopyObjsAndFilesToExistingBucketSubDir(self):
 718     """Tests recursive copy of objects and files to existing bucket subdir."""
 719     src_bucket_uri = self.CreateBucket(test_objects=['f0', 'nested/f1'])
 720     dst_bucket_uri = self.CreateBucket(test_objects=[
 721         'dst_subdir0/existing_obj', 'dst_subdir1/existing_obj'])
 722     src_dir = self.CreateTempDir(test_files=['f2', ('nested', 'f3')])
 723     # Test with and without final slash on dest subdir.
 724     for i, final_char in enumerate(('/', '')):
 725       self.RunCommand(
 726           'cp', ['-R', suri(src_bucket_uri), src_dir,
 727                  suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_char])
 728       actual = set(str(u) for u in self._test_wildcard_iterator(
 729           suri(dst_bucket_uri, 'dst_subdir%d' % i, '**')).IterAll(
 730               expand_top_level_buckets=True))
 731       tmp_dirname = os.path.split(src_dir)[1]
 732       bucketname = src_bucket_uri.bucket_name
 733       expected = set([
 734           suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing_obj'),
 735           suri(dst_bucket_uri, 'dst_subdir%d' % i, bucketname, 'f0'),
 736           suri(dst_bucket_uri, 'dst_subdir%d' % i, bucketname, 'nested', 'f1'),
 737           suri(dst_bucket_uri, 'dst_subdir%d' % i, tmp_dirname, 'f2'),
 738           suri(dst_bucket_uri, 'dst_subdir%d' % i, tmp_dirname, 'nested', 'f3')
 739       ])
 740       self.assertEqual(expected, actual)
 741
 742   # @PerformsFileToObjectUpload
 743   def testRecursiveCopyObjsAndFilesToNonExistentBucketSubDir(self):
 744     """Tests recursive copy of objs + files to non-existent bucket subdir."""
 745     src_bucket_uri = self.CreateBucket(test_objects=['f0', 'nested/f1'])
 746     src_dir = self.CreateTempDir(test_files=['f2', ('nested', 'f3')])
 747     dst_bucket_uri = self.CreateBucket()
 748     self.RunCommand('cp', ['-R', src_dir, suri(src_bucket_uri),
 749                            suri(dst_bucket_uri, 'dst_subdir')])
 750     actual = set(str(u) for u in self._test_wildcard_iterator(
 751         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 752     expected = set([suri(dst_bucket_uri, 'dst_subdir', 'f0'),
 753                     suri(dst_bucket_uri, 'dst_subdir', 'nested', 'f1'),
 754                     suri(dst_bucket_uri, 'dst_subdir', 'f2'),
 755                     suri(dst_bucket_uri, 'dst_subdir', 'nested', 'f3')])
 756     self.assertEqual(expected, actual)
 757
 758   def testCopyingBucketSubDirToDir(self):
 759     """Tests copying a bucket subdir to a directory."""
 760     src_bucket_uri = self.CreateBucket(test_objects=['src_subdir/obj'])
 761     dst_dir = self.CreateTempDir()
 762     # Test with and without final slash on dest subdir.
 763     for (final_src_char, final_dst_char) in (
 764         ('', ''), ('', '/'), ('/', ''), ('/', '/')):
 765       # Mock objects don't support hash digestion.
 766       with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]):
 767         self.RunCommand(
 768             'cp', ['-R', suri(src_bucket_uri, 'src_subdir') + final_src_char,
 769                    dst_dir + final_dst_char])
 770       actual = set(str(u) for u in self._test_wildcard_iterator(
 771           '%s%s**' % (dst_dir, os.sep)).IterAll(expand_top_level_buckets=True))
 772       expected = set([suri(dst_dir, 'src_subdir', 'obj')])
 773       self.assertEqual(expected, actual)
 774
 775   def testCopyingWildcardSpecifiedBucketSubDirToExistingDir(self):
 776     """Tests copying a wildcard-specified bucket subdir to a directory."""
 777     src_bucket_uri = self.CreateBucket(
 778         test_objects=['src_sub0dir/foo', 'src_sub1dir/foo', 'src_sub2dir/foo',
 779                       'src_sub3dir/foo'])
 780     dst_dir = self.CreateTempDir()
 781     # Test with and without final slash on dest subdir.
 782     for i, (final_src_char, final_dst_char) in enumerate((
 783         ('', ''), ('', '/'), ('/', ''), ('/', '/'))):
 784       # Mock objects don't support hash digestion.
 785       with SetBotoConfigForTest([('GSUtil', 'check_hashes', 'never')]):
 786         self.RunCommand(
 787             'cp', ['-R', suri(src_bucket_uri, 'src_sub%d*' % i) +
 788                    final_src_char, dst_dir + final_dst_char])
 789       actual = set(str(u) for u in self._test_wildcard_iterator(
 790           os.path.join(dst_dir, 'src_sub%ddir' % i, '**')).IterAll(
 791               expand_top_level_buckets=True))
 792       expected = set([suri(dst_dir, 'src_sub%ddir' % i, 'foo')])
 793       self.assertEqual(expected, actual)
 794
 795   def testCopyingBucketSubDirToDirFailsWithoutMinusR(self):
 796     """Tests for failure when attempting bucket subdir copy without -R."""
 797     src_bucket_uri = self.CreateBucket(test_objects=['src_subdir/obj'])
 798     dst_dir = self.CreateTempDir()
 799     try:
 800       self.RunCommand(
 801           'cp', [suri(src_bucket_uri, 'src_subdir'), dst_dir])
 802       self.fail('Did not get expected CommandException')
 803     except CommandException, e:
 804       self.assertIn('No URLs matched', e.reason)
 805
 806   def testCopyingBucketSubDirToBucketSubDir(self):
 807     """Tests copying a bucket subdir to another bucket subdir."""
 808     src_bucket_uri = self.CreateBucket(
 809         test_objects=['src_subdir_%d/obj' % i for i in range(4)])
 810     dst_bucket_uri = self.CreateBucket(
 811         test_objects=['dst_subdir_%d/obj2' % i for i in range(4)])
 812     # Test with and without final slash on dest subdir.
 813     for i, (final_src_char, final_dst_char) in enumerate((
 814         ('', ''), ('', '/'), ('/', ''), ('/', '/'))):
 815       self.RunCommand(
 816           'cp', ['-R',
 817                  suri(src_bucket_uri, 'src_subdir_%d' % i) + final_src_char,
 818                  suri(dst_bucket_uri, 'dst_subdir_%d' % i) + final_dst_char])
 819       actual = set(str(u) for u in self._test_wildcard_iterator(
 820           suri(dst_bucket_uri, 'dst_subdir_%d' % i, '**')).IterAll(
 821               expand_top_level_buckets=True))
 822       expected = set([suri(dst_bucket_uri, 'dst_subdir_%d' % i,
 823                            'src_subdir_%d' % i, 'obj'),
 824                       suri(dst_bucket_uri, 'dst_subdir_%d' % i, 'obj2')])
 825       self.assertEqual(expected, actual)
 826
 827   def testCopyingBucketSubDirToBucketSubDirWithNested(self):
 828     """Tests copying a bucket subdir to another bucket subdir with nesting."""
 829     src_bucket_uri = self.CreateBucket(
 830         test_objects=['src_subdir_%d/obj' % i for i in range(4)] +
 831         ['src_subdir_%d/nested/obj' % i for i in range(4)])
 832     dst_bucket_uri = self.CreateBucket(
 833         test_objects=['dst_subdir_%d/obj2' % i for i in range(4)])
 834     # Test with and without final slash on dest subdir.
 835     for i, (final_src_char, final_dst_char) in enumerate((
 836         ('', ''), ('', '/'), ('/', ''), ('/', '/'))):
 837       self.RunCommand(
 838           'cp', ['-R',
 839                  suri(src_bucket_uri, 'src_subdir_%d' % i) + final_src_char,
 840                  suri(dst_bucket_uri, 'dst_subdir_%d' % i) + final_dst_char])
 841       actual = set(str(u) for u in self._test_wildcard_iterator(
 842           suri(dst_bucket_uri, 'dst_subdir_%d' % i, '**')).IterAll(
 843               expand_top_level_buckets=True))
 844       expected = set([suri(dst_bucket_uri, 'dst_subdir_%d' % i,
 845                            'src_subdir_%d' % i, 'obj'),
 846                       suri(dst_bucket_uri, 'dst_subdir_%d' % i,
 847                            'src_subdir_%d' % i, 'nested', 'obj'),
 848                       suri(dst_bucket_uri, 'dst_subdir_%d' % i, 'obj2')])
 849       self.assertEqual(expected, actual)
 850
 851   def testMovingBucketSubDirToExistingBucketSubDir(self):
 852     """Tests moving a bucket subdir to a existing bucket subdir."""
 853     src_objs = ['foo']
 854     for i in range(4):
 855       src_objs.extend(['src_subdir%d/foo2' % i, 'src_subdir%d/nested/foo3' % i])
 856     src_bucket_uri = self.CreateBucket(test_objects=src_objs)
 857     dst_bucket_uri = self.CreateBucket(
 858         test_objects=['dst_subdir%d/existing' % i for i in range(4)])
 859     # Test with and without final slash on dest subdir.
 860     for i, (final_src_char, final_dst_char) in enumerate((
 861         ('', ''), ('', '/'), ('/', ''), ('/', '/'))):
 862       self.RunCommand(
 863           'mv', [suri(src_bucket_uri, 'src_subdir%d' % i) + final_src_char,
 864                  suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_dst_char])
 865
 866     actual = set(str(u) for u in self._test_wildcard_iterator(
 867         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 868     expected = set()
 869     for i in range(4):
 870       expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'existing'))
 871       expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'src_subdir%d' %i,
 872                         'foo2'))
 873       expected.add(suri(dst_bucket_uri, 'dst_subdir%d' % i, 'src_subdir%d' %i,
 874                         'nested', 'foo3'))
 875     self.assertEqual(expected, actual)
 876
 877   def testCopyingObjectToBucketSubDir(self):
 878     """Tests copying an object to a bucket subdir."""
 879     src_bucket_uri = self.CreateBucket(test_objects=['obj0'])
 880     dst_bucket_uri = self.CreateBucket(test_objects=['dir0/existing',
 881                                                      'dir1/existing'])
 882     # Test with and without final slash on dest subdir.
 883     for i, final_dst_char in enumerate(('', '/')):
 884       self.RunCommand('cp', [
 885           suri(src_bucket_uri, 'obj0'),
 886           suri(dst_bucket_uri, 'dir%d' % i) + final_dst_char])
 887       actual = set(str(u) for u in self._test_wildcard_iterator(
 888           suri(dst_bucket_uri, 'dir%d' % i, '**')).IterAll(
 889               expand_top_level_buckets=True))
 890       expected = set([suri(dst_bucket_uri, 'dir%d' % i, 'obj0'),
 891                       suri(dst_bucket_uri, 'dir%d' % i, 'existing')])
 892       self.assertEqual(expected, actual)
 893
 894   # @PerformsFileToObjectUpload
 895   def testCopyingWildcardedFilesToBucketSubDir(self):
 896     """Tests copying wildcarded files to a bucket subdir."""
 897     dst_bucket_uri = self.CreateBucket(test_objects=['subdir0/existing',
 898                                                      'subdir1/existing'])
 899     src_dir = self.CreateTempDir(test_files=['f0', 'f1', 'f2'])
 900     # Test with and without final slash on dest subdir.
 901     for i, final_dst_char in enumerate(('', '/')):
 902       self.RunCommand(
 903           'cp', [os.path.join(src_dir, 'f?'),
 904                  suri(dst_bucket_uri, 'subdir%d' % i) + final_dst_char])
 905       actual = set(str(u) for u in self._test_wildcard_iterator(
 906           suri(dst_bucket_uri, 'subdir%d' % i, '**')).IterAll(
 907               expand_top_level_buckets=True))
 908       expected = set([suri(dst_bucket_uri, 'subdir%d' % i, 'existing'),
 909                       suri(dst_bucket_uri, 'subdir%d' % i, 'f0'),
 910                       suri(dst_bucket_uri, 'subdir%d' % i, 'f1'),
 911                       suri(dst_bucket_uri, 'subdir%d' % i, 'f2')])
 912       self.assertEqual(expected, actual)
 913
 914   # @PerformsFileToObjectUpload
 915   def testCopyingOneNestedFileToBucketSubDir(self):
 916     """Tests copying one nested file to a bucket subdir."""
 917     dst_bucket_uri = self.CreateBucket(test_objects=['d0/placeholder',
 918                                                      'd1/placeholder'])
 919     src_dir = self.CreateTempDir(test_files=[('d3', 'd4', 'nested', 'f1')])
 920     # Test with and without final slash on dest subdir.
 921     for i, final_dst_char in enumerate(('', '/')):
 922       self.RunCommand('cp', ['-r', suri(src_dir, 'd3'),
 923                              suri(dst_bucket_uri, 'd%d' % i) + final_dst_char])
 924       actual = set(str(u) for u in self._test_wildcard_iterator(
 925           suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 926     expected = set([
 927         suri(dst_bucket_uri, 'd0', 'placeholder'),
 928         suri(dst_bucket_uri, 'd1', 'placeholder'),
 929         suri(dst_bucket_uri, 'd0', 'd3', 'd4', 'nested', 'f1'),
 930         suri(dst_bucket_uri, 'd1', 'd3', 'd4', 'nested', 'f1')])
 931     self.assertEqual(expected, actual)
 932
 933   def testMovingWildcardedFilesToNonExistentBucketSubDir(self):
 934     """Tests moving files to a non-existent bucket subdir."""
 935     # This tests for how we allow users to do something like:
 936     #   gsutil cp *.txt gs://bucket/dir
 937     # where *.txt matches more than 1 file and gs://bucket/dir
 938     # doesn't exist as a subdir.
 939     #
 940     src_bucket_uri = self.CreateBucket(test_objects=[
 941         'f0f0', 'f0f1', 'f1f0', 'f1f1'])
 942     dst_bucket_uri = self.CreateBucket(test_objects=[
 943         'dst_subdir0/existing_obj', 'dst_subdir1/existing_obj'])
 944     # Test with and without final slash on dest subdir.
 945     for i, final_dst_char in enumerate(('', '/')):
 946       # Copy some files into place in dst bucket.
 947       self.RunCommand(
 948           'cp', [suri(src_bucket_uri, 'f%df*' % i),
 949                  suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_dst_char])
 950       # Now do the move test.
 951       self.RunCommand(
 952           'mv', [suri(src_bucket_uri, 'f%d*' % i),
 953                  suri(dst_bucket_uri, 'nonexisting%d' % i) + final_dst_char])
 954
 955     actual = set(str(u) for u in self._test_wildcard_iterator(
 956         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 957     expected = set([
 958         suri(dst_bucket_uri, 'dst_subdir0', 'existing_obj'),
 959         suri(dst_bucket_uri, 'dst_subdir0', 'f0f0'),
 960         suri(dst_bucket_uri, 'dst_subdir0', 'f0f1'),
 961         suri(dst_bucket_uri, 'nonexisting0', 'f0f0'),
 962         suri(dst_bucket_uri, 'nonexisting0', 'f0f1'),
 963         suri(dst_bucket_uri, 'dst_subdir1', 'existing_obj'),
 964         suri(dst_bucket_uri, 'dst_subdir1', 'f1f0'),
 965         suri(dst_bucket_uri, 'dst_subdir1', 'f1f1'),
 966         suri(dst_bucket_uri, 'nonexisting1', 'f1f0'),
 967         suri(dst_bucket_uri, 'nonexisting1', 'f1f1')])
 968     self.assertEqual(expected, actual)
 969
 970   def testMovingObjectToBucketSubDir(self):
 971     """Tests moving an object to a bucket subdir."""
 972     src_bucket_uri = self.CreateBucket(test_objects=['obj0', 'obj1'])
 973     dst_bucket_uri = self.CreateBucket(test_objects=[
 974         'dst_subdir0/existing_obj', 'dst_subdir1/existing_obj'])
 975     # Test with and without final slash on dest subdir.
 976     for i, final_dst_char in enumerate(('', '/')):
 977       self.RunCommand(
 978           'mv', [suri(src_bucket_uri, 'obj%d' % i),
 979                  suri(dst_bucket_uri, 'dst_subdir%d' % i) + final_dst_char])
 980
 981     actual = set(str(u) for u in self._test_wildcard_iterator(
 982         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 983     expected = set([
 984         suri(dst_bucket_uri, 'dst_subdir0', 'existing_obj'),
 985         suri(dst_bucket_uri, 'dst_subdir0', 'obj0'),
 986         suri(dst_bucket_uri, 'dst_subdir1', 'existing_obj'),
 987         suri(dst_bucket_uri, 'dst_subdir1', 'obj1')])
 988     self.assertEqual(expected, actual)
 989
 990     actual = set(str(u) for u in self._test_wildcard_iterator(
 991         suri(src_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
 992     self.assertEqual(actual, set())
 993
 994   def testWildcardSrcSubDirMoveDisallowed(self):
 995     """Tests moving a bucket subdir specified by wildcard is disallowed."""
 996     src_bucket_uri = self.CreateBucket(test_objects=['dir/foo1'])
 997     dst_bucket_uri = self.CreateBucket(test_objects=['dir/foo2'])
 998     try:
 999       self.RunCommand(
1000           'mv', [suri(src_bucket_uri, 'dir*'), suri(dst_bucket_uri, 'dir')])
1001       self.fail('Did not get expected CommandException')
1002     except CommandException, e:
1003       self.assertIn('mv command disallows naming', e.reason)
1004
1005   def testMovingBucketSubDirToNonExistentBucketSubDir(self):
1006     """Tests moving a bucket subdir to a non-existent bucket subdir."""
1007     src_bucket = self.CreateBucket(test_objects=[
1008         'foo', 'src_subdir0/foo2', 'src_subdir0/nested/foo3',
1009         'src_subdir1/foo2', 'src_subdir1/nested/foo3'])
1010     dst_bucket = self.CreateBucket()
1011     # Test with and without final slash on dest subdir.
1012     for i, final_src_char in enumerate(('', '/')):
1013       self.RunCommand(
1014           'mv', [suri(src_bucket, 'src_subdir%d' % i) + final_src_char,
1015                  suri(dst_bucket, 'dst_subdir%d' % i)])
1016
1017     actual = set(str(u) for u in self._test_wildcard_iterator(
1018         suri(dst_bucket, '**')).IterAll(expand_top_level_buckets=True))
1019     # Unlike the case with copying, with mv we expect renaming to occur
1020     # at the level of the src subdir, vs appending that subdir beneath the
1021     # dst subdir like is done for copying.
1022     expected = set([suri(dst_bucket, 'dst_subdir0', 'foo2'),
1023                     suri(dst_bucket, 'dst_subdir1', 'foo2'),
1024                     suri(dst_bucket, 'dst_subdir0', 'nested', 'foo3'),
1025                     suri(dst_bucket, 'dst_subdir1', 'nested', 'foo3')])
1026     self.assertEqual(expected, actual)
1027
1028   def testRemovingBucketSubDir(self):
1029     """Tests removing a bucket subdir."""
1030     dst_bucket_uri = self.CreateBucket(test_objects=[
1031         'f0', 'dir0/f1', 'dir0/nested/f2', 'dir1/f1', 'dir1/nested/f2'])
1032     # Test with and without final slash on dest subdir.
1033     for i, final_src_char in enumerate(('', '/')):
1034       # Test removing bucket subdir.
1035       self.RunCommand(
1036           'rm', ['-R', suri(dst_bucket_uri, 'dir%d' % i) + final_src_char])
1037     actual = set(str(u) for u in self._test_wildcard_iterator(
1038         suri(dst_bucket_uri, '**')).IterAll(expand_top_level_buckets=True))
1039     expected = set([suri(dst_bucket_uri, 'f0')])
1040     self.assertEqual(expected, actual)
1041
1042   def testRecursiveRemoveObjsInBucket(self):
1043     """Tests removing all objects in bucket via rm -R gs://bucket."""
1044     bucket_uris = [
1045         self.CreateBucket(test_objects=['f0', 'dir/f1', 'dir/nested/f2']),
1046         self.CreateBucket(test_objects=['f0', 'dir/f1', 'dir/nested/f2'])]
1047     # Test with and without final slash on dest subdir.
1048     for i, final_src_char in enumerate(('', '/')):
1049       # Test removing all objects via rm -R.
1050       self.RunCommand('rm', ['-R', suri(bucket_uris[i]) + final_src_char])
1051       try:
1052         self.RunCommand('ls', [suri(bucket_uris[i])])
1053         # Ensure exception is raised.
1054         self.assertTrue(False)
1055       except NotFoundException, e:
1056         self.assertEqual(e.status, 404)
1057
1058   def testUnicodeArgs(self):
1059     """Tests that you can list an object with unicode characters."""
1060     object_name = u'フォ'
1061     bucket_uri = self.CreateBucket()
1062     self.CreateObject(bucket_uri=bucket_uri, object_name=object_name,
1063                       contents='foo')
1064     object_name_bytes = object_name.encode(UTF8)
1065     stdout = self.RunCommand('ls', [suri(bucket_uri, object_name_bytes)],
1066                              return_stdout=True)
1067     self.assertIn(object_name_bytes, stdout)
1068
1069   def testRecursiveListTrailingSlash(self):
1070     bucket_uri = self.CreateBucket()
1071     obj_uri = self.CreateObject(
1072         bucket_uri=bucket_uri, object_name='/', contents='foo')
1073     stdout = self.RunCommand('ls', ['-R', suri(bucket_uri)], return_stdout=True)
1074     # Note: The suri function normalizes the URI, so the double slash gets
1075     # removed.
1076     self.assertEqual(stdout.splitlines(), [suri(obj_uri) + '/:',
1077                                            suri(obj_uri) + '/'])
1078
1079   def FinalObjNameComponent(self, uri):
1080     """For gs://bucket/abc/def/ghi returns ghi."""
1081     return uri.uri.rpartition('/')[-1]
1082
1083   def testFileContainingColon(self):
1084     url_str = 'abc:def'
1085     url = StorageUrlFromString(url_str)
1086     self.assertEqual('file', url.scheme)
1087     self.assertEqual('file://%s' % url_str, url.url_string)
1088
1089
1090 # TODO: These should all be moved to their own test_*.py testing files.
1091 class GsUtilCommandTests(testcase.GsUtilUnitTestCase):
1092   """Basic sanity check tests to make sure commands run."""
1093
1094   def testDisableLoggingCommandRuns(self):
1095     """Test that the 'logging set off' command basically runs."""
1096     src_bucket_uri = self.CreateBucket()
1097     self.RunCommand('logging', ['set', 'off', suri(src_bucket_uri)])
1098
1099   def testEnableLoggingCommandRuns(self):
1100     """Test that the 'logging set on' command basically runs."""
1101     src_bucket_uri = self.CreateBucket()
1102     self.RunCommand('logging', ['set', 'on', '-b', 'gs://log_bucket',
1103                                 suri(src_bucket_uri)])
1104
1105   def testHelpCommandDoesntRaise(self):
1106     """Test that the help command doesn't raise (sanity checks all help)."""
1107     # Unset PAGER if defined, so help output paginating into $PAGER doesn't
1108     # cause test to pause.
1109     if 'PAGER' in os.environ:
1110       del os.environ['PAGER']
1111     self.RunCommand('help', [])
1112
1113   def testCatCommandRuns(self):
1114     """Test that the cat command basically runs."""
1115     src_uri = self.CreateObject(contents='foo')
1116     stdout = self.RunCommand('cat', [suri(src_uri)], return_stdout=True)
1117     self.assertEqual(stdout, 'foo')
1118
1119   def testGetLoggingCommandRuns(self):
1120     """Test that the 'logging get' command basically runs."""
1121     src_bucket_uri = self.CreateBucket()
1122     self.RunCommand('logging', ['get', suri(src_bucket_uri)])
1123
1124   def testMakeBucketsCommand(self):
1125     """Test mb on existing bucket."""
1126     dst_bucket_uri = self.CreateBucket()
1127     try:
1128       self.RunCommand('mb', [suri(dst_bucket_uri)])
1129       self.fail('Did not get expected StorageCreateError')
1130     except ServiceException, e:
1131       self.assertEqual(e.status, 409)
1132
1133   def testRemoveBucketsCommand(self):
1134     """Test rb on non-existent bucket."""
1135     dst_bucket_uri = self.CreateBucket()
1136     try:
1137       self.RunCommand(
1138           'rb', ['gs://no_exist_%s' % dst_bucket_uri.bucket_name])
1139       self.fail('Did not get expected NotFoundException')
1140     except NotFoundException, e:
1141       self.assertEqual(e.status, 404)
1142
1143   def testRemoveObjsCommand(self):
1144     """Test rm command on non-existent object."""
1145     dst_bucket_uri = self.CreateBucket()
1146     try:
1147       self.RunCommand('rm', [suri(dst_bucket_uri, 'non_existent')])
1148       self.fail('Did not get expected CommandException')
1149     except CommandException, e:
1150       self.assertIn('No URLs matched', e.reason)
1151
1152   # Now that gsutil ver computes a checksum it adds 1-3 seconds to test run
1153   # time (for in memory mocked tests that otherwise take ~ 0.1 seconds). Since
1154   # it provides very little test value, we're leaving this test commented out.
1155   # def testVerCommmandRuns(self):
1156   #   """Test that the Ver command basically runs"""
1157   #   self.RunCommand('ver', [])