native_client_sdk/src/build_tools/tests/sdktools_commands_test.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import os
   7 import sys
   8 import re
   9 import tarfile
  10 import tempfile
  11 import unittest
  12 from sdktools_test import SdkToolsTestCase
  13
  14 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
  15 BUILD_TOOLS_DIR = os.path.dirname(SCRIPT_DIR)
  16 TOOLS_DIR = os.path.join(os.path.dirname(BUILD_TOOLS_DIR), 'tools')
  17
  18 sys.path.extend([BUILD_TOOLS_DIR, TOOLS_DIR])
  19 import manifest_util
  20 import oshelpers
  21
  22
  23 class TestCommands(SdkToolsTestCase):
  24   def setUp(self):
  25     self.SetupDefault()
  26
  27   def _AddDummyBundle(self, manifest, bundle_name):
  28     bundle = manifest_util.Bundle(bundle_name)
  29     bundle.revision = 1337
  30     bundle.version = 23
  31     bundle.description = bundle_name
  32     bundle.stability = 'beta'
  33     bundle.recommended = 'no'
  34     bundle.repath = bundle_name
  35     archive = self._MakeDummyArchive(bundle_name)
  36     bundle.AddArchive(archive)
  37     manifest.SetBundle(bundle)
  38
  39     # Need to get the bundle from the manifest -- it doesn't use the one we
  40     # gave it.
  41     return manifest.GetBundle(bundle_name)
  42
  43   def _MakeDummyArchive(self, bundle_name, tarname=None, filename='dummy.txt'):
  44     tarname = (tarname or bundle_name) + '.tar.bz2'
  45     temp_dir = tempfile.mkdtemp(prefix='archive')
  46     try:
  47       dummy_path = os.path.join(temp_dir, filename)
  48       with open(dummy_path, 'w') as stream:
  49         stream.write('Dummy stuff for %s' % bundle_name)
  50
  51       # Build the tarfile directly into the server's directory.
  52       tar_path = os.path.join(self.basedir, tarname)
  53       tarstream = tarfile.open(tar_path, 'w:bz2')
  54       try:
  55         tarstream.add(dummy_path, os.path.join(bundle_name, filename))
  56       finally:
  57         tarstream.close()
  58
  59       with open(tar_path, 'rb') as archive_stream:
  60         sha1, size = manifest_util.DownloadAndComputeHash(archive_stream)
  61
  62       archive = manifest_util.Archive(manifest_util.GetHostOS())
  63       archive.url = self.server.GetURL(os.path.basename(tar_path))
  64       archive.size = size
  65       archive.checksum = sha1
  66       return archive
  67     finally:
  68       oshelpers.Remove(['-rf', temp_dir])
  69
  70   def testInfoBasic(self):
  71     """The info command should display information about the given bundle."""
  72     self._WriteManifest()
  73     output = self._Run(['info', 'sdk_tools'])
  74     # Make sure basic information is there
  75     bundle = self.manifest.GetBundle('sdk_tools')
  76     archive = bundle.GetHostOSArchive();
  77     self.assertTrue(bundle.name in output)
  78     self.assertTrue(bundle.description in output)
  79     self.assertTrue(str(bundle.revision) in output)
  80     self.assertTrue(str(archive.size) in output)
  81     self.assertTrue(archive.checksum in output)
  82     self.assertTrue(bundle.stability in output)
  83
  84   def testInfoUnknownBundle(self):
  85     """The info command should notify the user of unknown bundles."""
  86     self._WriteManifest()
  87     bogus_bundle = 'foobar'
  88     output = self._Run(['info', bogus_bundle])
  89     self.assertTrue(re.search(r'[uU]nknown', output))
  90     self.assertTrue(bogus_bundle in output)
  91
  92   def testInfoMultipleBundles(self):
  93     """The info command should support listing multiple bundles."""
  94     self._AddDummyBundle(self.manifest, 'pepper_23')
  95     self._AddDummyBundle(self.manifest, 'pepper_24')
  96     self._WriteManifest()
  97     output = self._Run(['info', 'pepper_23', 'pepper_24'])
  98     self.assertTrue('pepper_23' in output)
  99     self.assertTrue('pepper_24' in output)
 100     self.assertFalse(re.search(r'[uU]nknown', output))
 101
 102   def testInfoMultipleArchives(self):
 103     """The info command should display multiple archives."""
 104     bundle = self._AddDummyBundle(self.manifest, 'pepper_26')
 105     archive2 = self._MakeDummyArchive('pepper_26', tarname='pepper_26_more',
 106                                       filename='dummy2.txt')
 107     archive2.host_os = 'all'
 108     bundle.AddArchive(archive2)
 109     self._WriteManifest()
 110     output = self._Run(['info', 'pepper_26'])
 111     self.assertTrue('pepper_26' in output)
 112     self.assertTrue('pepper_26_more' in output)
 113
 114   def testListBasic(self):
 115     """The list command should display basic information about remote
 116     bundles."""
 117     self._WriteManifest()
 118     output = self._Run(['list'])
 119     self.assertTrue(re.search('I.*?sdk_tools.*?stable', output, re.MULTILINE))
 120     # This line is important (it's used by the updater to determine if the
 121     # sdk_tools bundle needs to be updated), so let's be explicit.
 122     self.assertTrue('All installed bundles are up-to-date.')
 123
 124   def testListMultiple(self):
 125     """The list command should display multiple bundles."""
 126     self._AddDummyBundle(self.manifest, 'pepper_23')
 127     self._WriteManifest()
 128     output = self._Run(['list'])
 129     # Added pepper_23 to the remote manifest not the local manifest, so it
 130     # shouldn't be installed.
 131     self.assertTrue(re.search('^[^I]*pepper_23', output, re.MULTILINE))
 132     self.assertTrue('sdk_tools' in output)
 133
 134   def testListWithRevision(self):
 135     """The list command should display the revision, if desired."""
 136     self._AddDummyBundle(self.manifest, 'pepper_23')
 137     self._WriteManifest()
 138     output = self._Run(['list', '-r'])
 139     self.assertTrue(re.search('pepper_23.*?r1337', output))
 140
 141   def testListWithUpdatedRevision(self):
 142     """The list command should display when there is an update available."""
 143     p23bundle = self._AddDummyBundle(self.manifest, 'pepper_23')
 144     self._WriteCacheManifest(self.manifest)
 145     # Modify the remote manifest to have a newer revision.
 146     p23bundle.revision += 1
 147     self._WriteManifest()
 148     output = self._Run(['list', '-r'])
 149     # We should see a display like this:  I* pepper_23 (r1337 -> r1338)
 150     # The star indicates the bundle has an update.
 151     self.assertTrue(re.search('I\*\s+pepper_23.*?r1337.*?r1338', output))
 152
 153   def testListLocalVersionNotOnRemote(self):
 154     """The list command should tell the user if they have a bundle installed
 155     that doesn't exist in the remote manifest."""
 156     self._WriteManifest()
 157     p23bundle = self._AddDummyBundle(self.manifest, 'pepper_23')
 158     self._WriteCacheManifest(self.manifest)
 159     output = self._Run(['list', '-r'])
 160     message = 'Bundles installed locally that are not available remotely:'
 161     message_loc = output.find(message)
 162     self.assertNotEqual(message_loc, -1)
 163     # Make sure pepper_23 is listed after the message above.
 164     self.assertTrue('pepper_23' in output[message_loc:])
 165
 166   def testSources(self):
 167     """The sources command should allow adding/listing/removing of sources.
 168     When a source is added, it will provide an additional set of bundles."""
 169     other_manifest = manifest_util.SDKManifest()
 170     self._AddDummyBundle(other_manifest, 'naclmono_23')
 171     with open(os.path.join(self.basedir, 'source.json'), 'w') as stream:
 172       stream.write(other_manifest.GetDataAsString())
 173
 174     source_json_url = self.server.GetURL('source.json')
 175     self._WriteManifest()
 176     output = self._Run(['sources', '--list'])
 177     self.assertTrue('No external sources installed.' in output)
 178     output = self._Run(['sources', '--add', source_json_url])
 179     output = self._Run(['sources', '--list'])
 180     self.assertTrue(source_json_url in output)
 181
 182     # Should be able to get info about that bundle.
 183     output = self._Run(['info', 'naclmono_23'])
 184     self.assertTrue('Unknown bundle' not in output)
 185
 186     self._Run(['sources', '--remove', source_json_url])
 187     output = self._Run(['sources', '--list'])
 188     self.assertTrue('No external sources installed.' in output)
 189
 190   def testUpdateBasic(self):
 191     """The update command should install the contents of a bundle to the SDK."""
 192     self._AddDummyBundle(self.manifest, 'pepper_23')
 193     self._WriteManifest()
 194     self._Run(['update', 'pepper_23'])
 195     self.assertTrue(os.path.exists(
 196         os.path.join(self.basedir, 'nacl_sdk', 'pepper_23', 'dummy.txt')))
 197
 198   def testUpdateInCacheButDirectoryRemoved(self):
 199     """The update command should update if the bundle directory does not exist,
 200     even if the bundle is already in the cache manifest."""
 201     self._AddDummyBundle(self.manifest, 'pepper_23')
 202     self._WriteCacheManifest(self.manifest)
 203     self._WriteManifest()
 204     self._Run(['update', 'pepper_23'])
 205     self.assertTrue(os.path.exists(
 206         os.path.join(self.basedir, 'nacl_sdk', 'pepper_23', 'dummy.txt')))
 207
 208   def testUpdateNoNewVersion(self):
 209     """The update command should do nothing if the bundle is already up-to-date.
 210     """
 211     self._AddDummyBundle(self.manifest, 'pepper_23')
 212     self._WriteManifest()
 213     self._Run(['update', 'pepper_23'])
 214     output = self._Run(['update', 'pepper_23'])
 215     self.assertTrue('is already up-to-date.' in output)
 216
 217   def testUpdateWithNewVersion(self):
 218     """The update command should update to a new version if it exists."""
 219     bundle = self._AddDummyBundle(self.manifest, 'pepper_23')
 220     self._WriteManifest()
 221     self._Run(['update', 'pepper_23'])
 222
 223     bundle.revision += 1
 224     self._WriteManifest()
 225     output = self._Run(['update', 'pepper_23'])
 226     self.assertTrue('already exists, but has an update available' in output)
 227
 228     # Now update using --force.
 229     output = self._Run(['update', 'pepper_23', '--force'])
 230     self.assertTrue('Updating bundle' in output)
 231
 232     cache_manifest = self._ReadCacheManifest()
 233     num_archives = len(cache_manifest.GetBundle('pepper_23').GetArchives())
 234     self.assertEqual(num_archives, 1)
 235
 236   def testUpdateUnknownBundles(self):
 237     """The update command should ignore unknown bundles and notify the user."""
 238     self._WriteManifest()
 239     output = self._Run(['update', 'foobar'])
 240     self.assertTrue('unknown bundle' in output)
 241
 242   def testUpdateRecommended(self):
 243     """The update command should update only recommended bundles when run
 244     without args.
 245     """
 246     bundle_25 = self._AddDummyBundle(self.manifest, 'pepper_25')
 247     bundle_25.recommended = 'no'
 248     bundle_26 = self._AddDummyBundle(self.manifest, 'pepper_26')
 249     bundle_26.recommended = 'yes'
 250
 251     self._WriteManifest()
 252     output = self._Run(['update'])
 253
 254     # Should not try to update sdk_tools (even though it is recommended)
 255     self.assertTrue('Ignoring manual update request.' not in output)
 256     self.assertFalse(os.path.exists(
 257         os.path.join(self.basedir, 'nacl_sdk', 'pepper_25')))
 258     self.assertTrue(os.path.exists(
 259         os.path.join(self.basedir, 'nacl_sdk', 'pepper_26', 'dummy.txt')))
 260
 261   def testUpdateCanary(self):
 262     """The update command should create the correct directory name for repath'd
 263     bundles.
 264     """
 265     bundle = self._AddDummyBundle(self.manifest, 'pepper_26')
 266     bundle.name = 'pepper_canary'
 267     self._WriteManifest()
 268     output = self._Run(['update', 'pepper_canary'])
 269     self.assertTrue(os.path.exists(
 270         os.path.join(self.basedir, 'nacl_sdk', 'pepper_canary', 'dummy.txt')))
 271
 272   def testUpdateMultiArchive(self):
 273     """The update command should include download/untar multiple archives
 274     specified in the bundle.
 275     """
 276     bundle = self._AddDummyBundle(self.manifest, 'pepper_26')
 277     archive2 = self._MakeDummyArchive('pepper_26', tarname='pepper_26_more',
 278                                       filename='dummy2.txt')
 279     archive2.host_os = 'all'
 280     bundle.AddArchive(archive2)
 281     self._WriteManifest()
 282     output = self._Run(['update', 'pepper_26'])
 283     self.assertTrue(os.path.exists(
 284         os.path.join(self.basedir, 'nacl_sdk', 'pepper_26', 'dummy.txt')))
 285     self.assertTrue(os.path.exists(
 286         os.path.join(self.basedir, 'nacl_sdk', 'pepper_26', 'dummy2.txt')))
 287
 288   def testUpdateBadSize(self):
 289     """If an archive has a bad size, print an error.
 290     """
 291     bundle = self._AddDummyBundle(self.manifest, 'pepper_26')
 292     archive = bundle.GetHostOSArchive();
 293     archive.size = -1
 294     self._WriteManifest()
 295     stdout = self._Run(['update', 'pepper_26'], expect_error=True)
 296     self.assertTrue('Size mismatch' in stdout)
 297
 298   def testUpdateBadSHA(self):
 299     """If an archive has a bad SHA, print an error.
 300     """
 301     bundle = self._AddDummyBundle(self.manifest, 'pepper_26')
 302     archive = bundle.GetHostOSArchive();
 303     archive.checksum = 0
 304     self._WriteManifest()
 305     stdout = self._Run(['update', 'pepper_26'], expect_error=True)
 306     self.assertTrue('SHA1 checksum mismatch' in stdout)
 307
 308   def testUninstall(self):
 309     """The uninstall command should remove the installed bundle, if it
 310     exists.
 311     """
 312     # First install the bundle.
 313     self._AddDummyBundle(self.manifest, 'pepper_23')
 314     self._WriteManifest()
 315     output = self._Run(['update', 'pepper_23'])
 316     self.assertTrue(os.path.exists(
 317         os.path.join(self.basedir, 'nacl_sdk', 'pepper_23', 'dummy.txt')))
 318
 319     # Now remove it.
 320     self._Run(['uninstall', 'pepper_23'])
 321     self.assertFalse(os.path.exists(
 322         os.path.join(self.basedir, 'nacl_sdk', 'pepper_23')))
 323
 324     # The bundle should not be marked as installed.
 325     output = self._Run(['list'])
 326     self.assertTrue(re.search('^[^I]*pepper_23', output, re.MULTILINE))
 327
 328   def testReinstall(self):
 329     """The reinstall command should remove, then install, the specified
 330     bundles.
 331     """
 332     # First install the bundle.
 333     self._AddDummyBundle(self.manifest, 'pepper_23')
 334     self._WriteManifest()
 335     output = self._Run(['update', 'pepper_23'])
 336     dummy_txt = os.path.join(self.basedir, 'nacl_sdk', 'pepper_23', 'dummy.txt')
 337     self.assertTrue(os.path.exists(dummy_txt))
 338     with open(dummy_txt) as f:
 339       self.assertEqual(f.read(), 'Dummy stuff for pepper_23')
 340
 341     # Change some files.
 342     foo_txt = os.path.join(self.basedir, 'nacl_sdk', 'pepper_23', 'foo.txt')
 343     with open(foo_txt, 'w') as f:
 344       f.write('Another dummy file. This one is not part of the bundle.')
 345     with open(dummy_txt, 'w') as f:
 346       f.write('changed dummy.txt')
 347
 348     # Reinstall the bundle.
 349     self._Run(['reinstall', 'pepper_23'])
 350
 351     self.assertFalse(os.path.exists(foo_txt))
 352     self.assertTrue(os.path.exists(dummy_txt))
 353     with open(dummy_txt) as f:
 354       self.assertEqual(f.read(), 'Dummy stuff for pepper_23')
 355
 356     cache_manifest = self._ReadCacheManifest()
 357     num_archives = len(cache_manifest.GetBundle('pepper_23').GetArchives())
 358     self.assertEqual(num_archives, 1)
 359
 360   def testReinstallWithDuplicatedArchives(self):
 361     """The reinstall command should only use the most recent archive if there
 362     are duplicated archives.
 363
 364     NOTE: There was a bug where the sdk_cache/naclsdk_manifest2.json file was
 365     duplicating archives from different revisions. Make sure that reinstall
 366     ignores old archives in the bundle.
 367     """
 368     # First install the bundle.
 369     self._AddDummyBundle(self.manifest, 'pepper_23')
 370     self._WriteManifest()
 371     self._Run(['update', 'pepper_23'])
 372
 373     manifest = self._ReadCacheManifest()
 374     bundle = manifest.GetBundle('pepper_23')
 375     self.assertEqual(len(bundle.GetArchives()), 1)
 376
 377     # Now add a bogus duplicate archive
 378     archive2 = self._MakeDummyArchive('pepper_23', tarname='pepper_23',
 379                                       filename='dummy2.txt')
 380     bundle.AddArchive(archive2)
 381     self._WriteCacheManifest(manifest)
 382
 383     output = self._Run(['reinstall', 'pepper_23'])
 384     # When updating just one file, there is no (file 1/2 - "...") output.
 385     self.assertFalse('file 1/' in output)
 386     # Should be using the last archive.
 387     self.assertFalse(os.path.exists(
 388         os.path.join(self.basedir, 'nacl_sdk', 'pepper_23', 'dummy.txt')))
 389     self.assertTrue(os.path.exists(
 390         os.path.join(self.basedir, 'nacl_sdk', 'pepper_23', 'dummy2.txt')))
 391
 392   def testReinstallDoesntUpdate(self):
 393     """The reinstall command should not update a bundle that has an update."""
 394     # First install the bundle.
 395     bundle = self._AddDummyBundle(self.manifest, 'pepper_23')
 396     self._WriteManifest()
 397     self._Run(['update', 'pepper_23'])
 398     dummy_txt = os.path.join(self.basedir, 'nacl_sdk', 'pepper_23', 'dummy.txt')
 399     self.assertTrue(os.path.exists(dummy_txt))
 400     with open(dummy_txt) as f:
 401       self.assertEqual(f.read(), 'Dummy stuff for pepper_23')
 402
 403     # Update the revision.
 404     bundle.revision += 1
 405     self._WriteManifest()
 406
 407     # Change the file.
 408     foo_txt = os.path.join(self.basedir, 'nacl_sdk', 'pepper_23', 'foo.txt')
 409     with open(dummy_txt, 'w') as f:
 410       f.write('changed dummy.txt')
 411
 412     # Reinstall.
 413     self._Run(['reinstall', 'pepper_23'])
 414
 415     # The data has been reinstalled.
 416     self.assertTrue(os.path.exists(dummy_txt))
 417     with open(dummy_txt) as f:
 418       self.assertEqual(f.read(), 'Dummy stuff for pepper_23')
 419
 420     # ... but the version hasn't been updated.
 421     output = self._Run(['list', '-r'])
 422     self.assertTrue(re.search('I\*\s+pepper_23.*?r1337.*?r1338', output))
 423
 424   def testArchiveCacheBasic(self):
 425     """Downloaded archives should be stored in the cache by default."""
 426     self._AddDummyBundle(self.manifest, 'pepper_23')
 427     self._WriteManifest()
 428     self._Run(['update', 'pepper_23'])
 429     archive_cache = os.path.join(self.cache_dir, 'archives')
 430     cache_contents = os.listdir(archive_cache)
 431     self.assertEqual(cache_contents, ['pepper_23'])
 432     cache_contents = os.listdir(os.path.join(archive_cache, 'pepper_23'))
 433     self.assertEqual(cache_contents, ['pepper_23.tar.bz2'])
 434
 435   def testArchiveCacheEviction(self):
 436     archive_cache = os.path.join(self.cache_dir, 'archives')
 437     self._AddDummyBundle(self.manifest, 'pepper_23')
 438     self._AddDummyBundle(self.manifest, 'pepper_22')
 439     self._WriteManifest()
 440
 441     # First install pepper_23
 442     self._Run(['update', 'pepper_23'])
 443     archive = os.path.join(archive_cache, 'pepper_23', 'pepper_23.tar.bz2')
 444     archive_size = os.path.getsize(archive)
 445
 446     # Set the mtime on the pepper_23 bundle to be a few seconds in the past.
 447     # This is needed so that the two bundles don't end up with the same
 448     # timestamp which can happen on systems that don't report sub-second
 449     # timestamps.
 450     atime = os.path.getatime(archive)
 451     mtime = os.path.getmtime(archive)
 452     os.utime(archive, (atime, mtime-10))
 453
 454     # Set cache limit to size of pepper archive * 1.5
 455     self._WriteConfig('{ "cache_max": %d }' % int(archive_size * 1.5))
 456
 457     # Now install pepper_22, which should cause pepper_23 to be evicted
 458     self._Run(['update', 'pepper_22'])
 459     cache_contents = os.listdir(archive_cache)
 460     self.assertEqual(cache_contents, ['pepper_22'])
 461
 462   def testArchiveCacheZero(self):
 463     """Archives should not be cached when cache_max is zero."""
 464     self._AddDummyBundle(self.manifest, 'pepper_23')
 465     self._WriteConfig('{ "cache_max": 0 }')
 466     self._AddDummyBundle(self.manifest, 'pepper_23')
 467     self._WriteManifest()
 468     self._Run(['update', 'pepper_23'])
 469     archive_cache = os.path.join(self.cache_dir, 'archives')
 470     # Archive folder should be completely remove by cache cleanup
 471     self.assertFalse(os.path.exists(archive_cache))
 472
 473 if __name__ == '__main__':
 474   unittest.main()