native_client_sdk/src/build_tools/sdk_tools/command/update.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import hashlib
   6 import copy
   7 import logging
   8 import os
   9 import subprocess
  10 import sys
  11 import urlparse
  12 import urllib2
  13
  14 import command_common
  15 import download
  16 from sdk_update_common import Error
  17 import sdk_update_common
  18
  19 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
  20 PARENT_DIR = os.path.dirname(SCRIPT_DIR)
  21 sys.path.append(PARENT_DIR)
  22 try:
  23   import cygtar
  24 except ImportError:
  25   # Try to find this in the Chromium repo.
  26   CHROME_SRC_DIR = os.path.abspath(
  27       os.path.join(PARENT_DIR, '..', '..', '..', '..'))
  28   sys.path.append(os.path.join(CHROME_SRC_DIR, 'native_client', 'build'))
  29   import cygtar
  30
  31
  32 RECOMMENDED = 'recommended'
  33 SDK_TOOLS = 'sdk_tools'
  34 HTTP_CONTENT_LENGTH = 'Content-Length'  # HTTP Header field for content length
  35 DEFAULT_CACHE_SIZE = 512 * 1024 * 1024  # 1/2 Gb cache by default
  36
  37
  38 class UpdateDelegate(object):
  39   def BundleDirectoryExists(self, bundle_name):
  40     raise NotImplementedError()
  41
  42   def DownloadToFile(self, url, dest_filename):
  43     raise NotImplementedError()
  44
  45   def ExtractArchives(self, archives, extract_dir, rename_from_dir,
  46                       rename_to_dir):
  47     raise NotImplementedError()
  48
  49
  50 class RealUpdateDelegate(UpdateDelegate):
  51   def __init__(self, user_data_dir, install_dir, cfg):
  52     UpdateDelegate.__init__(self)
  53     self.archive_cache = os.path.join(user_data_dir, 'archives')
  54     self.install_dir = install_dir
  55     self.cache_max = getattr(cfg, 'cache_max', DEFAULT_CACHE_SIZE)
  56
  57   def BundleDirectoryExists(self, bundle_name):
  58     bundle_path = os.path.join(self.install_dir, bundle_name)
  59     return os.path.isdir(bundle_path)
  60
  61   def VerifyDownload(self, filename, archive):
  62     """Verify that a local filename in the cache matches the given
  63     online archive.
  64
  65     Returns True if both size and sha1 match, False otherwise.
  66     """
  67     filename = os.path.join(self.archive_cache, filename)
  68     if not os.path.exists(filename):
  69       logging.info('File does not exist: %s.' % filename)
  70       return False
  71     size = os.path.getsize(filename)
  72     if size != archive.size:
  73       logging.info('File size does not match (%d vs %d): %s.' % (size,
  74           archive.size, filename))
  75       return False
  76     sha1_hash = hashlib.sha1()
  77     with open(filename) as f:
  78       sha1_hash.update(f.read())
  79     if sha1_hash.hexdigest() != archive.GetChecksum():
  80       logging.info('File hash does not match: %s.' % filename)
  81       return False
  82     return True
  83
  84   def BytesUsedInCache(self):
  85     """Determine number of bytes currently be in local archive cache."""
  86     total = 0
  87     for root, _, files in os.walk(self.archive_cache):
  88       for filename in files:
  89         total += os.path.getsize(os.path.join(root, filename))
  90     return total
  91
  92   def CleanupCache(self):
  93     """Remove archives from the local filesystem cache until the
  94     total size is below cache_max.
  95
  96     This is done my deleting the oldest archive files until the
  97     condition is satisfied.  If cache_max is zero then the entire
  98     cache will be removed.
  99     """
 100     used = self.BytesUsedInCache()
 101     logging.info('Cache usage: %d / %d' % (used, self.cache_max))
 102     if used <= self.cache_max:
 103       return
 104     clean_bytes = used - self.cache_max
 105
 106     logging.info('Clearing %d bytes in archive cache' % clean_bytes)
 107     file_timestamps = []
 108     for root, _, files in os.walk(self.archive_cache):
 109       for filename in files:
 110         fullname = os.path.join(root, filename)
 111         file_timestamps.append((os.path.getmtime(fullname), fullname))
 112
 113     file_timestamps.sort()
 114     while clean_bytes > 0:
 115       assert(file_timestamps)
 116       filename_to_remove = file_timestamps[0][1]
 117       clean_bytes -= os.path.getsize(filename_to_remove)
 118       logging.info('Removing from cache: %s' % filename_to_remove)
 119       os.remove(filename_to_remove)
 120       # Also remove resulting empty parent directory structure
 121       while True:
 122         filename_to_remove = os.path.dirname(filename_to_remove)
 123         if not os.listdir(filename_to_remove):
 124           os.rmdir(filename_to_remove)
 125         else:
 126           break
 127       file_timestamps = file_timestamps[1:]
 128
 129   def DownloadToFile(self, url, dest_filename):
 130     dest_path = os.path.join(self.archive_cache, dest_filename)
 131     sdk_update_common.MakeDirs(os.path.dirname(dest_path))
 132
 133     out_stream = None
 134     url_stream = None
 135     try:
 136       out_stream = open(dest_path, 'wb')
 137       url_stream = download.UrlOpen(url)
 138       content_length = int(url_stream.info()[HTTP_CONTENT_LENGTH])
 139       progress = download.MakeProgressFunction(content_length)
 140       sha1, size = download.DownloadAndComputeHash(url_stream, out_stream,
 141                                                    progress)
 142       return sha1, size
 143     except urllib2.URLError as e:
 144       raise Error('Unable to read from URL "%s".\n  %s' % (url, e))
 145     except IOError as e:
 146       raise Error('Unable to write to file "%s".\n  %s' % (dest_filename, e))
 147     finally:
 148       if url_stream:
 149         url_stream.close()
 150       if out_stream:
 151         out_stream.close()
 152
 153   def ExtractArchives(self, archives, extract_dir, rename_from_dir,
 154                       rename_to_dir):
 155     tar_file = None
 156
 157     extract_path = os.path.join(self.install_dir, extract_dir)
 158     rename_from_path = os.path.join(self.install_dir, rename_from_dir)
 159     rename_to_path = os.path.join(self.install_dir, rename_to_dir)
 160
 161     # Extract to extract_dir, usually "<bundle name>_update".
 162     # This way if the extraction fails, we haven't blown away the old bundle
 163     # (if it exists).
 164     sdk_update_common.RemoveDir(extract_path)
 165     sdk_update_common.MakeDirs(extract_path)
 166     curpath = os.getcwd()
 167     tar_file = None
 168
 169     try:
 170       try:
 171         logging.info('Changing the directory to %s' % (extract_path,))
 172         os.chdir(extract_path)
 173       except Exception as e:
 174         raise Error('Unable to chdir into "%s".\n  %s' % (extract_path, e))
 175
 176       for i, archive in enumerate(archives):
 177         archive_path = os.path.join(self.archive_cache, archive)
 178
 179         if len(archives) > 1:
 180           print '(file %d/%d - "%s")' % (
 181              i + 1, len(archives), os.path.basename(archive_path))
 182         logging.info('Extracting to %s' % (extract_path,))
 183
 184         if sys.platform == 'win32':
 185           try:
 186             logging.info('Opening file %s (%d/%d).' % (archive_path, i + 1,
 187                 len(archives)))
 188             try:
 189               tar_file = cygtar.CygTar(archive_path, 'r', verbose=True)
 190             except Exception as e:
 191               raise Error("Can't open archive '%s'.\n  %s" % (archive_path, e))
 192
 193             tar_file.Extract()
 194           finally:
 195             if tar_file:
 196               tar_file.Close()
 197         else:
 198           try:
 199             subprocess.check_call(['tar', 'xf', archive_path])
 200           except subprocess.CalledProcessError:
 201             raise Error('Error extracting archive: %s' % archive_path)
 202
 203       logging.info('Changing the directory to %s' % (curpath,))
 204       os.chdir(curpath)
 205
 206       logging.info('Renaming %s->%s' % (rename_from_path, rename_to_path))
 207       sdk_update_common.RenameDir(rename_from_path, rename_to_path)
 208     finally:
 209       # Change the directory back so we can remove the update directory.
 210       os.chdir(curpath)
 211
 212       # Clean up the ..._update directory.
 213       try:
 214         sdk_update_common.RemoveDir(extract_path)
 215       except Exception as e:
 216         logging.error('Failed to remove directory \"%s\".  %s' % (
 217             extract_path, e))
 218
 219
 220 def Update(delegate, remote_manifest, local_manifest, bundle_names, force):
 221   valid_bundles = set([bundle.name for bundle in remote_manifest.GetBundles()])
 222   requested_bundles = _GetRequestedBundleNamesFromArgs(remote_manifest,
 223                                                        bundle_names)
 224   invalid_bundles = requested_bundles - valid_bundles
 225   if invalid_bundles:
 226     logging.warn('Ignoring unknown bundle(s): %s' % (
 227         ', '.join(invalid_bundles)))
 228     requested_bundles -= invalid_bundles
 229
 230   if SDK_TOOLS in requested_bundles:
 231     logging.warn('Updating sdk_tools happens automatically. '
 232                  'Ignoring manual update request.')
 233     requested_bundles.discard(SDK_TOOLS)
 234
 235   if requested_bundles:
 236     for bundle_name in requested_bundles:
 237       logging.info('Trying to update %s' % (bundle_name,))
 238       UpdateBundleIfNeeded(delegate, remote_manifest, local_manifest,
 239           bundle_name, force)
 240   else:
 241     logging.warn('No bundles to update.')
 242
 243
 244 def Reinstall(delegate, local_manifest, bundle_names):
 245   valid_bundles, invalid_bundles = \
 246       command_common.GetValidBundles(local_manifest, bundle_names)
 247   if invalid_bundles:
 248     logging.warn('Unknown bundle(s): %s\n' % (', '.join(invalid_bundles)))
 249
 250   if not valid_bundles:
 251     logging.warn('No bundles to reinstall.')
 252     return
 253
 254   for bundle_name in valid_bundles:
 255     bundle = copy.deepcopy(local_manifest.GetBundle(bundle_name))
 256
 257     # HACK(binji): There was a bug where we'd merge the bundles from the old
 258     # archive and the new archive when updating. As a result, some users may
 259     # have a cache manifest that contains duplicate archives. Remove all
 260     # archives with the same basename except for the most recent.
 261     # Because the archives are added to a list, we know the most recent is at
 262     # the end.
 263     archives = {}
 264     for archive in bundle.GetArchives():
 265       url = archive.url
 266       path = urlparse.urlparse(url)[2]
 267       basename = os.path.basename(path)
 268       archives[basename] = archive
 269
 270     # Update the bundle with these new archives.
 271     bundle.RemoveAllArchives()
 272     for _, archive in archives.iteritems():
 273       bundle.AddArchive(archive)
 274
 275     _UpdateBundle(delegate, bundle, local_manifest)
 276
 277
 278 def UpdateBundleIfNeeded(delegate, remote_manifest, local_manifest,
 279                          bundle_name, force):
 280   bundle = remote_manifest.GetBundle(bundle_name)
 281   if bundle:
 282     if _BundleNeedsUpdate(delegate, local_manifest, bundle):
 283       # TODO(binji): It would be nicer to detect whether the user has any
 284       # modifications to the bundle. If not, we could update with impunity.
 285       if not force and delegate.BundleDirectoryExists(bundle_name):
 286         print ('%s already exists, but has an update available.\n'
 287             'Run update with the --force option to overwrite the '
 288             'existing directory.\nWarning: This will overwrite any '
 289             'modifications you have made within this directory.'
 290             % (bundle_name,))
 291         return
 292
 293       _UpdateBundle(delegate, bundle, local_manifest)
 294     else:
 295       print '%s is already up-to-date.' % (bundle.name,)
 296   else:
 297     logging.error('Bundle %s does not exist.' % (bundle_name,))
 298
 299
 300 def _GetRequestedBundleNamesFromArgs(remote_manifest, requested_bundles):
 301   requested_bundles = set(requested_bundles)
 302   if RECOMMENDED in requested_bundles:
 303     requested_bundles.discard(RECOMMENDED)
 304     requested_bundles |= set(_GetRecommendedBundleNames(remote_manifest))
 305
 306   return requested_bundles
 307
 308
 309 def _GetRecommendedBundleNames(remote_manifest):
 310   result = []
 311   for bundle in remote_manifest.GetBundles():
 312     if bundle.recommended == 'yes' and bundle.name != SDK_TOOLS:
 313       result.append(bundle.name)
 314   return result
 315
 316
 317 def _BundleNeedsUpdate(delegate, local_manifest, bundle):
 318   # Always update the bundle if the directory doesn't exist;
 319   # the user may have deleted it.
 320   if not delegate.BundleDirectoryExists(bundle.name):
 321     return True
 322
 323   return local_manifest.BundleNeedsUpdate(bundle)
 324
 325
 326 def _UpdateBundle(delegate, bundle, local_manifest):
 327   archives = bundle.GetHostOSArchives()
 328   if not archives:
 329     logging.warn('Bundle %s does not exist for this platform.' % (bundle.name,))
 330     return
 331
 332   archive_filenames = []
 333
 334   shown_banner = False
 335   for i, archive in enumerate(archives):
 336     archive_filename = _GetFilenameFromURL(archive.url)
 337     archive_filename = os.path.join(bundle.name, archive_filename)
 338
 339     if not delegate.VerifyDownload(archive_filename, archive):
 340       if not shown_banner:
 341         shown_banner = True
 342         print 'Downloading bundle %s' % (bundle.name,)
 343       if len(archives) > 1:
 344         print '(file %d/%d - "%s")' % (
 345             i + 1, len(archives), os.path.basename(archive.url))
 346       sha1, size = delegate.DownloadToFile(archive.url, archive_filename)
 347       _ValidateArchive(archive, sha1, size)
 348
 349     archive_filenames.append(archive_filename)
 350
 351   print 'Updating bundle %s to version %s, revision %s' % (
 352       bundle.name, bundle.version, bundle.revision)
 353   extract_dir = bundle.name + '_update'
 354
 355   repath_dir = bundle.get('repath', None)
 356   if repath_dir:
 357     # If repath is specified:
 358     # The files are extracted to nacl_sdk/<bundle.name>_update/<repath>/...
 359     # The destination directory is nacl_sdk/<bundle.name>/...
 360     rename_from_dir = os.path.join(extract_dir, repath_dir)
 361   else:
 362     # If no repath is specified:
 363     # The files are extracted to nacl_sdk/<bundle.name>_update/...
 364     # The destination directory is nacl_sdk/<bundle.name>/...
 365     rename_from_dir = extract_dir
 366
 367   rename_to_dir = bundle.name
 368
 369   delegate.ExtractArchives(archive_filenames, extract_dir, rename_from_dir,
 370                            rename_to_dir)
 371
 372   logging.info('Updating local manifest to include bundle %s' % (bundle.name))
 373   local_manifest.RemoveBundle(bundle.name)
 374   local_manifest.SetBundle(bundle)
 375   delegate.CleanupCache()
 376
 377
 378 def _GetFilenameFromURL(url):
 379   path = urlparse.urlparse(url)[2]
 380   return os.path.basename(path)
 381
 382
 383 def _ValidateArchive(archive, actual_sha1, actual_size):
 384   if actual_size != archive.size:
 385     raise Error('Size mismatch on "%s".  Expected %s but got %s bytes' % (
 386         archive.url, archive.size, actual_size))
 387   if actual_sha1 != archive.GetChecksum():
 388     raise Error('SHA1 checksum mismatch on "%s".  Expected %s but got %s' % (
 389         archive.url, archive.GetChecksum(), actual_sha1))