build/download_nacl_irt.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2011 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import hashlib
   7 import optparse
   8 import os
   9 import urllib2
  10 import sys
  11 import time
  12
  13
  14 # Print a dot every time this number of bytes is read.
  15 PROGRESS_SPACING = 128 * 1024
  16
  17
  18 def ReadFile(filename):
  19   fh = open(filename, 'r')
  20   try:
  21     return fh.read()
  22   finally:
  23     fh.close()
  24
  25
  26 def WriteFile(filename, data):
  27   fh = open(filename, 'w')
  28   try:
  29     fh.write(data)
  30   finally:
  31     fh.close()
  32
  33
  34 def HashFile(filename):
  35   hasher = hashlib.sha1()
  36   fh = open(filename, 'rb')
  37   try:
  38     while True:
  39       data = fh.read(4096)
  40       if len(data) == 0:
  41         break
  42       hasher.update(data)
  43   finally:
  44     fh.close()
  45   return hasher.hexdigest()
  46
  47
  48 def CopyStream(input_stream, output_stream):
  49   """Copies the contents of input_stream to output_stream.  Prints
  50   dots to indicate progress.
  51   """
  52   bytes_read = 0
  53   dots_printed = 0
  54   while True:
  55     data = input_stream.read(4096)
  56     if len(data) == 0:
  57       break
  58     output_stream.write(data)
  59     bytes_read += len(data)
  60     if bytes_read / PROGRESS_SPACING > dots_printed:
  61       sys.stdout.write('.')
  62       sys.stdout.flush()
  63       dots_printed += 1
  64
  65
  66 def RenameWithRetry(old_path, new_path):
  67   # Renames of files that have recently been closed are known to be
  68   # unreliable on Windows, because virus checkers like to keep the
  69   # file open for a little while longer.  This tends to happen more
  70   # for files that look like Windows executables, which does not apply
  71   # to our files, but we retry the rename here just in case.
  72   if sys.platform in ('win32', 'cygwin'):
  73     for i in range(5):
  74       try:
  75         if os.path.exists(new_path):
  76           os.remove(new_path)
  77         os.rename(old_path, new_path)
  78         return
  79       except Exception, exn:
  80         sys.stdout.write('Rename failed with %r.  Retrying...\n' % str(exn))
  81         sys.stdout.flush()
  82         time.sleep(1)
  83     raise Exception('Unabled to rename irt file')
  84   else:
  85     os.rename(old_path, new_path)
  86
  87
  88 def DownloadFile(dest_path, url):
  89   url_path = '%s.url' % dest_path
  90   temp_path = '%s.temp' % dest_path
  91   if os.path.exists(url_path) and ReadFile(url_path).strip() == url:
  92     # The URL matches that of the file we previously downloaded, so
  93     # there should be nothing to do.
  94     return
  95   sys.stdout.write('Downloading %r to %r\n' % (url, dest_path))
  96   output_fh = open(temp_path, 'wb')
  97   stream = urllib2.urlopen(url)
  98   CopyStream(stream, output_fh)
  99   output_fh.close()
 100   sys.stdout.write(' done\n')
 101   if os.path.exists(url_path):
 102     os.unlink(url_path)
 103   RenameWithRetry(temp_path, dest_path)
 104   WriteFile(url_path, url + '\n')
 105   stream.close()
 106
 107
 108 def DownloadFileWithRetry(dest_path, url):
 109   for i in range(5):
 110     try:
 111       DownloadFile(dest_path, url)
 112       break
 113     except urllib2.HTTPError, exn:
 114       if exn.getcode() == 404:
 115         raise
 116       sys.stdout.write('Download failed with error %r.  Retrying...\n'
 117                        % str(exn))
 118       sys.stdout.flush()
 119       time.sleep(1)
 120
 121
 122 def EvalDepsFile(path):
 123   scope = {'Var': lambda name: scope['vars'][name]}
 124   execfile(path, {}, scope)
 125   return scope
 126
 127
 128 def Main():
 129   parser = optparse.OptionParser()
 130   parser.add_option(
 131       '--base_url', dest='base_url',
 132       # For a view of this site that includes directory listings, see:
 133       # http://gsdview.appspot.com/nativeclient-archive2/
 134       # (The trailing slash is required.)
 135       default=('http://commondatastorage.googleapis.com/'
 136                'nativeclient-archive2/irt'),
 137       help='Base URL from which to download.')
 138   parser.add_option(
 139       '--nacl_revision', dest='nacl_revision',
 140       help='Download an IRT binary that was built from this '
 141         'SVN revision of Native Client.')
 142   parser.add_option(
 143       '--file_hash', dest='file_hashes', action='append', nargs=2, default=[],
 144       metavar='ARCH HASH',
 145       help='ARCH gives the name of the architecture (e.g. "x86_32") for '
 146         'which to download an IRT binary.  '
 147         'HASH gives the expected SHA1 hash of the file.')
 148   options, args = parser.parse_args()
 149   if len(args) != 0:
 150     parser.error('Unexpected arguments: %r' % args)
 151
 152   if options.nacl_revision is None and len(options.file_hashes) == 0:
 153     # The script must have been invoked directly with no arguments,
 154     # rather than being invoked by gclient.  In this case, read the
 155     # DEPS file ourselves rather than having gclient pass us values
 156     # from DEPS.
 157     deps_data = EvalDepsFile(os.path.join('src', 'DEPS'))
 158     options.nacl_revision = deps_data['vars']['nacl_revision']
 159     options.file_hashes = [
 160         ('x86_32', deps_data['vars']['nacl_irt_hash_x86_32']),
 161         ('x86_64', deps_data['vars']['nacl_irt_hash_x86_64']),
 162         ]
 163
 164   nacl_dir = os.path.join('src', 'native_client')
 165   if not os.path.exists(nacl_dir):
 166     # If "native_client" is not present, this might be because the
 167     # developer has put '"src/native_client": None' in their
 168     # '.gclient' file, because they don't want to build Chromium with
 169     # Native Client support.  So don't create 'src/native_client',
 170     # because that would interfere with checking it out from SVN
 171     # later.
 172     sys.stdout.write(
 173         'The directory %r does not exist: skipping downloading binaries '
 174         'for Native Client\'s IRT library\n' % nacl_dir)
 175     return
 176   if len(options.file_hashes) == 0:
 177     sys.stdout.write('No --file_hash arguments given: nothing to update\n')
 178
 179   new_deps = []
 180   for arch, expected_hash in options.file_hashes:
 181     url = '%s/r%s/irt_%s.nexe' % (options.base_url,
 182                                   options.nacl_revision,
 183                                   arch)
 184     dest_dir = os.path.join(nacl_dir, 'irt_binaries')
 185     if not os.path.exists(dest_dir):
 186       os.makedirs(dest_dir)
 187     dest_path = os.path.join(dest_dir, 'nacl_irt_%s.nexe' % arch)
 188     DownloadFileWithRetry(dest_path, url)
 189     downloaded_hash = HashFile(dest_path)
 190     if downloaded_hash != expected_hash:
 191       sys.stdout.write(
 192           'Hash mismatch: the file downloaded from URL %r had hash %r, '
 193           'but we expected %r\n' % (url, downloaded_hash, expected_hash))
 194       new_deps.append('  "nacl_irt_hash_%s": "%s",\n'
 195                       % (arch, downloaded_hash))
 196
 197   if len(new_deps) > 0:
 198     sys.stdout.write('\nIf you have changed nacl_revision, the DEPS file '
 199                      'probably needs to be updated with the following:\n%s\n'
 200                      % ''.join(new_deps))
 201     sys.exit(1)
 202
 203
 204 if __name__ == '__main__':
 205   Main()