native_client_sdk/src/build_tools/update_nacl_manifest.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Script that reads omahaproxy and gsutil to determine version of SDK to put
   7 in manifest.
   8 """
   9
  10 # pylint is convinced the email module is missing attributes
  11 # pylint: disable=E1101
  12
  13 import buildbot_common
  14 import csv
  15 import cStringIO
  16 import difflib
  17 import email
  18 import logging
  19 import logging.handlers
  20 import manifest_util
  21 import optparse
  22 import os
  23 import posixpath
  24 import re
  25 import smtplib
  26 import subprocess
  27 import sys
  28 import time
  29 import traceback
  30 import urllib2
  31
  32 MANIFEST_BASENAME = 'naclsdk_manifest2.json'
  33 SCRIPT_DIR = os.path.dirname(__file__)
  34 REPO_MANIFEST = os.path.join(SCRIPT_DIR, 'json', MANIFEST_BASENAME)
  35 GS_BUCKET_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/'
  36 GS_SDK_MANIFEST = GS_BUCKET_PATH + MANIFEST_BASENAME
  37 GS_SDK_MANIFEST_LOG = GS_BUCKET_PATH + MANIFEST_BASENAME + '.log'
  38 GS_MANIFEST_BACKUP_DIR = GS_BUCKET_PATH + 'manifest_backups/'
  39
  40 CANARY_BUNDLE_NAME = 'pepper_canary'
  41 BIONIC_CANARY_BUNDLE_NAME = 'bionic_canary'
  42 CANARY = 'canary'
  43 NACLPORTS_ARCHIVE_NAME = 'naclports.tar.bz2'
  44
  45
  46 logger = logging.getLogger(__name__)
  47
  48
  49 def SplitVersion(version_string):
  50   """Split a version string (e.g. "18.0.1025.163") into its components.
  51
  52   e.g.
  53   SplitVersion("trunk.123456") => ("trunk", "123456")
  54   SplitVersion("18.0.1025.163") => (18, 0, 1025, 163)
  55   """
  56   parts = version_string.split('.')
  57   if parts[0] == 'trunk':
  58     return (parts[0], int(parts[1]))
  59   return tuple([int(p) for p in parts])
  60
  61
  62 def GetMajorVersion(version_string):
  63   """Get the major version number from a version string (e.g. "18.0.1025.163").
  64
  65   e.g.
  66   GetMajorVersion("trunk.123456") => "trunk"
  67   GetMajorVersion("18.0.1025.163") => 18
  68   """
  69   return SplitVersion(version_string)[0]
  70
  71
  72 def CompareVersions(version1, version2):
  73   """Compare two version strings and return -1, 0, 1 (similar to cmp).
  74
  75   Versions can only be compared if they are both trunk versions, or neither is.
  76
  77   e.g.
  78   CompareVersions("trunk.123", "trunk.456") => -1
  79   CompareVersions("18.0.1025.163", "37.0.2054.3") => -1
  80   CompareVersions("trunk.123", "18.0.1025.163") => Error
  81
  82   """
  83   split1 = SplitVersion(version1)
  84   split2 = SplitVersion(version2)
  85   if split1[0] == split2[0]:
  86     return cmp(split1[1:], split2[1:])
  87
  88   if split1 == 'trunk' or split2 == 'trunk':
  89     raise Exception("Unable to compare versions %s and %s" % (
  90       version1, version2))
  91
  92   return cmp(split1, split2)
  93
  94
  95 def JoinVersion(version_tuple):
  96   """Create a string from a version tuple.
  97
  98   The tuple should be of the form (18, 0, 1025, 163).
  99   """
 100   assert len(version_tuple) == 4
 101   assert version_tuple[0] != 'trunk'
 102   return '.'.join(map(str, version_tuple))
 103
 104
 105 def GetTimestampManifestName():
 106   """Create a manifest name with a timestamp.
 107
 108   Returns:
 109     A manifest name with an embedded date. This should make it easier to roll
 110     back if necessary.
 111   """
 112   return time.strftime('naclsdk_manifest2.%Y_%m_%d_%H_%M_%S.json',
 113       time.gmtime())
 114
 115
 116 def GetPlatformArchiveName(platform):
 117   """Get the basename of an archive given a platform string.
 118
 119   Args:
 120     platform: One of ('win', 'mac', 'linux').
 121
 122   Returns:
 123     The basename of the sdk archive for that platform.
 124   """
 125   return 'naclsdk_%s.tar.bz2' % platform
 126
 127
 128 def GetBionicArchiveName():
 129   """Get the basename of an archive. Currently this is linux-only"""
 130   return 'naclsdk_bionic.tar.bz2'
 131
 132
 133 def GetCanonicalArchiveName(url):
 134   """Get the canonical name of an archive given its URL.
 135
 136   This will convert "naclsdk_linux.bz2" -> "naclsdk_linux.tar.bz2", and also
 137   remove everything but the filename of the URL.
 138
 139   This is used below to determine if an expected bundle is found in an version
 140   directory; the archives all have the same name, but may not exist for a given
 141   version.
 142
 143   Args:
 144     url: The url to parse.
 145
 146   Returns:
 147     The canonical name as described above.
 148   """
 149   name = posixpath.basename(url)
 150   match = re.match(r'naclsdk_(.*?)(?:\.tar)?\.bz2', name)
 151   if match:
 152     return 'naclsdk_%s.tar.bz2' % match.group(1)
 153
 154   return name
 155
 156
 157 class Delegate(object):
 158   """Delegate all external access; reading/writing to filesystem, gsutil etc."""
 159
 160   def GetRepoManifest(self):
 161     """Read the manifest file from the NaCl SDK repository.
 162
 163     This manifest is used as a template for the auto updater; only pepper
 164     bundles with no archives are considered for auto updating.
 165
 166     Returns:
 167       A manifest_util.SDKManifest object read from the NaCl SDK repo."""
 168     raise NotImplementedError()
 169
 170   def GetHistory(self):
 171     """Read Chrome release history from omahaproxy.appspot.com
 172
 173     Here is an example of data from this URL:
 174       cros,stable,18.0.1025.168,2012-05-01 17:04:05.962578\n
 175       win,canary,20.0.1123.0,2012-05-01 13:59:31.703020\n
 176       mac,canary,20.0.1123.0,2012-05-01 11:54:13.041875\n
 177       win,stable,18.0.1025.168,2012-04-30 20:34:56.078490\n
 178       mac,stable,18.0.1025.168,2012-04-30 20:34:55.231141\n
 179       ...
 180     Where each line has comma separated values in the following format:
 181     platform, channel, version, date/time\n
 182
 183     Returns:
 184       A list where each element is a line from the document, represented as a
 185       tuple."""
 186     raise NotImplementedError()
 187
 188   def GsUtil_ls(self, url):
 189     """Runs gsutil ls |url|
 190
 191     Args:
 192       url: The cloud storage url to list.
 193     Returns:
 194       A list of URLs, all with the gs:// schema."""
 195     raise NotImplementedError()
 196
 197   def GsUtil_cat(self, url):
 198     """Runs gsutil cat |url|
 199
 200     Args:
 201       url: The cloud storage url to read from.
 202     Returns:
 203       A string with the contents of the file at |url|."""
 204     raise NotImplementedError()
 205
 206   def GsUtil_cp(self, src, dest, stdin=None):
 207     """Runs gsutil cp |src| |dest|
 208
 209     Args:
 210       src: The file path or url to copy from.
 211       dest: The file path or url to copy to.
 212       stdin: If src is '-', this is used as the stdin to give to gsutil. The
 213           effect is that text in stdin is copied to |dest|."""
 214     raise NotImplementedError()
 215
 216   def SendMail(self, subject, text):
 217     """Send an email.
 218
 219     Args:
 220       subject: The subject of the email.
 221       text: The text of the email.
 222     """
 223     raise NotImplementedError()
 224
 225
 226 class RealDelegate(Delegate):
 227   def __init__(self, dryrun=False, gsutil=None, mailfrom=None, mailto=None):
 228     super(RealDelegate, self).__init__()
 229     self.dryrun = dryrun
 230     self.mailfrom = mailfrom
 231     self.mailto = mailto
 232     if gsutil:
 233       self.gsutil = gsutil
 234     else:
 235       self.gsutil = buildbot_common.GetGsutil()
 236
 237   def GetRepoManifest(self):
 238     """See Delegate.GetRepoManifest"""
 239     with open(REPO_MANIFEST, 'r') as sdk_stream:
 240       sdk_json_string = sdk_stream.read()
 241
 242     manifest = manifest_util.SDKManifest()
 243     manifest.LoadDataFromString(sdk_json_string, add_missing_info=True)
 244     return manifest
 245
 246   def GetHistory(self):
 247     """See Delegate.GetHistory"""
 248     url_stream = urllib2.urlopen('https://omahaproxy.appspot.com/history')
 249     history = [(platform, channel, version, date)
 250         for platform, channel, version, date in csv.reader(url_stream)]
 251
 252     # The first line of this URL is the header:
 253     #   os,channel,version,timestamp
 254     return history[1:]
 255
 256   def GsUtil_ls(self, url):
 257     """See Delegate.GsUtil_ls"""
 258     try:
 259       stdout = self._RunGsUtil(None, False, 'ls', url)
 260     except subprocess.CalledProcessError:
 261       return []
 262
 263     # filter out empty lines
 264     return filter(None, stdout.split('\n'))
 265
 266   def GsUtil_cat(self, url):
 267     """See Delegate.GsUtil_cat"""
 268     return self._RunGsUtil(None, True, 'cat', url)
 269
 270   def GsUtil_cp(self, src, dest, stdin=None):
 271     """See Delegate.GsUtil_cp"""
 272     if self.dryrun:
 273       logger.info("Skipping upload: %s -> %s" % (src, dest))
 274       if src == '-':
 275         logger.info('  contents = """%s"""' % stdin)
 276       return
 277
 278     return self._RunGsUtil(stdin, True, 'cp', '-a', 'public-read', src, dest)
 279
 280   def SendMail(self, subject, text):
 281     """See Delegate.SendMail"""
 282     if self.mailfrom and self.mailto:
 283       msg = email.MIMEMultipart.MIMEMultipart()
 284       msg['From'] = self.mailfrom
 285       msg['To'] = ', '.join(self.mailto)
 286       msg['Date'] = email.Utils.formatdate(localtime=True)
 287       msg['Subject'] = subject
 288       msg.attach(email.MIMEText.MIMEText(text))
 289       smtp_obj = smtplib.SMTP('localhost')
 290       smtp_obj.sendmail(self.mailfrom, self.mailto, msg.as_string())
 291       smtp_obj.close()
 292
 293   def _RunGsUtil(self, stdin, log_errors, *args):
 294     """Run gsutil as a subprocess.
 295
 296     Args:
 297       stdin: If non-None, used as input to the process.
 298       log_errors: If True, write errors to stderr.
 299       *args: Arguments to pass to gsutil. The first argument should be an
 300           operation such as ls, cp or cat.
 301     Returns:
 302       The stdout from the process."""
 303     cmd = [self.gsutil] + list(args)
 304     logger.debug("Running: %s" % str(cmd))
 305     if stdin:
 306       stdin_pipe = subprocess.PIPE
 307     else:
 308       stdin_pipe = None
 309
 310     try:
 311       process = subprocess.Popen(cmd, stdin=stdin_pipe, stdout=subprocess.PIPE,
 312           stderr=subprocess.PIPE)
 313       stdout, stderr = process.communicate(stdin)
 314     except OSError as e:
 315       raise manifest_util.Error("Unable to run '%s': %s" % (cmd[0], str(e)))
 316
 317     if process.returncode:
 318       if log_errors:
 319         logger.error(stderr)
 320       raise subprocess.CalledProcessError(process.returncode, ' '.join(cmd))
 321     return stdout
 322
 323
 324 class GsutilLoggingHandler(logging.handlers.BufferingHandler):
 325   def __init__(self, delegate):
 326     logging.handlers.BufferingHandler.__init__(self, capacity=0)
 327     self.delegate = delegate
 328
 329   def shouldFlush(self, record):
 330     # BufferingHandler.shouldFlush automatically flushes if the length of the
 331     # buffer is greater than self.capacity. We don't want that behavior, so
 332     # return False here.
 333     return False
 334
 335   def flush(self):
 336     # Do nothing here. We want to be explicit about uploading the log.
 337     pass
 338
 339   def upload(self):
 340     output_list = []
 341     for record in self.buffer:
 342       output_list.append(self.format(record))
 343     output = '\n'.join(output_list)
 344     self.delegate.GsUtil_cp('-', GS_SDK_MANIFEST_LOG, stdin=output)
 345
 346     logging.handlers.BufferingHandler.flush(self)
 347
 348
 349 class NoSharedVersionException(Exception):
 350   pass
 351
 352
 353 class VersionFinder(object):
 354   """Finds a version of a pepper bundle that all desired platforms share.
 355
 356   Args:
 357     delegate: See Delegate class above.
 358     platforms: A sequence of platforms to consider, e.g.
 359         ('mac', 'linux', 'win')
 360     extra_archives: A sequence of tuples: (archive_basename, minimum_version),
 361         e.g. [('foo.tar.bz2', '18.0.1000.0'), ('bar.tar.bz2', '19.0.1100.20')]
 362         These archives must exist to consider a version for inclusion, as
 363         long as that version is greater than the archive's minimum version.
 364     is_bionic: True if we are searching for bionic archives.
 365   """
 366   def __init__(self, delegate, platforms, extra_archives=None, is_bionic=False):
 367     self.delegate = delegate
 368     self.history = delegate.GetHistory()
 369     self.platforms = platforms
 370     self.extra_archives = extra_archives
 371     self.is_bionic = is_bionic
 372
 373   def GetMostRecentSharedVersion(self, major_version):
 374     """Returns the most recent version of a pepper bundle that exists on all
 375     given platforms.
 376
 377     Specifically, the resulting version should be the most recently released
 378     (meaning closest to the top of the listing on
 379     omahaproxy.appspot.com/history) version that has a Chrome release on all
 380     given platforms, and has a pepper bundle archive for each platform as well.
 381
 382     Args:
 383       major_version: The major version of the pepper bundle, e.g. 19.
 384     Returns:
 385       A tuple (version, channel, archives). The version is a string such as
 386       "19.0.1084.41". The channel is one of ('stable', 'beta', or 'dev').
 387       |archives| is a list of archive URLs."""
 388     def GetPlatformHistory(platform):
 389       return self._GetPlatformMajorVersionHistory(major_version, platform)
 390
 391     shared_version_generator = self._FindNextSharedVersion(self.platforms,
 392                                                            GetPlatformHistory)
 393     return self._DoGetMostRecentSharedVersion(shared_version_generator)
 394
 395   def GetMostRecentSharedCanary(self):
 396     """Returns the most recent version of a canary pepper bundle that exists on
 397     all given platforms.
 398
 399     Canary is special-cased because we don't care about its major version; we
 400     always use the most recent canary, regardless of major version.
 401
 402     Returns:
 403       A tuple (version, channel, archives). The version is a string such as
 404       "trunk.123456". The channel is always 'canary'. |archives| is a list of
 405       archive URLs."""
 406     version_generator = self._FindNextTrunkVersion()
 407     return self._DoGetMostRecentSharedVersion(version_generator)
 408
 409   def GetAvailablePlatformArchivesFor(self, version):
 410     """Returns a sequence of archives that exist for a given version, on the
 411     given platforms.
 412
 413     The second element of the returned tuple is a list of all platforms that do
 414     not have an archive for the given version.
 415
 416     Args:
 417       version: The version to find archives for. (e.g. "18.0.1025.164")
 418     Returns:
 419       A tuple (archives, missing_archives). |archives| is a list of archive
 420       URLs, |missing_archives| is a list of archive names.
 421     """
 422     archive_urls = self._GetAvailableArchivesFor(version)
 423
 424     if self.is_bionic:
 425       # Bionic currently is Linux-only.
 426       expected_archives = set([GetBionicArchiveName()])
 427     else:
 428       expected_archives = set(GetPlatformArchiveName(p) for p in self.platforms)
 429
 430     if self.extra_archives:
 431       for extra_archive, extra_archive_min_version in self.extra_archives:
 432         if CompareVersions(version, extra_archive_min_version) >= 0:
 433           expected_archives.add(extra_archive)
 434     found_archives = set(GetCanonicalArchiveName(a) for a in archive_urls)
 435     missing_archives = expected_archives - found_archives
 436
 437     # Only return archives that are "expected".
 438     def IsExpected(url):
 439       return GetCanonicalArchiveName(url) in expected_archives
 440
 441     expected_archive_urls = [u for u in archive_urls if IsExpected(u)]
 442     return expected_archive_urls, missing_archives
 443
 444   def _DoGetMostRecentSharedVersion(self, shared_version_generator):
 445     """Returns the most recent version of a pepper bundle that exists on all
 446     given platforms.
 447
 448     This function does the real work for the public GetMostRecentShared* above.
 449
 450     Args:
 451       shared_version_generator: A generator that will yield (version, channel)
 452           tuples in order of most recent to least recent.
 453     Returns:
 454       A tuple (version, channel, archives). The version is a string such as
 455       "19.0.1084.41". The channel is one of ('stable', 'beta', 'dev',
 456       'canary'). |archives| is a list of archive URLs."""
 457     version = None
 458     skipped_versions = []
 459     channel = ''
 460     while True:
 461       try:
 462         version, channel = shared_version_generator.next()
 463       except StopIteration:
 464         msg = 'No shared version for platforms: %s\n' % (
 465             ', '.join(self.platforms))
 466         msg += 'Last version checked = %s.\n' % (version,)
 467         if skipped_versions:
 468           msg += 'Versions skipped due to missing archives:\n'
 469           for version, channel, missing_archives in skipped_versions:
 470             archive_msg = '(missing %s)' % (', '.join(missing_archives))
 471           msg += '  %s (%s) %s\n' % (version, channel, archive_msg)
 472         raise NoSharedVersionException(msg)
 473
 474       logger.info('Found shared version: %s, channel: %s' % (
 475           version, channel))
 476
 477       archives, missing_archives = self.GetAvailablePlatformArchivesFor(version)
 478
 479       if not missing_archives:
 480         return version, channel, archives
 481
 482       logger.info('  skipping. Missing archives: %s' % (
 483           ', '.join(missing_archives)))
 484
 485       skipped_versions.append((version, channel, missing_archives))
 486
 487   def _GetPlatformMajorVersionHistory(self, with_major_version, with_platform):
 488     """Yields Chrome history for a given platform and major version.
 489
 490     Args:
 491       with_major_version: The major version to filter for. If 0, match all
 492           versions.
 493       with_platform: The name of the platform to filter for.
 494     Returns:
 495       A generator that yields a tuple (channel, version) for each version that
 496       matches the platform and major version. The version returned is a tuple as
 497       returned from SplitVersion.
 498     """
 499     for platform, channel, version, _ in self.history:
 500       version = SplitVersion(version)
 501       if (with_platform == platform and
 502           (with_major_version == 0 or with_major_version == version[0])):
 503         yield channel, version
 504
 505   def _FindNextSharedVersion(self, platforms, generator_func):
 506     """Yields versions of Chrome that exist on all given platforms, in order of
 507        newest to oldest.
 508
 509     Versions are compared in reverse order of release. That is, the most
 510     recently updated version will be tested first.
 511
 512     Args:
 513       platforms: A sequence of platforms to consider, e.g.
 514           ('mac', 'linux', 'win')
 515       generator_func: A function which takes a platform and returns a
 516           generator that yields (channel, version) tuples.
 517     Returns:
 518       A generator that yields a tuple (version, channel) for each version that
 519       matches all platforms and the major version. The version returned is a
 520       string (e.g. "18.0.1025.164").
 521     """
 522     platform_generators = []
 523     for platform in platforms:
 524       platform_generators.append(generator_func(platform))
 525
 526     shared_version = None
 527     platform_versions = []
 528     for platform_gen in platform_generators:
 529       platform_versions.append(platform_gen.next())
 530
 531     while True:
 532       if logger.isEnabledFor(logging.INFO):
 533         msg_info = []
 534         for i, platform in enumerate(platforms):
 535           msg_info.append('%s: %s' % (
 536               platform, JoinVersion(platform_versions[i][1])))
 537         logger.info('Checking versions: %s' % ', '.join(msg_info))
 538
 539       shared_version = min((v for c, v in platform_versions))
 540
 541       if all(v == shared_version for c, v in platform_versions):
 542         # grab the channel from an arbitrary platform
 543         first_platform = platform_versions[0]
 544         channel = first_platform[0]
 545         yield JoinVersion(shared_version), channel
 546
 547         # force increment to next version for all platforms
 548         shared_version = None
 549
 550       # Find the next version for any platform that isn't at the shared version.
 551       try:
 552         for i, platform_gen in enumerate(platform_generators):
 553           if platform_versions[i][1] != shared_version:
 554             platform_versions[i] = platform_gen.next()
 555       except StopIteration:
 556         return
 557
 558
 559   def _FindNextTrunkVersion(self):
 560     """Yields all trunk versions that exist in the cloud storage bucket, newest
 561     to oldest.
 562
 563     Returns:
 564       A generator that yields a tuple (version, channel) for each version that
 565       matches all platforms and the major version. The version returned is a
 566       string (e.g. "trunk.123456").
 567     """
 568     files = self.delegate.GsUtil_ls(GS_BUCKET_PATH)
 569     assert all(f.startswith('gs://') for f in files)
 570
 571     trunk_versions = []
 572     for f in files:
 573       match = re.search(r'(trunk\.\d+)', f)
 574       if match:
 575         trunk_versions.append(match.group(1))
 576
 577     trunk_versions.sort(reverse=True)
 578
 579     for version in trunk_versions:
 580       yield version, 'canary'
 581
 582
 583   def _GetAvailableArchivesFor(self, version_string):
 584     """Downloads a list of all available archives for a given version.
 585
 586     Args:
 587       version_string: The version to find archives for. (e.g. "18.0.1025.164")
 588     Returns:
 589       A list of strings, each of which is a platform-specific archive URL. (e.g.
 590       "gs://nativeclient_mirror/nacl/nacl_sdk/18.0.1025.164/"
 591       "naclsdk_linux.tar.bz2").
 592
 593       All returned URLs will use the gs:// schema."""
 594     files = self.delegate.GsUtil_ls(GS_BUCKET_PATH + version_string)
 595
 596     assert all(f.startswith('gs://') for f in files)
 597
 598     archives = [f for f in files if not f.endswith('.json')]
 599     manifests = [f for f in files if f.endswith('.json')]
 600
 601     # don't include any archives that don't have an associated manifest.
 602     return filter(lambda a: a + '.json' in manifests, archives)
 603
 604
 605 class UnknownLockedBundleException(Exception):
 606   pass
 607
 608
 609 class Updater(object):
 610   def __init__(self, delegate):
 611     self.delegate = delegate
 612     self.versions_to_update = []
 613     self.locked_bundles = []
 614     self.online_manifest = manifest_util.SDKManifest()
 615     self._FetchOnlineManifest()
 616
 617   def AddVersionToUpdate(self, bundle_name, version, channel, archives):
 618     """Add a pepper version to update in the uploaded manifest.
 619
 620     Args:
 621       bundle_name: The name of the pepper bundle, e.g. 'pepper_18'
 622       version: The version of the pepper bundle, e.g. '18.0.1025.64'
 623       channel: The stability of the pepper bundle, e.g. 'beta'
 624       archives: A sequence of archive URLs for this bundle."""
 625     self.versions_to_update.append((bundle_name, version, channel, archives))
 626
 627   def AddLockedBundle(self, bundle_name):
 628     """Add a "locked" bundle to the updater.
 629
 630     A locked bundle is a bundle that wasn't found in the history. When this
 631     happens, the bundle is now "locked" to whatever was last found. We want to
 632     ensure that the online manifest has this bundle.
 633
 634     Args:
 635       bundle_name: The name of the locked bundle.
 636     """
 637     self.locked_bundles.append(bundle_name)
 638
 639   def Update(self, manifest):
 640     """Update a manifest and upload it.
 641
 642     Note that bundles will not be updated if the current version is newer.
 643     That is, the updater will never automatically update to an older version of
 644     a bundle.
 645
 646     Args:
 647       manifest: The manifest used as a template for updating. Only pepper
 648       bundles that contain no archives will be considered for auto-updating."""
 649     # Make sure there is only one stable branch: the one with the max version.
 650     # All others are post-stable.
 651     stable_major_versions = [GetMajorVersion(version) for _, version, channel, _
 652                              in self.versions_to_update if channel == 'stable']
 653     # Add 0 in case there are no stable versions.
 654     max_stable_version = max([0] + stable_major_versions)
 655
 656     # Ensure that all locked bundles exist in the online manifest.
 657     for bundle_name in self.locked_bundles:
 658       online_bundle = self.online_manifest.GetBundle(bundle_name)
 659       if online_bundle:
 660         manifest.SetBundle(online_bundle)
 661       else:
 662         msg = ('Attempted to update bundle "%s", but no shared versions were '
 663             'found, and there is no online bundle with that name.')
 664         raise UnknownLockedBundleException(msg % bundle_name)
 665
 666     if self.locked_bundles:
 667       # Send a nagging email that we shouldn't be wasting time looking for
 668       # bundles that are no longer in the history.
 669       scriptname = os.path.basename(sys.argv[0])
 670       subject = '[%s] Reminder: remove bundles from %s' % (scriptname,
 671                                                            MANIFEST_BASENAME)
 672       text = 'These bundles are not in the omahaproxy history anymore: ' + \
 673               ', '.join(self.locked_bundles)
 674       self.delegate.SendMail(subject, text)
 675
 676
 677     # Update all versions.
 678     logger.info('>>> Updating bundles...')
 679     for bundle_name, version, channel, archives in self.versions_to_update:
 680       logger.info('Updating %s to %s...' % (bundle_name, version))
 681       bundle = manifest.GetBundle(bundle_name)
 682       for archive in archives:
 683         platform_bundle = self._GetPlatformArchiveBundle(archive)
 684         # Normally the manifest snippet's bundle name matches our bundle name.
 685         # pepper_canary, however is called "pepper_###" in the manifest
 686         # snippet.
 687         platform_bundle.name = bundle_name
 688         bundle.MergeWithBundle(platform_bundle)
 689
 690       # Fix the stability and recommended values
 691       major_version = GetMajorVersion(version)
 692       if major_version < max_stable_version:
 693         bundle.stability = 'post_stable'
 694       else:
 695         bundle.stability = channel
 696       # We always recommend the stable version.
 697       if bundle.stability == 'stable':
 698         bundle.recommended = 'yes'
 699       else:
 700         bundle.recommended = 'no'
 701
 702       # Check to ensure this bundle is newer than the online bundle.
 703       online_bundle = self.online_manifest.GetBundle(bundle_name)
 704       if online_bundle:
 705         # This test used to be online_bundle.revision >= bundle.revision.
 706         # That doesn't do quite what we want: sometimes the metadata changes
 707         # but the revision stays the same -- we still want to push those
 708         # changes.
 709         if online_bundle.revision > bundle.revision or online_bundle == bundle:
 710           logger.info(
 711               '  Revision %s is not newer than than online revision %s. '
 712               'Skipping.' % (bundle.revision, online_bundle.revision))
 713
 714           manifest.SetBundle(online_bundle)
 715           continue
 716     self._UploadManifest(manifest)
 717     logger.info('Done.')
 718
 719   def _GetPlatformArchiveBundle(self, archive):
 720     """Downloads the manifest "snippet" for an archive, and reads it as a
 721        Bundle.
 722
 723     Args:
 724       archive: A full URL of a platform-specific archive, using the gs schema.
 725     Returns:
 726       An object of type manifest_util.Bundle, read from a JSON file storing
 727       metadata for this archive.
 728     """
 729     stdout = self.delegate.GsUtil_cat(archive + '.json')
 730     bundle = manifest_util.Bundle('')
 731     bundle.LoadDataFromString(stdout)
 732     # Some snippets were uploaded with revisions and versions as strings. Fix
 733     # those here.
 734     bundle.revision = int(bundle.revision)
 735     bundle.version = int(bundle.version)
 736
 737     # HACK. The naclports archive specifies host_os as linux. Change it to all.
 738     for archive in bundle.GetArchives():
 739       if NACLPORTS_ARCHIVE_NAME in archive.url:
 740         archive.host_os = 'all'
 741     return bundle
 742
 743   def _UploadManifest(self, manifest):
 744     """Upload a serialized manifest_util.SDKManifest object.
 745
 746     Upload one copy to gs://<BUCKET_PATH>/naclsdk_manifest2.json, and a copy to
 747     gs://<BUCKET_PATH>/manifest_backups/naclsdk_manifest2.<TIMESTAMP>.json.
 748
 749     Args:
 750       manifest: The new manifest to upload.
 751     """
 752     new_manifest_string = manifest.GetDataAsString()
 753     online_manifest_string = self.online_manifest.GetDataAsString()
 754
 755     if self.delegate.dryrun:
 756       logger.info(''.join(list(difflib.unified_diff(
 757           online_manifest_string.splitlines(1),
 758           new_manifest_string.splitlines(1)))))
 759       return
 760     else:
 761       online_manifest = manifest_util.SDKManifest()
 762       online_manifest.LoadDataFromString(online_manifest_string)
 763
 764       if online_manifest == manifest:
 765         logger.info('New manifest doesn\'t differ from online manifest.'
 766             'Skipping upload.')
 767         return
 768
 769     timestamp_manifest_path = GS_MANIFEST_BACKUP_DIR + \
 770         GetTimestampManifestName()
 771     self.delegate.GsUtil_cp('-', timestamp_manifest_path,
 772         stdin=manifest.GetDataAsString())
 773
 774     # copy from timestampped copy over the official manifest.
 775     self.delegate.GsUtil_cp(timestamp_manifest_path, GS_SDK_MANIFEST)
 776
 777   def _FetchOnlineManifest(self):
 778     try:
 779       online_manifest_string = self.delegate.GsUtil_cat(GS_SDK_MANIFEST)
 780     except subprocess.CalledProcessError:
 781       # It is not a failure if the online manifest doesn't exist.
 782       online_manifest_string = ''
 783
 784     if online_manifest_string:
 785       self.online_manifest.LoadDataFromString(online_manifest_string)
 786
 787
 788 def Run(delegate, platforms, extra_archives, fixed_bundle_versions=None):
 789   """Entry point for the auto-updater.
 790
 791   Args:
 792     delegate: The Delegate object to use for reading Urls, files, etc.
 793     platforms: A sequence of platforms to consider, e.g.
 794         ('mac', 'linux', 'win')
 795       extra_archives: A sequence of tuples: (archive_basename, minimum_version),
 796           e.g. [('foo.tar.bz2', '18.0.1000.0'), ('bar.tar.bz2', '19.0.1100.20')]
 797           These archives must exist to consider a version for inclusion, as
 798           long as that version is greater than the archive's minimum version.
 799     fixed_bundle_versions: A sequence of tuples (bundle_name, version_string).
 800         e.g. ('pepper_21', '21.0.1145.0')
 801   """
 802   if fixed_bundle_versions:
 803     fixed_bundle_versions = dict(fixed_bundle_versions)
 804   else:
 805     fixed_bundle_versions = {}
 806
 807   manifest = delegate.GetRepoManifest()
 808   auto_update_bundles = []
 809   for bundle in manifest.GetBundles():
 810     if not bundle.name.startswith(('pepper_', 'bionic_')):
 811       continue
 812     archives = bundle.GetArchives()
 813     if not archives:
 814       auto_update_bundles.append(bundle)
 815
 816   if not auto_update_bundles:
 817     logger.info('No versions need auto-updating.')
 818     return
 819
 820   updater = Updater(delegate)
 821
 822   for bundle in auto_update_bundles:
 823     try:
 824       if bundle.name == BIONIC_CANARY_BUNDLE_NAME:
 825         logger.info('>>> Looking for most recent bionic_canary...')
 826         # Ignore extra_archives on bionic; There is no naclports bundle yet.
 827         version_finder = VersionFinder(delegate, platforms, None,
 828                                        is_bionic=True)
 829         version, channel, archives = version_finder.GetMostRecentSharedCanary()
 830       elif bundle.name == CANARY_BUNDLE_NAME:
 831         logger.info('>>> Looking for most recent pepper_canary...')
 832         version_finder = VersionFinder(delegate, platforms, extra_archives)
 833         version, channel, archives = version_finder.GetMostRecentSharedCanary()
 834       else:
 835         logger.info('>>> Looking for most recent pepper_%s...' %
 836             bundle.version)
 837         version_finder = VersionFinder(delegate, platforms, extra_archives)
 838         version, channel, archives = version_finder.GetMostRecentSharedVersion(
 839             bundle.version)
 840     except NoSharedVersionException:
 841       # If we can't find a shared version, make sure that there is an uploaded
 842       # bundle with that name already.
 843       updater.AddLockedBundle(bundle.name)
 844       continue
 845
 846     if bundle.name in fixed_bundle_versions:
 847       # Ensure this version is valid for all platforms.
 848       # If it is, use the channel found above (because the channel for this
 849       # version may not be in the history.)
 850       version = fixed_bundle_versions[bundle.name]
 851       logger.info('Fixed bundle version: %s, %s' % (bundle.name, version))
 852       archives, missing = \
 853           version_finder.GetAvailablePlatformArchivesFor(version)
 854       if missing:
 855         logger.warn(
 856             'Some archives for version %s of bundle %s don\'t exist: '
 857             'Missing %s' % (version, bundle.name, ', '.join(missing)))
 858         return
 859
 860     updater.AddVersionToUpdate(bundle.name, version, channel, archives)
 861
 862   updater.Update(manifest)
 863
 864
 865 class CapturedFile(object):
 866   """A file-like object that captures text written to it, but also passes it
 867   through to an underlying file-like object."""
 868   def __init__(self, passthrough):
 869     self.passthrough = passthrough
 870     self.written = cStringIO.StringIO()
 871
 872   def write(self, s):
 873     self.written.write(s)
 874     if self.passthrough:
 875       self.passthrough.write(s)
 876
 877   def getvalue(self):
 878     return self.written.getvalue()
 879
 880
 881 def main(args):
 882   parser = optparse.OptionParser()
 883   parser.add_option('--gsutil', help='path to gsutil.')
 884   parser.add_option('-d', '--debug', help='run in debug mode.',
 885       action='store_true')
 886   parser.add_option('--mailfrom', help='email address of sender.')
 887   parser.add_option('--mailto', help='send error mails to...', action='append')
 888   parser.add_option('-n', '--dryrun', help="don't upload the manifest.",
 889       action='store_true')
 890   parser.add_option('-v', '--verbose', help='print more diagnotic messages. '
 891       'Use more than once for more info.',
 892       action='count')
 893   parser.add_option('--log-file', metavar='FILE', help='log to FILE')
 894   parser.add_option('--upload-log', help='Upload log alongside the manifest.',
 895                     action='store_true')
 896   parser.add_option('--bundle-version',
 897       help='Manually set a bundle version. This can be passed more than once. '
 898       'format: --bundle-version pepper_24=24.0.1312.25', action='append')
 899   options, args = parser.parse_args(args[1:])
 900
 901   if (options.mailfrom is None) != (not options.mailto):
 902     options.mailfrom = None
 903     options.mailto = None
 904     logger.warning('Disabling email, one of --mailto or --mailfrom '
 905         'was missing.\n')
 906
 907   if options.verbose >= 2:
 908     logging.basicConfig(level=logging.DEBUG, filename=options.log_file)
 909   elif options.verbose:
 910     logging.basicConfig(level=logging.INFO, filename=options.log_file)
 911   else:
 912     logging.basicConfig(level=logging.WARNING, filename=options.log_file)
 913
 914   # Parse bundle versions.
 915   fixed_bundle_versions = {}
 916   if options.bundle_version:
 917     for bundle_version_string in options.bundle_version:
 918       bundle_name, version = bundle_version_string.split('=')
 919       fixed_bundle_versions[bundle_name] = version
 920
 921   if options.mailfrom and options.mailto:
 922     # Capture stderr so it can be emailed, if necessary.
 923     sys.stderr = CapturedFile(sys.stderr)
 924
 925   try:
 926     try:
 927       delegate = RealDelegate(options.dryrun, options.gsutil,
 928                               options.mailfrom, options.mailto)
 929
 930       if options.upload_log:
 931         gsutil_logging_handler = GsutilLoggingHandler(delegate)
 932         logger.addHandler(gsutil_logging_handler)
 933
 934       # Only look for naclports archives >= 27. The old ports bundles don't
 935       # include license information.
 936       extra_archives = [('naclports.tar.bz2', '27.0.0.0')]
 937       Run(delegate, ('mac', 'win', 'linux'), extra_archives,
 938           fixed_bundle_versions)
 939       return 0
 940     except Exception:
 941       if options.mailfrom and options.mailto:
 942         traceback.print_exc()
 943         scriptname = os.path.basename(sys.argv[0])
 944         subject = '[%s] Failed to update manifest' % (scriptname,)
 945         text = '%s failed.\n\nSTDERR:\n%s\n' % (scriptname,
 946                                                 sys.stderr.getvalue())
 947         delegate.SendMail(subject, text)
 948         return 1
 949       else:
 950         raise
 951     finally:
 952       if options.upload_log:
 953         gsutil_logging_handler.upload()
 954   except manifest_util.Error as e:
 955     if options.debug:
 956       raise
 957     sys.stderr.write(str(e) + '\n')
 958     return 1
 959
 960
 961 if __name__ == '__main__':
 962   sys.exit(main(sys.argv))