native_client_sdk/src/build_tools/buildbot_common.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Common utilities for all buildbot scripts that specifically don't rely
   6 on having a full chromium checkout.
   7 """
   8
   9 import os
  10 import subprocess
  11 import sys
  12
  13 from build_paths import SDK_SRC_DIR, NACL_DIR
  14
  15 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  16 import oshelpers
  17 import getos
  18
  19
  20 verbose = True
  21
  22
  23 def IsSDKBuilder():
  24   """Returns True if this script is running on an SDK builder.
  25
  26   False means it is either running on a trybot, or a user's machine.
  27
  28   Trybot names:
  29     (win|mac|linux)_nacl_sdk
  30
  31   Build-only Trybot names:
  32     (win|mac|linux)_nacl_sdk_build
  33
  34   Builder names:
  35     (windows|mac|linux)-sdk-multi(bionic)(rel)?"""
  36   bot =  os.getenv('BUILDBOT_BUILDERNAME', '')
  37   return '-sdk-multi' in bot or '-sdk-bionic-multi' in bot
  38
  39
  40 def IsSDKTrybot():
  41   """Returns True if this script is running on an SDK trybot.
  42
  43   False means it is either running on an SDK builder, or a user's machine.
  44
  45   See IsSDKBuilder above for trybot/buildbot names."""
  46   return '_nacl_sdk' in os.getenv('BUILDBOT_BUILDERNAME', '')
  47
  48
  49 def ErrorExit(msg):
  50   """Write and error to stderr, then exit with 1 signaling failure."""
  51   sys.stderr.write(str(msg) + '\n')
  52   sys.exit(1)
  53
  54
  55 def Trace(msg):
  56   if verbose:
  57     sys.stderr.write(str(msg) + '\n')
  58
  59
  60 def GetWindowsEnvironment():
  61   sys.path.append(os.path.join(NACL_DIR, 'buildbot'))
  62   import buildbot_standard
  63
  64   # buildbot_standard.SetupWindowsEnvironment expects a "context" object. We'll
  65   # fake enough of that here to work.
  66   class FakeContext(object):
  67     def __init__(self):
  68       self.env = os.environ
  69
  70     def GetEnv(self, key):
  71       return self.env[key]
  72
  73     def __getitem__(self, key):
  74       return self.env[key]
  75
  76     def SetEnv(self, key, value):
  77       self.env[key] = value
  78
  79     def __setitem__(self, key, value):
  80       self.env[key] = value
  81
  82   context = FakeContext()
  83   context['gyp_vars'] = []
  84   buildbot_standard.SetupWindowsEnvironment(context)
  85
  86   # buildbot_standard.SetupWindowsEnvironment adds the directory which contains
  87   # vcvarsall.bat to the path, but not the directory which contains cl.exe,
  88   # link.exe, etc.
  89   # Running vcvarsall.bat adds the correct directories to the path, which we
  90   # extract below.
  91   process = subprocess.Popen('vcvarsall.bat x86 > NUL && set',
  92       stdout=subprocess.PIPE, env=context.env, shell=True)
  93   stdout, _ = process.communicate()
  94
  95   # Parse environment from "set" command above.
  96   # It looks like this:
  97   # KEY1=VALUE1\r\n
  98   # KEY2=VALUE2\r\n
  99   # ...
 100   return dict(line.split('=') for line in stdout.split('\r\n')[:-1])
 101
 102
 103 def BuildStep(name):
 104   """Annotate a buildbot build step."""
 105   sys.stdout.flush()
 106   sys.stderr.write('\n@@@BUILD_STEP %s@@@\n' % name)
 107
 108
 109 def Run(args, cwd=None, env=None, shell=False):
 110   """Start a process with the provided arguments.
 111
 112   Starts a process in the provided directory given the provided arguments. If
 113   shell is not False, the process is launched via the shell to provide shell
 114   interpretation of the arguments.  Shell behavior can differ between platforms
 115   so this should be avoided when not using platform dependent shell scripts."""
 116
 117   # We need to modify the environment to build host on Windows.
 118   if not env and getos.GetPlatform() == 'win':
 119     env = GetWindowsEnvironment()
 120
 121   Trace('Running: ' + ' '.join(args))
 122   sys.stdout.flush()
 123   sys.stderr.flush()
 124   try:
 125     subprocess.check_call(args, cwd=cwd, env=env, shell=shell)
 126   except subprocess.CalledProcessError as e:
 127     sys.stdout.flush()
 128     sys.stderr.flush()
 129     ErrorExit('buildbot_common: %s' % e)
 130
 131   sys.stdout.flush()
 132   sys.stderr.flush()
 133
 134
 135 def CopyDir(src, dst, excludes=('.svn', '*/.svn')):
 136   """Recursively copy a directory using."""
 137   args = ['-r', src, dst]
 138   for exc in excludes:
 139     args.append('--exclude=' + exc)
 140   Trace('cp -r %s %s' % (src, dst))
 141   if os.path.abspath(src) == os.path.abspath(dst):
 142     ErrorExit('ERROR: Copying directory onto itself: ' + src)
 143   oshelpers.Copy(args)
 144
 145
 146 def CopyFile(src, dst):
 147   Trace('cp %s %s' % (src, dst))
 148   if os.path.abspath(src) == os.path.abspath(dst):
 149     ErrorExit('ERROR: Copying file onto itself: ' + src)
 150   args = [src, dst]
 151   oshelpers.Copy(args)
 152
 153
 154 def RemoveDir(dst):
 155   """Remove the provided path."""
 156   Trace('rm -fr ' + dst)
 157   oshelpers.Remove(['-fr', dst])
 158
 159
 160 def MakeDir(dst):
 161   """Create the path including all parent directories as needed."""
 162   Trace('mkdir -p ' + dst)
 163   oshelpers.Mkdir(['-p', dst])
 164
 165
 166 def Move(src, dst):
 167   """Move the path src to dst."""
 168   Trace('mv -f %s %s' % (src, dst))
 169   oshelpers.Move(['-f', src, dst])
 170
 171
 172 def RemoveFile(dst):
 173   """Remove the provided file."""
 174   Trace('rm ' + dst)
 175   oshelpers.Remove(['-f', dst])
 176
 177
 178 BOT_GSUTIL = '/b/build/scripts/slave/gsutil'
 179 # On Windows, the current working directory may be on a different drive than
 180 # gsutil.
 181 WIN_BOT_GSUTIL = 'E:' + BOT_GSUTIL
 182 LOCAL_GSUTIL = 'gsutil'
 183
 184
 185 def GetGsutil():
 186   if os.environ.get('BUILDBOT_BUILDERNAME') \
 187      and not os.environ.get('BUILDBOT_FAKE'):
 188     if getos.GetPlatform() == 'win':
 189       return WIN_BOT_GSUTIL
 190     return BOT_GSUTIL
 191   else:
 192     return LOCAL_GSUTIL
 193
 194
 195 def Archive(filename, bucket_path, cwd=None, step_link=True):
 196   """Upload the given filename to Google Store."""
 197   full_dst = 'gs://%s/%s' % (bucket_path, filename)
 198
 199   # Since GetGsutil() might just return 'gsutil' and expect it to be looked
 200   # up in the PATH, we must pass shell=True on windows.
 201   # Without shell=True the windows implementation of subprocess.call will not
 202   # search the PATH for the executable: http://bugs.python.org/issue8557
 203   shell = getos.GetPlatform() == 'win'
 204
 205   cmd = [GetGsutil(), 'cp', '-a', 'public-read', filename, full_dst]
 206   Run(cmd, shell=shell, cwd=cwd)
 207   url = 'https://storage.googleapis.com/%s/%s' % (bucket_path, filename)
 208   if step_link:
 209     sys.stdout.flush()
 210     sys.stderr.write('@@@STEP_LINK@download@%s@@@\n' % url)