native_client_sdk/src/build_tools/buildbot_common.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Common utilities for all buildbot scripts that specifically don't rely
   6 on having a full chromium checkout.
   7 """
   8
   9 import os
  10 import subprocess
  11 import sys
  12
  13 from build_paths import SDK_SRC_DIR, NACL_DIR, SRC_DIR
  14
  15 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  16 import oshelpers
  17 import getos
  18
  19
  20 verbose = True
  21
  22
  23 def IsSDKBuilder():
  24   """Returns True if this script is running on an SDK builder.
  25
  26   False means it is either running on a trybot, or a user's machine.
  27
  28   Trybot names:
  29     (win|mac|linux)_nacl_sdk
  30
  31   Build-only Trybot names:
  32     (win|mac|linux)_nacl_sdk_build
  33
  34   Builder names:
  35     (windows|mac|linux)-sdk-multi(bionic)(rel)?"""
  36   bot =  os.getenv('BUILDBOT_BUILDERNAME', '')
  37   return '-sdk-multi' in bot or '-sdk-bionic-multi' in bot
  38
  39
  40 def ErrorExit(msg):
  41   """Write and error to stderr, then exit with 1 signaling failure."""
  42   sys.stderr.write(str(msg) + '\n')
  43   sys.exit(1)
  44
  45
  46 def Trace(msg):
  47   if verbose:
  48     sys.stderr.write(str(msg) + '\n')
  49
  50
  51 def GetWindowsEnvironment():
  52   if oshelpers.FindExeInPath('cl.exe') is not None:
  53     # cl.exe is already in the path, let's just use that.
  54     return os.environ
  55
  56   sys.path.append(os.path.join(NACL_DIR, 'buildbot'))
  57   import buildbot_standard
  58
  59   # buildbot_standard.SetupWindowsEnvironment expects a "context" object. We'll
  60   # fake enough of that here to work.
  61   class FakeContext(object):
  62     def __init__(self):
  63       self.env = os.environ
  64
  65     def GetEnv(self, key):
  66       return self.env[key]
  67
  68     def __getitem__(self, key):
  69       # The nacl side script now needs gyp_vars to return a list.
  70       if key == 'gyp_vars':
  71         return []
  72       return self.env[key]
  73
  74     def SetEnv(self, key, value):
  75       self.env[key] = value
  76
  77     def __setitem__(self, key, value):
  78       self.env[key] = value
  79
  80   context = FakeContext()
  81   buildbot_standard.SetupWindowsEnvironment(context)
  82
  83   env_script = 'vcvarsall.bat'
  84
  85   if not oshelpers.FindExeInPath(env_script):
  86     # This might happen if Visual Studio is not installed. Check to see if
  87     # vs2013 is in depot_tools.
  88
  89     # Find depot_tools by looking for gclient.bat.
  90     gclient_bat = oshelpers.FindExeInPath('gclient.bat')
  91     if gclient_bat is None:
  92       ErrorExit('gclient.bat is not in the path. Where is depot_tools?')
  93
  94     depot_tools_dir = os.path.dirname(gclient_bat)
  95     vs2013_dir = os.path.join(depot_tools_dir, 'win_toolchain', 'vs2013_files')
  96     if not os.path.exists(vs2013_dir):
  97       ErrorExit('Visual Studio not installed normally or in depot_tools.')
  98
  99     # The depot_tools vs2013 toolchain has its own batch file (not
 100     # vcvarsall.bat) for setting the environment variables needed by vs2013.
 101     env_script = os.path.join(vs2013_dir, 'win8sdk', 'bin', 'SetEnv.cmd')
 102
 103   # Running the env_script adds the correct directories to the path for
 104   # executables (e.g. cl.exe, link.exe), include paths, lib directories, etc,
 105   # which we extract below.
 106   process = subprocess.Popen(env_script + ' x86 > NUL && set',
 107       stdout=subprocess.PIPE, env=context.env, shell=True)
 108   stdout, _ = process.communicate()
 109
 110   # Parse environment from "set" command above.
 111   # It looks like this:
 112   # KEY1=VALUE1\r\n
 113   # KEY2=VALUE2\r\n
 114   # ...
 115   return dict(line.split('=', 1) for line in stdout.split('\r\n')[:-1])
 116
 117
 118 def BuildStep(name):
 119   """Annotate a buildbot build step."""
 120   sys.stdout.flush()
 121   sys.stderr.write('\n@@@BUILD_STEP %s@@@\n' % name)
 122
 123
 124 def Run(args, cwd=None, env=None, shell=False):
 125   """Start a process with the provided arguments.
 126
 127   Starts a process in the provided directory given the provided arguments. If
 128   shell is not False, the process is launched via the shell to provide shell
 129   interpretation of the arguments.  Shell behavior can differ between platforms
 130   so this should be avoided when not using platform dependent shell scripts."""
 131
 132   # We need to modify the environment to build host on Windows.
 133   if not env and getos.GetPlatform() == 'win':
 134     env = GetWindowsEnvironment()
 135
 136   Trace('Running: ' + ' '.join(args))
 137   sys.stdout.flush()
 138   sys.stderr.flush()
 139   try:
 140     subprocess.check_call(args, cwd=cwd, env=env, shell=shell)
 141   except subprocess.CalledProcessError as e:
 142     sys.stdout.flush()
 143     sys.stderr.flush()
 144     ErrorExit('buildbot_common: %s' % e)
 145
 146   sys.stdout.flush()
 147   sys.stderr.flush()
 148
 149
 150 def ShortFilename(filename):
 151   drive = os.path.splitdrive(filename)[0]
 152   if drive and drive != os.path.splitdrive(SRC_DIR)[0]:
 153     return filename
 154   return os.path.relpath(filename, SRC_DIR)
 155
 156
 157 def CopyDir(src, dst, excludes=('.svn', '*/.svn')):
 158   """Recursively copy a directory using."""
 159   args = ['-r', src, dst]
 160   for exc in excludes:
 161     args.append('--exclude=' + exc)
 162   Trace('cp -r %s %s' % (ShortFilename(src), ShortFilename(dst)))
 163   if os.path.abspath(src) == os.path.abspath(dst):
 164     ErrorExit('ERROR: Copying directory onto itself: ' + src)
 165   oshelpers.Copy(args)
 166
 167
 168 def CopyFile(src, dst):
 169   Trace('cp %s %s' % (ShortFilename(src), ShortFilename(dst)))
 170   if os.path.abspath(src) == os.path.abspath(dst):
 171     ErrorExit('ERROR: Copying file onto itself: ' + src)
 172   args = [src, dst]
 173   oshelpers.Copy(args)
 174
 175
 176 def RemoveDir(dst):
 177   """Remove the provided path."""
 178   Trace('rm -fr ' + ShortFilename(dst))
 179   oshelpers.Remove(['-fr', dst])
 180
 181
 182 def MakeDir(dst):
 183   """Create the path including all parent directories as needed."""
 184   Trace('mkdir -p ' + ShortFilename(dst))
 185   oshelpers.Mkdir(['-p', dst])
 186
 187
 188 def Move(src, dst):
 189   """Move the path src to dst."""
 190   Trace('mv -f %s %s' % (ShortFilename(src), ShortFilename(dst)))
 191   oshelpers.Move(['-f', src, dst])
 192
 193
 194 def RemoveFile(dst):
 195   """Remove the provided file."""
 196   Trace('rm ' + ShortFilename(dst))
 197   oshelpers.Remove(['-f', dst])
 198
 199
 200 BOT_GSUTIL = '/b/build/scripts/slave/gsutil'
 201 # On Windows, the current working directory may be on a different drive than
 202 # gsutil.
 203 WIN_BOT_GSUTIL = 'E:' + BOT_GSUTIL
 204 LOCAL_GSUTIL = 'gsutil'
 205
 206
 207 def GetGsutil():
 208   if os.environ.get('BUILDBOT_BUILDERNAME') \
 209      and not os.environ.get('BUILDBOT_FAKE'):
 210     if getos.GetPlatform() == 'win':
 211       return WIN_BOT_GSUTIL
 212     return BOT_GSUTIL
 213   else:
 214     return LOCAL_GSUTIL
 215
 216
 217 def Archive(filename, bucket_path, cwd=None, step_link=True):
 218   """Upload the given filename to Google Store."""
 219   full_dst = 'gs://%s/%s' % (bucket_path, filename)
 220
 221   # Since GetGsutil() might just return 'gsutil' and expect it to be looked
 222   # up in the PATH, we must pass shell=True on windows.
 223   # Without shell=True the windows implementation of subprocess.call will not
 224   # search the PATH for the executable: http://bugs.python.org/issue8557
 225   shell = getos.GetPlatform() == 'win'
 226
 227   cmd = [GetGsutil(), 'cp', '-a', 'public-read', filename, full_dst]
 228   Run(cmd, shell=shell, cwd=cwd)
 229   url = 'https://storage.googleapis.com/%s/%s' % (bucket_path, filename)
 230   if step_link:
 231     sys.stdout.flush()
 232     sys.stderr.write('@@@STEP_LINK@download@%s@@@\n' % url)