native_client_sdk/src/build_tools/buildbot_common.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Common utilities for all buildbot scripts that specifically don't rely
   6 on having a full chromium checkout.
   7 """
   8
   9 import os
  10 import subprocess
  11 import sys
  12
  13 from build_paths import SDK_SRC_DIR, NACL_DIR, SRC_DIR
  14
  15 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  16 import oshelpers
  17 import getos
  18
  19
  20 verbose = True
  21
  22
  23 def IsSDKBuilder():
  24   """Returns True if this script is running on an SDK builder.
  25
  26   False means it is either running on a trybot, or a user's machine.
  27
  28   Trybot names:
  29     (win|mac|linux)_nacl_sdk
  30
  31   Build-only Trybot names:
  32     (win|mac|linux)_nacl_sdk_build
  33
  34   Builder names:
  35     (windows|mac|linux)-sdk-multi(bionic)(rel)?"""
  36   bot =  os.getenv('BUILDBOT_BUILDERNAME', '')
  37   return '-sdk-multi' in bot or '-sdk-bionic-multi' in bot
  38
  39
  40 def IsSDKTrybot():
  41   """Returns True if this script is running on an SDK trybot.
  42
  43   False means it is either running on an SDK builder, or a user's machine.
  44
  45   See IsSDKBuilder above for trybot/buildbot names."""
  46   return '_nacl_sdk' in os.getenv('BUILDBOT_BUILDERNAME', '')
  47
  48
  49 def ErrorExit(msg):
  50   """Write and error to stderr, then exit with 1 signaling failure."""
  51   sys.stderr.write(str(msg) + '\n')
  52   sys.exit(1)
  53
  54
  55 def Trace(msg):
  56   if verbose:
  57     sys.stderr.write(str(msg) + '\n')
  58
  59
  60 def GetWindowsEnvironment():
  61   if oshelpers.FindExeInPath('cl.exe') is not None:
  62     # cl.exe is already in the path, let's just use that.
  63     return os.environ
  64
  65   sys.path.append(os.path.join(NACL_DIR, 'buildbot'))
  66   import buildbot_standard
  67
  68   # buildbot_standard.SetupWindowsEnvironment expects a "context" object. We'll
  69   # fake enough of that here to work.
  70   class FakeContext(object):
  71     def __init__(self):
  72       self.env = os.environ
  73
  74     def GetEnv(self, key):
  75       return self.env[key]
  76
  77     def __getitem__(self, key):
  78       # The nacl side script now needs gyp_vars to return a list.
  79       if key == 'gyp_vars':
  80         return []
  81       return self.env[key]
  82
  83     def SetEnv(self, key, value):
  84       self.env[key] = value
  85
  86     def __setitem__(self, key, value):
  87       self.env[key] = value
  88
  89   context = FakeContext()
  90   buildbot_standard.SetupWindowsEnvironment(context)
  91
  92   env_script = 'vcvarsall.bat'
  93
  94   if not oshelpers.FindExeInPath(env_script):
  95     # This might happen if Visual Studio is not installed. Check to see if
  96     # vs2013 is in depot_tools.
  97
  98     # Find depot_tools by looking for gclient.bat.
  99     gclient_bat = oshelpers.FindExeInPath('gclient.bat')
 100     if gclient_bat is None:
 101       ErrorExit('gclient.bat is not in the path. Where is depot_tools?')
 102
 103     depot_tools_dir = os.path.dirname(gclient_bat)
 104     vs2013_dir = os.path.join(depot_tools_dir, 'win_toolchain', 'vs2013_files')
 105     if not os.path.exists(vs2013_dir):
 106       ErrorExit('Visual Studio not installed normally or in depot_tools.')
 107
 108     # The depot_tools vs2013 toolchain has its own batch file (not
 109     # vcvarsall.bat) for setting the environment variables needed by vs2013.
 110     env_script = os.path.join(vs2013_dir, 'win8sdk', 'bin', 'SetEnv.cmd')
 111
 112   # Running the env_script adds the correct directories to the path for
 113   # executables (e.g. cl.exe, link.exe), include paths, lib directories, etc,
 114   # which we extract below.
 115   process = subprocess.Popen(env_script + ' x86 > NUL && set',
 116       stdout=subprocess.PIPE, env=context.env, shell=True)
 117   stdout, _ = process.communicate()
 118
 119   # Parse environment from "set" command above.
 120   # It looks like this:
 121   # KEY1=VALUE1\r\n
 122   # KEY2=VALUE2\r\n
 123   # ...
 124   return dict(line.split('=', 1) for line in stdout.split('\r\n')[:-1])
 125
 126
 127 def BuildStep(name):
 128   """Annotate a buildbot build step."""
 129   sys.stdout.flush()
 130   sys.stderr.write('\n@@@BUILD_STEP %s@@@\n' % name)
 131
 132
 133 def Run(args, cwd=None, env=None, shell=False):
 134   """Start a process with the provided arguments.
 135
 136   Starts a process in the provided directory given the provided arguments. If
 137   shell is not False, the process is launched via the shell to provide shell
 138   interpretation of the arguments.  Shell behavior can differ between platforms
 139   so this should be avoided when not using platform dependent shell scripts."""
 140
 141   # We need to modify the environment to build host on Windows.
 142   if not env and getos.GetPlatform() == 'win':
 143     env = GetWindowsEnvironment()
 144
 145   Trace('Running: ' + ' '.join(args))
 146   sys.stdout.flush()
 147   sys.stderr.flush()
 148   try:
 149     subprocess.check_call(args, cwd=cwd, env=env, shell=shell)
 150   except subprocess.CalledProcessError as e:
 151     sys.stdout.flush()
 152     sys.stderr.flush()
 153     ErrorExit('buildbot_common: %s' % e)
 154
 155   sys.stdout.flush()
 156   sys.stderr.flush()
 157
 158
 159 def ShortFilename(filename):
 160   drive = os.path.splitdrive(filename)[0]
 161   if drive and drive != os.path.splitdrive(SRC_DIR)[0]:
 162     return filename
 163   return os.path.relpath(filename, SRC_DIR)
 164
 165
 166 def CopyDir(src, dst, excludes=('.svn', '*/.svn')):
 167   """Recursively copy a directory using."""
 168   args = ['-r', src, dst]
 169   for exc in excludes:
 170     args.append('--exclude=' + exc)
 171   Trace('cp -r %s %s' % (ShortFilename(src), ShortFilename(dst)))
 172   if os.path.abspath(src) == os.path.abspath(dst):
 173     ErrorExit('ERROR: Copying directory onto itself: ' + src)
 174   oshelpers.Copy(args)
 175
 176
 177 def CopyFile(src, dst):
 178   Trace('cp %s %s' % (ShortFilename(src), ShortFilename(dst)))
 179   if os.path.abspath(src) == os.path.abspath(dst):
 180     ErrorExit('ERROR: Copying file onto itself: ' + src)
 181   args = [src, dst]
 182   oshelpers.Copy(args)
 183
 184
 185 def RemoveDir(dst):
 186   """Remove the provided path."""
 187   Trace('rm -fr ' + ShortFilename(dst))
 188   oshelpers.Remove(['-fr', dst])
 189
 190
 191 def MakeDir(dst):
 192   """Create the path including all parent directories as needed."""
 193   Trace('mkdir -p ' + ShortFilename(dst))
 194   oshelpers.Mkdir(['-p', dst])
 195
 196
 197 def Move(src, dst):
 198   """Move the path src to dst."""
 199   Trace('mv -f %s %s' % (ShortFilename(src), ShortFilename(dst)))
 200   oshelpers.Move(['-f', src, dst])
 201
 202
 203 def RemoveFile(dst):
 204   """Remove the provided file."""
 205   Trace('rm ' + ShortFilename(dst))
 206   oshelpers.Remove(['-f', dst])
 207
 208
 209 BOT_GSUTIL = '/b/build/scripts/slave/gsutil'
 210 # On Windows, the current working directory may be on a different drive than
 211 # gsutil.
 212 WIN_BOT_GSUTIL = 'E:' + BOT_GSUTIL
 213 LOCAL_GSUTIL = 'gsutil'
 214
 215
 216 def GetGsutil():
 217   if os.environ.get('BUILDBOT_BUILDERNAME') \
 218      and not os.environ.get('BUILDBOT_FAKE'):
 219     if getos.GetPlatform() == 'win':
 220       return WIN_BOT_GSUTIL
 221     return BOT_GSUTIL
 222   else:
 223     return LOCAL_GSUTIL
 224
 225
 226 def Archive(filename, bucket_path, cwd=None, step_link=True):
 227   """Upload the given filename to Google Store."""
 228   full_dst = 'gs://%s/%s' % (bucket_path, filename)
 229
 230   # Since GetGsutil() might just return 'gsutil' and expect it to be looked
 231   # up in the PATH, we must pass shell=True on windows.
 232   # Without shell=True the windows implementation of subprocess.call will not
 233   # search the PATH for the executable: http://bugs.python.org/issue8557
 234   shell = getos.GetPlatform() == 'win'
 235
 236   cmd = [GetGsutil(), 'cp', '-a', 'public-read', filename, full_dst]
 237   Run(cmd, shell=shell, cwd=cwd)
 238   url = 'https://storage.googleapis.com/%s/%s' % (bucket_path, filename)
 239   if step_link:
 240     sys.stdout.flush()
 241     sys.stderr.write('@@@STEP_LINK@download@%s@@@\n' % url)