native_client_sdk/src/build_tools/buildbot_common.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Common utilities for all buildbot scripts that specifically don't rely
   6 on having a full chromium checkout.
   7 """
   8
   9 import os
  10 import subprocess
  11 import sys
  12
  13 from build_paths import SDK_SRC_DIR, NACL_DIR
  14
  15 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  16 import oshelpers
  17 import getos
  18
  19
  20 verbose = True
  21
  22
  23 def IsSDKBuilder():
  24   """Returns True if this script is running on an SDK builder.
  25
  26   False means it is either running on a trybot, or a user's machine.
  27
  28   Trybot names:
  29     (win|mac|linux)_nacl_sdk
  30
  31   Build-only Trybot names:
  32     (win|mac|linux)_nacl_sdk_build
  33
  34   Builder names:
  35     (windows|mac|linux)-sdk-multi(bionic)(rel)?"""
  36   bot =  os.getenv('BUILDBOT_BUILDERNAME', '')
  37   return '-sdk-multi' in bot or '-sdk-bionic-multi' in bot
  38
  39
  40 def IsSDKTrybot():
  41   """Returns True if this script is running on an SDK trybot.
  42
  43   False means it is either running on an SDK builder, or a user's machine.
  44
  45   See IsSDKBuilder above for trybot/buildbot names."""
  46   return '_nacl_sdk' in os.getenv('BUILDBOT_BUILDERNAME', '')
  47
  48
  49 def ErrorExit(msg):
  50   """Write and error to stderr, then exit with 1 signaling failure."""
  51   sys.stderr.write(str(msg) + '\n')
  52   sys.exit(1)
  53
  54
  55 def Trace(msg):
  56   if verbose:
  57     sys.stderr.write(str(msg) + '\n')
  58
  59
  60 def GetWindowsEnvironment():
  61   sys.path.append(os.path.join(NACL_DIR, 'buildbot'))
  62   import buildbot_standard
  63
  64   # buildbot_standard.SetupWindowsEnvironment expects a "context" object. We'll
  65   # fake enough of that here to work.
  66   class FakeContext(object):
  67     def __init__(self):
  68       self.env = os.environ
  69
  70     def GetEnv(self, key):
  71       return self.env[key]
  72
  73     def __getitem__(self, key):
  74       # The nacl side script now needs gyp_vars to return a list.
  75       if key == 'gyp_vars':
  76         return []
  77       return self.env[key]
  78
  79     def SetEnv(self, key, value):
  80       self.env[key] = value
  81
  82     def __setitem__(self, key, value):
  83       self.env[key] = value
  84
  85   context = FakeContext()
  86   buildbot_standard.SetupWindowsEnvironment(context)
  87
  88   # buildbot_standard.SetupWindowsEnvironment adds the directory which contains
  89   # vcvarsall.bat to the path, but not the directory which contains cl.exe,
  90   # link.exe, etc.
  91   # Running vcvarsall.bat adds the correct directories to the path, which we
  92   # extract below.
  93   process = subprocess.Popen('vcvarsall.bat x86 > NUL && set',
  94       stdout=subprocess.PIPE, env=context.env, shell=True)
  95   stdout, _ = process.communicate()
  96
  97   # Parse environment from "set" command above.
  98   # It looks like this:
  99   # KEY1=VALUE1\r\n
 100   # KEY2=VALUE2\r\n
 101   # ...
 102   return dict(line.split('=', 1) for line in stdout.split('\r\n')[:-1])
 103
 104
 105 def BuildStep(name):
 106   """Annotate a buildbot build step."""
 107   sys.stdout.flush()
 108   sys.stderr.write('\n@@@BUILD_STEP %s@@@\n' % name)
 109
 110
 111 def Run(args, cwd=None, env=None, shell=False):
 112   """Start a process with the provided arguments.
 113
 114   Starts a process in the provided directory given the provided arguments. If
 115   shell is not False, the process is launched via the shell to provide shell
 116   interpretation of the arguments.  Shell behavior can differ between platforms
 117   so this should be avoided when not using platform dependent shell scripts."""
 118
 119   # We need to modify the environment to build host on Windows.
 120   if not env and getos.GetPlatform() == 'win':
 121     env = GetWindowsEnvironment()
 122
 123   Trace('Running: ' + ' '.join(args))
 124   sys.stdout.flush()
 125   sys.stderr.flush()
 126   try:
 127     subprocess.check_call(args, cwd=cwd, env=env, shell=shell)
 128   except subprocess.CalledProcessError as e:
 129     sys.stdout.flush()
 130     sys.stderr.flush()
 131     ErrorExit('buildbot_common: %s' % e)
 132
 133   sys.stdout.flush()
 134   sys.stderr.flush()
 135
 136
 137 def CopyDir(src, dst, excludes=('.svn', '*/.svn')):
 138   """Recursively copy a directory using."""
 139   args = ['-r', src, dst]
 140   for exc in excludes:
 141     args.append('--exclude=' + exc)
 142   Trace('cp -r %s %s' % (src, dst))
 143   if os.path.abspath(src) == os.path.abspath(dst):
 144     ErrorExit('ERROR: Copying directory onto itself: ' + src)
 145   oshelpers.Copy(args)
 146
 147
 148 def CopyFile(src, dst):
 149   Trace('cp %s %s' % (src, dst))
 150   if os.path.abspath(src) == os.path.abspath(dst):
 151     ErrorExit('ERROR: Copying file onto itself: ' + src)
 152   args = [src, dst]
 153   oshelpers.Copy(args)
 154
 155
 156 def RemoveDir(dst):
 157   """Remove the provided path."""
 158   Trace('rm -fr ' + dst)
 159   oshelpers.Remove(['-fr', dst])
 160
 161
 162 def MakeDir(dst):
 163   """Create the path including all parent directories as needed."""
 164   Trace('mkdir -p ' + dst)
 165   oshelpers.Mkdir(['-p', dst])
 166
 167
 168 def Move(src, dst):
 169   """Move the path src to dst."""
 170   Trace('mv -f %s %s' % (src, dst))
 171   oshelpers.Move(['-f', src, dst])
 172
 173
 174 def RemoveFile(dst):
 175   """Remove the provided file."""
 176   Trace('rm ' + dst)
 177   oshelpers.Remove(['-f', dst])
 178
 179
 180 BOT_GSUTIL = '/b/build/scripts/slave/gsutil'
 181 # On Windows, the current working directory may be on a different drive than
 182 # gsutil.
 183 WIN_BOT_GSUTIL = 'E:' + BOT_GSUTIL
 184 LOCAL_GSUTIL = 'gsutil'
 185
 186
 187 def GetGsutil():
 188   if os.environ.get('BUILDBOT_BUILDERNAME') \
 189      and not os.environ.get('BUILDBOT_FAKE'):
 190     if getos.GetPlatform() == 'win':
 191       return WIN_BOT_GSUTIL
 192     return BOT_GSUTIL
 193   else:
 194     return LOCAL_GSUTIL
 195
 196
 197 def Archive(filename, bucket_path, cwd=None, step_link=True):
 198   """Upload the given filename to Google Store."""
 199   full_dst = 'gs://%s/%s' % (bucket_path, filename)
 200
 201   # Since GetGsutil() might just return 'gsutil' and expect it to be looked
 202   # up in the PATH, we must pass shell=True on windows.
 203   # Without shell=True the windows implementation of subprocess.call will not
 204   # search the PATH for the executable: http://bugs.python.org/issue8557
 205   shell = getos.GetPlatform() == 'win'
 206
 207   cmd = [GetGsutil(), 'cp', '-a', 'public-read', filename, full_dst]
 208   Run(cmd, shell=shell, cwd=cwd)
 209   url = 'https://storage.googleapis.com/%s/%s' % (bucket_path, filename)
 210   if step_link:
 211     sys.stdout.flush()
 212     sys.stderr.write('@@@STEP_LINK@download@%s@@@\n' % url)