native_client_sdk/src/build_tools/buildbot_common.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Common utilities for all buildbot scripts that specifically don't rely
   6 on having a full chromium checkout.
   7 """
   8
   9 import os
  10 import subprocess
  11 import sys
  12
  13 from build_paths import SDK_SRC_DIR, NACL_DIR
  14
  15 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  16 import oshelpers
  17 import getos
  18
  19
  20 verbose = True
  21
  22
  23 def IsSDKBuilder():
  24   """Returns True if this script is running on an SDK builder.
  25
  26   False means it is either running on a trybot, or a user's machine.
  27
  28   Trybot names:
  29     (win|mac|linux)_nacl_sdk
  30
  31   Build-only Trybot names:
  32     (win|mac|linux)_nacl_sdk_build
  33
  34   Builder names:
  35     (windows|mac|linux)-sdk-multi(bionic)(rel)?"""
  36   bot =  os.getenv('BUILDBOT_BUILDERNAME', '')
  37   return '-sdk-multi' in bot or '-sdk-bionic-multi' in bot
  38
  39
  40 def IsSDKTrybot():
  41   """Returns True if this script is running on an SDK trybot.
  42
  43   False means it is either running on an SDK builder, or a user's machine.
  44
  45   See IsSDKBuilder above for trybot/buildbot names."""
  46   return '_nacl_sdk' in os.getenv('BUILDBOT_BUILDERNAME', '')
  47
  48
  49 def ErrorExit(msg):
  50   """Write and error to stderr, then exit with 1 signaling failure."""
  51   sys.stderr.write(str(msg) + '\n')
  52   sys.exit(1)
  53
  54
  55 def Trace(msg):
  56   if verbose:
  57     sys.stderr.write(str(msg) + '\n')
  58
  59
  60 def GetWindowsEnvironment():
  61   sys.path.append(os.path.join(NACL_DIR, 'buildbot'))
  62   import buildbot_standard
  63
  64   # buildbot_standard.SetupWindowsEnvironment expects a "context" object. We'll
  65   # fake enough of that here to work.
  66   class FakeContext(object):
  67     def __init__(self):
  68       self.env = os.environ
  69
  70     def GetEnv(self, key):
  71       return self.env[key]
  72
  73     def __getitem__(self, key):
  74       return self.env[key]
  75
  76     def SetEnv(self, key, value):
  77       self.env[key] = value
  78
  79     def __setitem__(self, key, value):
  80       self.env[key] = value
  81
  82   context = FakeContext()
  83   buildbot_standard.SetupWindowsEnvironment(context)
  84
  85   # buildbot_standard.SetupWindowsEnvironment adds the directory which contains
  86   # vcvarsall.bat to the path, but not the directory which contains cl.exe,
  87   # link.exe, etc.
  88   # Running vcvarsall.bat adds the correct directories to the path, which we
  89   # extract below.
  90   process = subprocess.Popen('vcvarsall.bat x86 > NUL && set',
  91       stdout=subprocess.PIPE, env=context.env, shell=True)
  92   stdout, _ = process.communicate()
  93
  94   # Parse environment from "set" command above.
  95   # It looks like this:
  96   # KEY1=VALUE1\r\n
  97   # KEY2=VALUE2\r\n
  98   # ...
  99   return dict(line.split('=') for line in stdout.split('\r\n')[:-1])
 100
 101
 102 def BuildStep(name):
 103   """Annotate a buildbot build step."""
 104   sys.stdout.flush()
 105   sys.stderr.write('\n@@@BUILD_STEP %s@@@\n' % name)
 106
 107
 108 def Run(args, cwd=None, env=None, shell=False):
 109   """Start a process with the provided arguments.
 110
 111   Starts a process in the provided directory given the provided arguments. If
 112   shell is not False, the process is launched via the shell to provide shell
 113   interpretation of the arguments.  Shell behavior can differ between platforms
 114   so this should be avoided when not using platform dependent shell scripts."""
 115
 116   # We need to modify the environment to build host on Windows.
 117   if not env and getos.GetPlatform() == 'win':
 118     env = GetWindowsEnvironment()
 119
 120   Trace('Running: ' + ' '.join(args))
 121   sys.stdout.flush()
 122   sys.stderr.flush()
 123   try:
 124     subprocess.check_call(args, cwd=cwd, env=env, shell=shell)
 125   except subprocess.CalledProcessError as e:
 126     sys.stdout.flush()
 127     sys.stderr.flush()
 128     ErrorExit('buildbot_common: %s' % e)
 129
 130   sys.stdout.flush()
 131   sys.stderr.flush()
 132
 133
 134 def CopyDir(src, dst, excludes=('.svn', '*/.svn')):
 135   """Recursively copy a directory using."""
 136   args = ['-r', src, dst]
 137   for exc in excludes:
 138     args.append('--exclude=' + exc)
 139   Trace('cp -r %s %s' % (src, dst))
 140   if os.path.abspath(src) == os.path.abspath(dst):
 141     ErrorExit('ERROR: Copying directory onto itself: ' + src)
 142   oshelpers.Copy(args)
 143
 144
 145 def CopyFile(src, dst):
 146   Trace('cp %s %s' % (src, dst))
 147   if os.path.abspath(src) == os.path.abspath(dst):
 148     ErrorExit('ERROR: Copying file onto itself: ' + src)
 149   args = [src, dst]
 150   oshelpers.Copy(args)
 151
 152
 153 def RemoveDir(dst):
 154   """Remove the provided path."""
 155   Trace('rm -fr ' + dst)
 156   oshelpers.Remove(['-fr', dst])
 157
 158
 159 def MakeDir(dst):
 160   """Create the path including all parent directories as needed."""
 161   Trace('mkdir -p ' + dst)
 162   oshelpers.Mkdir(['-p', dst])
 163
 164
 165 def Move(src, dst):
 166   """Move the path src to dst."""
 167   Trace('mv -f %s %s' % (src, dst))
 168   oshelpers.Move(['-f', src, dst])
 169
 170
 171 def RemoveFile(dst):
 172   """Remove the provided file."""
 173   Trace('rm ' + dst)
 174   oshelpers.Remove(['-f', dst])
 175
 176
 177 BOT_GSUTIL = '/b/build/scripts/slave/gsutil'
 178 # On Windows, the current working directory may be on a different drive than
 179 # gsutil.
 180 WIN_BOT_GSUTIL = 'E:' + BOT_GSUTIL
 181 LOCAL_GSUTIL = 'gsutil'
 182
 183
 184 def GetGsutil():
 185   if os.environ.get('BUILDBOT_BUILDERNAME') \
 186      and not os.environ.get('BUILDBOT_FAKE'):
 187     if getos.GetPlatform() == 'win':
 188       return WIN_BOT_GSUTIL
 189     return BOT_GSUTIL
 190   else:
 191     return LOCAL_GSUTIL
 192
 193
 194 def Archive(filename, bucket_path, cwd=None, step_link=True):
 195   """Upload the given filename to Google Store."""
 196   full_dst = 'gs://%s/%s' % (bucket_path, filename)
 197
 198   # Since GetGsutil() might just return 'gsutil' and expect it to be looked
 199   # up in the PATH, we must pass shell=True on windows.
 200   # Without shell=True the windows implementation of subprocess.call will not
 201   # search the PATH for the executable: http://bugs.python.org/issue8557
 202   shell = getos.GetPlatform() == 'win'
 203
 204   cmd = [GetGsutil(), 'cp', '-a', 'public-read', filename, full_dst]
 205   Run(cmd, shell=shell, cwd=cwd)
 206   url = 'https://storage.googleapis.com/%s/%s' % (bucket_path, filename)
 207   if step_link:
 208     sys.stdout.flush()
 209     sys.stderr.write('@@@STEP_LINK@download@%s@@@\n' % url)