native_client_sdk/src/build_tools/build_sdk.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Entry point for both build and try bots.
   7
   8 This script is invoked from XXX, usually without arguments
   9 to package an SDK. It automatically determines whether
  10 this SDK is for mac, win, linux.
  11
  12 The script inspects the following environment variables:
  13
  14 BUILDBOT_BUILDERNAME to determine whether the script is run locally
  15 and whether it should upload an SDK to file storage (GSTORE)
  16 """
  17
  18 # pylint: disable=W0621
  19
  20 # std python includes
  21 import datetime
  22 import glob
  23 import optparse
  24 import os
  25 import re
  26 import sys
  27
  28 if sys.version_info < (2, 6, 0):
  29   sys.stderr.write("python 2.6 or later is required run this script\n")
  30   sys.exit(1)
  31
  32 # local includes
  33 import buildbot_common
  34 import build_projects
  35 import build_updater
  36 import build_version
  37 import generate_notice
  38 import manifest_util
  39 import parse_dsc
  40 import verify_filelist
  41
  42 from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR
  43 from build_paths import NACLPORTS_DIR, GSTORE, GONACL_APPENGINE_SRC_DIR
  44
  45 # Add SDK make tools scripts to the python path.
  46 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  47 sys.path.append(os.path.join(NACL_DIR, 'build'))
  48
  49 import getos
  50 import oshelpers
  51
  52 BUILD_DIR = os.path.join(NACL_DIR, 'build')
  53 NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
  54 NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
  55
  56 CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
  57 PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
  58
  59 NACLPORTS_URL = 'https://naclports.googlecode.com/svn/trunk/src'
  60 NACLPORTS_REV = 1293
  61
  62 GYPBUILD_DIR = 'gypbuild'
  63
  64 options = None
  65
  66   # Map of: ToolchainName: (PackageName, SDKDir).
  67 TOOLCHAIN_PACKAGE_MAP = {
  68     'newlib': ('nacl_x86_newlib', '%(platform)s_x86_newlib'),
  69     'bionic': ('nacl_arm_bionic', '%(platform)s_arm_bionic'),
  70     'arm': ('nacl_arm_newlib', '%(platform)s_arm_newlib'),
  71     'glibc': ('nacl_x86_glibc', '%(platform)s_x86_glibc'),
  72     'pnacl': ('pnacl_newlib', '%(platform)s_pnacl')
  73     }
  74
  75
  76 def GetToolchainNaClInclude(tcname, tcpath, arch):
  77   if arch == 'x86':
  78     if tcname == 'pnacl':
  79       return os.path.join(tcpath, 'sdk', 'include')
  80     return os.path.join(tcpath, 'x86_64-nacl', 'include')
  81   elif arch == 'arm':
  82     return os.path.join(tcpath, 'arm-nacl', 'include')
  83   else:
  84     buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
  85
  86
  87 def GetGypGenDir(xarch):
  88   if xarch == 'arm':
  89     build_dir = GYPBUILD_DIR + '-arm'
  90   else:
  91     build_dir = GYPBUILD_DIR
  92   return os.path.join(OUT_DIR, build_dir, 'Release', 'gen')
  93
  94
  95 def GetGypBuiltLib(tcname, xarch=None):
  96   if tcname == 'pnacl':
  97     tcname = 'pnacl_newlib'
  98   if not xarch:
  99     xarch = ''
 100   return os.path.join(GetGypGenDir(xarch), 'tc_' + tcname, 'lib' + xarch)
 101
 102
 103 def GetToolchainNaClLib(tcname, tcpath, xarch):
 104   if tcname == 'pnacl':
 105     return os.path.join(tcpath, 'sdk', 'lib')
 106   elif xarch == '32':
 107     return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
 108   elif xarch == '64':
 109     return os.path.join(tcpath, 'x86_64-nacl', 'lib')
 110   elif xarch == 'arm':
 111     return os.path.join(tcpath, 'arm-nacl', 'lib')
 112
 113
 114 def GetToolchainDirName(tcname, xarch):
 115   if tcname == 'pnacl':
 116     return '%s_%s' % (getos.GetPlatform(), tcname)
 117   elif xarch == 'arm':
 118     return '%s_arm_%s' % (getos.GetPlatform(), tcname)
 119   else:
 120     return '%s_x86_%s' % (getos.GetPlatform(), tcname)
 121
 122
 123 def GetGypToolchainLib(tcname, xarch):
 124   if xarch == 'arm':
 125     toolchain = xarch
 126   else:
 127     toolchain = tcname
 128
 129   tcpath = os.path.join(GetGypGenDir(xarch), 'sdk',
 130                         '%s_x86' % getos.GetPlatform(),
 131                         TOOLCHAIN_PACKAGE_MAP[toolchain][0])
 132   return GetToolchainNaClLib(tcname, tcpath, xarch)
 133
 134
 135 def GetOutputToolchainLib(pepperdir, tcname, xarch):
 136   tcpath = os.path.join(pepperdir, 'toolchain',
 137                         GetToolchainDirName(tcname, xarch))
 138   return GetToolchainNaClLib(tcname, tcpath, xarch)
 139
 140
 141 def GetPNaClNativeLib(tcpath, arch):
 142   if arch not in ['arm', 'x86-32', 'x86-64']:
 143     buildbot_common.ErrorExit('Unknown architecture %s.' % arch)
 144   return os.path.join(tcpath, 'lib-' + arch)
 145
 146
 147 def BuildStepDownloadToolchains(toolchains):
 148   buildbot_common.BuildStep('Running package_version.py')
 149   args = [sys.executable, PKGVER, '--exclude', 'arm_trusted']
 150   if 'bionic' in toolchains:
 151     build_platform = '%s_x86' % getos.GetPlatform()
 152     args.extend(['--append', os.path.join(build_platform, 'nacl_arm_bionic')])
 153   args.append('sync')
 154   buildbot_common.Run(args, cwd=NACL_DIR)
 155
 156
 157 def BuildStepCleanPepperDirs(pepperdir, pepperdir_old):
 158   buildbot_common.BuildStep('Clean Pepper Dirs')
 159   buildbot_common.RemoveDir(pepperdir_old)
 160   buildbot_common.RemoveDir(pepperdir)
 161   buildbot_common.MakeDir(pepperdir)
 162
 163
 164 def BuildStepMakePepperDirs(pepperdir, subdirs):
 165   for subdir in subdirs:
 166     buildbot_common.MakeDir(os.path.join(pepperdir, subdir))
 167
 168 TEXT_FILES = [
 169   'AUTHORS',
 170   'COPYING',
 171   'LICENSE',
 172   'README.Makefiles',
 173   'getting_started/README',
 174 ]
 175
 176 def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision,
 177                            nacl_revision):
 178   buildbot_common.BuildStep('Add Text Files')
 179   InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES)
 180
 181   # Replace a few placeholders in README
 182   readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read()
 183   readme_text = readme_text.replace('${VERSION}', pepper_ver)
 184   readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision)
 185   readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision)
 186
 187   # Year/Month/Day Hour:Minute:Second
 188   time_format = '%Y/%m/%d %H:%M:%S'
 189   readme_text = readme_text.replace('${DATE}',
 190       datetime.datetime.now().strftime(time_format))
 191
 192   open(os.path.join(pepperdir, 'README'), 'w').write(readme_text)
 193
 194
 195 def BuildStepUntarToolchains(pepperdir, toolchains):
 196   buildbot_common.BuildStep('Untar Toolchains')
 197   platform = getos.GetPlatform()
 198   build_platform = '%s_x86' % platform
 199   tmpdir = os.path.join(OUT_DIR, 'tc_temp')
 200   buildbot_common.RemoveDir(tmpdir)
 201   buildbot_common.MakeDir(tmpdir)
 202
 203   # Create a list of extract packages tuples, the first part should be
 204   # "$PACKAGE_TARGET/$PACKAGE". The second part should be the destination
 205   # directory relative to pepperdir/toolchain.
 206   extract_packages = []
 207   for toolchain in toolchains:
 208     toolchain_map = TOOLCHAIN_PACKAGE_MAP.get(toolchain, None)
 209     if toolchain_map:
 210       package_name, tcname = toolchain_map
 211       package_tuple = (os.path.join(build_platform, package_name),
 212                        tcname % {'platform': platform})
 213       extract_packages.append(package_tuple)
 214
 215   if extract_packages:
 216     # Extract all of the packages into the temp directory.
 217     package_names = [package_tuple[0] for package_tuple in extract_packages]
 218     buildbot_common.Run([sys.executable, PKGVER,
 219                            '--packages', ','.join(package_names),
 220                            '--tar-dir', NACL_TOOLCHAINTARS_DIR,
 221                            '--dest-dir', tmpdir,
 222                            'extract'])
 223
 224     # Move all the packages we extracted to the correct destination.
 225     for package_name, dest_dir in extract_packages:
 226       full_src_dir = os.path.join(tmpdir, package_name)
 227       full_dst_dir = os.path.join(pepperdir, 'toolchain', dest_dir)
 228       buildbot_common.Move(full_src_dir, full_dst_dir)
 229
 230   # Cleanup the temporary directory we are no longer using.
 231   buildbot_common.RemoveDir(tmpdir)
 232
 233
 234 # List of toolchain headers to install.
 235 # Source is relative to top of Chromium tree, destination is relative
 236 # to the toolchain header directory.
 237 NACL_HEADER_MAP = {
 238   'newlib': [
 239       ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
 240       ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
 241       ('native_client/src/untrusted/irt/irt.h', ''),
 242       ('native_client/src/untrusted/irt/irt_dev.h', ''),
 243       ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
 244       ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
 245       ('native_client/src/untrusted/pthread/pthread.h', ''),
 246       ('native_client/src/untrusted/pthread/semaphore.h', ''),
 247       ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
 248       ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
 249   ],
 250   'glibc': [
 251       ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
 252       ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
 253       ('native_client/src/untrusted/irt/irt.h', ''),
 254       ('native_client/src/untrusted/irt/irt_dev.h', ''),
 255       ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
 256       ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
 257       ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
 258       ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
 259   ],
 260   'host': []
 261 }
 262
 263 def InstallFiles(src_root, dest_root, file_list):
 264   """Copy a set of files from src_root to dest_root according
 265   to the given mapping.  This allows files to be copied from
 266   to a location in the destination tree that is different to the
 267   location in the source tree.
 268
 269   If the destination mapping ends with a '/' then the destination
 270   basename is inherited from the the source file.
 271
 272   Wildcards can be used in the source list but it is not recommended
 273   as this can end up adding things to the SDK unintentionally.
 274   """
 275   for file_spec in file_list:
 276     # The list of files to install can be a simple list of
 277     # strings or a list of pairs, where each pair corresponds
 278     # to a mapping from source to destination names.
 279     if type(file_spec) == str:
 280       src_file = dest_file = file_spec
 281     else:
 282       src_file, dest_file = file_spec
 283
 284     src_file = os.path.join(src_root, src_file)
 285
 286     # Expand sources files using glob.
 287     sources = glob.glob(src_file)
 288     if not sources:
 289       sources = [src_file]
 290
 291     if len(sources) > 1 and not dest_file.endswith('/'):
 292       buildbot_common.ErrorExit("Target file must end in '/' when "
 293                                 "using globbing to install multiple files")
 294
 295     for source in sources:
 296       if dest_file.endswith('/'):
 297         dest = os.path.join(dest_file, os.path.basename(source))
 298       else:
 299         dest = dest_file
 300       dest = os.path.join(dest_root, dest)
 301       if not os.path.isdir(os.path.dirname(dest)):
 302         buildbot_common.MakeDir(os.path.dirname(dest))
 303       buildbot_common.CopyFile(source, dest)
 304
 305
 306 def InstallNaClHeaders(tc_dst_inc, tc_name):
 307   """Copies NaCl headers to expected locations in the toolchain."""
 308   if tc_name == 'arm':
 309     # arm toolchain header should be the same as the x86 newlib
 310     # ones
 311     tc_name = 'newlib'
 312
 313   InstallFiles(SRC_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name])
 314
 315
 316 def MakeNinjaRelPath(path):
 317   return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
 318
 319
 320 TOOLCHAIN_LIBS = {
 321   'bionic' : [
 322     'libminidump_generator.a',
 323     'libnacl_dyncode.a',
 324     'libnacl_exception.a',
 325     'libnacl_list_mappings.a',
 326     'libppapi.a',
 327   ],
 328   'newlib' : [
 329     'crti.o',
 330     'crtn.o',
 331     'libminidump_generator.a',
 332     'libnacl.a',
 333     'libnacl_dyncode.a',
 334     'libnacl_exception.a',
 335     'libnacl_list_mappings.a',
 336     'libnosys.a',
 337     'libppapi.a',
 338     'libppapi_stub.a',
 339     'libpthread.a',
 340   ],
 341   'glibc': [
 342     'libminidump_generator.a',
 343     'libminidump_generator.so',
 344     'libnacl.a',
 345     'libnacl_dyncode.a',
 346     'libnacl_dyncode.so',
 347     'libnacl_exception.a',
 348     'libnacl_exception.so',
 349     'libnacl_list_mappings.a',
 350     'libnacl_list_mappings.so',
 351     'libppapi.a',
 352     'libppapi.so',
 353     'libppapi_stub.a',
 354   ],
 355   'pnacl': [
 356     'libminidump_generator.a',
 357     'libnacl.a',
 358     'libnacl_dyncode.a',
 359     'libnacl_exception.a',
 360     'libnacl_list_mappings.a',
 361     'libnosys.a',
 362     'libppapi.a',
 363     'libppapi_stub.a',
 364     'libpthread.a',
 365   ]
 366 }
 367
 368
 369 def GypNinjaInstall(pepperdir, toolchains):
 370   build_dir = GYPBUILD_DIR
 371   ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release')
 372   tools_files = [
 373     ['sel_ldr', 'sel_ldr_x86_32'],
 374     ['ncval_new', 'ncval'],
 375     ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'],
 376     ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'],
 377   ]
 378
 379   platform = getos.GetPlatform()
 380
 381   # TODO(binji): dump_syms doesn't currently build on Windows. See
 382   # http://crbug.com/245456
 383   if platform != 'win':
 384     tools_files += [
 385       ['dump_syms', 'dump_syms'],
 386       ['minidump_dump', 'minidump_dump'],
 387       ['minidump_stackwalk', 'minidump_stackwalk']
 388     ]
 389
 390   tools_files.append(['sel_ldr64', 'sel_ldr_x86_64'])
 391
 392   if platform == 'linux':
 393     tools_files.append(['nacl_helper_bootstrap',
 394                         'nacl_helper_bootstrap_x86_32'])
 395     tools_files.append(['nacl_helper_bootstrap64',
 396                         'nacl_helper_bootstrap_x86_64'])
 397
 398   buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
 399
 400   # Add .exe extensions to all windows tools
 401   for pair in tools_files:
 402     if platform == 'win' and not pair[0].endswith('.nexe'):
 403       pair[0] += '.exe'
 404       pair[1] += '.exe'
 405
 406   InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files)
 407
 408   # Add ARM binaries
 409   if platform == 'linux':
 410     tools_files = [
 411       ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
 412       ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
 413       ['sel_ldr', 'sel_ldr_arm'],
 414       ['nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm']
 415     ]
 416     ninja_out_dir = os.path.join(OUT_DIR, build_dir + '-arm', 'Release')
 417     InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files)
 418
 419   for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']):
 420     if tc == 'pnacl':
 421       xarches = (None,)
 422     else:
 423       xarches = ('arm', '32', '64')
 424
 425     for xarch in xarches:
 426       if tc == 'glibc' and xarch == 'arm':
 427         continue
 428
 429       src_dir = GetGypBuiltLib(tc, xarch)
 430       dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch)
 431       InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc])
 432
 433       # Copy ARM newlib components to bionic
 434       if tc == 'newlib' and xarch == 'arm' and 'bionic' in toolchains:
 435         bionic_dir = GetOutputToolchainLib(pepperdir, 'bionic', xarch)
 436         InstallFiles(src_dir, bionic_dir, TOOLCHAIN_LIBS['bionic'])
 437
 438       if tc != 'pnacl':
 439         src_dir = GetGypToolchainLib(tc, xarch)
 440         InstallFiles(src_dir, dst_dir, ['crt1.o'])
 441
 442
 443 def GypNinjaBuild_NaCl(rel_out_dir):
 444   gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl')
 445   nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
 446   all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp')
 447
 448   out_dir = MakeNinjaRelPath(rel_out_dir)
 449   out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm')
 450   GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir)
 451   GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm)
 452   GypNinjaBuild('ia32', gyp_py, all_gyp, 'ncval_new', out_dir)
 453
 454   platform = getos.GetPlatform()
 455   if platform == 'win':
 456     NinjaBuild('sel_ldr64', out_dir)
 457   else:
 458     out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-64')
 459     GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'sel_ldr', out_dir_64)
 460
 461     # We only need sel_ldr from the 64-bit out directory.
 462     # sel_ldr needs to be renamed, so we'll call it sel_ldr64.
 463     files_to_copy = [('sel_ldr', 'sel_ldr64')]
 464     if platform == 'linux':
 465       files_to_copy.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap64'))
 466
 467     for src, dst in files_to_copy:
 468       buildbot_common.CopyFile(
 469           os.path.join(SRC_DIR, out_dir_64, 'Release', src),
 470           os.path.join(SRC_DIR, out_dir, 'Release', dst))
 471
 472
 473 def GypNinjaBuild_Breakpad(rel_out_dir):
 474   # TODO(binji): dump_syms doesn't currently build on Windows. See
 475   # http://crbug.com/245456
 476   if getos.GetPlatform() == 'win':
 477     return
 478
 479   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 480   out_dir = MakeNinjaRelPath(rel_out_dir)
 481   gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp')
 482   build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
 483   GypNinjaBuild('ia32', gyp_py, gyp_file, build_list, out_dir)
 484
 485
 486 def GypNinjaBuild_PPAPI(arch, rel_out_dir):
 487   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 488   out_dir = MakeNinjaRelPath(rel_out_dir)
 489   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client',
 490                           'native_client.gyp')
 491   GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir)
 492
 493
 494 def GypNinjaBuild_Pnacl(rel_out_dir, target_arch):
 495   # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the
 496   # Chromium build, and once here. When we move more of the SDK build process
 497   # to gyp, we can remove this.
 498   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 499
 500   out_dir = MakeNinjaRelPath(rel_out_dir)
 501   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src',
 502                           'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
 503   targets = ['aot']
 504   GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir, False)
 505
 506
 507 def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets,
 508                   out_dir, force_arm_gcc=True):
 509   gyp_env = dict(os.environ)
 510   gyp_env['GYP_GENERATORS'] = 'ninja'
 511   gyp_defines = []
 512   if options.mac_sdk:
 513     gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
 514   if arch:
 515     gyp_defines.append('target_arch=%s' % arch)
 516     if arch == 'arm':
 517       if getos.GetPlatform() == 'linux':
 518         gyp_env['CC'] = 'arm-linux-gnueabihf-gcc'
 519         gyp_env['CXX'] = 'arm-linux-gnueabihf-g++'
 520         gyp_env['AR'] = 'arm-linux-gnueabihf-ar'
 521         gyp_env['AS'] = 'arm-linux-gnueabihf-as'
 522         gyp_env['CC_host'] = 'cc'
 523         gyp_env['CXX_host'] = 'c++'
 524       gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1',
 525           'arm_float_abi=hard']
 526       if force_arm_gcc:
 527         gyp_defines.append('nacl_enable_arm_gcc=1')
 528   if getos.GetPlatform() == 'mac':
 529     gyp_defines.append('clang=1')
 530
 531   gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
 532   for key in ['GYP_GENERATORS', 'GYP_DEFINES', 'CC']:
 533     value = gyp_env.get(key)
 534     if value is not None:
 535       print '%s="%s"' % (key, value)
 536   gyp_generator_flags = ['-G', 'output_dir=%s' % (out_dir,)]
 537   gyp_depth = '--depth=.'
 538   buildbot_common.Run(
 539       [sys.executable, gyp_py_script, gyp_file, gyp_depth] + \
 540           gyp_generator_flags,
 541       cwd=SRC_DIR,
 542       env=gyp_env)
 543   NinjaBuild(targets, out_dir)
 544
 545
 546 def NinjaBuild(targets, out_dir):
 547   if type(targets) is not list:
 548     targets = [targets]
 549   out_config_dir = os.path.join(out_dir, 'Release')
 550   buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
 551
 552
 553 def BuildStepBuildToolchains(pepperdir, toolchains):
 554   buildbot_common.BuildStep('SDK Items')
 555
 556   GypNinjaBuild_NaCl(GYPBUILD_DIR)
 557   GypNinjaBuild_Breakpad(GYPBUILD_DIR)
 558
 559   platform = getos.GetPlatform()
 560   newlibdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_newlib')
 561   glibcdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_glibc')
 562   armdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_newlib')
 563   pnacldir = os.path.join(pepperdir, 'toolchain', platform + '_pnacl')
 564
 565   if set(toolchains) & set(['glibc', 'newlib']):
 566     GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR)
 567
 568   if 'arm' in toolchains:
 569     GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm')
 570
 571   GypNinjaInstall(pepperdir, toolchains)
 572
 573   if 'newlib' in toolchains:
 574     InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
 575                        'newlib')
 576
 577   if 'glibc' in toolchains:
 578     InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
 579                        'glibc')
 580
 581   if 'arm' in toolchains:
 582     InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'),
 583                        'arm')
 584
 585   if 'pnacl' in toolchains:
 586     # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
 587     for arch in ('ia32', 'arm'):
 588       # Fill in the latest native pnacl shim library from the chrome build.
 589       build_dir = GYPBUILD_DIR + '-pnacl-' + arch
 590       GypNinjaBuild_Pnacl(build_dir, arch)
 591       if arch == 'ia32':
 592         nacl_arches = ['x86-32', 'x86-64']
 593       elif arch == 'arm':
 594         nacl_arches = ['arm']
 595       else:
 596         buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
 597       for nacl_arch in nacl_arches:
 598         release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release',
 599                                          'gen', 'tc_pnacl_translate',
 600                                          'lib-' + nacl_arch)
 601
 602         buildbot_common.CopyFile(
 603             os.path.join(release_build_dir, 'libpnacl_irt_shim.a'),
 604             GetPNaClNativeLib(pnacldir, nacl_arch))
 605
 606     InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
 607                        'newlib')
 608
 609
 610 def MakeDirectoryOrClobber(pepperdir, dirname, clobber):
 611   dirpath = os.path.join(pepperdir, dirname)
 612   if clobber:
 613     buildbot_common.RemoveDir(dirpath)
 614   buildbot_common.MakeDir(dirpath)
 615
 616   return dirpath
 617
 618
 619 def BuildStepUpdateHelpers(pepperdir, clobber):
 620   buildbot_common.BuildStep('Update project helpers')
 621   build_projects.UpdateHelpers(pepperdir, clobber=clobber)
 622
 623
 624 def BuildStepUpdateUserProjects(pepperdir, toolchains,
 625                                 build_experimental, clobber):
 626   buildbot_common.BuildStep('Update examples and libraries')
 627
 628   filters = {}
 629   if not build_experimental:
 630     filters['EXPERIMENTAL'] = False
 631   if toolchains:
 632     toolchains = toolchains[:]
 633
 634     # arm isn't a valid toolchain for build_projects
 635     if 'arm' in toolchains:
 636       toolchains.remove('arm')
 637
 638     if 'host' in toolchains:
 639       toolchains.remove('host')
 640       toolchains.append(getos.GetPlatform())
 641
 642     filters['TOOLS'] = toolchains
 643
 644   # Update examples and libraries
 645   filters['DEST'] = [
 646     'getting_started',
 647     'examples/api',
 648     'examples/demo',
 649     'examples/tutorial',
 650     'src'
 651   ]
 652
 653   tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
 654   build_projects.UpdateProjects(pepperdir, tree, clobber=clobber,
 655                                 toolchains=toolchains)
 656
 657
 658 def BuildStepMakeAll(pepperdir, directory, step_name,
 659                      deps=True, clean=False, config='Debug', args=None):
 660   buildbot_common.BuildStep(step_name)
 661   build_projects.BuildProjectsBranch(pepperdir, directory, clean,
 662                                      deps, config, args)
 663
 664
 665 def BuildStepBuildLibraries(pepperdir, directory):
 666   BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug',
 667       clean=True, config='Debug')
 668   BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release',
 669       clean=True, config='Release')
 670
 671   # Cleanup .pyc file generated while building libraries.  Without
 672   # this we would end up shipping the pyc in the SDK tarball.
 673   buildbot_common.RemoveFile(os.path.join(pepperdir, 'tools', '*.pyc'))
 674
 675
 676 def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None):
 677   # Look for LICENSE files
 678   license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT')
 679
 680   license_files = []
 681   for root, _, files in os.walk(fileroot):
 682     for filename in files:
 683       if license_filenames_re.match(filename):
 684         path = os.path.join(root, filename)
 685         license_files.append(path)
 686
 687   if extra_files:
 688     license_files += [os.path.join(fileroot, f) for f in extra_files]
 689   print '\n'.join(license_files)
 690
 691   if not os.path.isabs(output_filename):
 692     output_filename = os.path.join(fileroot, output_filename)
 693   generate_notice.Generate(output_filename, fileroot, license_files)
 694
 695
 696 def BuildStepVerifyFilelist(pepperdir):
 697   buildbot_common.BuildStep('Verify SDK Files')
 698   file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list')
 699   try:
 700     verify_filelist.Verify(file_list_path, pepperdir)
 701     print 'OK'
 702   except verify_filelist.ParseException, e:
 703     buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e)
 704   except verify_filelist.VerifyException, e:
 705     file_list_rel = os.path.relpath(file_list_path)
 706     verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py'
 707     verify_filelist_py = os.path.relpath(verify_filelist_py)
 708     pepperdir_rel = os.path.relpath(pepperdir)
 709
 710     msg = """\
 711 SDK verification failed:
 712
 713 %s
 714 Add/remove files from %s to fix.
 715
 716 Run:
 717     ./%s %s %s
 718 to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel,
 719                pepperdir_rel)
 720     buildbot_common.ErrorExit(msg)
 721
 722
 723 def BuildStepTarBundle(pepper_ver, tarfile):
 724   buildbot_common.BuildStep('Tar Pepper Bundle')
 725   buildbot_common.MakeDir(os.path.dirname(tarfile))
 726   buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
 727        'pepper_' + pepper_ver], cwd=NACL_DIR)
 728
 729
 730 def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile,
 731                       archive_url):
 732   with open(tarfile, 'rb') as tarfile_stream:
 733     archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
 734         tarfile_stream)
 735
 736   archive = manifest_util.Archive(manifest_util.GetHostOS())
 737   archive.url = archive_url
 738   archive.size = archive_size
 739   archive.checksum = archive_sha1
 740
 741   bundle = manifest_util.Bundle('pepper_' + pepper_ver)
 742   bundle.revision = int(chrome_revision)
 743   bundle.repath = 'pepper_' + pepper_ver
 744   bundle.version = int(pepper_ver)
 745   bundle.description = (
 746       'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % (
 747             pepper_ver, chrome_revision, nacl_revision))
 748   bundle.stability = 'dev'
 749   bundle.recommended = 'no'
 750   bundle.archives = [archive]
 751   return bundle
 752
 753
 754 def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision,
 755                            tarfile):
 756   buildbot_common.BuildStep('Archive %s' % name)
 757   bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
 758       build_version.ChromeVersion(),)
 759   tarname = os.path.basename(tarfile)
 760   tarfile_dir = os.path.dirname(tarfile)
 761   buildbot_common.Archive(tarname, bucket_path, tarfile_dir)
 762
 763   # generate "manifest snippet" for this archive.
 764   archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
 765       build_version.ChromeVersion(), tarname)
 766   bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision,
 767                              tarfile, archive_url)
 768
 769   manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
 770   with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
 771     manifest_snippet_stream.write(bundle.GetDataAsString())
 772
 773   buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
 774                           step_link=False)
 775
 776
 777 def BuildStepArchiveSDKTools():
 778   # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot.
 779   builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
 780   if builder_name == 'linux-sdk-multi':
 781     buildbot_common.BuildStep('Build SDK Tools')
 782     build_updater.BuildUpdater(OUT_DIR)
 783
 784     buildbot_common.BuildStep('Archive SDK Tools')
 785     bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
 786         build_version.ChromeVersion(),)
 787     buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR,
 788                             step_link=False)
 789     buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR,
 790                             step_link=False)
 791
 792
 793 def BuildStepSyncNaClPorts():
 794   """Pull the pinned revision of naclports from SVN."""
 795   buildbot_common.BuildStep('Sync naclports')
 796   if not os.path.exists(NACLPORTS_DIR):
 797     # checkout new copy of naclports
 798     cmd = ['svn', 'checkout', '-q', '-r', str(NACLPORTS_REV), NACLPORTS_URL,
 799            'naclports']
 800     buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR))
 801   else:
 802     # sync existing copy to pinned revision.
 803     cmd = ['svn', 'update', '-r', str(NACLPORTS_REV)]
 804     buildbot_common.Run(cmd, cwd=NACLPORTS_DIR)
 805
 806
 807 def BuildStepBuildNaClPorts(pepper_ver, pepperdir):
 808   """Build selected naclports in all configurations."""
 809   # TODO(sbc): currently naclports doesn't know anything about
 810   # Debug builds so the Debug subfolders are all empty.
 811
 812   env = dict(os.environ)
 813   env['NACL_SDK_ROOT'] = pepperdir
 814   env['PEPPER_DIR'] = os.path.basename(pepperdir)  # pepper_NN
 815   env['NACLPORTS_NO_ANNOTATE'] = "1"
 816   env['NACLPORTS_NO_UPLOAD'] = "1"
 817   env['BUILDBOT_GOT_REVISION'] = str(NACLPORTS_REV)
 818
 819   build_script = 'build_tools/buildbot_sdk_bundle.sh'
 820   buildbot_common.BuildStep('Build naclports')
 821
 822   bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
 823   out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver)
 824
 825   # Remove the sdk_bundle directory to remove stale files from previous builds.
 826   buildbot_common.RemoveDir(bundle_dir)
 827
 828   buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR)
 829
 830   # Some naclports do not include a standalone LICENSE/COPYING file
 831   # so we explicitly list those here for inclusion.
 832   extra_licenses = ('tinyxml/readme.txt',
 833                     'jpeg-8d/README',
 834                     'zlib-1.2.3/README')
 835   src_root = os.path.join(NACLPORTS_DIR, 'out', 'build')
 836   output_license = os.path.join(out_dir, 'ports', 'LICENSE')
 837   GenerateNotice(src_root , output_license, extra_licenses)
 838   readme = os.path.join(out_dir, 'ports', 'README')
 839   oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme])
 840
 841
 842 def BuildStepTarNaClPorts(pepper_ver, tarfile):
 843   """Create tar archive containing headers and libs from naclports build."""
 844   buildbot_common.BuildStep('Tar naclports Bundle')
 845   buildbot_common.MakeDir(os.path.dirname(tarfile))
 846   pepper_dir = 'pepper_%s' % pepper_ver
 847   archive_dirs = [os.path.join(pepper_dir, 'ports')]
 848
 849   ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
 850   cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile]
 851   cmd += archive_dirs
 852   buildbot_common.Run(cmd, cwd=NACL_DIR)
 853
 854
 855 def BuildStepBuildAppEngine(pepperdir, chrome_revision):
 856   """Build the projects found in src/gonacl_appengine/src"""
 857   buildbot_common.BuildStep('Build GoNaCl AppEngine Projects')
 858   cmd = ['make', 'upload', 'REVISION=%s' % chrome_revision]
 859   env = dict(os.environ)
 860   env['NACL_SDK_ROOT'] = pepperdir
 861   env['NACLPORTS_NO_ANNOTATE'] = "1"
 862   buildbot_common.Run(cmd, env=env, cwd=GONACL_APPENGINE_SRC_DIR)
 863
 864
 865 def main(args):
 866   parser = optparse.OptionParser(description=__doc__)
 867   parser.add_option('--nacl-tree-path',
 868       help='Path to native client tree for bionic build.',
 869       dest='nacl_tree_path')
 870   parser.add_option('--qemu', help='Add qemu for ARM.',
 871       action='store_true')
 872   parser.add_option('--bionic', help='Add bionic build.',
 873       action='store_true')
 874   parser.add_option('--tar', help='Force the tar step.',
 875       action='store_true')
 876   parser.add_option('--archive', help='Force the archive step.',
 877       action='store_true')
 878   parser.add_option('--release', help='PPAPI release version.',
 879       dest='release', default=None)
 880   parser.add_option('--build-ports',
 881       help='Build naclport bundle.', action='store_true')
 882   parser.add_option('--build-app-engine',
 883       help='Build AppEngine demos.', action='store_true')
 884   parser.add_option('--experimental',
 885       help='build experimental examples and libraries', action='store_true',
 886       dest='build_experimental')
 887   parser.add_option('--skip-toolchain', help='Skip toolchain untar',
 888       action='store_true')
 889   parser.add_option('--mac-sdk',
 890       help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
 891
 892   # To setup bash completion for this command first install optcomplete
 893   # and then add this line to your .bashrc:
 894   #  complete -F _optcomplete build_sdk.py
 895   try:
 896     import optcomplete
 897     optcomplete.autocomplete(parser)
 898   except ImportError:
 899     pass
 900
 901   global options
 902   options, args = parser.parse_args(args[1:])
 903   if args:
 904     parser.error("Unexpected arguments: %s" % str(args))
 905
 906   if options.nacl_tree_path:
 907     options.bionic = True
 908     toolchain_build = os.path.join(options.nacl_tree_path, 'toolchain_build')
 909     print 'WARNING: Building bionic toolchain from NaCl checkout.'
 910     print 'This option builds bionic from the sources currently in the'
 911     print 'provided NativeClient checkout, and the results instead of '
 912     print 'downloading a toolchain from the builder. This may result in a'
 913     print 'NaCl SDK that can not run on ToT chrome.'
 914     print 'NOTE:  To clobber you will need to run toolchain_build_bionic.py'
 915     print 'directly from the NativeClient checkout.'
 916     print ''
 917     response = raw_input("Type 'y' and hit enter to continue.\n")
 918     if response != 'y' and response != 'Y':
 919       print 'Aborting.'
 920       return 1
 921
 922     # Get head version of NativeClient tree
 923     buildbot_common.BuildStep('Build bionic toolchain.')
 924     buildbot_common.Run([sys.executable, 'toolchain_build_bionic.py', '-f'],
 925                         cwd=toolchain_build)
 926   else:
 927     toolchain_build = None
 928
 929   if buildbot_common.IsSDKBuilder():
 930     options.archive = True
 931     options.build_ports = True
 932     options.build_app_engine = True
 933     options.tar = True
 934
 935   toolchains = ['newlib', 'glibc', 'arm', 'pnacl', 'host']
 936
 937   # Changes for experimental bionic builder
 938   if options.bionic:
 939     toolchains.append('bionic')
 940     options.build_ports = False
 941     options.build_app_engine = False
 942
 943   print 'Building: ' + ' '.join(toolchains)
 944
 945   if options.archive and not options.tar:
 946     parser.error('Incompatible arguments with archive.')
 947
 948   chrome_version = int(build_version.ChromeMajorVersion())
 949   chrome_revision = build_version.ChromeRevision()
 950   nacl_revision = build_version.NaClRevision()
 951   pepper_ver = str(chrome_version)
 952   pepper_old = str(chrome_version - 1)
 953   pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
 954   pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old)
 955   if options.bionic:
 956     tarname = 'naclsdk_bionic.tar.bz2'
 957   else:
 958     tarname = 'naclsdk_' + getos.GetPlatform() + '.tar.bz2'
 959   tarfile = os.path.join(OUT_DIR, tarname)
 960
 961   if options.release:
 962     pepper_ver = options.release
 963   print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision)
 964
 965   if 'NACL_SDK_ROOT' in os.environ:
 966     # We don't want the currently configured NACL_SDK_ROOT to have any effect
 967     # of the build.
 968     del os.environ['NACL_SDK_ROOT']
 969
 970   if not options.skip_toolchain:
 971     BuildStepCleanPepperDirs(pepperdir, pepperdir_old)
 972     BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools'])
 973     BuildStepDownloadToolchains(toolchains)
 974     if options.nacl_tree_path:
 975       # Instead of untarring, copy the raw bionic toolchain
 976       not_bionic = [i for i in toolchains if i != 'bionic']
 977       BuildStepUntarToolchains(pepperdir, not_bionic)
 978       tcname = GetToolchainDirName('bionic', 'arm')
 979       srcdir = os.path.join(toolchain_build, 'out', tcname)
 980       bionicdir = os.path.join(pepperdir, 'toolchain', tcname)
 981       oshelpers.Copy(['-r', srcdir, bionicdir])
 982     else:
 983       BuildStepUntarToolchains(pepperdir, toolchains)
 984     BuildStepBuildToolchains(pepperdir, toolchains)
 985
 986   BuildStepUpdateHelpers(pepperdir, True)
 987   BuildStepUpdateUserProjects(pepperdir, toolchains,
 988                               options.build_experimental, True)
 989
 990   BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision)
 991
 992   # Ship with libraries prebuilt, so run that first.
 993   BuildStepBuildLibraries(pepperdir, 'src')
 994   GenerateNotice(pepperdir)
 995
 996   # Verify the SDK contains what we expect.
 997   if not options.bionic:
 998     BuildStepVerifyFilelist(pepperdir)
 999
1000   if options.tar:
1001     BuildStepTarBundle(pepper_ver, tarfile)
1002
1003   if options.build_ports and getos.GetPlatform() == 'linux':
1004     ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2')
1005     BuildStepSyncNaClPorts()
1006     BuildStepBuildNaClPorts(pepper_ver, pepperdir)
1007     if options.tar:
1008       BuildStepTarNaClPorts(pepper_ver, ports_tarfile)
1009
1010   if options.build_app_engine and getos.GetPlatform() == 'linux':
1011     BuildStepBuildAppEngine(pepperdir, chrome_revision)
1012
1013   if options.qemu:
1014     qemudir = os.path.join(NACL_DIR, 'toolchain', 'linux_arm-trusted')
1015     oshelpers.Copy(['-r', qemudir, pepperdir])
1016
1017   # Archive on non-trybots.
1018   if options.archive:
1019     BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision,
1020                            tarfile)
1021     if options.build_ports and getos.GetPlatform() == 'linux':
1022       BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision,
1023                              nacl_revision, ports_tarfile)
1024     BuildStepArchiveSDKTools()
1025
1026   return 0
1027
1028
1029 if __name__ == '__main__':
1030   try:
1031     sys.exit(main(sys.argv))
1032   except KeyboardInterrupt:
1033     buildbot_common.ErrorExit('build_sdk: interrupted')