native_client_sdk/src/build_tools/build_sdk.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Entry point for both build and try bots.
   7
   8 This script is invoked from XXX, usually without arguments
   9 to package an SDK. It automatically determines whether
  10 this SDK is for mac, win, linux.
  11
  12 The script inspects the following environment variables:
  13
  14 BUILDBOT_BUILDERNAME to determine whether the script is run locally
  15 and whether it should upload an SDK to file storage (GSTORE)
  16 """
  17
  18 # pylint: disable=W0621
  19
  20 # std python includes
  21 import datetime
  22 import glob
  23 import optparse
  24 import os
  25 import re
  26 import sys
  27
  28 if sys.version_info < (2, 6, 0):
  29   sys.stderr.write("python 2.6 or later is required run this script\n")
  30   sys.exit(1)
  31
  32 # local includes
  33 import buildbot_common
  34 import build_projects
  35 import build_updater
  36 import build_version
  37 import generate_notice
  38 import manifest_util
  39 import parse_dsc
  40 import verify_filelist
  41
  42 from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR
  43 from build_paths import NACLPORTS_DIR, GSTORE, GONACL_APPENGINE_SRC_DIR
  44
  45 # Add SDK make tools scripts to the python path.
  46 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  47 sys.path.append(os.path.join(NACL_DIR, 'build'))
  48
  49 import getos
  50 import oshelpers
  51
  52 BUILD_DIR = os.path.join(NACL_DIR, 'build')
  53 NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
  54 NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
  55
  56 CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
  57 PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
  58
  59 NACLPORTS_URL = 'https://naclports.googlecode.com/svn/trunk/src'
  60 NACLPORTS_REV = 1293
  61
  62 GYPBUILD_DIR = 'gypbuild'
  63
  64 options = None
  65
  66 # Map of: ToolchainName: (PackageName, SDKDir).
  67 TOOLCHAIN_PACKAGE_MAP = {
  68     'newlib': ('nacl_x86_newlib', '%(platform)s_x86_newlib'),
  69     'bionic': ('nacl_arm_bionic', '%(platform)s_arm_bionic'),
  70     'arm': ('nacl_arm_newlib', '%(platform)s_arm_newlib'),
  71     'glibc': ('nacl_x86_glibc', '%(platform)s_x86_glibc'),
  72     'pnacl': ('pnacl_newlib', '%(platform)s_pnacl')
  73     }
  74
  75
  76 def GetToolchainNaClInclude(tcname, tcpath, arch):
  77   if arch == 'x86':
  78     if tcname == 'pnacl':
  79       return os.path.join(tcpath, 'sdk', 'include')
  80     return os.path.join(tcpath, 'x86_64-nacl', 'include')
  81   elif arch == 'arm':
  82     return os.path.join(tcpath, 'arm-nacl', 'include')
  83   else:
  84     buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
  85
  86
  87 def GetGypGenDir(xarch):
  88   if xarch == 'arm':
  89     build_dir = GYPBUILD_DIR + '-arm'
  90   else:
  91     build_dir = GYPBUILD_DIR
  92   return os.path.join(OUT_DIR, build_dir, 'Release', 'gen')
  93
  94
  95 def GetGypBuiltLib(tcname, xarch=None):
  96   if tcname == 'pnacl':
  97     tcname = 'pnacl_newlib'
  98   if not xarch:
  99     xarch = ''
 100   return os.path.join(GetGypGenDir(xarch), 'tc_' + tcname, 'lib' + xarch)
 101
 102
 103 def GetToolchainNaClLib(tcname, tcpath, xarch):
 104   if tcname == 'pnacl':
 105     return os.path.join(tcpath, 'sdk', 'lib')
 106   elif xarch == '32':
 107     return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
 108   elif xarch == '64':
 109     return os.path.join(tcpath, 'x86_64-nacl', 'lib')
 110   elif xarch == 'arm':
 111     return os.path.join(tcpath, 'arm-nacl', 'lib')
 112
 113
 114 def GetToolchainDirName(tcname, xarch):
 115   if tcname == 'pnacl':
 116     return '%s_%s' % (getos.GetPlatform(), tcname)
 117   elif xarch == 'arm':
 118     return '%s_arm_%s' % (getos.GetPlatform(), tcname)
 119   else:
 120     return '%s_x86_%s' % (getos.GetPlatform(), tcname)
 121
 122
 123 def GetGypToolchainLib(tcname, xarch):
 124   if xarch == 'arm':
 125     toolchain = xarch
 126   else:
 127     toolchain = tcname
 128
 129   tcpath = os.path.join(GetGypGenDir(xarch), 'sdk',
 130                         '%s_x86' % getos.GetPlatform(),
 131                         TOOLCHAIN_PACKAGE_MAP[toolchain][0])
 132   return GetToolchainNaClLib(tcname, tcpath, xarch)
 133
 134
 135 def GetOutputToolchainLib(pepperdir, tcname, xarch):
 136   tcpath = os.path.join(pepperdir, 'toolchain',
 137                         GetToolchainDirName(tcname, xarch))
 138   return GetToolchainNaClLib(tcname, tcpath, xarch)
 139
 140
 141 def GetPNaClNativeLib(tcpath, arch):
 142   if arch not in ['arm', 'x86-32', 'x86-64']:
 143     buildbot_common.ErrorExit('Unknown architecture %s.' % arch)
 144   return os.path.join(tcpath, 'lib-' + arch)
 145
 146
 147 def BuildStepDownloadToolchains(toolchains):
 148   buildbot_common.BuildStep('Running package_version.py')
 149   args = [sys.executable, PKGVER, '--exclude', 'arm_trusted']
 150   if 'bionic' in toolchains:
 151     build_platform = '%s_x86' % getos.GetPlatform()
 152     args.extend(['--append', os.path.join(build_platform, 'nacl_arm_bionic')])
 153   args.append('sync')
 154   buildbot_common.Run(args, cwd=NACL_DIR)
 155
 156
 157 def BuildStepCleanPepperDirs(pepperdir, pepperdir_old):
 158   buildbot_common.BuildStep('Clean Pepper Dirs')
 159   buildbot_common.RemoveDir(pepperdir_old)
 160   buildbot_common.RemoveDir(pepperdir)
 161   buildbot_common.MakeDir(pepperdir)
 162
 163
 164 def BuildStepMakePepperDirs(pepperdir, subdirs):
 165   for subdir in subdirs:
 166     buildbot_common.MakeDir(os.path.join(pepperdir, subdir))
 167
 168 TEXT_FILES = [
 169   'AUTHORS',
 170   'COPYING',
 171   'LICENSE',
 172   'README.Makefiles',
 173   'getting_started/README',
 174 ]
 175
 176 def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision,
 177                            nacl_revision):
 178   buildbot_common.BuildStep('Add Text Files')
 179   InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES)
 180
 181   # Replace a few placeholders in README
 182   readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read()
 183   readme_text = readme_text.replace('${VERSION}', pepper_ver)
 184   readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision)
 185   readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision)
 186
 187   # Year/Month/Day Hour:Minute:Second
 188   time_format = '%Y/%m/%d %H:%M:%S'
 189   readme_text = readme_text.replace('${DATE}',
 190       datetime.datetime.now().strftime(time_format))
 191
 192   open(os.path.join(pepperdir, 'README'), 'w').write(readme_text)
 193
 194
 195 def BuildStepUntarToolchains(pepperdir, toolchains):
 196   buildbot_common.BuildStep('Untar Toolchains')
 197   platform = getos.GetPlatform()
 198   build_platform = '%s_x86' % platform
 199   tmpdir = os.path.join(OUT_DIR, 'tc_temp')
 200   buildbot_common.RemoveDir(tmpdir)
 201   buildbot_common.MakeDir(tmpdir)
 202
 203   # Create a list of extract packages tuples, the first part should be
 204   # "$PACKAGE_TARGET/$PACKAGE". The second part should be the destination
 205   # directory relative to pepperdir/toolchain.
 206   extract_packages = []
 207   for toolchain in toolchains:
 208     toolchain_map = TOOLCHAIN_PACKAGE_MAP.get(toolchain, None)
 209     if toolchain_map:
 210       package_name, tcname = toolchain_map
 211       package_tuple = (os.path.join(build_platform, package_name),
 212                        tcname % {'platform': platform})
 213       extract_packages.append(package_tuple)
 214
 215   if extract_packages:
 216     # Extract all of the packages into the temp directory.
 217     package_names = [package_tuple[0] for package_tuple in extract_packages]
 218     buildbot_common.Run([sys.executable, PKGVER,
 219                            '--packages', ','.join(package_names),
 220                            '--tar-dir', NACL_TOOLCHAINTARS_DIR,
 221                            '--dest-dir', tmpdir,
 222                            'extract'])
 223
 224     # Move all the packages we extracted to the correct destination.
 225     for package_name, dest_dir in extract_packages:
 226       full_src_dir = os.path.join(tmpdir, package_name)
 227       full_dst_dir = os.path.join(pepperdir, 'toolchain', dest_dir)
 228       buildbot_common.Move(full_src_dir, full_dst_dir)
 229
 230   # Cleanup the temporary directory we are no longer using.
 231   buildbot_common.RemoveDir(tmpdir)
 232
 233
 234 # List of toolchain headers to install.
 235 # Source is relative to top of Chromium tree, destination is relative
 236 # to the toolchain header directory.
 237 NACL_HEADER_MAP = {
 238   'newlib': [
 239       ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
 240       ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
 241       ('native_client/src/untrusted/irt/irt.h', ''),
 242       ('native_client/src/untrusted/irt/irt_dev.h', ''),
 243       ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
 244       ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
 245       ('native_client/src/untrusted/pthread/pthread.h', ''),
 246       ('native_client/src/untrusted/pthread/semaphore.h', ''),
 247       ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
 248       ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
 249   ],
 250   'glibc': [
 251       ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
 252       ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
 253       ('native_client/src/untrusted/irt/irt.h', ''),
 254       ('native_client/src/untrusted/irt/irt_dev.h', ''),
 255       ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
 256       ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
 257       ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
 258       ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
 259   ],
 260   'host': []
 261 }
 262
 263 def InstallFiles(src_root, dest_root, file_list):
 264   """Copy a set of files from src_root to dest_root according
 265   to the given mapping.  This allows files to be copied from
 266   to a location in the destination tree that is different to the
 267   location in the source tree.
 268
 269   If the destination mapping ends with a '/' then the destination
 270   basename is inherited from the the source file.
 271
 272   Wildcards can be used in the source list but it is not recommended
 273   as this can end up adding things to the SDK unintentionally.
 274   """
 275   for file_spec in file_list:
 276     # The list of files to install can be a simple list of
 277     # strings or a list of pairs, where each pair corresponds
 278     # to a mapping from source to destination names.
 279     if type(file_spec) == str:
 280       src_file = dest_file = file_spec
 281     else:
 282       src_file, dest_file = file_spec
 283
 284     src_file = os.path.join(src_root, src_file)
 285
 286     # Expand sources files using glob.
 287     sources = glob.glob(src_file)
 288     if not sources:
 289       sources = [src_file]
 290
 291     if len(sources) > 1 and not dest_file.endswith('/'):
 292       buildbot_common.ErrorExit("Target file must end in '/' when "
 293                                 "using globbing to install multiple files")
 294
 295     for source in sources:
 296       if dest_file.endswith('/'):
 297         dest = os.path.join(dest_file, os.path.basename(source))
 298       else:
 299         dest = dest_file
 300       dest = os.path.join(dest_root, dest)
 301       if not os.path.isdir(os.path.dirname(dest)):
 302         buildbot_common.MakeDir(os.path.dirname(dest))
 303       buildbot_common.CopyFile(source, dest)
 304
 305
 306 def InstallNaClHeaders(tc_dst_inc, tc_name):
 307   """Copies NaCl headers to expected locations in the toolchain."""
 308   if tc_name == 'arm':
 309     # arm toolchain header should be the same as the x86 newlib
 310     # ones
 311     tc_name = 'newlib'
 312
 313   InstallFiles(SRC_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name])
 314
 315
 316 def MakeNinjaRelPath(path):
 317   return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
 318
 319
 320 TOOLCHAIN_LIBS = {
 321   'bionic' : [
 322     'libminidump_generator.a',
 323     'libnacl_dyncode.a',
 324     'libnacl_exception.a',
 325     'libnacl_list_mappings.a',
 326     'libppapi.a',
 327   ],
 328   'newlib' : [
 329     'crti.o',
 330     'crtn.o',
 331     'libminidump_generator.a',
 332     'libnacl.a',
 333     'libnacl_dyncode.a',
 334     'libnacl_exception.a',
 335     'libnacl_list_mappings.a',
 336     'libnosys.a',
 337     'libppapi.a',
 338     'libppapi_stub.a',
 339     'libpthread.a',
 340   ],
 341   'glibc': [
 342     'libminidump_generator.a',
 343     'libminidump_generator.so',
 344     'libnacl.a',
 345     'libnacl_dyncode.a',
 346     'libnacl_dyncode.so',
 347     'libnacl_exception.a',
 348     'libnacl_exception.so',
 349     'libnacl_list_mappings.a',
 350     'libnacl_list_mappings.so',
 351     'libppapi.a',
 352     'libppapi.so',
 353     'libppapi_stub.a',
 354   ],
 355   'pnacl': [
 356     'libminidump_generator.a',
 357     'libnacl.a',
 358     'libnacl_dyncode.a',
 359     'libnacl_exception.a',
 360     'libnacl_list_mappings.a',
 361     'libnosys.a',
 362     'libppapi.a',
 363     'libppapi_stub.a',
 364     'libpthread.a',
 365   ]
 366 }
 367
 368
 369 def GypNinjaInstall(pepperdir, toolchains):
 370   build_dir = GYPBUILD_DIR
 371   ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release')
 372   tools_files = [
 373     ['sel_ldr', 'sel_ldr_x86_32'],
 374     ['ncval_new', 'ncval'],
 375     ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'],
 376     ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'],
 377   ]
 378
 379   platform = getos.GetPlatform()
 380
 381   # TODO(binji): dump_syms doesn't currently build on Windows. See
 382   # http://crbug.com/245456
 383   if platform != 'win':
 384     tools_files += [
 385       ['dump_syms', 'dump_syms'],
 386       ['minidump_dump', 'minidump_dump'],
 387       ['minidump_stackwalk', 'minidump_stackwalk']
 388     ]
 389
 390   tools_files.append(['sel_ldr64', 'sel_ldr_x86_64'])
 391
 392   if platform == 'linux':
 393     tools_files.append(['nacl_helper_bootstrap',
 394                         'nacl_helper_bootstrap_x86_32'])
 395     tools_files.append(['nacl_helper_bootstrap64',
 396                         'nacl_helper_bootstrap_x86_64'])
 397
 398   buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
 399
 400   # Add .exe extensions to all windows tools
 401   for pair in tools_files:
 402     if platform == 'win' and not pair[0].endswith('.nexe'):
 403       pair[0] += '.exe'
 404       pair[1] += '.exe'
 405
 406   InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files)
 407
 408   # Add ARM binaries
 409   if platform == 'linux' and not options.no_arm_trusted:
 410     tools_files = [
 411       ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
 412       ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
 413       ['sel_ldr', 'sel_ldr_arm'],
 414       ['nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm']
 415     ]
 416     ninja_out_dir = os.path.join(OUT_DIR, build_dir + '-arm', 'Release')
 417     InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files)
 418
 419   for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']):
 420     if tc == 'pnacl':
 421       xarches = (None,)
 422     else:
 423       xarches = ('arm', '32', '64')
 424
 425     for xarch in xarches:
 426       if tc == 'glibc' and xarch == 'arm':
 427         continue
 428
 429       src_dir = GetGypBuiltLib(tc, xarch)
 430       dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch)
 431       InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc])
 432
 433       # Copy ARM newlib components to bionic
 434       if tc == 'newlib' and xarch == 'arm' and 'bionic' in toolchains:
 435         bionic_dir = GetOutputToolchainLib(pepperdir, 'bionic', xarch)
 436         InstallFiles(src_dir, bionic_dir, TOOLCHAIN_LIBS['bionic'])
 437
 438       if tc != 'pnacl':
 439         src_dir = GetGypToolchainLib(tc, xarch)
 440         InstallFiles(src_dir, dst_dir, ['crt1.o'])
 441
 442
 443 def GypNinjaBuild_NaCl(rel_out_dir):
 444   gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl')
 445   nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
 446   all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp')
 447
 448   out_dir = MakeNinjaRelPath(rel_out_dir)
 449   out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm')
 450   GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir)
 451   GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm)
 452   GypNinjaBuild('ia32', gyp_py, all_gyp, 'ncval_new', out_dir)
 453
 454   platform = getos.GetPlatform()
 455   if platform == 'win':
 456     NinjaBuild('sel_ldr64', out_dir)
 457   else:
 458     out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-64')
 459     GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'sel_ldr', out_dir_64)
 460
 461     # We only need sel_ldr from the 64-bit out directory.
 462     # sel_ldr needs to be renamed, so we'll call it sel_ldr64.
 463     files_to_copy = [('sel_ldr', 'sel_ldr64')]
 464     if platform == 'linux':
 465       files_to_copy.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap64'))
 466
 467     for src, dst in files_to_copy:
 468       buildbot_common.CopyFile(
 469           os.path.join(SRC_DIR, out_dir_64, 'Release', src),
 470           os.path.join(SRC_DIR, out_dir, 'Release', dst))
 471
 472
 473 def GypNinjaBuild_Breakpad(rel_out_dir):
 474   # TODO(binji): dump_syms doesn't currently build on Windows. See
 475   # http://crbug.com/245456
 476   if getos.GetPlatform() == 'win':
 477     return
 478
 479   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 480   out_dir = MakeNinjaRelPath(rel_out_dir)
 481   gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp')
 482   build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
 483   GypNinjaBuild('ia32', gyp_py, gyp_file, build_list, out_dir)
 484
 485
 486 def GypNinjaBuild_PPAPI(arch, rel_out_dir):
 487   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 488   out_dir = MakeNinjaRelPath(rel_out_dir)
 489   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client',
 490                           'native_client.gyp')
 491   GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir)
 492
 493
 494 def GypNinjaBuild_Pnacl(rel_out_dir, target_arch):
 495   # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the
 496   # Chromium build, and once here. When we move more of the SDK build process
 497   # to gyp, we can remove this.
 498   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 499
 500   out_dir = MakeNinjaRelPath(rel_out_dir)
 501   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src',
 502                           'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
 503   targets = ['aot']
 504   GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir, False)
 505
 506
 507 def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets,
 508                   out_dir, force_arm_gcc=True):
 509   gyp_env = dict(os.environ)
 510   gyp_env['GYP_GENERATORS'] = 'ninja'
 511   gyp_defines = []
 512   if options.mac_sdk:
 513     gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
 514   if arch:
 515     gyp_defines.append('target_arch=%s' % arch)
 516     if arch == 'arm':
 517       if getos.GetPlatform() == 'linux':
 518         gyp_env['CC'] = 'arm-linux-gnueabihf-gcc'
 519         gyp_env['CXX'] = 'arm-linux-gnueabihf-g++'
 520         gyp_env['AR'] = 'arm-linux-gnueabihf-ar'
 521         gyp_env['AS'] = 'arm-linux-gnueabihf-as'
 522         gyp_env['CC_host'] = 'cc'
 523         gyp_env['CXX_host'] = 'c++'
 524       gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1',
 525           'arm_float_abi=hard']
 526       if force_arm_gcc:
 527         gyp_defines.append('nacl_enable_arm_gcc=1')
 528       if options.no_arm_trusted:
 529         gyp_defines.append('disable_cross_trusted=1')
 530   if getos.GetPlatform() == 'mac':
 531     gyp_defines.append('clang=1')
 532
 533   gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
 534   for key in ['GYP_GENERATORS', 'GYP_DEFINES', 'CC']:
 535     value = gyp_env.get(key)
 536     if value is not None:
 537       print '%s="%s"' % (key, value)
 538   gyp_generator_flags = ['-G', 'output_dir=%s' % (out_dir,)]
 539   gyp_depth = '--depth=.'
 540   buildbot_common.Run(
 541       [sys.executable, gyp_py_script, gyp_file, gyp_depth] + \
 542           gyp_generator_flags,
 543       cwd=SRC_DIR,
 544       env=gyp_env)
 545   NinjaBuild(targets, out_dir)
 546
 547
 548 def NinjaBuild(targets, out_dir):
 549   if type(targets) is not list:
 550     targets = [targets]
 551   out_config_dir = os.path.join(out_dir, 'Release')
 552   buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
 553
 554
 555 def BuildStepBuildToolchains(pepperdir, toolchains):
 556   buildbot_common.BuildStep('SDK Items')
 557
 558   GypNinjaBuild_NaCl(GYPBUILD_DIR)
 559   GypNinjaBuild_Breakpad(GYPBUILD_DIR)
 560
 561   platform = getos.GetPlatform()
 562   newlibdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_newlib')
 563   glibcdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_glibc')
 564   armdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_newlib')
 565   pnacldir = os.path.join(pepperdir, 'toolchain', platform + '_pnacl')
 566
 567   if set(toolchains) & set(['glibc', 'newlib']):
 568     GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR)
 569
 570   if 'arm' in toolchains:
 571     GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm')
 572
 573   GypNinjaInstall(pepperdir, toolchains)
 574
 575   if 'newlib' in toolchains:
 576     InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
 577                        'newlib')
 578
 579   if 'glibc' in toolchains:
 580     InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
 581                        'glibc')
 582
 583   if 'arm' in toolchains:
 584     InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'),
 585                        'arm')
 586
 587   if 'pnacl' in toolchains:
 588     # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
 589     for arch in ('ia32', 'arm'):
 590       # Fill in the latest native pnacl shim library from the chrome build.
 591       build_dir = GYPBUILD_DIR + '-pnacl-' + arch
 592       GypNinjaBuild_Pnacl(build_dir, arch)
 593       if arch == 'ia32':
 594         nacl_arches = ['x86-32', 'x86-64']
 595       elif arch == 'arm':
 596         nacl_arches = ['arm']
 597       else:
 598         buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
 599       for nacl_arch in nacl_arches:
 600         release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release',
 601                                          'gen', 'tc_pnacl_translate',
 602                                          'lib-' + nacl_arch)
 603
 604         buildbot_common.CopyFile(
 605             os.path.join(release_build_dir, 'libpnacl_irt_shim.a'),
 606             GetPNaClNativeLib(pnacldir, nacl_arch))
 607
 608     InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
 609                        'newlib')
 610
 611
 612 def MakeDirectoryOrClobber(pepperdir, dirname, clobber):
 613   dirpath = os.path.join(pepperdir, dirname)
 614   if clobber:
 615     buildbot_common.RemoveDir(dirpath)
 616   buildbot_common.MakeDir(dirpath)
 617
 618   return dirpath
 619
 620
 621 def BuildStepUpdateHelpers(pepperdir, clobber):
 622   buildbot_common.BuildStep('Update project helpers')
 623   build_projects.UpdateHelpers(pepperdir, clobber=clobber)
 624
 625
 626 def BuildStepUpdateUserProjects(pepperdir, toolchains,
 627                                 build_experimental, clobber):
 628   buildbot_common.BuildStep('Update examples and libraries')
 629
 630   filters = {}
 631   if not build_experimental:
 632     filters['EXPERIMENTAL'] = False
 633   if toolchains:
 634     toolchains = toolchains[:]
 635
 636     # arm isn't a valid toolchain for build_projects
 637     if 'arm' in toolchains:
 638       toolchains.remove('arm')
 639
 640     if 'host' in toolchains:
 641       toolchains.remove('host')
 642       toolchains.append(getos.GetPlatform())
 643
 644     filters['TOOLS'] = toolchains
 645
 646   # Update examples and libraries
 647   filters['DEST'] = [
 648     'getting_started',
 649     'examples/api',
 650     'examples/demo',
 651     'examples/tutorial',
 652     'src'
 653   ]
 654
 655   tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
 656   build_projects.UpdateProjects(pepperdir, tree, clobber=clobber,
 657                                 toolchains=toolchains)
 658
 659
 660 def BuildStepMakeAll(pepperdir, directory, step_name,
 661                      deps=True, clean=False, config='Debug', args=None):
 662   buildbot_common.BuildStep(step_name)
 663   build_projects.BuildProjectsBranch(pepperdir, directory, clean,
 664                                      deps, config, args)
 665
 666
 667 def BuildStepBuildLibraries(pepperdir, directory):
 668   BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug',
 669       clean=True, config='Debug')
 670   BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release',
 671       clean=True, config='Release')
 672
 673   # Cleanup .pyc file generated while building libraries.  Without
 674   # this we would end up shipping the pyc in the SDK tarball.
 675   buildbot_common.RemoveFile(os.path.join(pepperdir, 'tools', '*.pyc'))
 676
 677
 678 def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None):
 679   # Look for LICENSE files
 680   license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT')
 681
 682   license_files = []
 683   for root, _, files in os.walk(fileroot):
 684     for filename in files:
 685       if license_filenames_re.match(filename):
 686         path = os.path.join(root, filename)
 687         license_files.append(path)
 688
 689   if extra_files:
 690     license_files += [os.path.join(fileroot, f) for f in extra_files]
 691   print '\n'.join(license_files)
 692
 693   if not os.path.isabs(output_filename):
 694     output_filename = os.path.join(fileroot, output_filename)
 695   generate_notice.Generate(output_filename, fileroot, license_files)
 696
 697
 698 def BuildStepVerifyFilelist(pepperdir):
 699   buildbot_common.BuildStep('Verify SDK Files')
 700   file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list')
 701   try:
 702     verify_filelist.Verify(file_list_path, pepperdir)
 703     print 'OK'
 704   except verify_filelist.ParseException, e:
 705     buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e)
 706   except verify_filelist.VerifyException, e:
 707     file_list_rel = os.path.relpath(file_list_path)
 708     verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py'
 709     verify_filelist_py = os.path.relpath(verify_filelist_py)
 710     pepperdir_rel = os.path.relpath(pepperdir)
 711
 712     msg = """\
 713 SDK verification failed:
 714
 715 %s
 716 Add/remove files from %s to fix.
 717
 718 Run:
 719     ./%s %s %s
 720 to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel,
 721                pepperdir_rel)
 722     buildbot_common.ErrorExit(msg)
 723
 724
 725 def BuildStepTarBundle(pepper_ver, tarfile):
 726   buildbot_common.BuildStep('Tar Pepper Bundle')
 727   buildbot_common.MakeDir(os.path.dirname(tarfile))
 728   buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
 729        'pepper_' + pepper_ver], cwd=NACL_DIR)
 730
 731
 732 def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile,
 733                       archive_url):
 734   with open(tarfile, 'rb') as tarfile_stream:
 735     archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
 736         tarfile_stream)
 737
 738   archive = manifest_util.Archive(manifest_util.GetHostOS())
 739   archive.url = archive_url
 740   archive.size = archive_size
 741   archive.checksum = archive_sha1
 742
 743   bundle = manifest_util.Bundle('pepper_' + pepper_ver)
 744   bundle.revision = int(chrome_revision)
 745   bundle.repath = 'pepper_' + pepper_ver
 746   bundle.version = int(pepper_ver)
 747   bundle.description = (
 748       'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % (
 749             pepper_ver, chrome_revision, nacl_revision))
 750   bundle.stability = 'dev'
 751   bundle.recommended = 'no'
 752   bundle.archives = [archive]
 753   return bundle
 754
 755
 756 def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision,
 757                            tarfile):
 758   buildbot_common.BuildStep('Archive %s' % name)
 759   bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
 760       build_version.ChromeVersion(),)
 761   tarname = os.path.basename(tarfile)
 762   tarfile_dir = os.path.dirname(tarfile)
 763   buildbot_common.Archive(tarname, bucket_path, tarfile_dir)
 764
 765   # generate "manifest snippet" for this archive.
 766   archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
 767       build_version.ChromeVersion(), tarname)
 768   bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision,
 769                              tarfile, archive_url)
 770
 771   manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
 772   with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
 773     manifest_snippet_stream.write(bundle.GetDataAsString())
 774
 775   buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
 776                           step_link=False)
 777
 778
 779 def BuildStepArchiveSDKTools():
 780   # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot.
 781   builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
 782   if builder_name == 'linux-sdk-multi':
 783     buildbot_common.BuildStep('Build SDK Tools')
 784     build_updater.BuildUpdater(OUT_DIR)
 785
 786     buildbot_common.BuildStep('Archive SDK Tools')
 787     bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
 788         build_version.ChromeVersion(),)
 789     buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR,
 790                             step_link=False)
 791     buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR,
 792                             step_link=False)
 793
 794
 795 def BuildStepSyncNaClPorts():
 796   """Pull the pinned revision of naclports from SVN."""
 797   buildbot_common.BuildStep('Sync naclports')
 798   if not os.path.exists(NACLPORTS_DIR):
 799     # checkout new copy of naclports
 800     cmd = ['svn', 'checkout', '-q', '-r', str(NACLPORTS_REV), NACLPORTS_URL,
 801            'naclports']
 802     buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR))
 803   else:
 804     # sync existing copy to pinned revision.
 805     cmd = ['svn', 'update', '-r', str(NACLPORTS_REV)]
 806     buildbot_common.Run(cmd, cwd=NACLPORTS_DIR)
 807
 808
 809 def BuildStepBuildNaClPorts(pepper_ver, pepperdir):
 810   """Build selected naclports in all configurations."""
 811   # TODO(sbc): currently naclports doesn't know anything about
 812   # Debug builds so the Debug subfolders are all empty.
 813
 814   env = dict(os.environ)
 815   env['NACL_SDK_ROOT'] = pepperdir
 816   env['PEPPER_DIR'] = os.path.basename(pepperdir)  # pepper_NN
 817   env['NACLPORTS_NO_ANNOTATE'] = "1"
 818   env['NACLPORTS_NO_UPLOAD'] = "1"
 819   env['BUILDBOT_GOT_REVISION'] = str(NACLPORTS_REV)
 820
 821   build_script = 'build_tools/buildbot_sdk_bundle.sh'
 822   buildbot_common.BuildStep('Build naclports')
 823
 824   bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
 825   out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver)
 826
 827   # Remove the sdk_bundle directory to remove stale files from previous builds.
 828   buildbot_common.RemoveDir(bundle_dir)
 829
 830   buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR)
 831
 832   # Some naclports do not include a standalone LICENSE/COPYING file
 833   # so we explicitly list those here for inclusion.
 834   extra_licenses = ('tinyxml/readme.txt',
 835                     'jpeg-8d/README',
 836                     'zlib-1.2.3/README')
 837   src_root = os.path.join(NACLPORTS_DIR, 'out', 'build')
 838   output_license = os.path.join(out_dir, 'ports', 'LICENSE')
 839   GenerateNotice(src_root , output_license, extra_licenses)
 840   readme = os.path.join(out_dir, 'ports', 'README')
 841   oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme])
 842
 843
 844 def BuildStepTarNaClPorts(pepper_ver, tarfile):
 845   """Create tar archive containing headers and libs from naclports build."""
 846   buildbot_common.BuildStep('Tar naclports Bundle')
 847   buildbot_common.MakeDir(os.path.dirname(tarfile))
 848   pepper_dir = 'pepper_%s' % pepper_ver
 849   archive_dirs = [os.path.join(pepper_dir, 'ports')]
 850
 851   ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
 852   cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile]
 853   cmd += archive_dirs
 854   buildbot_common.Run(cmd, cwd=NACL_DIR)
 855
 856
 857 def BuildStepBuildAppEngine(pepperdir, chrome_revision):
 858   """Build the projects found in src/gonacl_appengine/src"""
 859   buildbot_common.BuildStep('Build GoNaCl AppEngine Projects')
 860   cmd = ['make', 'upload', 'REVISION=%s' % chrome_revision]
 861   env = dict(os.environ)
 862   env['NACL_SDK_ROOT'] = pepperdir
 863   env['NACLPORTS_NO_ANNOTATE'] = "1"
 864   buildbot_common.Run(cmd, env=env, cwd=GONACL_APPENGINE_SRC_DIR)
 865
 866
 867 def main(args):
 868   parser = optparse.OptionParser(description=__doc__)
 869   parser.add_option('--nacl-tree-path',
 870       help='Path to native client tree for bionic build.',
 871       dest='nacl_tree_path')
 872   parser.add_option('--qemu', help='Add qemu for ARM.',
 873       action='store_true')
 874   parser.add_option('--bionic', help='Add bionic build.',
 875       action='store_true')
 876   parser.add_option('--tar', help='Force the tar step.',
 877       action='store_true')
 878   parser.add_option('--archive', help='Force the archive step.',
 879       action='store_true')
 880   parser.add_option('--release', help='PPAPI release version.',
 881       dest='release', default=None)
 882   parser.add_option('--build-ports',
 883       help='Build naclport bundle.', action='store_true')
 884   parser.add_option('--build-app-engine',
 885       help='Build AppEngine demos.', action='store_true')
 886   parser.add_option('--experimental',
 887       help='build experimental examples and libraries', action='store_true',
 888       dest='build_experimental')
 889   parser.add_option('--skip-toolchain', help='Skip toolchain untar',
 890       action='store_true')
 891   parser.add_option('--mac-sdk',
 892       help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
 893   parser.add_option('--no-arm-trusted', action='store_true',
 894       help='Disable building of ARM trusted components (sel_ldr, etc).')
 895
 896   # To setup bash completion for this command first install optcomplete
 897   # and then add this line to your .bashrc:
 898   #  complete -F _optcomplete build_sdk.py
 899   try:
 900     import optcomplete
 901     optcomplete.autocomplete(parser)
 902   except ImportError:
 903     pass
 904
 905   global options
 906   options, args = parser.parse_args(args[1:])
 907   if args:
 908     parser.error("Unexpected arguments: %s" % str(args))
 909
 910   if options.nacl_tree_path:
 911     options.bionic = True
 912     toolchain_build = os.path.join(options.nacl_tree_path, 'toolchain_build')
 913     print 'WARNING: Building bionic toolchain from NaCl checkout.'
 914     print 'This option builds bionic from the sources currently in the'
 915     print 'provided NativeClient checkout, and the results instead of '
 916     print 'downloading a toolchain from the builder. This may result in a'
 917     print 'NaCl SDK that can not run on ToT chrome.'
 918     print 'NOTE:  To clobber you will need to run toolchain_build_bionic.py'
 919     print 'directly from the NativeClient checkout.'
 920     print ''
 921     response = raw_input("Type 'y' and hit enter to continue.\n")
 922     if response != 'y' and response != 'Y':
 923       print 'Aborting.'
 924       return 1
 925
 926     # Get head version of NativeClient tree
 927     buildbot_common.BuildStep('Build bionic toolchain.')
 928     buildbot_common.Run([sys.executable, 'toolchain_build_bionic.py', '-f'],
 929                         cwd=toolchain_build)
 930   else:
 931     toolchain_build = None
 932
 933   if buildbot_common.IsSDKBuilder():
 934     options.archive = True
 935     options.build_ports = True
 936     options.build_app_engine = True
 937     options.tar = True
 938
 939   toolchains = ['newlib', 'glibc', 'arm', 'pnacl', 'host']
 940
 941   # Changes for experimental bionic builder
 942   if options.bionic:
 943     toolchains.append('bionic')
 944     options.build_ports = False
 945     options.build_app_engine = False
 946
 947   print 'Building: ' + ' '.join(toolchains)
 948
 949   if options.archive and not options.tar:
 950     parser.error('Incompatible arguments with archive.')
 951
 952   chrome_version = int(build_version.ChromeMajorVersion())
 953   chrome_revision = build_version.ChromeRevision()
 954   nacl_revision = build_version.NaClRevision()
 955   pepper_ver = str(chrome_version)
 956   pepper_old = str(chrome_version - 1)
 957   pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
 958   pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old)
 959   if options.bionic:
 960     tarname = 'naclsdk_bionic.tar.bz2'
 961   else:
 962     tarname = 'naclsdk_' + getos.GetPlatform() + '.tar.bz2'
 963   tarfile = os.path.join(OUT_DIR, tarname)
 964
 965   if options.release:
 966     pepper_ver = options.release
 967   print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision)
 968
 969   if 'NACL_SDK_ROOT' in os.environ:
 970     # We don't want the currently configured NACL_SDK_ROOT to have any effect
 971     # of the build.
 972     del os.environ['NACL_SDK_ROOT']
 973
 974   if not options.skip_toolchain:
 975     BuildStepCleanPepperDirs(pepperdir, pepperdir_old)
 976     BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools'])
 977     BuildStepDownloadToolchains(toolchains)
 978     if options.nacl_tree_path:
 979       # Instead of untarring, copy the raw bionic toolchain
 980       not_bionic = [i for i in toolchains if i != 'bionic']
 981       BuildStepUntarToolchains(pepperdir, not_bionic)
 982       tcname = GetToolchainDirName('bionic', 'arm')
 983       srcdir = os.path.join(toolchain_build, 'out', tcname)
 984       bionicdir = os.path.join(pepperdir, 'toolchain', tcname)
 985       oshelpers.Copy(['-r', srcdir, bionicdir])
 986     else:
 987       BuildStepUntarToolchains(pepperdir, toolchains)
 988
 989     BuildStepBuildToolchains(pepperdir, toolchains)
 990
 991   BuildStepUpdateHelpers(pepperdir, True)
 992   BuildStepUpdateUserProjects(pepperdir, toolchains,
 993                               options.build_experimental, True)
 994
 995   BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision)
 996
 997   # Ship with libraries prebuilt, so run that first.
 998   BuildStepBuildLibraries(pepperdir, 'src')
 999   GenerateNotice(pepperdir)
1000
1001   # Verify the SDK contains what we expect.
1002   if not options.bionic:
1003     BuildStepVerifyFilelist(pepperdir)
1004
1005   if options.tar:
1006     BuildStepTarBundle(pepper_ver, tarfile)
1007
1008   if options.build_ports and getos.GetPlatform() == 'linux':
1009     ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2')
1010     BuildStepSyncNaClPorts()
1011     BuildStepBuildNaClPorts(pepper_ver, pepperdir)
1012     if options.tar:
1013       BuildStepTarNaClPorts(pepper_ver, ports_tarfile)
1014
1015   if options.build_app_engine and getos.GetPlatform() == 'linux':
1016     BuildStepBuildAppEngine(pepperdir, chrome_revision)
1017
1018   if options.qemu:
1019     qemudir = os.path.join(NACL_DIR, 'toolchain', 'linux_arm-trusted')
1020     oshelpers.Copy(['-r', qemudir, pepperdir])
1021
1022   # Archive on non-trybots.
1023   if options.archive:
1024     BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision,
1025                            tarfile)
1026     if options.build_ports and getos.GetPlatform() == 'linux':
1027       BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision,
1028                              nacl_revision, ports_tarfile)
1029     BuildStepArchiveSDKTools()
1030
1031   return 0
1032
1033
1034 if __name__ == '__main__':
1035   try:
1036     sys.exit(main(sys.argv))
1037   except KeyboardInterrupt:
1038     buildbot_common.ErrorExit('build_sdk: interrupted')