native_client_sdk/src/build_tools/build_sdk.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Entry point for both build and try bots.
   7
   8 This script is invoked from XXX, usually without arguments
   9 to package an SDK. It automatically determines whether
  10 this SDK is for mac, win, linux.
  11
  12 The script inspects the following environment variables:
  13
  14 BUILDBOT_BUILDERNAME to determine whether the script is run locally
  15 and whether it should upload an SDK to file storage (GSTORE)
  16 """
  17
  18 # pylint: disable=W0621
  19
  20 # std python includes
  21 import argparse
  22 import datetime
  23 import glob
  24 import os
  25 import re
  26 import sys
  27
  28 if sys.version_info < (2, 7, 0):
  29   sys.stderr.write("python 2.7 or later is required run this script\n")
  30   sys.exit(1)
  31
  32 # local includes
  33 import buildbot_common
  34 import build_projects
  35 import build_updater
  36 import build_version
  37 import generate_notice
  38 import manifest_util
  39 import parse_dsc
  40 import verify_filelist
  41
  42 from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR
  43 from build_paths import NACLPORTS_DIR, GSTORE, GONACL_APPENGINE_SRC_DIR
  44
  45 # Add SDK make tools scripts to the python path.
  46 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  47 sys.path.append(os.path.join(NACL_DIR, 'build'))
  48
  49 import getos
  50 import oshelpers
  51
  52 BUILD_DIR = os.path.join(NACL_DIR, 'build')
  53 NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
  54 NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
  55
  56 CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
  57 PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
  58
  59 NACLPORTS_URL = 'https://chromium.googlesource.com/external/naclports.git'
  60 NACLPORTS_REV = 'e53078c33d99b0b3cbadbbbbb92cccf7a48d5dc1'
  61
  62 GYPBUILD_DIR = 'gypbuild'
  63
  64 options = None
  65
  66 # Map of: ToolchainName: (PackageName, SDKDir).
  67 TOOLCHAIN_PACKAGE_MAP = {
  68     'newlib': ('nacl_x86_newlib', '%(platform)s_x86_newlib'),
  69     'bionic': ('nacl_arm_bionic', '%(platform)s_arm_bionic'),
  70     'arm': ('nacl_arm_newlib', '%(platform)s_arm_newlib'),
  71     'glibc': ('nacl_x86_glibc', '%(platform)s_x86_glibc'),
  72     'pnacl': ('pnacl_newlib', '%(platform)s_pnacl')
  73     }
  74
  75
  76 def GetToolchainNaClInclude(tcname, tcpath, arch):
  77   if arch == 'x86':
  78     return os.path.join(tcpath, 'x86_64-nacl', 'include')
  79   elif arch == 'pnacl':
  80     return os.path.join(tcpath, 'le32-nacl', 'include')
  81   elif arch == 'arm':
  82     return os.path.join(tcpath, 'arm-nacl', 'include')
  83   else:
  84     buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
  85
  86
  87 def GetConfigDir(arch):
  88   if arch.endswith('x64') and getos.GetPlatform() == 'win':
  89     return 'Release_x64'
  90   else:
  91     return 'Release'
  92
  93
  94 def GetNinjaOutDir(arch):
  95   return os.path.join(OUT_DIR, GYPBUILD_DIR + '-' + arch, GetConfigDir(arch))
  96
  97
  98 def GetGypBuiltLib(tcname, arch):
  99   if arch == 'ia32':
 100     lib_suffix = '32'
 101   elif arch == 'x64':
 102     lib_suffix = '64'
 103   elif arch == 'arm':
 104     lib_suffix = 'arm'
 105   else:
 106     lib_suffix = ''
 107
 108   if tcname == 'pnacl':
 109     print arch
 110     if arch is None:
 111       arch = 'x64'
 112       tcname = 'pnacl_newlib'
 113     else:
 114       arch = 'clang-' + arch
 115       tcname = 'newlib'
 116
 117   return os.path.join(GetNinjaOutDir(arch),
 118                       'gen',
 119                       'tc_' + tcname,
 120                       'lib' + lib_suffix)
 121
 122
 123 def GetToolchainNaClLib(tcname, tcpath, arch):
 124   if arch == 'ia32':
 125     return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
 126   elif arch == 'x64':
 127     return os.path.join(tcpath, 'x86_64-nacl', 'lib')
 128   elif arch == 'arm':
 129     return os.path.join(tcpath, 'arm-nacl', 'lib')
 130   elif tcname == 'pnacl':
 131     return os.path.join(tcpath, 'le32-nacl', 'lib')
 132
 133
 134 def GetToolchainDirName(tcname, arch):
 135   if tcname == 'pnacl':
 136     return '%s_%s' % (getos.GetPlatform(), tcname)
 137   elif arch == 'arm':
 138     return '%s_arm_%s' % (getos.GetPlatform(), tcname)
 139   else:
 140     return '%s_x86_%s' % (getos.GetPlatform(), tcname)
 141
 142
 143 def GetGypToolchainLib(tcname, arch):
 144   if arch == 'arm':
 145     toolchain = arch
 146   else:
 147     toolchain = tcname
 148
 149   tcpath = os.path.join(GetNinjaOutDir(arch), 'gen', 'sdk',
 150                         '%s_x86' % getos.GetPlatform(),
 151                         TOOLCHAIN_PACKAGE_MAP[toolchain][0])
 152   return GetToolchainNaClLib(tcname, tcpath, arch)
 153
 154
 155 def GetOutputToolchainLib(pepperdir, tcname, arch):
 156   tcpath = os.path.join(pepperdir, 'toolchain',
 157                         GetToolchainDirName(tcname, arch))
 158   return GetToolchainNaClLib(tcname, tcpath, arch)
 159
 160
 161 def GetPNaClTranslatorLib(tcpath, arch):
 162   if arch not in ['arm', 'x86-32', 'x86-64']:
 163     buildbot_common.ErrorExit('Unknown architecture %s.' % arch)
 164   return os.path.join(tcpath, 'translator', arch, 'lib')
 165
 166
 167 def BuildStepDownloadToolchains(toolchains):
 168   buildbot_common.BuildStep('Running package_version.py')
 169   args = [sys.executable, PKGVER, '--mode', 'nacl_core_sdk']
 170   if 'bionic' in toolchains:
 171     build_platform = '%s_x86' % getos.GetPlatform()
 172     args.extend(['--append', os.path.join(build_platform, 'nacl_arm_bionic')])
 173   if getos.GetPlatform() == 'linux':
 174     # TODO(sbc): remove this once this change makes it into chrome
 175     # https://codereview.chromium.org/1080513003/
 176     args.extend(['--append', 'arm_trusted'])
 177   args.extend(['sync', '--extract'])
 178   buildbot_common.Run(args, cwd=NACL_DIR)
 179
 180
 181 def BuildStepCleanPepperDirs(pepperdir, pepperdir_old):
 182   buildbot_common.BuildStep('Clean Pepper Dirs')
 183   dirs_to_remove = (
 184       pepperdir,
 185       pepperdir_old,
 186       os.path.join(OUT_DIR, 'arm_trusted')
 187   )
 188   for dirname in dirs_to_remove:
 189     if os.path.exists(dirname):
 190       buildbot_common.RemoveDir(dirname)
 191   buildbot_common.MakeDir(pepperdir)
 192
 193
 194 def BuildStepMakePepperDirs(pepperdir, subdirs):
 195   for subdir in subdirs:
 196     buildbot_common.MakeDir(os.path.join(pepperdir, subdir))
 197
 198 TEXT_FILES = [
 199   'AUTHORS',
 200   'COPYING',
 201   'LICENSE',
 202   'README.Makefiles',
 203   'getting_started/README',
 204 ]
 205
 206 def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision,
 207                            nacl_revision):
 208   buildbot_common.BuildStep('Add Text Files')
 209   InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES)
 210
 211   # Replace a few placeholders in README
 212   readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read()
 213   readme_text = readme_text.replace('${VERSION}', pepper_ver)
 214   readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision)
 215   readme_text = readme_text.replace('${CHROME_COMMIT_POSITION}',
 216                                     build_version.ChromeCommitPosition())
 217   readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision)
 218
 219   # Year/Month/Day Hour:Minute:Second
 220   time_format = '%Y/%m/%d %H:%M:%S'
 221   readme_text = readme_text.replace('${DATE}',
 222       datetime.datetime.now().strftime(time_format))
 223
 224   open(os.path.join(pepperdir, 'README'), 'w').write(readme_text)
 225
 226
 227 def BuildStepUntarToolchains(pepperdir, toolchains):
 228   buildbot_common.BuildStep('Untar Toolchains')
 229   platform = getos.GetPlatform()
 230   build_platform = '%s_x86' % platform
 231   tmpdir = os.path.join(OUT_DIR, 'tc_temp')
 232   buildbot_common.RemoveDir(tmpdir)
 233   buildbot_common.MakeDir(tmpdir)
 234
 235   # Create a list of extract packages tuples, the first part should be
 236   # "$PACKAGE_TARGET/$PACKAGE". The second part should be the destination
 237   # directory relative to pepperdir/toolchain.
 238   extract_packages = []
 239   for toolchain in toolchains:
 240     toolchain_map = TOOLCHAIN_PACKAGE_MAP.get(toolchain, None)
 241     if toolchain_map:
 242       package_name, tcname = toolchain_map
 243       package_tuple = (os.path.join(build_platform, package_name),
 244                        tcname % {'platform': platform})
 245       extract_packages.append(package_tuple)
 246
 247
 248   # On linux we also want to extract the arm_trusted package which contains
 249   # the ARM libraries we ship in support of sel_ldr_arm.
 250   if platform == 'linux':
 251     extract_packages.append((os.path.join(build_platform, 'arm_trusted'),
 252                             'arm_trusted'))
 253   if extract_packages:
 254     # Extract all of the packages into the temp directory.
 255     package_names = [package_tuple[0] for package_tuple in extract_packages]
 256     buildbot_common.Run([sys.executable, PKGVER,
 257                            '--packages', ','.join(package_names),
 258                            '--tar-dir', NACL_TOOLCHAINTARS_DIR,
 259                            '--dest-dir', tmpdir,
 260                            'extract'])
 261
 262     # Move all the packages we extracted to the correct destination.
 263     for package_name, dest_dir in extract_packages:
 264       full_src_dir = os.path.join(tmpdir, package_name)
 265       full_dst_dir = os.path.join(pepperdir, 'toolchain', dest_dir)
 266       buildbot_common.Move(full_src_dir, full_dst_dir)
 267
 268   # Cleanup the temporary directory we are no longer using.
 269   buildbot_common.RemoveDir(tmpdir)
 270
 271
 272 # List of toolchain headers to install.
 273 # Source is relative to top of Chromium tree, destination is relative
 274 # to the toolchain header directory.
 275 NACL_HEADER_MAP = {
 276   'newlib': [
 277       ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
 278       ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
 279       ('native_client/src/untrusted/irt/irt.h', ''),
 280       ('native_client/src/untrusted/irt/irt_dev.h', ''),
 281       ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
 282       ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
 283       ('native_client/src/untrusted/pthread/pthread.h', ''),
 284       ('native_client/src/untrusted/pthread/semaphore.h', ''),
 285       ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
 286       ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
 287   ],
 288   'glibc': [
 289       ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
 290       ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
 291       ('native_client/src/untrusted/irt/irt.h', ''),
 292       ('native_client/src/untrusted/irt/irt_dev.h', ''),
 293       ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
 294       ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
 295       ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
 296       ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
 297   ],
 298   'bionic': [
 299       ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
 300   ],
 301   'host': []
 302 }
 303
 304 def InstallFiles(src_root, dest_root, file_list):
 305   """Copy a set of files from src_root to dest_root according
 306   to the given mapping.  This allows files to be copied from
 307   to a location in the destination tree that is different to the
 308   location in the source tree.
 309
 310   If the destination mapping ends with a '/' then the destination
 311   basename is inherited from the the source file.
 312
 313   Wildcards can be used in the source list but it is not recommended
 314   as this can end up adding things to the SDK unintentionally.
 315   """
 316   for file_spec in file_list:
 317     # The list of files to install can be a simple list of
 318     # strings or a list of pairs, where each pair corresponds
 319     # to a mapping from source to destination names.
 320     if type(file_spec) == str:
 321       src_file = dest_file = file_spec
 322     else:
 323       src_file, dest_file = file_spec
 324
 325     src_file = os.path.join(src_root, src_file)
 326
 327     # Expand sources files using glob.
 328     sources = glob.glob(src_file)
 329     if not sources:
 330       sources = [src_file]
 331
 332     if len(sources) > 1 and not dest_file.endswith('/'):
 333       buildbot_common.ErrorExit("Target file must end in '/' when "
 334                                 "using globbing to install multiple files")
 335
 336     for source in sources:
 337       if dest_file.endswith('/'):
 338         dest = os.path.join(dest_file, os.path.basename(source))
 339       else:
 340         dest = dest_file
 341       dest = os.path.join(dest_root, dest)
 342       if not os.path.isdir(os.path.dirname(dest)):
 343         buildbot_common.MakeDir(os.path.dirname(dest))
 344       buildbot_common.CopyFile(source, dest)
 345
 346
 347 def InstallNaClHeaders(tc_dst_inc, tc_name):
 348   """Copies NaCl headers to expected locations in the toolchain."""
 349   if tc_name in ('arm', 'pnacl'):
 350     # arm and pnacl toolchain headers should be the same as the newlib
 351     # ones
 352     tc_name = 'newlib'
 353
 354   InstallFiles(SRC_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name])
 355
 356
 357 def MakeNinjaRelPath(path):
 358   return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
 359
 360
 361 # TODO(ncbray): stop building and copying libraries into the SDK that are
 362 # already provided by the toolchain.
 363 TOOLCHAIN_LIBS = {
 364   'bionic' : [
 365     'libminidump_generator.a',
 366     'libnacl_dyncode.a',
 367     'libnacl_exception.a',
 368     'libnacl_list_mappings.a',
 369     'libppapi.a',
 370   ],
 371   'newlib' : [
 372     'libminidump_generator.a',
 373     'libnacl.a',
 374     'libnacl_dyncode.a',
 375     'libnacl_exception.a',
 376     'libnacl_list_mappings.a',
 377     'libnosys.a',
 378     'libppapi.a',
 379     'libppapi_stub.a',
 380     'libpthread.a',
 381   ],
 382   'glibc': [
 383     'libminidump_generator.a',
 384     'libminidump_generator.so',
 385     'libnacl.a',
 386     'libnacl_dyncode.a',
 387     'libnacl_dyncode.so',
 388     'libnacl_exception.a',
 389     'libnacl_exception.so',
 390     'libnacl_list_mappings.a',
 391     'libnacl_list_mappings.so',
 392     'libppapi.a',
 393     'libppapi.so',
 394     'libppapi_stub.a',
 395   ],
 396   'pnacl': [
 397     'libminidump_generator.a',
 398     'libnacl.a',
 399     'libnacl_dyncode.a',
 400     'libnacl_exception.a',
 401     'libnacl_list_mappings.a',
 402     'libnosys.a',
 403     'libppapi.a',
 404     'libppapi_stub.a',
 405     'libpthread.a',
 406   ]
 407 }
 408
 409
 410 def GypNinjaInstall(pepperdir, toolchains):
 411   tools_files_32 = [
 412     ['sel_ldr', 'sel_ldr_x86_32'],
 413     ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'],
 414     ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'],
 415   ]
 416
 417   tools_files_64 = []
 418
 419   platform = getos.GetPlatform()
 420
 421   # TODO(binji): dump_syms doesn't currently build on Windows. See
 422   # http://crbug.com/245456
 423   if platform != 'win':
 424     tools_files_64 += [
 425       ['dump_syms', 'dump_syms'],
 426       ['minidump_dump', 'minidump_dump'],
 427       ['minidump_stackwalk', 'minidump_stackwalk']
 428     ]
 429
 430   tools_files_64.append(['sel_ldr', 'sel_ldr_x86_64'])
 431   tools_files_64.append(['ncval_new', 'ncval'])
 432
 433   if platform == 'linux':
 434     tools_files_32.append(['nacl_helper_bootstrap',
 435                            'nacl_helper_bootstrap_x86_32'])
 436     tools_files_64.append(['nacl_helper_bootstrap',
 437                            'nacl_helper_bootstrap_x86_64'])
 438     tools_files_32.append(['nonsfi_loader_newlib_x32_nonsfi.nexe',
 439                            'nonsfi_loader_x86_32'])
 440
 441   tools_dir = os.path.join(pepperdir, 'tools')
 442   buildbot_common.MakeDir(tools_dir)
 443
 444   # Add .exe extensions to all windows tools
 445   for pair in tools_files_32 + tools_files_64:
 446     if platform == 'win' and not pair[0].endswith('.nexe'):
 447       pair[0] += '.exe'
 448       pair[1] += '.exe'
 449
 450   InstallFiles(GetNinjaOutDir('x64'), tools_dir, tools_files_64)
 451   InstallFiles(GetNinjaOutDir('ia32'), tools_dir, tools_files_32)
 452
 453   # Add ARM binaries
 454   if platform == 'linux' and not options.no_arm_trusted:
 455     arm_files = [
 456       ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
 457       ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
 458       ['nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm'],
 459       ['nonsfi_loader_newlib_arm_nonsfi.nexe', 'nonsfi_loader_arm'],
 460       ['sel_ldr', 'sel_ldr_arm']
 461     ]
 462     InstallFiles(GetNinjaOutDir('arm'), tools_dir, arm_files)
 463
 464   for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']):
 465     if tc == 'pnacl':
 466       xarches = (None, 'ia32', 'x64', 'arm')
 467     elif tc == 'glibc':
 468       xarches = ('ia32', 'x64')
 469     else:
 470       xarches = ('arm', 'ia32', 'x64')
 471
 472     for xarch in xarches:
 473       src_dir = GetGypBuiltLib(tc, xarch)
 474       dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch)
 475       InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc])
 476
 477       # Copy ARM newlib components to bionic
 478       if tc == 'newlib' and xarch == 'arm' and 'bionic' in toolchains:
 479         bionic_dir = GetOutputToolchainLib(pepperdir, 'bionic', xarch)
 480         InstallFiles(src_dir, bionic_dir, TOOLCHAIN_LIBS['bionic'])
 481
 482
 483 def GypNinjaBuild_NaCl(rel_out_dir):
 484   # TODO(binji): gyp_nacl doesn't build properly on Windows anymore; it only
 485   # can use VS2010, not VS2013 which is now required by the Chromium repo. NaCl
 486   # needs to be updated to perform the same logic as Chromium in detecting VS,
 487   # which can now exist in the depot_tools directory.
 488   # See https://code.google.com/p/nativeclient/issues/detail?id=4022
 489   #
 490   # For now, let's use gyp_chromium to build these components.
 491 #  gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl')
 492   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 493   nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
 494   all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp')
 495
 496   out_dir_32 = MakeNinjaRelPath(rel_out_dir + '-ia32')
 497   out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-x64')
 498   out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm')
 499   out_dir_clang_32 = MakeNinjaRelPath(rel_out_dir + '-clang-ia32')
 500   out_dir_clang_64 = MakeNinjaRelPath(rel_out_dir + '-clang-x64')
 501   out_dir_clang_arm = MakeNinjaRelPath(rel_out_dir + '-clang-arm')
 502
 503   GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_32)
 504   GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_64)
 505   GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm)
 506   GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk',
 507       out_dir_clang_32, gyp_defines=['use_nacl_clang=1'])
 508   GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk',
 509       out_dir_clang_64, gyp_defines=['use_nacl_clang=1'])
 510   GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk',
 511       out_dir_clang_arm, gyp_defines=['use_nacl_clang=1'])
 512   GypNinjaBuild('x64', gyp_py, all_gyp, 'ncval_new', out_dir_64)
 513
 514
 515 def GypNinjaBuild_Breakpad(rel_out_dir):
 516   # TODO(binji): dump_syms doesn't currently build on Windows. See
 517   # http://crbug.com/245456
 518   if getos.GetPlatform() == 'win':
 519     return
 520
 521   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 522   out_dir = MakeNinjaRelPath(rel_out_dir)
 523   gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp')
 524   build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
 525   GypNinjaBuild('x64', gyp_py, gyp_file, build_list, out_dir)
 526
 527
 528 def GypNinjaBuild_PPAPI(arch, rel_out_dir, gyp_defines=None):
 529   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 530   out_dir = MakeNinjaRelPath(rel_out_dir)
 531   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client',
 532                           'native_client.gyp')
 533   GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir,
 534                 gyp_defines=gyp_defines)
 535
 536
 537 def GypNinjaBuild_Pnacl(rel_out_dir, target_arch):
 538   # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the
 539   # Chromium build, and once here. When we move more of the SDK build process
 540   # to gyp, we can remove this.
 541   gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
 542
 543   out_dir = MakeNinjaRelPath(rel_out_dir)
 544   gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src',
 545                           'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
 546   targets = ['aot']
 547   GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir)
 548
 549
 550 def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets,
 551                   out_dir, force_arm_gcc=True, gyp_defines=None):
 552   gyp_env = dict(os.environ)
 553   gyp_env['GYP_GENERATORS'] = 'ninja'
 554   gyp_defines = gyp_defines or []
 555   gyp_defines.append('nacl_allow_thin_archives=0')
 556   if options.mac_sdk:
 557     gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
 558
 559   if arch is not None:
 560     gyp_defines.append('target_arch=%s' % arch)
 561     if arch == 'arm':
 562       gyp_env['GYP_CROSSCOMPILE'] = '1'
 563       gyp_defines.append('arm_float_abi=hard')
 564       if options.no_arm_trusted:
 565         gyp_defines.append('disable_cross_trusted=1')
 566   if getos.GetPlatform() == 'mac':
 567     gyp_defines.append('clang=1')
 568
 569   gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
 570   # We can't use windows path separators in GYP_GENERATOR_FLAGS since
 571   # gyp uses shlex to parse them and treats '\' as an escape char.
 572   gyp_env['GYP_GENERATOR_FLAGS'] = 'output_dir=%s' % out_dir.replace('\\', '/')
 573
 574   # Print relevant environment variables
 575   for key, value in gyp_env.iteritems():
 576     if key.startswith('GYP') or key in ('CC',):
 577       print '  %s="%s"' % (key, value)
 578
 579   buildbot_common.Run(
 580       [sys.executable, gyp_py_script, gyp_file, '--depth=.'],
 581       cwd=SRC_DIR,
 582       env=gyp_env)
 583
 584   NinjaBuild(targets, out_dir, arch)
 585
 586
 587 def NinjaBuild(targets, out_dir, arch):
 588   if type(targets) is not list:
 589     targets = [targets]
 590   out_config_dir = os.path.join(out_dir, GetConfigDir(arch))
 591   buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
 592
 593
 594 def BuildStepBuildToolchains(pepperdir, toolchains, build, clean):
 595   buildbot_common.BuildStep('SDK Items')
 596
 597   if clean:
 598     for dirname in glob.glob(os.path.join(OUT_DIR, GYPBUILD_DIR + '*')):
 599       buildbot_common.RemoveDir(dirname)
 600     build = True
 601
 602   if build:
 603     GypNinjaBuild_NaCl(GYPBUILD_DIR)
 604     GypNinjaBuild_Breakpad(GYPBUILD_DIR + '-x64')
 605
 606     if set(toolchains) & set(['glibc', 'newlib']):
 607       GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR + '-ia32')
 608       GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR + '-x64')
 609
 610     if 'arm' in toolchains:
 611       GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm')
 612
 613     if 'pnacl' in toolchains:
 614       GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR + '-clang-ia32',
 615                           ['use_nacl_clang=1'])
 616       GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR + '-clang-x64',
 617                           ['use_nacl_clang=1'])
 618       GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-clang-arm',
 619                           ['use_nacl_clang=1'])
 620
 621       # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
 622       for arch in ('ia32', 'arm'):
 623         # Fill in the latest native pnacl shim library from the chrome build.
 624         build_dir = GYPBUILD_DIR + '-pnacl-' + arch
 625         GypNinjaBuild_Pnacl(build_dir, arch)
 626
 627   GypNinjaInstall(pepperdir, toolchains)
 628
 629   platform = getos.GetPlatform()
 630   newlibdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_newlib')
 631   glibcdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_glibc')
 632   armdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_newlib')
 633   pnacldir = os.path.join(pepperdir, 'toolchain', platform + '_pnacl')
 634   bionicdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_bionic')
 635
 636
 637   if 'newlib' in toolchains:
 638     InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
 639                        'newlib')
 640
 641   if 'glibc' in toolchains:
 642     InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
 643                        'glibc')
 644
 645   if 'arm' in toolchains:
 646     InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'),
 647                        'arm')
 648
 649   if 'bionic' in toolchains:
 650     InstallNaClHeaders(GetToolchainNaClInclude('bionic', bionicdir, 'arm'),
 651                        'bionic')
 652
 653   if 'pnacl' in toolchains:
 654     # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
 655     for arch in ('ia32', 'arm'):
 656       # Fill in the latest native pnacl shim library from the chrome build.
 657       build_dir = GYPBUILD_DIR + '-pnacl-' + arch
 658       if arch == 'ia32':
 659         nacl_arches = ['x86-32', 'x86-64']
 660       elif arch == 'arm':
 661         nacl_arches = ['arm']
 662       else:
 663         buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
 664       for nacl_arch in nacl_arches:
 665         release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release',
 666                                          'gen', 'tc_pnacl_translate',
 667                                          'lib-' + nacl_arch)
 668
 669         pnacl_translator_lib_dir = GetPNaClTranslatorLib(pnacldir, nacl_arch)
 670         if not os.path.isdir(pnacl_translator_lib_dir):
 671           buildbot_common.ErrorExit('Expected %s directory to exist.' %
 672                                     pnacl_translator_lib_dir)
 673
 674         buildbot_common.CopyFile(
 675             os.path.join(release_build_dir, 'libpnacl_irt_shim.a'),
 676             pnacl_translator_lib_dir)
 677
 678     InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'pnacl'),
 679                        'pnacl')
 680     InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
 681                        'pnacl')
 682     InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'arm'),
 683                        'pnacl')
 684
 685
 686 def MakeDirectoryOrClobber(pepperdir, dirname, clobber):
 687   dirpath = os.path.join(pepperdir, dirname)
 688   if clobber:
 689     buildbot_common.RemoveDir(dirpath)
 690   buildbot_common.MakeDir(dirpath)
 691
 692   return dirpath
 693
 694
 695 def BuildStepUpdateHelpers(pepperdir, clobber):
 696   buildbot_common.BuildStep('Update project helpers')
 697   build_projects.UpdateHelpers(pepperdir, clobber=clobber)
 698
 699
 700 def BuildStepUpdateUserProjects(pepperdir, toolchains,
 701                                 build_experimental, clobber):
 702   buildbot_common.BuildStep('Update examples and libraries')
 703
 704   filters = {}
 705   if not build_experimental:
 706     filters['EXPERIMENTAL'] = False
 707   if toolchains:
 708     toolchains = toolchains[:]
 709
 710     # arm isn't a valid toolchain for build_projects
 711     if 'arm' in toolchains:
 712       toolchains.remove('arm')
 713
 714     if 'host' in toolchains:
 715       toolchains.remove('host')
 716       toolchains.append(getos.GetPlatform())
 717
 718     filters['TOOLS'] = toolchains
 719
 720   # Update examples and libraries
 721   filters['DEST'] = [
 722     'getting_started',
 723     'examples/api',
 724     'examples/demo',
 725     'examples/tutorial',
 726     'src'
 727   ]
 728
 729   tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
 730   build_projects.UpdateProjects(pepperdir, tree, clobber=clobber,
 731                                 toolchains=toolchains)
 732
 733
 734 def BuildStepMakeAll(pepperdir, directory, step_name,
 735                      deps=True, clean=False, config='Debug', args=None):
 736   buildbot_common.BuildStep(step_name)
 737   build_projects.BuildProjectsBranch(pepperdir, directory, clean,
 738                                      deps, config, args)
 739
 740
 741 def BuildStepBuildLibraries(pepperdir, directory):
 742   BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug',
 743       clean=True, config='Debug')
 744   BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release',
 745       clean=True, config='Release')
 746
 747   # Cleanup .pyc file generated while building libraries.  Without
 748   # this we would end up shipping the pyc in the SDK tarball.
 749   buildbot_common.RemoveFile(os.path.join(pepperdir, 'tools', '*.pyc'))
 750
 751
 752 def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None):
 753   # Look for LICENSE files
 754   license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT')
 755
 756   license_files = []
 757   for root, _, files in os.walk(fileroot):
 758     for filename in files:
 759       if license_filenames_re.match(filename):
 760         path = os.path.join(root, filename)
 761         license_files.append(path)
 762
 763   if extra_files:
 764     license_files += [os.path.join(fileroot, f) for f in extra_files]
 765   print '\n'.join(license_files)
 766
 767   if not os.path.isabs(output_filename):
 768     output_filename = os.path.join(fileroot, output_filename)
 769   generate_notice.Generate(output_filename, fileroot, license_files)
 770
 771
 772 def BuildStepVerifyFilelist(pepperdir):
 773   buildbot_common.BuildStep('Verify SDK Files')
 774   file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list')
 775   try:
 776     print 'SDK directory: %s' % pepperdir
 777     verify_filelist.Verify(file_list_path, pepperdir)
 778     print 'OK'
 779   except verify_filelist.ParseException, e:
 780     buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e)
 781   except verify_filelist.VerifyException, e:
 782     file_list_rel = os.path.relpath(file_list_path)
 783     verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py'
 784     verify_filelist_py = os.path.relpath(verify_filelist_py)
 785     pepperdir_rel = os.path.relpath(pepperdir)
 786
 787     msg = """\
 788 SDK verification failed:
 789
 790 %s
 791 Add/remove files from %s to fix.
 792
 793 Run:
 794     ./%s %s %s
 795 to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel,
 796                pepperdir_rel)
 797     buildbot_common.ErrorExit(msg)
 798
 799
 800 def BuildStepTarBundle(pepper_ver, tarfile):
 801   buildbot_common.BuildStep('Tar Pepper Bundle')
 802   buildbot_common.MakeDir(os.path.dirname(tarfile))
 803   buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
 804        'pepper_' + pepper_ver], cwd=NACL_DIR)
 805
 806
 807 def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile,
 808                       archive_url):
 809   with open(tarfile, 'rb') as tarfile_stream:
 810     archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
 811         tarfile_stream)
 812
 813   archive = manifest_util.Archive(manifest_util.GetHostOS())
 814   archive.url = archive_url
 815   archive.size = archive_size
 816   archive.checksum = archive_sha1
 817
 818   bundle = manifest_util.Bundle('pepper_' + pepper_ver)
 819   bundle.revision = int(chrome_revision)
 820   bundle.repath = 'pepper_' + pepper_ver
 821   bundle.version = int(pepper_ver)
 822   bundle.description = (
 823       'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % (
 824             pepper_ver, chrome_revision, nacl_revision))
 825   bundle.stability = 'dev'
 826   bundle.recommended = 'no'
 827   bundle.archives = [archive]
 828   return bundle
 829
 830
 831 def Archive(filename, from_directory, step_link=True):
 832   if buildbot_common.IsSDKBuilder():
 833     bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/'
 834   else:
 835     bucket_path = 'nativeclient-mirror/nacl/nacl_sdk_test/'
 836   bucket_path += build_version.ChromeVersion()
 837   buildbot_common.Archive(filename, bucket_path, from_directory, step_link)
 838
 839
 840 def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision,
 841                            tarfile):
 842   buildbot_common.BuildStep('Archive %s' % name)
 843   tarname = os.path.basename(tarfile)
 844   tarfile_dir = os.path.dirname(tarfile)
 845   Archive(tarname, tarfile_dir)
 846
 847   # generate "manifest snippet" for this archive.
 848   archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
 849       build_version.ChromeVersion(), tarname)
 850   bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision,
 851                              tarfile, archive_url)
 852
 853   manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
 854   with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
 855     manifest_snippet_stream.write(bundle.GetDataAsString())
 856
 857   Archive(tarname + '.json', OUT_DIR, step_link=False)
 858
 859
 860 def BuildStepBuildPNaClComponent(version, revision):
 861   # Sadly revision can go backwords for a given version since when a version
 862   # is built from master, revision will be a huge number (in the hundreds of
 863   # thousands.  Once the branch happens the revision will reset to zero.
 864   # TODO(sbc): figure out how to compensate for this in some way such that
 865   # revisions always go forward for a given version.
 866   buildbot_common.BuildStep('PNaCl Component')
 867   if len(revision) > 4:
 868     rev_minor = revision[-4:]
 869     rev_major = revision[:-4]
 870     version = "0.%s.%s.%s" % (version, rev_major, rev_minor)
 871   else:
 872     version = "0.%s.0.%s" % (version, revision)
 873   buildbot_common.Run(['./make_pnacl_component.sh', version], cwd=SCRIPT_DIR)
 874
 875
 876 def BuildStepArchivePNaClComponent():
 877   buildbot_common.BuildStep('Archive PNaCl Component')
 878   Archive('pnacl_multicrx.zip', OUT_DIR)
 879
 880
 881 def BuildStepArchiveSDKTools():
 882   buildbot_common.BuildStep('Build SDK Tools')
 883   build_updater.BuildUpdater(OUT_DIR)
 884
 885   buildbot_common.BuildStep('Archive SDK Tools')
 886   Archive('sdk_tools.tgz', OUT_DIR, step_link=False)
 887   Archive('nacl_sdk.zip', OUT_DIR, step_link=False)
 888
 889
 890 def BuildStepSyncNaClPorts():
 891   """Pull the pinned revision of naclports from SVN."""
 892   buildbot_common.BuildStep('Sync naclports')
 893
 894   # In case a previous svn checkout exists, remove it.
 895   # TODO(sbc): remove this once all the build machines
 896   # have removed the old checkout
 897   if (os.path.exists(NACLPORTS_DIR) and
 898       not os.path.exists(os.path.join(NACLPORTS_DIR, '.git'))):
 899     buildbot_common.RemoveDir(NACLPORTS_DIR)
 900
 901   if not os.path.exists(NACLPORTS_DIR):
 902     # checkout new copy of naclports
 903     cmd = ['git', 'clone', NACLPORTS_URL, 'naclports']
 904     buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR))
 905   else:
 906     # checkout new copy of naclports
 907     buildbot_common.Run(['git', 'fetch'], cwd=NACLPORTS_DIR)
 908
 909   # sync to required revision
 910   cmd = ['git', 'checkout', str(NACLPORTS_REV)]
 911   buildbot_common.Run(cmd, cwd=NACLPORTS_DIR)
 912
 913
 914 def BuildStepBuildNaClPorts(pepper_ver, pepperdir):
 915   """Build selected naclports in all configurations."""
 916   # TODO(sbc): currently naclports doesn't know anything about
 917   # Debug builds so the Debug subfolders are all empty.
 918
 919   env = dict(os.environ)
 920   env['NACL_SDK_ROOT'] = pepperdir
 921   env['PEPPER_DIR'] = os.path.basename(pepperdir)  # pepper_NN
 922   env['NACLPORTS_NO_ANNOTATE'] = "1"
 923   env['NACLPORTS_NO_UPLOAD'] = "1"
 924   env['BUILDBOT_GOT_REVISION'] = str(NACLPORTS_REV)
 925
 926   build_script = 'build_tools/buildbot_sdk_bundle.sh'
 927   buildbot_common.BuildStep('Build naclports')
 928
 929   bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
 930   out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver)
 931
 932   # Remove the sdk_bundle directory to remove stale files from previous builds.
 933   buildbot_common.RemoveDir(bundle_dir)
 934
 935   buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR)
 936
 937   # Some naclports do not include a standalone LICENSE/COPYING file
 938   # so we explicitly list those here for inclusion.
 939   extra_licenses = ('tinyxml/readme.txt',
 940                     'jpeg-8d/README',
 941                     'zlib-1.2.3/README')
 942   src_root = os.path.join(NACLPORTS_DIR, 'out', 'build')
 943   output_license = os.path.join(out_dir, 'ports', 'LICENSE')
 944   GenerateNotice(src_root, output_license, extra_licenses)
 945   readme = os.path.join(out_dir, 'ports', 'README')
 946   oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme])
 947
 948
 949 def BuildStepTarNaClPorts(pepper_ver, tarfile):
 950   """Create tar archive containing headers and libs from naclports build."""
 951   buildbot_common.BuildStep('Tar naclports Bundle')
 952   buildbot_common.MakeDir(os.path.dirname(tarfile))
 953   pepper_dir = 'pepper_%s' % pepper_ver
 954   archive_dirs = [os.path.join(pepper_dir, 'ports')]
 955
 956   ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
 957   cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile]
 958   cmd += archive_dirs
 959   buildbot_common.Run(cmd, cwd=NACL_DIR)
 960
 961
 962 def BuildStepBuildAppEngine(pepperdir, chrome_revision):
 963   """Build the projects found in src/gonacl_appengine/src"""
 964   buildbot_common.BuildStep('Build GoNaCl AppEngine Projects')
 965   cmd = ['make', 'upload', 'REVISION=%s' % chrome_revision]
 966   env = dict(os.environ)
 967   env['NACL_SDK_ROOT'] = pepperdir
 968   env['NACLPORTS_NO_ANNOTATE'] = "1"
 969   buildbot_common.Run(cmd, env=env, cwd=GONACL_APPENGINE_SRC_DIR)
 970
 971
 972 def main(args):
 973   parser = argparse.ArgumentParser(description=__doc__)
 974   parser.add_argument('--nacl-tree-path',
 975       help='Path to native client tree for bionic build.',
 976       dest='nacl_tree_path')
 977   parser.add_argument('--qemu', help='Add qemu for ARM.',
 978       action='store_true')
 979   parser.add_argument('--bionic', help='Add bionic build.',
 980       action='store_true')
 981   parser.add_argument('--tar', help='Force the tar step.',
 982       action='store_true')
 983   parser.add_argument('--archive', help='Force the archive step.',
 984       action='store_true')
 985   parser.add_argument('--release', help='PPAPI release version.',
 986       dest='release', default=None)
 987   parser.add_argument('--build-ports',
 988       help='Build naclport bundle.', action='store_true')
 989   parser.add_argument('--build-app-engine',
 990       help='Build AppEngine demos.', action='store_true')
 991   parser.add_argument('--experimental',
 992       help='build experimental examples and libraries', action='store_true',
 993       dest='build_experimental')
 994   parser.add_argument('--skip-toolchain', help='Skip toolchain untar',
 995       action='store_true')
 996   parser.add_argument('--no-clean', dest='clean', action='store_false',
 997       help="Don't clean gypbuild directories")
 998   parser.add_argument('--mac-sdk',
 999       help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
1000   parser.add_argument('--no-arm-trusted', action='store_true',
1001       help='Disable building of ARM trusted components (sel_ldr, etc).')
1002
1003   # To setup bash completion for this command first install optcomplete
1004   # and then add this line to your .bashrc:
1005   #  complete -F _optcomplete build_sdk.py
1006   try:
1007     import optcomplete
1008     optcomplete.autocomplete(parser)
1009   except ImportError:
1010     pass
1011
1012   global options
1013   options = parser.parse_args(args)
1014
1015   if options.nacl_tree_path:
1016     options.bionic = True
1017     toolchain_build = os.path.join(options.nacl_tree_path, 'toolchain_build')
1018     print 'WARNING: Building bionic toolchain from NaCl checkout.'
1019     print 'This option builds bionic from the sources currently in the'
1020     print 'provided NativeClient checkout, and the results instead of '
1021     print 'downloading a toolchain from the builder. This may result in a'
1022     print 'NaCl SDK that can not run on ToT chrome.'
1023     print 'NOTE:  To clobber you will need to run toolchain_build_bionic.py'
1024     print 'directly from the NativeClient checkout.'
1025     print ''
1026     response = raw_input("Type 'y' and hit enter to continue.\n")
1027     if response != 'y' and response != 'Y':
1028       print 'Aborting.'
1029       return 1
1030
1031     # Get head version of NativeClient tree
1032     buildbot_common.BuildStep('Build bionic toolchain.')
1033     buildbot_common.Run([sys.executable, 'toolchain_build_bionic.py', '-f'],
1034                         cwd=toolchain_build)
1035   else:
1036     toolchain_build = None
1037
1038   if buildbot_common.IsSDKBuilder():
1039     options.archive = True
1040     options.build_ports = True
1041     # TODO(binji): re-enable app_engine build when the linux builder stops
1042     # breaking when trying to git clone from github.
1043     # See http://crbug.com/412969.
1044     options.build_app_engine = False
1045     options.tar = True
1046
1047   # NOTE: order matters here. This will be the order that is specified in the
1048   # Makefiles; the first toolchain will be the default.
1049   toolchains = ['pnacl', 'newlib', 'glibc', 'arm', 'clang-newlib', 'host']
1050
1051   # Changes for experimental bionic builder
1052   if options.bionic:
1053     toolchains.append('bionic')
1054     options.build_ports = False
1055     options.build_app_engine = False
1056
1057   print 'Building: ' + ' '.join(toolchains)
1058   platform = getos.GetPlatform()
1059
1060   if options.archive and not options.tar:
1061     parser.error('Incompatible arguments with archive.')
1062
1063   chrome_version = int(build_version.ChromeMajorVersion())
1064   chrome_revision = build_version.ChromeRevision()
1065   nacl_revision = build_version.NaClRevision()
1066   pepper_ver = str(chrome_version)
1067   pepper_old = str(chrome_version - 1)
1068   pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
1069   pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old)
1070   if options.bionic:
1071     tarname = 'naclsdk_bionic.tar.bz2'
1072   else:
1073     tarname = 'naclsdk_%s.tar.bz2' % platform
1074   tarfile = os.path.join(OUT_DIR, tarname)
1075
1076   if options.release:
1077     pepper_ver = options.release
1078   print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision)
1079
1080   if 'NACL_SDK_ROOT' in os.environ:
1081     # We don't want the currently configured NACL_SDK_ROOT to have any effect
1082     # of the build.
1083     del os.environ['NACL_SDK_ROOT']
1084
1085   if not options.skip_toolchain:
1086     BuildStepCleanPepperDirs(pepperdir, pepperdir_old)
1087     BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools'])
1088     BuildStepDownloadToolchains(toolchains)
1089     if options.nacl_tree_path:
1090       # Instead of untarring, copy the raw bionic toolchain
1091       not_bionic = [i for i in toolchains if i != 'bionic']
1092       BuildStepUntarToolchains(pepperdir, not_bionic)
1093       tcname = GetToolchainDirName('bionic', 'arm')
1094       srcdir = os.path.join(toolchain_build, 'out', tcname)
1095       bionicdir = os.path.join(pepperdir, 'toolchain', tcname)
1096       oshelpers.Copy(['-r', srcdir, bionicdir])
1097     else:
1098       BuildStepUntarToolchains(pepperdir, toolchains)
1099     if platform == 'linux':
1100       buildbot_common.Move(os.path.join(pepperdir, 'toolchain', 'arm_trusted'),
1101                            os.path.join(OUT_DIR, 'arm_trusted'))
1102
1103
1104   if platform == 'linux':
1105     # Linux-only: Copy arm libraries from the arm_trusted package.  These are
1106     # needed to be able to run sel_ldr_arm under qemu.
1107     arm_libs = [
1108       'lib/arm-linux-gnueabihf/librt.so.1',
1109       'lib/arm-linux-gnueabihf/libpthread.so.0',
1110       'lib/arm-linux-gnueabihf/libgcc_s.so.1',
1111       'lib/arm-linux-gnueabihf/libc.so.6',
1112       'lib/arm-linux-gnueabihf/ld-linux-armhf.so.3',
1113       'lib/arm-linux-gnueabihf/libm.so.6',
1114       'usr/lib/arm-linux-gnueabihf/libstdc++.so.6'
1115     ]
1116     arm_lib_dir = os.path.join(pepperdir, 'tools', 'lib', 'arm_trusted', 'lib')
1117     buildbot_common.MakeDir(arm_lib_dir)
1118     for arm_lib in arm_libs:
1119       arm_lib = os.path.join(OUT_DIR, 'arm_trusted', arm_lib)
1120       buildbot_common.CopyFile(arm_lib, arm_lib_dir)
1121     buildbot_common.CopyFile(os.path.join(OUT_DIR, 'arm_trusted', 'qemu-arm'),
1122                              os.path.join(pepperdir, 'tools'))
1123
1124
1125   BuildStepBuildToolchains(pepperdir, toolchains,
1126                            not options.skip_toolchain,
1127                            options.clean)
1128
1129   BuildStepUpdateHelpers(pepperdir, True)
1130   BuildStepUpdateUserProjects(pepperdir, toolchains,
1131                               options.build_experimental, True)
1132
1133   BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision)
1134
1135   # Ship with libraries prebuilt, so run that first.
1136   BuildStepBuildLibraries(pepperdir, 'src')
1137   GenerateNotice(pepperdir)
1138
1139   # Verify the SDK contains what we expect.
1140   if not options.bionic:
1141     BuildStepVerifyFilelist(pepperdir)
1142
1143   if options.tar:
1144     BuildStepTarBundle(pepper_ver, tarfile)
1145
1146   if platform == 'linux':
1147     BuildStepBuildPNaClComponent(pepper_ver, chrome_revision)
1148
1149     if options.build_ports:
1150       ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2')
1151       BuildStepSyncNaClPorts()
1152       BuildStepBuildNaClPorts(pepper_ver, pepperdir)
1153       if options.tar:
1154         BuildStepTarNaClPorts(pepper_ver, ports_tarfile)
1155
1156   if options.build_app_engine and platform == 'linux':
1157     BuildStepBuildAppEngine(pepperdir, chrome_revision)
1158
1159   if options.qemu:
1160     qemudir = os.path.join(NACL_DIR, 'toolchain', 'linux_arm-trusted')
1161     oshelpers.Copy(['-r', qemudir, pepperdir])
1162
1163   # Archive the results on Google Cloud Storage.
1164   if options.archive:
1165     BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision,
1166                            tarfile)
1167     # Only archive sdk_tools/naclport/pnacl_component on linux.
1168     if platform == 'linux':
1169       if options.build_ports:
1170         BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision,
1171                                nacl_revision, ports_tarfile)
1172       BuildStepArchiveSDKTools()
1173       BuildStepArchivePNaClComponent()
1174
1175   return 0
1176
1177
1178 if __name__ == '__main__':
1179   try:
1180     sys.exit(main(sys.argv[1:]))
1181   except KeyboardInterrupt:
1182     buildbot_common.ErrorExit('build_sdk: interrupted')