Explicitly add python-numpy dependency to install-build-deps.
[chromium-blink-merge.git] / native_client_sdk / src / build_tools / build_sdk.py
blob6a6e323ea7d3d79c80ff018a2afff463dc044d0d
1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Entry point for both build and try bots.
8 This script is invoked from XXX, usually without arguments
9 to package an SDK. It automatically determines whether
10 this SDK is for mac, win, linux.
12 The script inspects the following environment variables:
14 BUILDBOT_BUILDERNAME to determine whether the script is run locally
15 and whether it should upload an SDK to file storage (GSTORE)
16 """
18 # pylint: disable=W0621
20 # std python includes
21 import datetime
22 import glob
23 import optparse
24 import os
25 import re
26 import sys
28 if sys.version_info < (2, 7, 0):
29 sys.stderr.write("python 2.7 or later is required run this script\n")
30 sys.exit(1)
32 # local includes
33 import buildbot_common
34 import build_projects
35 import build_updater
36 import build_version
37 import generate_notice
38 import manifest_util
39 import parse_dsc
40 import verify_filelist
42 from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR
43 from build_paths import NACLPORTS_DIR, GSTORE, GONACL_APPENGINE_SRC_DIR
45 # Add SDK make tools scripts to the python path.
46 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
47 sys.path.append(os.path.join(NACL_DIR, 'build'))
49 import getos
50 import oshelpers
52 BUILD_DIR = os.path.join(NACL_DIR, 'build')
53 NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
54 NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
56 CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
57 PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
59 NACLPORTS_URL = 'https://chromium.googlesource.com/external/naclports.git'
60 NACLPORTS_REV = 'e53078c33d99b0b3cbadbbbbb92cccf7a48d5dc1'
62 GYPBUILD_DIR = 'gypbuild'
64 options = None
66 # Map of: ToolchainName: (PackageName, SDKDir).
67 TOOLCHAIN_PACKAGE_MAP = {
68 'newlib': ('nacl_x86_newlib', '%(platform)s_x86_newlib'),
69 'bionic': ('nacl_arm_bionic', '%(platform)s_arm_bionic'),
70 'arm': ('nacl_arm_newlib', '%(platform)s_arm_newlib'),
71 'glibc': ('nacl_x86_glibc', '%(platform)s_x86_glibc'),
72 'pnacl': ('pnacl_newlib', '%(platform)s_pnacl')
76 def GetToolchainNaClInclude(tcname, tcpath, arch):
77 if arch == 'x86':
78 if tcname == 'pnacl':
79 return os.path.join(tcpath, 'le32-nacl', 'include')
80 return os.path.join(tcpath, 'x86_64-nacl', 'include')
81 elif arch == 'arm':
82 return os.path.join(tcpath, 'arm-nacl', 'include')
83 else:
84 buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
87 def GetGypGenDir(xarch):
88 if xarch == 'arm':
89 build_dir = GYPBUILD_DIR + '-arm'
90 else:
91 build_dir = GYPBUILD_DIR
92 return os.path.join(OUT_DIR, build_dir, 'Release', 'gen')
95 def GetGypBuiltLib(tcname, xarch=None):
96 if tcname == 'pnacl':
97 tcname = 'pnacl_newlib'
98 if not xarch:
99 xarch = ''
100 return os.path.join(GetGypGenDir(xarch), 'tc_' + tcname, 'lib' + xarch)
103 def GetToolchainNaClLib(tcname, tcpath, xarch):
104 if tcname == 'pnacl':
105 return os.path.join(tcpath, 'le32-nacl', 'lib')
106 elif xarch == '32':
107 return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
108 elif xarch == '64':
109 return os.path.join(tcpath, 'x86_64-nacl', 'lib')
110 elif xarch == 'arm':
111 return os.path.join(tcpath, 'arm-nacl', 'lib')
114 def GetToolchainDirName(tcname, xarch):
115 if tcname == 'pnacl':
116 return '%s_%s' % (getos.GetPlatform(), tcname)
117 elif xarch == 'arm':
118 return '%s_arm_%s' % (getos.GetPlatform(), tcname)
119 else:
120 return '%s_x86_%s' % (getos.GetPlatform(), tcname)
123 def GetGypToolchainLib(tcname, xarch):
124 if xarch == 'arm':
125 toolchain = xarch
126 else:
127 toolchain = tcname
129 tcpath = os.path.join(GetGypGenDir(xarch), 'sdk',
130 '%s_x86' % getos.GetPlatform(),
131 TOOLCHAIN_PACKAGE_MAP[toolchain][0])
132 return GetToolchainNaClLib(tcname, tcpath, xarch)
135 def GetOutputToolchainLib(pepperdir, tcname, xarch):
136 tcpath = os.path.join(pepperdir, 'toolchain',
137 GetToolchainDirName(tcname, xarch))
138 return GetToolchainNaClLib(tcname, tcpath, xarch)
141 def GetPNaClTranslatorLib(tcpath, arch):
142 if arch not in ['arm', 'x86-32', 'x86-64']:
143 buildbot_common.ErrorExit('Unknown architecture %s.' % arch)
144 return os.path.join(tcpath, 'translator', arch, 'lib')
147 def BuildStepDownloadToolchains(toolchains):
148 buildbot_common.BuildStep('Running package_version.py')
149 args = [sys.executable, PKGVER, '--mode', 'nacl_core_sdk']
150 if 'bionic' in toolchains:
151 build_platform = '%s_x86' % getos.GetPlatform()
152 args.extend(['--append', os.path.join(build_platform, 'nacl_arm_bionic')])
153 args.append('sync')
154 buildbot_common.Run(args, cwd=NACL_DIR)
157 def BuildStepCleanPepperDirs(pepperdir, pepperdir_old):
158 buildbot_common.BuildStep('Clean Pepper Dirs')
159 buildbot_common.RemoveDir(pepperdir_old)
160 buildbot_common.RemoveDir(pepperdir)
161 buildbot_common.MakeDir(pepperdir)
164 def BuildStepMakePepperDirs(pepperdir, subdirs):
165 for subdir in subdirs:
166 buildbot_common.MakeDir(os.path.join(pepperdir, subdir))
168 TEXT_FILES = [
169 'AUTHORS',
170 'COPYING',
171 'LICENSE',
172 'README.Makefiles',
173 'getting_started/README',
176 def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision,
177 nacl_revision):
178 buildbot_common.BuildStep('Add Text Files')
179 InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES)
181 # Replace a few placeholders in README
182 readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read()
183 readme_text = readme_text.replace('${VERSION}', pepper_ver)
184 readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision)
185 readme_text = readme_text.replace('${CHROME_COMMIT_POSITION}',
186 build_version.ChromeCommitPosition())
187 readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision)
189 # Year/Month/Day Hour:Minute:Second
190 time_format = '%Y/%m/%d %H:%M:%S'
191 readme_text = readme_text.replace('${DATE}',
192 datetime.datetime.now().strftime(time_format))
194 open(os.path.join(pepperdir, 'README'), 'w').write(readme_text)
197 def BuildStepUntarToolchains(pepperdir, toolchains):
198 buildbot_common.BuildStep('Untar Toolchains')
199 platform = getos.GetPlatform()
200 build_platform = '%s_x86' % platform
201 tmpdir = os.path.join(OUT_DIR, 'tc_temp')
202 buildbot_common.RemoveDir(tmpdir)
203 buildbot_common.MakeDir(tmpdir)
205 # Create a list of extract packages tuples, the first part should be
206 # "$PACKAGE_TARGET/$PACKAGE". The second part should be the destination
207 # directory relative to pepperdir/toolchain.
208 extract_packages = []
209 for toolchain in toolchains:
210 toolchain_map = TOOLCHAIN_PACKAGE_MAP.get(toolchain, None)
211 if toolchain_map:
212 package_name, tcname = toolchain_map
213 package_tuple = (os.path.join(build_platform, package_name),
214 tcname % {'platform': platform})
215 extract_packages.append(package_tuple)
217 if extract_packages:
218 # Extract all of the packages into the temp directory.
219 package_names = [package_tuple[0] for package_tuple in extract_packages]
220 buildbot_common.Run([sys.executable, PKGVER,
221 '--packages', ','.join(package_names),
222 '--tar-dir', NACL_TOOLCHAINTARS_DIR,
223 '--dest-dir', tmpdir,
224 'extract'])
226 # Move all the packages we extracted to the correct destination.
227 for package_name, dest_dir in extract_packages:
228 full_src_dir = os.path.join(tmpdir, package_name)
229 full_dst_dir = os.path.join(pepperdir, 'toolchain', dest_dir)
230 buildbot_common.Move(full_src_dir, full_dst_dir)
232 # Cleanup the temporary directory we are no longer using.
233 buildbot_common.RemoveDir(tmpdir)
236 # List of toolchain headers to install.
237 # Source is relative to top of Chromium tree, destination is relative
238 # to the toolchain header directory.
239 NACL_HEADER_MAP = {
240 'newlib': [
241 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
242 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
243 ('native_client/src/untrusted/irt/irt.h', ''),
244 ('native_client/src/untrusted/irt/irt_dev.h', ''),
245 ('native_client/src/untrusted/irt/irt_extension.h', ''),
246 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
247 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
248 ('native_client/src/untrusted/pthread/pthread.h', ''),
249 ('native_client/src/untrusted/pthread/semaphore.h', ''),
250 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
251 ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
253 'glibc': [
254 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
255 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
256 ('native_client/src/untrusted/irt/irt.h', ''),
257 ('native_client/src/untrusted/irt/irt_dev.h', ''),
258 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
259 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
260 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
261 ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
263 'bionic': [
264 ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
266 'host': []
269 def InstallFiles(src_root, dest_root, file_list):
270 """Copy a set of files from src_root to dest_root according
271 to the given mapping. This allows files to be copied from
272 to a location in the destination tree that is different to the
273 location in the source tree.
275 If the destination mapping ends with a '/' then the destination
276 basename is inherited from the the source file.
278 Wildcards can be used in the source list but it is not recommended
279 as this can end up adding things to the SDK unintentionally.
281 for file_spec in file_list:
282 # The list of files to install can be a simple list of
283 # strings or a list of pairs, where each pair corresponds
284 # to a mapping from source to destination names.
285 if type(file_spec) == str:
286 src_file = dest_file = file_spec
287 else:
288 src_file, dest_file = file_spec
290 src_file = os.path.join(src_root, src_file)
292 # Expand sources files using glob.
293 sources = glob.glob(src_file)
294 if not sources:
295 sources = [src_file]
297 if len(sources) > 1 and not dest_file.endswith('/'):
298 buildbot_common.ErrorExit("Target file must end in '/' when "
299 "using globbing to install multiple files")
301 for source in sources:
302 if dest_file.endswith('/'):
303 dest = os.path.join(dest_file, os.path.basename(source))
304 else:
305 dest = dest_file
306 dest = os.path.join(dest_root, dest)
307 if not os.path.isdir(os.path.dirname(dest)):
308 buildbot_common.MakeDir(os.path.dirname(dest))
309 buildbot_common.CopyFile(source, dest)
312 def InstallNaClHeaders(tc_dst_inc, tc_name):
313 """Copies NaCl headers to expected locations in the toolchain."""
314 if tc_name == 'arm':
315 # arm toolchain header should be the same as the x86 newlib
316 # ones
317 tc_name = 'newlib'
319 InstallFiles(SRC_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name])
322 def MakeNinjaRelPath(path):
323 return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
326 TOOLCHAIN_LIBS = {
327 'bionic' : [
328 'libminidump_generator.a',
329 'libnacl_dyncode.a',
330 'libnacl_exception.a',
331 'libnacl_list_mappings.a',
332 'libppapi.a',
334 'newlib' : [
335 'crti.o',
336 'crtn.o',
337 'libminidump_generator.a',
338 'libnacl.a',
339 'libnacl_dyncode.a',
340 'libnacl_exception.a',
341 'libnacl_list_mappings.a',
342 'libnosys.a',
343 'libppapi.a',
344 'libppapi_stub.a',
345 'libpthread.a',
347 'glibc': [
348 'libminidump_generator.a',
349 'libminidump_generator.so',
350 'libnacl.a',
351 'libnacl_dyncode.a',
352 'libnacl_dyncode.so',
353 'libnacl_exception.a',
354 'libnacl_exception.so',
355 'libnacl_list_mappings.a',
356 'libnacl_list_mappings.so',
357 'libppapi.a',
358 'libppapi.so',
359 'libppapi_stub.a',
361 'pnacl': [
362 'libminidump_generator.a',
363 'libnacl.a',
364 'libnacl_dyncode.a',
365 'libnacl_exception.a',
366 'libnacl_list_mappings.a',
367 'libnosys.a',
368 'libppapi.a',
369 'libppapi_stub.a',
370 'libpthread.a',
375 def GypNinjaInstall(pepperdir, toolchains):
376 build_dir = GYPBUILD_DIR
377 ninja_out_dir = os.path.join(OUT_DIR, build_dir, 'Release')
378 tools_files = [
379 ['sel_ldr', 'sel_ldr_x86_32'],
380 ['ncval_new', 'ncval'],
381 ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'],
382 ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'],
385 platform = getos.GetPlatform()
387 # TODO(binji): dump_syms doesn't currently build on Windows. See
388 # http://crbug.com/245456
389 if platform != 'win':
390 tools_files += [
391 ['dump_syms', 'dump_syms'],
392 ['minidump_dump', 'minidump_dump'],
393 ['minidump_stackwalk', 'minidump_stackwalk']
396 tools_files.append(['sel_ldr64', 'sel_ldr_x86_64'])
398 if platform == 'linux':
399 tools_files.append(['nacl_helper_bootstrap',
400 'nacl_helper_bootstrap_x86_32'])
401 tools_files.append(['nacl_helper_bootstrap64',
402 'nacl_helper_bootstrap_x86_64'])
403 tools_files.append(['nonsfi_loader_newlib_x32_nonsfi.nexe',
404 'nonsfi_loader_x86_32'])
406 buildbot_common.MakeDir(os.path.join(pepperdir, 'tools'))
408 # Add .exe extensions to all windows tools
409 for pair in tools_files:
410 if platform == 'win' and not pair[0].endswith('.nexe'):
411 pair[0] += '.exe'
412 pair[1] += '.exe'
414 InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files)
416 # Add ARM binaries
417 if platform == 'linux' and not options.no_arm_trusted:
418 tools_files = [
419 ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
420 ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
421 ['sel_ldr', 'sel_ldr_arm'],
422 ['nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm']
424 ninja_out_dir = os.path.join(OUT_DIR, build_dir + '-arm', 'Release')
425 InstallFiles(ninja_out_dir, os.path.join(pepperdir, 'tools'), tools_files)
427 for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']):
428 if tc == 'pnacl':
429 xarches = (None,)
430 else:
431 xarches = ('arm', '32', '64')
433 for xarch in xarches:
434 if tc == 'glibc' and xarch == 'arm':
435 continue
437 src_dir = GetGypBuiltLib(tc, xarch)
438 dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch)
439 InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc])
441 # Copy ARM newlib components to bionic
442 if tc == 'newlib' and xarch == 'arm' and 'bionic' in toolchains:
443 bionic_dir = GetOutputToolchainLib(pepperdir, 'bionic', xarch)
444 InstallFiles(src_dir, bionic_dir, TOOLCHAIN_LIBS['bionic'])
446 if tc != 'pnacl':
447 src_dir = GetGypToolchainLib(tc, xarch)
448 InstallFiles(src_dir, dst_dir, ['crt1.o'])
451 def GypNinjaBuild_NaCl(rel_out_dir):
452 gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl')
453 nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
454 all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp')
456 out_dir = MakeNinjaRelPath(rel_out_dir)
457 out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm')
458 GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir)
459 GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm)
460 GypNinjaBuild(None, gyp_py, all_gyp, 'ncval_new', out_dir)
462 platform = getos.GetPlatform()
463 if platform == 'win':
464 NinjaBuild('sel_ldr64', out_dir)
465 else:
466 out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-64')
467 GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'sel_ldr', out_dir_64)
469 # We only need sel_ldr from the 64-bit out directory.
470 # sel_ldr needs to be renamed, so we'll call it sel_ldr64.
471 files_to_copy = [('sel_ldr', 'sel_ldr64')]
472 if platform == 'linux':
473 files_to_copy.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap64'))
475 for src, dst in files_to_copy:
476 buildbot_common.CopyFile(
477 os.path.join(SRC_DIR, out_dir_64, 'Release', src),
478 os.path.join(SRC_DIR, out_dir, 'Release', dst))
481 def GypNinjaBuild_Breakpad(rel_out_dir):
482 # TODO(binji): dump_syms doesn't currently build on Windows. See
483 # http://crbug.com/245456
484 if getos.GetPlatform() == 'win':
485 return
487 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
488 out_dir = MakeNinjaRelPath(rel_out_dir)
489 gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp')
490 build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
491 GypNinjaBuild(None, gyp_py, gyp_file, build_list, out_dir)
494 def GypNinjaBuild_PPAPI(arch, rel_out_dir):
495 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
496 out_dir = MakeNinjaRelPath(rel_out_dir)
497 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client',
498 'native_client.gyp')
499 GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir)
502 def GypNinjaBuild_Pnacl(rel_out_dir, target_arch):
503 # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the
504 # Chromium build, and once here. When we move more of the SDK build process
505 # to gyp, we can remove this.
506 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
508 out_dir = MakeNinjaRelPath(rel_out_dir)
509 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src',
510 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
511 targets = ['aot']
512 GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir, False)
515 def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets,
516 out_dir, force_arm_gcc=True):
517 gyp_env = dict(os.environ)
518 gyp_env['GYP_GENERATORS'] = 'ninja'
519 gyp_defines = []
520 if options.mac_sdk:
521 gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
522 if arch is not None:
523 gyp_defines.append('target_arch=%s' % arch)
524 if arch == 'arm':
525 if getos.GetPlatform() == 'linux':
526 gyp_env['CC'] = 'arm-linux-gnueabihf-gcc'
527 gyp_env['CXX'] = 'arm-linux-gnueabihf-g++'
528 gyp_env['AR'] = 'arm-linux-gnueabihf-ar'
529 gyp_env['AS'] = 'arm-linux-gnueabihf-as'
530 gyp_env['CC_host'] = 'cc'
531 gyp_env['CXX_host'] = 'c++'
532 gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1',
533 'arm_float_abi=hard']
534 if force_arm_gcc:
535 gyp_defines.append('nacl_enable_arm_gcc=1')
536 if options.no_arm_trusted:
537 gyp_defines.append('disable_cross_trusted=1')
538 if getos.GetPlatform() == 'mac':
539 gyp_defines.append('clang=1')
541 gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
542 for key in ['GYP_GENERATORS', 'GYP_DEFINES', 'CC']:
543 value = gyp_env.get(key)
544 if value is not None:
545 print '%s="%s"' % (key, value)
546 gyp_generator_flags = ['-G', 'output_dir=%s' % (out_dir,)]
547 gyp_depth = '--depth=.'
548 buildbot_common.Run(
549 [sys.executable, gyp_py_script, gyp_file, gyp_depth] + \
550 gyp_generator_flags,
551 cwd=SRC_DIR,
552 env=gyp_env)
553 NinjaBuild(targets, out_dir)
556 def NinjaBuild(targets, out_dir):
557 if type(targets) is not list:
558 targets = [targets]
559 out_config_dir = os.path.join(out_dir, 'Release')
560 buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
563 def BuildStepBuildToolchains(pepperdir, toolchains):
564 buildbot_common.BuildStep('SDK Items')
566 GypNinjaBuild_NaCl(GYPBUILD_DIR)
567 GypNinjaBuild_Breakpad(GYPBUILD_DIR)
569 platform = getos.GetPlatform()
570 newlibdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_newlib')
571 glibcdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_glibc')
572 armdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_newlib')
573 pnacldir = os.path.join(pepperdir, 'toolchain', platform + '_pnacl')
574 bionicdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_bionic')
576 if set(toolchains) & set(['glibc', 'newlib']):
577 GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR)
578 GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR)
580 if 'arm' in toolchains:
581 GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm')
583 GypNinjaInstall(pepperdir, toolchains)
585 if 'newlib' in toolchains:
586 InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
587 'newlib')
589 if 'glibc' in toolchains:
590 InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
591 'glibc')
593 if 'arm' in toolchains:
594 InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'),
595 'arm')
597 if 'bionic' in toolchains:
598 InstallNaClHeaders(GetToolchainNaClInclude('bionic', bionicdir, 'arm'),
599 'bionic')
601 if 'pnacl' in toolchains:
602 # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
603 for arch in ('ia32', 'arm'):
604 # Fill in the latest native pnacl shim library from the chrome build.
605 build_dir = GYPBUILD_DIR + '-pnacl-' + arch
606 GypNinjaBuild_Pnacl(build_dir, arch)
607 if arch == 'ia32':
608 nacl_arches = ['x86-32', 'x86-64']
609 elif arch == 'arm':
610 nacl_arches = ['arm']
611 else:
612 buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
613 for nacl_arch in nacl_arches:
614 release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release',
615 'gen', 'tc_pnacl_translate',
616 'lib-' + nacl_arch)
618 pnacl_translator_lib_dir = GetPNaClTranslatorLib(pnacldir, nacl_arch)
619 if not os.path.isdir(pnacl_translator_lib_dir):
620 buildbot_common.ErrorExit('Expected %s directory to exist.' %
621 pnacl_translator_lib_dir)
623 buildbot_common.CopyFile(
624 os.path.join(release_build_dir, 'libpnacl_irt_shim.a'),
625 pnacl_translator_lib_dir)
627 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
628 'newlib')
631 def MakeDirectoryOrClobber(pepperdir, dirname, clobber):
632 dirpath = os.path.join(pepperdir, dirname)
633 if clobber:
634 buildbot_common.RemoveDir(dirpath)
635 buildbot_common.MakeDir(dirpath)
637 return dirpath
640 def BuildStepUpdateHelpers(pepperdir, clobber):
641 buildbot_common.BuildStep('Update project helpers')
642 build_projects.UpdateHelpers(pepperdir, clobber=clobber)
645 def BuildStepUpdateUserProjects(pepperdir, toolchains,
646 build_experimental, clobber):
647 buildbot_common.BuildStep('Update examples and libraries')
649 filters = {}
650 if not build_experimental:
651 filters['EXPERIMENTAL'] = False
652 if toolchains:
653 toolchains = toolchains[:]
655 # arm isn't a valid toolchain for build_projects
656 if 'arm' in toolchains:
657 toolchains.remove('arm')
659 if 'host' in toolchains:
660 toolchains.remove('host')
661 toolchains.append(getos.GetPlatform())
663 filters['TOOLS'] = toolchains
665 # Update examples and libraries
666 filters['DEST'] = [
667 'getting_started',
668 'examples/api',
669 'examples/benchmarks',
670 'examples/demo',
671 'examples/tutorial',
672 'src'
675 tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
676 build_projects.UpdateProjects(pepperdir, tree, clobber=clobber,
677 toolchains=toolchains)
680 def BuildStepMakeAll(pepperdir, directory, step_name,
681 deps=True, clean=False, config='Debug', args=None):
682 buildbot_common.BuildStep(step_name)
683 build_projects.BuildProjectsBranch(pepperdir, directory, clean,
684 deps, config, args)
687 def BuildStepBuildLibraries(pepperdir, directory):
688 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug',
689 clean=True, config='Debug')
690 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release',
691 clean=True, config='Release')
693 # Cleanup .pyc file generated while building libraries. Without
694 # this we would end up shipping the pyc in the SDK tarball.
695 buildbot_common.RemoveFile(os.path.join(pepperdir, 'tools', '*.pyc'))
698 def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None):
699 # Look for LICENSE files
700 license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT')
702 license_files = []
703 for root, _, files in os.walk(fileroot):
704 for filename in files:
705 if license_filenames_re.match(filename):
706 path = os.path.join(root, filename)
707 license_files.append(path)
709 if extra_files:
710 license_files += [os.path.join(fileroot, f) for f in extra_files]
711 print '\n'.join(license_files)
713 if not os.path.isabs(output_filename):
714 output_filename = os.path.join(fileroot, output_filename)
715 generate_notice.Generate(output_filename, fileroot, license_files)
718 def BuildStepVerifyFilelist(pepperdir):
719 buildbot_common.BuildStep('Verify SDK Files')
720 file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list')
721 try:
722 verify_filelist.Verify(file_list_path, pepperdir)
723 print 'OK'
724 except verify_filelist.ParseException, e:
725 buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e)
726 except verify_filelist.VerifyException, e:
727 file_list_rel = os.path.relpath(file_list_path)
728 verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py'
729 verify_filelist_py = os.path.relpath(verify_filelist_py)
730 pepperdir_rel = os.path.relpath(pepperdir)
732 msg = """\
733 SDK verification failed:
736 Add/remove files from %s to fix.
738 Run:
739 ./%s %s %s
740 to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel,
741 pepperdir_rel)
742 buildbot_common.ErrorExit(msg)
745 def BuildStepTarBundle(pepper_ver, tarfile):
746 buildbot_common.BuildStep('Tar Pepper Bundle')
747 buildbot_common.MakeDir(os.path.dirname(tarfile))
748 buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
749 'pepper_' + pepper_ver], cwd=NACL_DIR)
752 def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile,
753 archive_url):
754 with open(tarfile, 'rb') as tarfile_stream:
755 archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
756 tarfile_stream)
758 archive = manifest_util.Archive(manifest_util.GetHostOS())
759 archive.url = archive_url
760 archive.size = archive_size
761 archive.checksum = archive_sha1
763 bundle = manifest_util.Bundle('pepper_' + pepper_ver)
764 bundle.revision = int(chrome_revision)
765 bundle.repath = 'pepper_' + pepper_ver
766 bundle.version = int(pepper_ver)
767 bundle.description = (
768 'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % (
769 pepper_ver, chrome_revision, nacl_revision))
770 bundle.stability = 'dev'
771 bundle.recommended = 'no'
772 bundle.archives = [archive]
773 return bundle
776 def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision,
777 tarfile):
778 buildbot_common.BuildStep('Archive %s' % name)
779 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
780 build_version.ChromeVersion(),)
781 tarname = os.path.basename(tarfile)
782 tarfile_dir = os.path.dirname(tarfile)
783 buildbot_common.Archive(tarname, bucket_path, tarfile_dir)
785 # generate "manifest snippet" for this archive.
786 archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
787 build_version.ChromeVersion(), tarname)
788 bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision,
789 tarfile, archive_url)
791 manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
792 with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
793 manifest_snippet_stream.write(bundle.GetDataAsString())
795 buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
796 step_link=False)
799 def BuildStepArchiveSDKTools():
800 # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot.
801 builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
802 if builder_name == 'linux-sdk-multi':
803 buildbot_common.BuildStep('Build SDK Tools')
804 build_updater.BuildUpdater(OUT_DIR)
806 buildbot_common.BuildStep('Archive SDK Tools')
807 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
808 build_version.ChromeVersion(),)
809 buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR,
810 step_link=False)
811 buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR,
812 step_link=False)
815 def BuildStepSyncNaClPorts():
816 """Pull the pinned revision of naclports from SVN."""
817 buildbot_common.BuildStep('Sync naclports')
819 # In case a previous svn checkout exists, remove it.
820 # TODO(sbc): remove this once all the build machines
821 # have removed the old checkout
822 if (os.path.exists(NACLPORTS_DIR) and
823 not os.path.exists(os.path.join(NACLPORTS_DIR, '.git'))):
824 buildbot_common.RemoveDir(NACLPORTS_DIR)
826 if not os.path.exists(NACLPORTS_DIR):
827 # checkout new copy of naclports
828 cmd = ['git', 'clone', NACLPORTS_URL, 'naclports']
829 buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR))
830 else:
831 # checkout new copy of naclports
832 buildbot_common.Run(['git', 'fetch'], cwd=NACLPORTS_DIR)
834 # sync to required revision
835 cmd = ['git', 'checkout', str(NACLPORTS_REV)]
836 buildbot_common.Run(cmd, cwd=NACLPORTS_DIR)
839 def BuildStepBuildNaClPorts(pepper_ver, pepperdir):
840 """Build selected naclports in all configurations."""
841 # TODO(sbc): currently naclports doesn't know anything about
842 # Debug builds so the Debug subfolders are all empty.
844 env = dict(os.environ)
845 env['NACL_SDK_ROOT'] = pepperdir
846 env['PEPPER_DIR'] = os.path.basename(pepperdir) # pepper_NN
847 env['NACLPORTS_NO_ANNOTATE'] = "1"
848 env['NACLPORTS_NO_UPLOAD'] = "1"
849 env['BUILDBOT_GOT_REVISION'] = str(NACLPORTS_REV)
851 build_script = 'build_tools/buildbot_sdk_bundle.sh'
852 buildbot_common.BuildStep('Build naclports')
854 bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
855 out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver)
857 # Remove the sdk_bundle directory to remove stale files from previous builds.
858 buildbot_common.RemoveDir(bundle_dir)
860 buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR)
862 # Some naclports do not include a standalone LICENSE/COPYING file
863 # so we explicitly list those here for inclusion.
864 extra_licenses = ('tinyxml/readme.txt',
865 'jpeg-8d/README',
866 'zlib-1.2.3/README')
867 src_root = os.path.join(NACLPORTS_DIR, 'out', 'build')
868 output_license = os.path.join(out_dir, 'ports', 'LICENSE')
869 GenerateNotice(src_root, output_license, extra_licenses)
870 readme = os.path.join(out_dir, 'ports', 'README')
871 oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme])
874 def BuildStepTarNaClPorts(pepper_ver, tarfile):
875 """Create tar archive containing headers and libs from naclports build."""
876 buildbot_common.BuildStep('Tar naclports Bundle')
877 buildbot_common.MakeDir(os.path.dirname(tarfile))
878 pepper_dir = 'pepper_%s' % pepper_ver
879 archive_dirs = [os.path.join(pepper_dir, 'ports')]
881 ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
882 cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile]
883 cmd += archive_dirs
884 buildbot_common.Run(cmd, cwd=NACL_DIR)
887 def BuildStepBuildAppEngine(pepperdir, chrome_revision):
888 """Build the projects found in src/gonacl_appengine/src"""
889 buildbot_common.BuildStep('Build GoNaCl AppEngine Projects')
890 cmd = ['make', 'upload', 'REVISION=%s' % chrome_revision]
891 env = dict(os.environ)
892 env['NACL_SDK_ROOT'] = pepperdir
893 env['NACLPORTS_NO_ANNOTATE'] = "1"
894 buildbot_common.Run(cmd, env=env, cwd=GONACL_APPENGINE_SRC_DIR)
897 def main(args):
898 parser = optparse.OptionParser(description=__doc__)
899 parser.add_option('--nacl-tree-path',
900 help='Path to native client tree for bionic build.',
901 dest='nacl_tree_path')
902 parser.add_option('--qemu', help='Add qemu for ARM.',
903 action='store_true')
904 parser.add_option('--bionic', help='Add bionic build.',
905 action='store_true')
906 parser.add_option('--tar', help='Force the tar step.',
907 action='store_true')
908 parser.add_option('--archive', help='Force the archive step.',
909 action='store_true')
910 parser.add_option('--release', help='PPAPI release version.',
911 dest='release', default=None)
912 parser.add_option('--build-ports',
913 help='Build naclport bundle.', action='store_true')
914 parser.add_option('--build-app-engine',
915 help='Build AppEngine demos.', action='store_true')
916 parser.add_option('--experimental',
917 help='build experimental examples and libraries', action='store_true',
918 dest='build_experimental')
919 parser.add_option('--skip-toolchain', help='Skip toolchain untar',
920 action='store_true')
921 parser.add_option('--mac-sdk',
922 help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
923 parser.add_option('--no-arm-trusted', action='store_true',
924 help='Disable building of ARM trusted components (sel_ldr, etc).')
926 # To setup bash completion for this command first install optcomplete
927 # and then add this line to your .bashrc:
928 # complete -F _optcomplete build_sdk.py
929 try:
930 import optcomplete
931 optcomplete.autocomplete(parser)
932 except ImportError:
933 pass
935 global options
936 options, args = parser.parse_args(args)
937 if args:
938 parser.error("Unexpected arguments: %s" % str(args))
940 if options.nacl_tree_path:
941 options.bionic = True
942 toolchain_build = os.path.join(options.nacl_tree_path, 'toolchain_build')
943 print 'WARNING: Building bionic toolchain from NaCl checkout.'
944 print 'This option builds bionic from the sources currently in the'
945 print 'provided NativeClient checkout, and the results instead of '
946 print 'downloading a toolchain from the builder. This may result in a'
947 print 'NaCl SDK that can not run on ToT chrome.'
948 print 'NOTE: To clobber you will need to run toolchain_build_bionic.py'
949 print 'directly from the NativeClient checkout.'
950 print ''
951 response = raw_input("Type 'y' and hit enter to continue.\n")
952 if response != 'y' and response != 'Y':
953 print 'Aborting.'
954 return 1
956 # Get head version of NativeClient tree
957 buildbot_common.BuildStep('Build bionic toolchain.')
958 buildbot_common.Run([sys.executable, 'toolchain_build_bionic.py', '-f'],
959 cwd=toolchain_build)
960 else:
961 toolchain_build = None
963 if buildbot_common.IsSDKBuilder():
964 options.archive = True
965 options.build_ports = True
966 # TODO(binji): re-enable app_engine build when the linux builder stops
967 # breaking when trying to git clone from github.
968 # See http://crbug.com/412969.
969 options.build_app_engine = False
970 options.tar = True
972 # NOTE: order matters here. This will be the order that is specified in the
973 # Makefiles; the first toolchain will be the default.
974 toolchains = ['pnacl', 'newlib', 'glibc', 'arm', 'host']
976 # Changes for experimental bionic builder
977 if options.bionic:
978 toolchains.append('bionic')
979 options.build_ports = False
980 options.build_app_engine = False
982 print 'Building: ' + ' '.join(toolchains)
984 if options.archive and not options.tar:
985 parser.error('Incompatible arguments with archive.')
987 chrome_version = int(build_version.ChromeMajorVersion())
988 chrome_revision = build_version.ChromeRevision()
989 nacl_revision = build_version.NaClRevision()
990 pepper_ver = str(chrome_version)
991 pepper_old = str(chrome_version - 1)
992 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
993 pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old)
994 if options.bionic:
995 tarname = 'naclsdk_bionic.tar.bz2'
996 else:
997 tarname = 'naclsdk_' + getos.GetPlatform() + '.tar.bz2'
998 tarfile = os.path.join(OUT_DIR, tarname)
1000 if options.release:
1001 pepper_ver = options.release
1002 print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision)
1004 if 'NACL_SDK_ROOT' in os.environ:
1005 # We don't want the currently configured NACL_SDK_ROOT to have any effect
1006 # of the build.
1007 del os.environ['NACL_SDK_ROOT']
1009 if not options.skip_toolchain:
1010 BuildStepCleanPepperDirs(pepperdir, pepperdir_old)
1011 BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools'])
1012 BuildStepDownloadToolchains(toolchains)
1013 if options.nacl_tree_path:
1014 # Instead of untarring, copy the raw bionic toolchain
1015 not_bionic = [i for i in toolchains if i != 'bionic']
1016 BuildStepUntarToolchains(pepperdir, not_bionic)
1017 tcname = GetToolchainDirName('bionic', 'arm')
1018 srcdir = os.path.join(toolchain_build, 'out', tcname)
1019 bionicdir = os.path.join(pepperdir, 'toolchain', tcname)
1020 oshelpers.Copy(['-r', srcdir, bionicdir])
1021 else:
1022 BuildStepUntarToolchains(pepperdir, toolchains)
1024 BuildStepBuildToolchains(pepperdir, toolchains)
1026 BuildStepUpdateHelpers(pepperdir, True)
1027 BuildStepUpdateUserProjects(pepperdir, toolchains,
1028 options.build_experimental, True)
1030 BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision)
1032 # Ship with libraries prebuilt, so run that first.
1033 BuildStepBuildLibraries(pepperdir, 'src')
1034 GenerateNotice(pepperdir)
1036 # Verify the SDK contains what we expect.
1037 if not options.bionic:
1038 BuildStepVerifyFilelist(pepperdir)
1040 if options.tar:
1041 BuildStepTarBundle(pepper_ver, tarfile)
1043 if options.build_ports and getos.GetPlatform() == 'linux':
1044 ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2')
1045 BuildStepSyncNaClPorts()
1046 BuildStepBuildNaClPorts(pepper_ver, pepperdir)
1047 if options.tar:
1048 BuildStepTarNaClPorts(pepper_ver, ports_tarfile)
1050 if options.build_app_engine and getos.GetPlatform() == 'linux':
1051 BuildStepBuildAppEngine(pepperdir, chrome_revision)
1053 if options.qemu:
1054 qemudir = os.path.join(NACL_DIR, 'toolchain', 'linux_arm-trusted')
1055 oshelpers.Copy(['-r', qemudir, pepperdir])
1057 # Archive on non-trybots.
1058 if options.archive:
1059 BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision,
1060 tarfile)
1061 if options.build_ports and getos.GetPlatform() == 'linux':
1062 BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision,
1063 nacl_revision, ports_tarfile)
1064 BuildStepArchiveSDKTools()
1066 return 0
1069 if __name__ == '__main__':
1070 try:
1071 sys.exit(main(sys.argv[1:]))
1072 except KeyboardInterrupt:
1073 buildbot_common.ErrorExit('build_sdk: interrupted')