ozone: evdev: Sync caps lock LED state to evdev
[chromium-blink-merge.git] / native_client_sdk / src / build_tools / build_sdk.py
blobffa456f8839dd17830da772e814742bec3cd56ca
1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Entry point for both build and try bots.
8 This script is invoked from XXX, usually without arguments
9 to package an SDK. It automatically determines whether
10 this SDK is for mac, win, linux.
12 The script inspects the following environment variables:
14 BUILDBOT_BUILDERNAME to determine whether the script is run locally
15 and whether it should upload an SDK to file storage (GSTORE)
16 """
18 # pylint: disable=W0621
20 # std python includes
21 import argparse
22 import datetime
23 import glob
24 import os
25 import re
26 import sys
28 if sys.version_info < (2, 7, 0):
29 sys.stderr.write("python 2.7 or later is required run this script\n")
30 sys.exit(1)
32 # local includes
33 import buildbot_common
34 import build_projects
35 import build_updater
36 import build_version
37 import generate_notice
38 import manifest_util
39 import parse_dsc
40 import verify_filelist
42 from build_paths import SCRIPT_DIR, SDK_SRC_DIR, SRC_DIR, NACL_DIR, OUT_DIR
43 from build_paths import NACLPORTS_DIR, GSTORE, GONACL_APPENGINE_SRC_DIR
45 # Add SDK make tools scripts to the python path.
46 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
47 sys.path.append(os.path.join(NACL_DIR, 'build'))
49 import getos
50 import oshelpers
52 BUILD_DIR = os.path.join(NACL_DIR, 'build')
53 NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
54 NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
56 CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
57 PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
59 NACLPORTS_URL = 'https://chromium.googlesource.com/external/naclports.git'
60 NACLPORTS_REV = 'e53078c33d99b0b3cbadbbbbb92cccf7a48d5dc1'
62 GYPBUILD_DIR = 'gypbuild'
64 options = None
66 # Map of: ToolchainName: (PackageName, SDKDir).
67 TOOLCHAIN_PACKAGE_MAP = {
68 'newlib': ('nacl_x86_newlib', '%(platform)s_x86_newlib'),
69 'bionic': ('nacl_arm_bionic', '%(platform)s_arm_bionic'),
70 'arm': ('nacl_arm_newlib', '%(platform)s_arm_newlib'),
71 'glibc': ('nacl_x86_glibc', '%(platform)s_x86_glibc'),
72 'pnacl': ('pnacl_newlib', '%(platform)s_pnacl')
76 def GetToolchainNaClInclude(tcname, tcpath, arch):
77 if arch == 'x86':
78 return os.path.join(tcpath, 'x86_64-nacl', 'include')
79 elif arch == 'pnacl':
80 return os.path.join(tcpath, 'le32-nacl', 'include')
81 elif arch == 'arm':
82 return os.path.join(tcpath, 'arm-nacl', 'include')
83 else:
84 buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
87 def GetConfigDir(arch):
88 if arch.endswith('x64') and getos.GetPlatform() == 'win':
89 return 'Release_x64'
90 else:
91 return 'Release'
94 def GetNinjaOutDir(arch):
95 return os.path.join(OUT_DIR, GYPBUILD_DIR + '-' + arch, GetConfigDir(arch))
98 def GetGypBuiltLib(tcname, arch):
99 if arch == 'ia32':
100 lib_suffix = '32'
101 elif arch == 'x64':
102 lib_suffix = '64'
103 elif arch == 'arm':
104 lib_suffix = 'arm'
105 else:
106 lib_suffix = ''
108 if tcname == 'pnacl':
109 print arch
110 if arch is None:
111 arch = 'x64'
112 tcname = 'pnacl_newlib'
113 else:
114 arch = 'clang-' + arch
115 tcname = 'newlib'
117 return os.path.join(GetNinjaOutDir(arch),
118 'gen',
119 'tc_' + tcname,
120 'lib' + lib_suffix)
123 def GetToolchainNaClLib(tcname, tcpath, arch):
124 if arch == 'ia32':
125 return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
126 elif arch == 'x64':
127 return os.path.join(tcpath, 'x86_64-nacl', 'lib')
128 elif arch == 'arm':
129 return os.path.join(tcpath, 'arm-nacl', 'lib')
130 elif tcname == 'pnacl':
131 return os.path.join(tcpath, 'le32-nacl', 'lib')
134 def GetToolchainDirName(tcname, arch):
135 if tcname == 'pnacl':
136 return '%s_%s' % (getos.GetPlatform(), tcname)
137 elif arch == 'arm':
138 return '%s_arm_%s' % (getos.GetPlatform(), tcname)
139 else:
140 return '%s_x86_%s' % (getos.GetPlatform(), tcname)
143 def GetGypToolchainLib(tcname, arch):
144 if arch == 'arm':
145 toolchain = arch
146 else:
147 toolchain = tcname
149 tcpath = os.path.join(GetNinjaOutDir(arch), 'gen', 'sdk',
150 '%s_x86' % getos.GetPlatform(),
151 TOOLCHAIN_PACKAGE_MAP[toolchain][0])
152 return GetToolchainNaClLib(tcname, tcpath, arch)
155 def GetOutputToolchainLib(pepperdir, tcname, arch):
156 tcpath = os.path.join(pepperdir, 'toolchain',
157 GetToolchainDirName(tcname, arch))
158 return GetToolchainNaClLib(tcname, tcpath, arch)
161 def GetPNaClTranslatorLib(tcpath, arch):
162 if arch not in ['arm', 'x86-32', 'x86-64']:
163 buildbot_common.ErrorExit('Unknown architecture %s.' % arch)
164 return os.path.join(tcpath, 'translator', arch, 'lib')
167 def BuildStepDownloadToolchains(toolchains):
168 buildbot_common.BuildStep('Running package_version.py')
169 args = [sys.executable, PKGVER, '--mode', 'nacl_core_sdk']
170 if 'bionic' in toolchains:
171 build_platform = '%s_x86' % getos.GetPlatform()
172 args.extend(['--append', os.path.join(build_platform, 'nacl_arm_bionic')])
173 args.extend(['sync', '--extract'])
174 buildbot_common.Run(args, cwd=NACL_DIR)
177 def BuildStepCleanPepperDirs(pepperdir, pepperdir_old):
178 buildbot_common.BuildStep('Clean Pepper Dirs')
179 buildbot_common.RemoveDir(pepperdir_old)
180 buildbot_common.RemoveDir(pepperdir)
181 buildbot_common.MakeDir(pepperdir)
184 def BuildStepMakePepperDirs(pepperdir, subdirs):
185 for subdir in subdirs:
186 buildbot_common.MakeDir(os.path.join(pepperdir, subdir))
188 TEXT_FILES = [
189 'AUTHORS',
190 'COPYING',
191 'LICENSE',
192 'README.Makefiles',
193 'getting_started/README',
196 def BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision,
197 nacl_revision):
198 buildbot_common.BuildStep('Add Text Files')
199 InstallFiles(SDK_SRC_DIR, pepperdir, TEXT_FILES)
201 # Replace a few placeholders in README
202 readme_text = open(os.path.join(SDK_SRC_DIR, 'README')).read()
203 readme_text = readme_text.replace('${VERSION}', pepper_ver)
204 readme_text = readme_text.replace('${CHROME_REVISION}', chrome_revision)
205 readme_text = readme_text.replace('${CHROME_COMMIT_POSITION}',
206 build_version.ChromeCommitPosition())
207 readme_text = readme_text.replace('${NACL_REVISION}', nacl_revision)
209 # Year/Month/Day Hour:Minute:Second
210 time_format = '%Y/%m/%d %H:%M:%S'
211 readme_text = readme_text.replace('${DATE}',
212 datetime.datetime.now().strftime(time_format))
214 open(os.path.join(pepperdir, 'README'), 'w').write(readme_text)
217 def BuildStepUntarToolchains(pepperdir, toolchains):
218 buildbot_common.BuildStep('Untar Toolchains')
219 platform = getos.GetPlatform()
220 build_platform = '%s_x86' % platform
221 tmpdir = os.path.join(OUT_DIR, 'tc_temp')
222 buildbot_common.RemoveDir(tmpdir)
223 buildbot_common.MakeDir(tmpdir)
225 # Create a list of extract packages tuples, the first part should be
226 # "$PACKAGE_TARGET/$PACKAGE". The second part should be the destination
227 # directory relative to pepperdir/toolchain.
228 extract_packages = []
229 for toolchain in toolchains:
230 toolchain_map = TOOLCHAIN_PACKAGE_MAP.get(toolchain, None)
231 if toolchain_map:
232 package_name, tcname = toolchain_map
233 package_tuple = (os.path.join(build_platform, package_name),
234 tcname % {'platform': platform})
235 extract_packages.append(package_tuple)
237 if extract_packages:
238 # Extract all of the packages into the temp directory.
239 package_names = [package_tuple[0] for package_tuple in extract_packages]
240 buildbot_common.Run([sys.executable, PKGVER,
241 '--packages', ','.join(package_names),
242 '--tar-dir', NACL_TOOLCHAINTARS_DIR,
243 '--dest-dir', tmpdir,
244 'extract'])
246 # Move all the packages we extracted to the correct destination.
247 for package_name, dest_dir in extract_packages:
248 full_src_dir = os.path.join(tmpdir, package_name)
249 full_dst_dir = os.path.join(pepperdir, 'toolchain', dest_dir)
250 buildbot_common.Move(full_src_dir, full_dst_dir)
252 # Cleanup the temporary directory we are no longer using.
253 buildbot_common.RemoveDir(tmpdir)
256 # List of toolchain headers to install.
257 # Source is relative to top of Chromium tree, destination is relative
258 # to the toolchain header directory.
259 NACL_HEADER_MAP = {
260 'newlib': [
261 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
262 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
263 ('native_client/src/untrusted/irt/irt.h', ''),
264 ('native_client/src/untrusted/irt/irt_dev.h', ''),
265 ('native_client/src/untrusted/irt/irt_extension.h', ''),
266 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
267 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
268 ('native_client/src/untrusted/pthread/pthread.h', ''),
269 ('native_client/src/untrusted/pthread/semaphore.h', ''),
270 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
271 ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
273 'glibc': [
274 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
275 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
276 ('native_client/src/untrusted/irt/irt.h', ''),
277 ('native_client/src/untrusted/irt/irt_dev.h', ''),
278 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
279 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
280 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
281 ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
283 'bionic': [
284 ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
286 'host': []
289 def InstallFiles(src_root, dest_root, file_list):
290 """Copy a set of files from src_root to dest_root according
291 to the given mapping. This allows files to be copied from
292 to a location in the destination tree that is different to the
293 location in the source tree.
295 If the destination mapping ends with a '/' then the destination
296 basename is inherited from the the source file.
298 Wildcards can be used in the source list but it is not recommended
299 as this can end up adding things to the SDK unintentionally.
301 for file_spec in file_list:
302 # The list of files to install can be a simple list of
303 # strings or a list of pairs, where each pair corresponds
304 # to a mapping from source to destination names.
305 if type(file_spec) == str:
306 src_file = dest_file = file_spec
307 else:
308 src_file, dest_file = file_spec
310 src_file = os.path.join(src_root, src_file)
312 # Expand sources files using glob.
313 sources = glob.glob(src_file)
314 if not sources:
315 sources = [src_file]
317 if len(sources) > 1 and not dest_file.endswith('/'):
318 buildbot_common.ErrorExit("Target file must end in '/' when "
319 "using globbing to install multiple files")
321 for source in sources:
322 if dest_file.endswith('/'):
323 dest = os.path.join(dest_file, os.path.basename(source))
324 else:
325 dest = dest_file
326 dest = os.path.join(dest_root, dest)
327 if not os.path.isdir(os.path.dirname(dest)):
328 buildbot_common.MakeDir(os.path.dirname(dest))
329 buildbot_common.CopyFile(source, dest)
332 def InstallNaClHeaders(tc_dst_inc, tc_name):
333 """Copies NaCl headers to expected locations in the toolchain."""
334 if tc_name in ('arm', 'pnacl'):
335 # arm and pnacl toolchain headers should be the same as the newlib
336 # ones
337 tc_name = 'newlib'
339 InstallFiles(SRC_DIR, tc_dst_inc, NACL_HEADER_MAP[tc_name])
342 def MakeNinjaRelPath(path):
343 return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
346 # TODO(ncbray): stop building and copying libraries into the SDK that are
347 # already provided by the toolchain.
348 TOOLCHAIN_LIBS = {
349 'bionic' : [
350 'libminidump_generator.a',
351 'libnacl_dyncode.a',
352 'libnacl_exception.a',
353 'libnacl_list_mappings.a',
354 'libppapi.a',
356 'newlib' : [
357 'libminidump_generator.a',
358 'libnacl.a',
359 'libnacl_dyncode.a',
360 'libnacl_exception.a',
361 'libnacl_list_mappings.a',
362 'libnosys.a',
363 'libppapi.a',
364 'libppapi_stub.a',
365 'libpthread.a',
367 'glibc': [
368 'libminidump_generator.a',
369 'libminidump_generator.so',
370 'libnacl.a',
371 'libnacl_dyncode.a',
372 'libnacl_dyncode.so',
373 'libnacl_exception.a',
374 'libnacl_exception.so',
375 'libnacl_list_mappings.a',
376 'libnacl_list_mappings.so',
377 'libppapi.a',
378 'libppapi.so',
379 'libppapi_stub.a',
381 'pnacl': [
382 'libminidump_generator.a',
383 'libnacl.a',
384 'libnacl_dyncode.a',
385 'libnacl_exception.a',
386 'libnacl_list_mappings.a',
387 'libnosys.a',
388 'libppapi.a',
389 'libppapi_stub.a',
390 'libpthread.a',
395 def GypNinjaInstall(pepperdir, toolchains):
396 tools_files_32 = [
397 ['sel_ldr', 'sel_ldr_x86_32'],
398 ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'],
399 ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'],
402 tools_files_64 = []
404 platform = getos.GetPlatform()
406 # TODO(binji): dump_syms doesn't currently build on Windows. See
407 # http://crbug.com/245456
408 if platform != 'win':
409 tools_files_64 += [
410 ['dump_syms', 'dump_syms'],
411 ['minidump_dump', 'minidump_dump'],
412 ['minidump_stackwalk', 'minidump_stackwalk']
415 tools_files_64.append(['sel_ldr', 'sel_ldr_x86_64'])
416 tools_files_64.append(['ncval_new', 'ncval'])
418 if platform == 'linux':
419 tools_files_32.append(['nacl_helper_bootstrap',
420 'nacl_helper_bootstrap_x86_32'])
421 tools_files_64.append(['nacl_helper_bootstrap',
422 'nacl_helper_bootstrap_x86_64'])
423 tools_files_32.append(['nonsfi_loader_newlib_x32_nonsfi.nexe',
424 'nonsfi_loader_x86_32'])
426 tools_dir = os.path.join(pepperdir, 'tools')
427 buildbot_common.MakeDir(tools_dir)
429 # Add .exe extensions to all windows tools
430 for pair in tools_files_32 + tools_files_64:
431 if platform == 'win' and not pair[0].endswith('.nexe'):
432 pair[0] += '.exe'
433 pair[1] += '.exe'
435 InstallFiles(GetNinjaOutDir('x64'), tools_dir, tools_files_64)
436 InstallFiles(GetNinjaOutDir('ia32'), tools_dir, tools_files_32)
438 # Add ARM binaries
439 if platform == 'linux' and not options.no_arm_trusted:
440 arm_files = [
441 ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
442 ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
443 ['nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm'],
444 ['nonsfi_loader_newlib_arm_nonsfi.nexe', 'nonsfi_loader_arm'],
445 ['sel_ldr', 'sel_ldr_arm']
447 InstallFiles(GetNinjaOutDir('arm'), tools_dir, arm_files)
449 for tc in set(toolchains) & set(['newlib', 'glibc', 'pnacl']):
450 if tc == 'pnacl':
451 xarches = (None, 'ia32', 'x64', 'arm')
452 elif tc == 'glibc':
453 xarches = ('ia32', 'x64')
454 else:
455 xarches = ('arm', 'ia32', 'x64')
457 for xarch in xarches:
458 src_dir = GetGypBuiltLib(tc, xarch)
459 dst_dir = GetOutputToolchainLib(pepperdir, tc, xarch)
460 InstallFiles(src_dir, dst_dir, TOOLCHAIN_LIBS[tc])
462 # Copy ARM newlib components to bionic
463 if tc == 'newlib' and xarch == 'arm' and 'bionic' in toolchains:
464 bionic_dir = GetOutputToolchainLib(pepperdir, 'bionic', xarch)
465 InstallFiles(src_dir, bionic_dir, TOOLCHAIN_LIBS['bionic'])
468 def GypNinjaBuild_NaCl(rel_out_dir):
469 # TODO(binji): gyp_nacl doesn't build properly on Windows anymore; it only
470 # can use VS2010, not VS2013 which is now required by the Chromium repo. NaCl
471 # needs to be updated to perform the same logic as Chromium in detecting VS,
472 # which can now exist in the depot_tools directory.
473 # See https://code.google.com/p/nativeclient/issues/detail?id=4022
475 # For now, let's use gyp_chromium to build these components.
476 # gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl')
477 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
478 nacl_core_sdk_gyp = os.path.join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
479 all_gyp = os.path.join(NACL_DIR, 'build', 'all.gyp')
481 out_dir_32 = MakeNinjaRelPath(rel_out_dir + '-ia32')
482 out_dir_64 = MakeNinjaRelPath(rel_out_dir + '-x64')
483 out_dir_arm = MakeNinjaRelPath(rel_out_dir + '-arm')
484 out_dir_clang_32 = MakeNinjaRelPath(rel_out_dir + '-clang-ia32')
485 out_dir_clang_64 = MakeNinjaRelPath(rel_out_dir + '-clang-x64')
486 out_dir_clang_arm = MakeNinjaRelPath(rel_out_dir + '-clang-arm')
488 GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_32)
489 GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_64)
490 GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk', out_dir_arm)
491 GypNinjaBuild('ia32', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk',
492 out_dir_clang_32, gyp_defines=['use_nacl_clang=1'])
493 GypNinjaBuild('x64', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk',
494 out_dir_clang_64, gyp_defines=['use_nacl_clang=1'])
495 GypNinjaBuild('arm', gyp_py, nacl_core_sdk_gyp, 'nacl_core_sdk',
496 out_dir_clang_arm, gyp_defines=['use_nacl_clang=1'])
497 GypNinjaBuild('x64', gyp_py, all_gyp, 'ncval_new', out_dir_64)
500 def GypNinjaBuild_Breakpad(rel_out_dir):
501 # TODO(binji): dump_syms doesn't currently build on Windows. See
502 # http://crbug.com/245456
503 if getos.GetPlatform() == 'win':
504 return
506 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
507 out_dir = MakeNinjaRelPath(rel_out_dir)
508 gyp_file = os.path.join(SRC_DIR, 'breakpad', 'breakpad.gyp')
509 build_list = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
510 GypNinjaBuild('x64', gyp_py, gyp_file, build_list, out_dir)
513 def GypNinjaBuild_PPAPI(arch, rel_out_dir, gyp_defines=None):
514 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
515 out_dir = MakeNinjaRelPath(rel_out_dir)
516 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client',
517 'native_client.gyp')
518 GypNinjaBuild(arch, gyp_py, gyp_file, 'ppapi_lib', out_dir,
519 gyp_defines=gyp_defines)
522 def GypNinjaBuild_Pnacl(rel_out_dir, target_arch):
523 # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the
524 # Chromium build, and once here. When we move more of the SDK build process
525 # to gyp, we can remove this.
526 gyp_py = os.path.join(SRC_DIR, 'build', 'gyp_chromium')
528 out_dir = MakeNinjaRelPath(rel_out_dir)
529 gyp_file = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'src',
530 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
531 targets = ['aot']
532 GypNinjaBuild(target_arch, gyp_py, gyp_file, targets, out_dir)
535 def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets,
536 out_dir, force_arm_gcc=True, gyp_defines=None):
537 gyp_env = dict(os.environ)
538 gyp_env['GYP_GENERATORS'] = 'ninja'
539 gyp_defines = gyp_defines or []
540 gyp_defines.append('nacl_allow_thin_archives=0')
541 if options.mac_sdk:
542 gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
544 if arch is not None:
545 gyp_defines.append('target_arch=%s' % arch)
546 if arch == 'arm':
547 gyp_env['GYP_CROSSCOMPILE'] = '1'
548 gyp_defines.append('arm_float_abi=hard')
549 if options.no_arm_trusted:
550 gyp_defines.append('disable_cross_trusted=1')
551 if getos.GetPlatform() == 'mac':
552 gyp_defines.append('clang=1')
554 gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
555 # We can't use windows path separators in GYP_GENERATOR_FLAGS since
556 # gyp uses shlex to parse them and treats '\' as an escape char.
557 gyp_env['GYP_GENERATOR_FLAGS'] = 'output_dir=%s' % out_dir.replace('\\', '/')
559 # Print relevant environment variables
560 for key, value in gyp_env.iteritems():
561 if key.startswith('GYP') or key in ('CC',):
562 print ' %s="%s"' % (key, value)
564 buildbot_common.Run(
565 [sys.executable, gyp_py_script, gyp_file, '--depth=.'],
566 cwd=SRC_DIR,
567 env=gyp_env)
569 NinjaBuild(targets, out_dir, arch)
572 def NinjaBuild(targets, out_dir, arch):
573 if type(targets) is not list:
574 targets = [targets]
575 out_config_dir = os.path.join(out_dir, GetConfigDir(arch))
576 buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
579 def BuildStepBuildToolchains(pepperdir, toolchains, build, clean):
580 buildbot_common.BuildStep('SDK Items')
582 if clean:
583 for dirname in glob.glob(os.path.join(OUT_DIR, GYPBUILD_DIR + '*')):
584 buildbot_common.RemoveDir(dirname)
586 if build:
587 GypNinjaBuild_NaCl(GYPBUILD_DIR)
588 GypNinjaBuild_Breakpad(GYPBUILD_DIR + '-x64')
590 if set(toolchains) & set(['glibc', 'newlib']):
591 GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR + '-ia32')
592 GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR + '-x64')
594 if 'arm' in toolchains:
595 GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-arm')
597 if 'pnacl' in toolchains:
598 GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR + '-clang-ia32',
599 ['use_nacl_clang=1'])
600 GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR + '-clang-x64',
601 ['use_nacl_clang=1'])
602 GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR + '-clang-arm',
603 ['use_nacl_clang=1'])
605 # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
606 for arch in ('ia32', 'arm'):
607 # Fill in the latest native pnacl shim library from the chrome build.
608 build_dir = GYPBUILD_DIR + '-pnacl-' + arch
609 GypNinjaBuild_Pnacl(build_dir, arch)
611 GypNinjaInstall(pepperdir, toolchains)
613 platform = getos.GetPlatform()
614 newlibdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_newlib')
615 glibcdir = os.path.join(pepperdir, 'toolchain', platform + '_x86_glibc')
616 armdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_newlib')
617 pnacldir = os.path.join(pepperdir, 'toolchain', platform + '_pnacl')
618 bionicdir = os.path.join(pepperdir, 'toolchain', platform + '_arm_bionic')
621 if 'newlib' in toolchains:
622 InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir, 'x86'),
623 'newlib')
625 if 'glibc' in toolchains:
626 InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir, 'x86'),
627 'glibc')
629 if 'arm' in toolchains:
630 InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir, 'arm'),
631 'arm')
633 if 'bionic' in toolchains:
634 InstallNaClHeaders(GetToolchainNaClInclude('bionic', bionicdir, 'arm'),
635 'bionic')
637 if 'pnacl' in toolchains:
638 # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
639 for arch in ('ia32', 'arm'):
640 # Fill in the latest native pnacl shim library from the chrome build.
641 build_dir = GYPBUILD_DIR + '-pnacl-' + arch
642 if arch == 'ia32':
643 nacl_arches = ['x86-32', 'x86-64']
644 elif arch == 'arm':
645 nacl_arches = ['arm']
646 else:
647 buildbot_common.ErrorExit('Unknown architecture: %s' % arch)
648 for nacl_arch in nacl_arches:
649 release_build_dir = os.path.join(OUT_DIR, build_dir, 'Release',
650 'gen', 'tc_pnacl_translate',
651 'lib-' + nacl_arch)
653 pnacl_translator_lib_dir = GetPNaClTranslatorLib(pnacldir, nacl_arch)
654 if not os.path.isdir(pnacl_translator_lib_dir):
655 buildbot_common.ErrorExit('Expected %s directory to exist.' %
656 pnacl_translator_lib_dir)
658 buildbot_common.CopyFile(
659 os.path.join(release_build_dir, 'libpnacl_irt_shim.a'),
660 pnacl_translator_lib_dir)
662 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'pnacl'),
663 'pnacl')
664 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'x86'),
665 'pnacl')
666 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir, 'arm'),
667 'pnacl')
670 def MakeDirectoryOrClobber(pepperdir, dirname, clobber):
671 dirpath = os.path.join(pepperdir, dirname)
672 if clobber:
673 buildbot_common.RemoveDir(dirpath)
674 buildbot_common.MakeDir(dirpath)
676 return dirpath
679 def BuildStepUpdateHelpers(pepperdir, clobber):
680 buildbot_common.BuildStep('Update project helpers')
681 build_projects.UpdateHelpers(pepperdir, clobber=clobber)
684 def BuildStepUpdateUserProjects(pepperdir, toolchains,
685 build_experimental, clobber):
686 buildbot_common.BuildStep('Update examples and libraries')
688 filters = {}
689 if not build_experimental:
690 filters['EXPERIMENTAL'] = False
691 if toolchains:
692 toolchains = toolchains[:]
694 # arm isn't a valid toolchain for build_projects
695 if 'arm' in toolchains:
696 toolchains.remove('arm')
698 if 'host' in toolchains:
699 toolchains.remove('host')
700 toolchains.append(getos.GetPlatform())
702 filters['TOOLS'] = toolchains
704 # Update examples and libraries
705 filters['DEST'] = [
706 'getting_started',
707 'examples/api',
708 'examples/benchmarks',
709 'examples/demo',
710 'examples/tutorial',
711 'src'
714 tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
715 build_projects.UpdateProjects(pepperdir, tree, clobber=clobber,
716 toolchains=toolchains)
719 def BuildStepMakeAll(pepperdir, directory, step_name,
720 deps=True, clean=False, config='Debug', args=None):
721 buildbot_common.BuildStep(step_name)
722 build_projects.BuildProjectsBranch(pepperdir, directory, clean,
723 deps, config, args)
726 def BuildStepBuildLibraries(pepperdir, directory):
727 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Debug',
728 clean=True, config='Debug')
729 BuildStepMakeAll(pepperdir, directory, 'Build Libraries Release',
730 clean=True, config='Release')
732 # Cleanup .pyc file generated while building libraries. Without
733 # this we would end up shipping the pyc in the SDK tarball.
734 buildbot_common.RemoveFile(os.path.join(pepperdir, 'tools', '*.pyc'))
737 def GenerateNotice(fileroot, output_filename='NOTICE', extra_files=None):
738 # Look for LICENSE files
739 license_filenames_re = re.compile('LICENSE|COPYING|COPYRIGHT')
741 license_files = []
742 for root, _, files in os.walk(fileroot):
743 for filename in files:
744 if license_filenames_re.match(filename):
745 path = os.path.join(root, filename)
746 license_files.append(path)
748 if extra_files:
749 license_files += [os.path.join(fileroot, f) for f in extra_files]
750 print '\n'.join(license_files)
752 if not os.path.isabs(output_filename):
753 output_filename = os.path.join(fileroot, output_filename)
754 generate_notice.Generate(output_filename, fileroot, license_files)
757 def BuildStepVerifyFilelist(pepperdir):
758 buildbot_common.BuildStep('Verify SDK Files')
759 file_list_path = os.path.join(SCRIPT_DIR, 'sdk_files.list')
760 try:
761 print 'SDK directory: %s' % pepperdir
762 verify_filelist.Verify(file_list_path, pepperdir)
763 print 'OK'
764 except verify_filelist.ParseException, e:
765 buildbot_common.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e)
766 except verify_filelist.VerifyException, e:
767 file_list_rel = os.path.relpath(file_list_path)
768 verify_filelist_py = os.path.splitext(verify_filelist.__file__)[0] + '.py'
769 verify_filelist_py = os.path.relpath(verify_filelist_py)
770 pepperdir_rel = os.path.relpath(pepperdir)
772 msg = """\
773 SDK verification failed:
776 Add/remove files from %s to fix.
778 Run:
779 ./%s %s %s
780 to test.""" % (e, file_list_rel, verify_filelist_py, file_list_rel,
781 pepperdir_rel)
782 buildbot_common.ErrorExit(msg)
785 def BuildStepTarBundle(pepper_ver, tarfile):
786 buildbot_common.BuildStep('Tar Pepper Bundle')
787 buildbot_common.MakeDir(os.path.dirname(tarfile))
788 buildbot_common.Run([sys.executable, CYGTAR, '-C', OUT_DIR, '-cjf', tarfile,
789 'pepper_' + pepper_ver], cwd=NACL_DIR)
792 def GetManifestBundle(pepper_ver, chrome_revision, nacl_revision, tarfile,
793 archive_url):
794 with open(tarfile, 'rb') as tarfile_stream:
795 archive_sha1, archive_size = manifest_util.DownloadAndComputeHash(
796 tarfile_stream)
798 archive = manifest_util.Archive(manifest_util.GetHostOS())
799 archive.url = archive_url
800 archive.size = archive_size
801 archive.checksum = archive_sha1
803 bundle = manifest_util.Bundle('pepper_' + pepper_ver)
804 bundle.revision = int(chrome_revision)
805 bundle.repath = 'pepper_' + pepper_ver
806 bundle.version = int(pepper_ver)
807 bundle.description = (
808 'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % (
809 pepper_ver, chrome_revision, nacl_revision))
810 bundle.stability = 'dev'
811 bundle.recommended = 'no'
812 bundle.archives = [archive]
813 return bundle
816 def BuildStepArchiveBundle(name, pepper_ver, chrome_revision, nacl_revision,
817 tarfile):
818 buildbot_common.BuildStep('Archive %s' % name)
819 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
820 build_version.ChromeVersion(),)
821 tarname = os.path.basename(tarfile)
822 tarfile_dir = os.path.dirname(tarfile)
823 buildbot_common.Archive(tarname, bucket_path, tarfile_dir)
825 # generate "manifest snippet" for this archive.
826 archive_url = GSTORE + 'nacl_sdk/%s/%s' % (
827 build_version.ChromeVersion(), tarname)
828 bundle = GetManifestBundle(pepper_ver, chrome_revision, nacl_revision,
829 tarfile, archive_url)
831 manifest_snippet_file = os.path.join(OUT_DIR, tarname + '.json')
832 with open(manifest_snippet_file, 'wb') as manifest_snippet_stream:
833 manifest_snippet_stream.write(bundle.GetDataAsString())
835 buildbot_common.Archive(tarname + '.json', bucket_path, OUT_DIR,
836 step_link=False)
839 def BuildStepArchiveSDKTools():
840 # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot.
841 builder_name = os.getenv('BUILDBOT_BUILDERNAME', '')
842 if builder_name == 'linux-sdk-multi':
843 buildbot_common.BuildStep('Build SDK Tools')
844 build_updater.BuildUpdater(OUT_DIR)
846 buildbot_common.BuildStep('Archive SDK Tools')
847 bucket_path = 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
848 build_version.ChromeVersion(),)
849 buildbot_common.Archive('sdk_tools.tgz', bucket_path, OUT_DIR,
850 step_link=False)
851 buildbot_common.Archive('nacl_sdk.zip', bucket_path, OUT_DIR,
852 step_link=False)
855 def BuildStepSyncNaClPorts():
856 """Pull the pinned revision of naclports from SVN."""
857 buildbot_common.BuildStep('Sync naclports')
859 # In case a previous svn checkout exists, remove it.
860 # TODO(sbc): remove this once all the build machines
861 # have removed the old checkout
862 if (os.path.exists(NACLPORTS_DIR) and
863 not os.path.exists(os.path.join(NACLPORTS_DIR, '.git'))):
864 buildbot_common.RemoveDir(NACLPORTS_DIR)
866 if not os.path.exists(NACLPORTS_DIR):
867 # checkout new copy of naclports
868 cmd = ['git', 'clone', NACLPORTS_URL, 'naclports']
869 buildbot_common.Run(cmd, cwd=os.path.dirname(NACLPORTS_DIR))
870 else:
871 # checkout new copy of naclports
872 buildbot_common.Run(['git', 'fetch'], cwd=NACLPORTS_DIR)
874 # sync to required revision
875 cmd = ['git', 'checkout', str(NACLPORTS_REV)]
876 buildbot_common.Run(cmd, cwd=NACLPORTS_DIR)
879 def BuildStepBuildNaClPorts(pepper_ver, pepperdir):
880 """Build selected naclports in all configurations."""
881 # TODO(sbc): currently naclports doesn't know anything about
882 # Debug builds so the Debug subfolders are all empty.
884 env = dict(os.environ)
885 env['NACL_SDK_ROOT'] = pepperdir
886 env['PEPPER_DIR'] = os.path.basename(pepperdir) # pepper_NN
887 env['NACLPORTS_NO_ANNOTATE'] = "1"
888 env['NACLPORTS_NO_UPLOAD'] = "1"
889 env['BUILDBOT_GOT_REVISION'] = str(NACLPORTS_REV)
891 build_script = 'build_tools/buildbot_sdk_bundle.sh'
892 buildbot_common.BuildStep('Build naclports')
894 bundle_dir = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
895 out_dir = os.path.join(bundle_dir, 'pepper_%s' % pepper_ver)
897 # Remove the sdk_bundle directory to remove stale files from previous builds.
898 buildbot_common.RemoveDir(bundle_dir)
900 buildbot_common.Run([build_script], env=env, cwd=NACLPORTS_DIR)
902 # Some naclports do not include a standalone LICENSE/COPYING file
903 # so we explicitly list those here for inclusion.
904 extra_licenses = ('tinyxml/readme.txt',
905 'jpeg-8d/README',
906 'zlib-1.2.3/README')
907 src_root = os.path.join(NACLPORTS_DIR, 'out', 'build')
908 output_license = os.path.join(out_dir, 'ports', 'LICENSE')
909 GenerateNotice(src_root, output_license, extra_licenses)
910 readme = os.path.join(out_dir, 'ports', 'README')
911 oshelpers.Copy(['-v', os.path.join(SDK_SRC_DIR, 'README.naclports'), readme])
914 def BuildStepTarNaClPorts(pepper_ver, tarfile):
915 """Create tar archive containing headers and libs from naclports build."""
916 buildbot_common.BuildStep('Tar naclports Bundle')
917 buildbot_common.MakeDir(os.path.dirname(tarfile))
918 pepper_dir = 'pepper_%s' % pepper_ver
919 archive_dirs = [os.path.join(pepper_dir, 'ports')]
921 ports_out = os.path.join(NACLPORTS_DIR, 'out', 'sdk_bundle')
922 cmd = [sys.executable, CYGTAR, '-C', ports_out, '-cjf', tarfile]
923 cmd += archive_dirs
924 buildbot_common.Run(cmd, cwd=NACL_DIR)
927 def BuildStepBuildAppEngine(pepperdir, chrome_revision):
928 """Build the projects found in src/gonacl_appengine/src"""
929 buildbot_common.BuildStep('Build GoNaCl AppEngine Projects')
930 cmd = ['make', 'upload', 'REVISION=%s' % chrome_revision]
931 env = dict(os.environ)
932 env['NACL_SDK_ROOT'] = pepperdir
933 env['NACLPORTS_NO_ANNOTATE'] = "1"
934 buildbot_common.Run(cmd, env=env, cwd=GONACL_APPENGINE_SRC_DIR)
937 def main(args):
938 parser = argparse.ArgumentParser(description=__doc__)
939 parser.add_argument('--nacl-tree-path',
940 help='Path to native client tree for bionic build.',
941 dest='nacl_tree_path')
942 parser.add_argument('--qemu', help='Add qemu for ARM.',
943 action='store_true')
944 parser.add_argument('--bionic', help='Add bionic build.',
945 action='store_true')
946 parser.add_argument('--tar', help='Force the tar step.',
947 action='store_true')
948 parser.add_argument('--archive', help='Force the archive step.',
949 action='store_true')
950 parser.add_argument('--release', help='PPAPI release version.',
951 dest='release', default=None)
952 parser.add_argument('--build-ports',
953 help='Build naclport bundle.', action='store_true')
954 parser.add_argument('--build-app-engine',
955 help='Build AppEngine demos.', action='store_true')
956 parser.add_argument('--experimental',
957 help='build experimental examples and libraries', action='store_true',
958 dest='build_experimental')
959 parser.add_argument('--skip-toolchain', help='Skip toolchain untar',
960 action='store_true')
961 parser.add_argument('--no-clean', dest='clean', action='store_false',
962 help="Don't clean gypbuild directories")
963 parser.add_argument('--mac-sdk',
964 help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
965 parser.add_argument('--no-arm-trusted', action='store_true',
966 help='Disable building of ARM trusted components (sel_ldr, etc).')
968 # To setup bash completion for this command first install optcomplete
969 # and then add this line to your .bashrc:
970 # complete -F _optcomplete build_sdk.py
971 try:
972 import optcomplete
973 optcomplete.autocomplete(parser)
974 except ImportError:
975 pass
977 global options
978 options = parser.parse_args(args)
980 if options.nacl_tree_path:
981 options.bionic = True
982 toolchain_build = os.path.join(options.nacl_tree_path, 'toolchain_build')
983 print 'WARNING: Building bionic toolchain from NaCl checkout.'
984 print 'This option builds bionic from the sources currently in the'
985 print 'provided NativeClient checkout, and the results instead of '
986 print 'downloading a toolchain from the builder. This may result in a'
987 print 'NaCl SDK that can not run on ToT chrome.'
988 print 'NOTE: To clobber you will need to run toolchain_build_bionic.py'
989 print 'directly from the NativeClient checkout.'
990 print ''
991 response = raw_input("Type 'y' and hit enter to continue.\n")
992 if response != 'y' and response != 'Y':
993 print 'Aborting.'
994 return 1
996 # Get head version of NativeClient tree
997 buildbot_common.BuildStep('Build bionic toolchain.')
998 buildbot_common.Run([sys.executable, 'toolchain_build_bionic.py', '-f'],
999 cwd=toolchain_build)
1000 else:
1001 toolchain_build = None
1003 if buildbot_common.IsSDKBuilder():
1004 options.archive = True
1005 options.build_ports = True
1006 # TODO(binji): re-enable app_engine build when the linux builder stops
1007 # breaking when trying to git clone from github.
1008 # See http://crbug.com/412969.
1009 options.build_app_engine = False
1010 options.tar = True
1012 # NOTE: order matters here. This will be the order that is specified in the
1013 # Makefiles; the first toolchain will be the default.
1014 toolchains = ['pnacl', 'newlib', 'glibc', 'arm', 'clang-newlib', 'host']
1016 # Changes for experimental bionic builder
1017 if options.bionic:
1018 toolchains.append('bionic')
1019 options.build_ports = False
1020 options.build_app_engine = False
1022 print 'Building: ' + ' '.join(toolchains)
1024 if options.archive and not options.tar:
1025 parser.error('Incompatible arguments with archive.')
1027 chrome_version = int(build_version.ChromeMajorVersion())
1028 chrome_revision = build_version.ChromeRevision()
1029 nacl_revision = build_version.NaClRevision()
1030 pepper_ver = str(chrome_version)
1031 pepper_old = str(chrome_version - 1)
1032 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
1033 pepperdir_old = os.path.join(OUT_DIR, 'pepper_' + pepper_old)
1034 if options.bionic:
1035 tarname = 'naclsdk_bionic.tar.bz2'
1036 else:
1037 tarname = 'naclsdk_' + getos.GetPlatform() + '.tar.bz2'
1038 tarfile = os.path.join(OUT_DIR, tarname)
1040 if options.release:
1041 pepper_ver = options.release
1042 print 'Building PEPPER %s at %s' % (pepper_ver, chrome_revision)
1044 if 'NACL_SDK_ROOT' in os.environ:
1045 # We don't want the currently configured NACL_SDK_ROOT to have any effect
1046 # of the build.
1047 del os.environ['NACL_SDK_ROOT']
1049 if not options.skip_toolchain:
1050 BuildStepCleanPepperDirs(pepperdir, pepperdir_old)
1051 BuildStepMakePepperDirs(pepperdir, ['include', 'toolchain', 'tools'])
1052 BuildStepDownloadToolchains(toolchains)
1053 if options.nacl_tree_path:
1054 # Instead of untarring, copy the raw bionic toolchain
1055 not_bionic = [i for i in toolchains if i != 'bionic']
1056 BuildStepUntarToolchains(pepperdir, not_bionic)
1057 tcname = GetToolchainDirName('bionic', 'arm')
1058 srcdir = os.path.join(toolchain_build, 'out', tcname)
1059 bionicdir = os.path.join(pepperdir, 'toolchain', tcname)
1060 oshelpers.Copy(['-r', srcdir, bionicdir])
1061 else:
1062 BuildStepUntarToolchains(pepperdir, toolchains)
1064 BuildStepBuildToolchains(pepperdir, toolchains,
1065 not options.skip_toolchain,
1066 options.clean)
1068 BuildStepUpdateHelpers(pepperdir, True)
1069 BuildStepUpdateUserProjects(pepperdir, toolchains,
1070 options.build_experimental, True)
1072 BuildStepCopyTextFiles(pepperdir, pepper_ver, chrome_revision, nacl_revision)
1074 # Ship with libraries prebuilt, so run that first.
1075 BuildStepBuildLibraries(pepperdir, 'src')
1076 GenerateNotice(pepperdir)
1078 # Verify the SDK contains what we expect.
1079 if not options.bionic:
1080 BuildStepVerifyFilelist(pepperdir)
1082 if options.tar:
1083 BuildStepTarBundle(pepper_ver, tarfile)
1085 if options.build_ports and getos.GetPlatform() == 'linux':
1086 ports_tarfile = os.path.join(OUT_DIR, 'naclports.tar.bz2')
1087 BuildStepSyncNaClPorts()
1088 BuildStepBuildNaClPorts(pepper_ver, pepperdir)
1089 if options.tar:
1090 BuildStepTarNaClPorts(pepper_ver, ports_tarfile)
1092 if options.build_app_engine and getos.GetPlatform() == 'linux':
1093 BuildStepBuildAppEngine(pepperdir, chrome_revision)
1095 if options.qemu:
1096 qemudir = os.path.join(NACL_DIR, 'toolchain', 'linux_arm-trusted')
1097 oshelpers.Copy(['-r', qemudir, pepperdir])
1099 # Archive on non-trybots.
1100 if options.archive:
1101 BuildStepArchiveBundle('build', pepper_ver, chrome_revision, nacl_revision,
1102 tarfile)
1103 if options.build_ports and getos.GetPlatform() == 'linux':
1104 BuildStepArchiveBundle('naclports', pepper_ver, chrome_revision,
1105 nacl_revision, ports_tarfile)
1106 BuildStepArchiveSDKTools()
1108 return 0
1111 if __name__ == '__main__':
1112 try:
1113 sys.exit(main(sys.argv[1:]))
1114 except KeyboardInterrupt:
1115 buildbot_common.ErrorExit('build_sdk: interrupted')