2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Entry point for both build and try bots.
8 This script is invoked from XXX, usually without arguments
9 to package an SDK. It automatically determines whether
10 this SDK is for mac, win, linux.
12 The script inspects the following environment variables:
14 BUILDBOT_BUILDERNAME to determine whether the script is run locally
15 and whether it should upload an SDK to file storage (GSTORE)
18 # pylint: disable=W0621
28 if sys
.version_info
< (2, 7, 0):
29 sys
.stderr
.write("python 2.7 or later is required run this script\n")
33 import buildbot_common
37 import generate_notice
40 import verify_filelist
42 from build_paths
import SCRIPT_DIR
, SDK_SRC_DIR
, SRC_DIR
, NACL_DIR
, OUT_DIR
43 from build_paths
import NACLPORTS_DIR
, GSTORE
, GONACL_APPENGINE_SRC_DIR
45 # Add SDK make tools scripts to the python path.
46 sys
.path
.append(os
.path
.join(SDK_SRC_DIR
, 'tools'))
47 sys
.path
.append(os
.path
.join(NACL_DIR
, 'build'))
52 BUILD_DIR
= os
.path
.join(NACL_DIR
, 'build')
53 NACL_TOOLCHAIN_DIR
= os
.path
.join(NACL_DIR
, 'toolchain')
54 NACL_TOOLCHAINTARS_DIR
= os
.path
.join(NACL_TOOLCHAIN_DIR
, '.tars')
56 CYGTAR
= os
.path
.join(BUILD_DIR
, 'cygtar.py')
57 PKGVER
= os
.path
.join(BUILD_DIR
, 'package_version', 'package_version.py')
59 NACLPORTS_URL
= 'https://chromium.googlesource.com/external/naclports.git'
60 NACLPORTS_REV
= 'e53078c33d99b0b3cbadbbbbb92cccf7a48d5dc1'
62 GYPBUILD_DIR
= 'gypbuild'
66 # Map of: ToolchainName: (PackageName, SDKDir).
67 TOOLCHAIN_PACKAGE_MAP
= {
68 'newlib': ('nacl_x86_newlib', '%(platform)s_x86_newlib'),
69 'bionic': ('nacl_arm_bionic', '%(platform)s_arm_bionic'),
70 'arm': ('nacl_arm_newlib', '%(platform)s_arm_newlib'),
71 'glibc': ('nacl_x86_glibc', '%(platform)s_x86_glibc'),
72 'pnacl': ('pnacl_newlib', '%(platform)s_pnacl')
76 def GetToolchainNaClInclude(tcname
, tcpath
, arch
):
78 return os
.path
.join(tcpath
, 'x86_64-nacl', 'include')
80 return os
.path
.join(tcpath
, 'le32-nacl', 'include')
82 return os
.path
.join(tcpath
, 'arm-nacl', 'include')
84 buildbot_common
.ErrorExit('Unknown architecture: %s' % arch
)
87 def GetConfigDir(arch
):
88 if arch
.endswith('x64') and getos
.GetPlatform() == 'win':
94 def GetNinjaOutDir(arch
):
95 return os
.path
.join(OUT_DIR
, GYPBUILD_DIR
+ '-' + arch
, GetConfigDir(arch
))
98 def GetGypBuiltLib(tcname
, arch
):
108 if tcname
== 'pnacl':
112 tcname
= 'pnacl_newlib'
114 arch
= 'clang-' + arch
117 return os
.path
.join(GetNinjaOutDir(arch
),
123 def GetToolchainNaClLib(tcname
, tcpath
, arch
):
125 return os
.path
.join(tcpath
, 'x86_64-nacl', 'lib32')
127 return os
.path
.join(tcpath
, 'x86_64-nacl', 'lib')
129 return os
.path
.join(tcpath
, 'arm-nacl', 'lib')
130 elif tcname
== 'pnacl':
131 return os
.path
.join(tcpath
, 'le32-nacl', 'lib')
134 def GetToolchainDirName(tcname
, arch
):
135 if tcname
== 'pnacl':
136 return '%s_%s' % (getos
.GetPlatform(), tcname
)
138 return '%s_arm_%s' % (getos
.GetPlatform(), tcname
)
140 return '%s_x86_%s' % (getos
.GetPlatform(), tcname
)
143 def GetGypToolchainLib(tcname
, arch
):
149 tcpath
= os
.path
.join(GetNinjaOutDir(arch
), 'gen', 'sdk',
150 '%s_x86' % getos
.GetPlatform(),
151 TOOLCHAIN_PACKAGE_MAP
[toolchain
][0])
152 return GetToolchainNaClLib(tcname
, tcpath
, arch
)
155 def GetOutputToolchainLib(pepperdir
, tcname
, arch
):
156 tcpath
= os
.path
.join(pepperdir
, 'toolchain',
157 GetToolchainDirName(tcname
, arch
))
158 return GetToolchainNaClLib(tcname
, tcpath
, arch
)
161 def GetPNaClTranslatorLib(tcpath
, arch
):
162 if arch
not in ['arm', 'x86-32', 'x86-64']:
163 buildbot_common
.ErrorExit('Unknown architecture %s.' % arch
)
164 return os
.path
.join(tcpath
, 'translator', arch
, 'lib')
167 def BuildStepDownloadToolchains(toolchains
):
168 buildbot_common
.BuildStep('Running package_version.py')
169 args
= [sys
.executable
, PKGVER
, '--mode', 'nacl_core_sdk']
170 if 'bionic' in toolchains
:
171 build_platform
= '%s_x86' % getos
.GetPlatform()
172 args
.extend(['--append', os
.path
.join(build_platform
, 'nacl_arm_bionic')])
173 args
.extend(['sync', '--extract'])
174 buildbot_common
.Run(args
, cwd
=NACL_DIR
)
177 def BuildStepCleanPepperDirs(pepperdir
, pepperdir_old
):
178 buildbot_common
.BuildStep('Clean Pepper Dirs')
179 buildbot_common
.RemoveDir(pepperdir_old
)
180 buildbot_common
.RemoveDir(pepperdir
)
181 buildbot_common
.MakeDir(pepperdir
)
184 def BuildStepMakePepperDirs(pepperdir
, subdirs
):
185 for subdir
in subdirs
:
186 buildbot_common
.MakeDir(os
.path
.join(pepperdir
, subdir
))
193 'getting_started/README',
196 def BuildStepCopyTextFiles(pepperdir
, pepper_ver
, chrome_revision
,
198 buildbot_common
.BuildStep('Add Text Files')
199 InstallFiles(SDK_SRC_DIR
, pepperdir
, TEXT_FILES
)
201 # Replace a few placeholders in README
202 readme_text
= open(os
.path
.join(SDK_SRC_DIR
, 'README')).read()
203 readme_text
= readme_text
.replace('${VERSION}', pepper_ver
)
204 readme_text
= readme_text
.replace('${CHROME_REVISION}', chrome_revision
)
205 readme_text
= readme_text
.replace('${CHROME_COMMIT_POSITION}',
206 build_version
.ChromeCommitPosition())
207 readme_text
= readme_text
.replace('${NACL_REVISION}', nacl_revision
)
209 # Year/Month/Day Hour:Minute:Second
210 time_format
= '%Y/%m/%d %H:%M:%S'
211 readme_text
= readme_text
.replace('${DATE}',
212 datetime
.datetime
.now().strftime(time_format
))
214 open(os
.path
.join(pepperdir
, 'README'), 'w').write(readme_text
)
217 def BuildStepUntarToolchains(pepperdir
, toolchains
):
218 buildbot_common
.BuildStep('Untar Toolchains')
219 platform
= getos
.GetPlatform()
220 build_platform
= '%s_x86' % platform
221 tmpdir
= os
.path
.join(OUT_DIR
, 'tc_temp')
222 buildbot_common
.RemoveDir(tmpdir
)
223 buildbot_common
.MakeDir(tmpdir
)
225 # Create a list of extract packages tuples, the first part should be
226 # "$PACKAGE_TARGET/$PACKAGE". The second part should be the destination
227 # directory relative to pepperdir/toolchain.
228 extract_packages
= []
229 for toolchain
in toolchains
:
230 toolchain_map
= TOOLCHAIN_PACKAGE_MAP
.get(toolchain
, None)
232 package_name
, tcname
= toolchain_map
233 package_tuple
= (os
.path
.join(build_platform
, package_name
),
234 tcname
% {'platform': platform
})
235 extract_packages
.append(package_tuple
)
238 # Extract all of the packages into the temp directory.
239 package_names
= [package_tuple
[0] for package_tuple
in extract_packages
]
240 buildbot_common
.Run([sys
.executable
, PKGVER
,
241 '--packages', ','.join(package_names
),
242 '--tar-dir', NACL_TOOLCHAINTARS_DIR
,
243 '--dest-dir', tmpdir
,
246 # Move all the packages we extracted to the correct destination.
247 for package_name
, dest_dir
in extract_packages
:
248 full_src_dir
= os
.path
.join(tmpdir
, package_name
)
249 full_dst_dir
= os
.path
.join(pepperdir
, 'toolchain', dest_dir
)
250 buildbot_common
.Move(full_src_dir
, full_dst_dir
)
252 # Cleanup the temporary directory we are no longer using.
253 buildbot_common
.RemoveDir(tmpdir
)
256 # List of toolchain headers to install.
257 # Source is relative to top of Chromium tree, destination is relative
258 # to the toolchain header directory.
261 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
262 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
263 ('native_client/src/untrusted/irt/irt.h', ''),
264 ('native_client/src/untrusted/irt/irt_dev.h', ''),
265 ('native_client/src/untrusted/irt/irt_extension.h', ''),
266 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
267 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
268 ('native_client/src/untrusted/pthread/pthread.h', ''),
269 ('native_client/src/untrusted/pthread/semaphore.h', ''),
270 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
271 ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
274 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
275 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
276 ('native_client/src/untrusted/irt/irt.h', ''),
277 ('native_client/src/untrusted/irt/irt_dev.h', ''),
278 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
279 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
280 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
281 ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
284 ('ppapi/nacl_irt/public/irt_ppapi.h', ''),
289 def InstallFiles(src_root
, dest_root
, file_list
):
290 """Copy a set of files from src_root to dest_root according
291 to the given mapping. This allows files to be copied from
292 to a location in the destination tree that is different to the
293 location in the source tree.
295 If the destination mapping ends with a '/' then the destination
296 basename is inherited from the the source file.
298 Wildcards can be used in the source list but it is not recommended
299 as this can end up adding things to the SDK unintentionally.
301 for file_spec
in file_list
:
302 # The list of files to install can be a simple list of
303 # strings or a list of pairs, where each pair corresponds
304 # to a mapping from source to destination names.
305 if type(file_spec
) == str:
306 src_file
= dest_file
= file_spec
308 src_file
, dest_file
= file_spec
310 src_file
= os
.path
.join(src_root
, src_file
)
312 # Expand sources files using glob.
313 sources
= glob
.glob(src_file
)
317 if len(sources
) > 1 and not dest_file
.endswith('/'):
318 buildbot_common
.ErrorExit("Target file must end in '/' when "
319 "using globbing to install multiple files")
321 for source
in sources
:
322 if dest_file
.endswith('/'):
323 dest
= os
.path
.join(dest_file
, os
.path
.basename(source
))
326 dest
= os
.path
.join(dest_root
, dest
)
327 if not os
.path
.isdir(os
.path
.dirname(dest
)):
328 buildbot_common
.MakeDir(os
.path
.dirname(dest
))
329 buildbot_common
.CopyFile(source
, dest
)
332 def InstallNaClHeaders(tc_dst_inc
, tc_name
):
333 """Copies NaCl headers to expected locations in the toolchain."""
334 if tc_name
in ('arm', 'pnacl'):
335 # arm and pnacl toolchain headers should be the same as the newlib
339 InstallFiles(SRC_DIR
, tc_dst_inc
, NACL_HEADER_MAP
[tc_name
])
342 def MakeNinjaRelPath(path
):
343 return os
.path
.join(os
.path
.relpath(OUT_DIR
, SRC_DIR
), path
)
346 # TODO(ncbray): stop building and copying libraries into the SDK that are
347 # already provided by the toolchain.
350 'libminidump_generator.a',
352 'libnacl_exception.a',
353 'libnacl_list_mappings.a',
357 'libminidump_generator.a',
360 'libnacl_exception.a',
361 'libnacl_list_mappings.a',
368 'libminidump_generator.a',
369 'libminidump_generator.so',
372 'libnacl_dyncode.so',
373 'libnacl_exception.a',
374 'libnacl_exception.so',
375 'libnacl_list_mappings.a',
376 'libnacl_list_mappings.so',
382 'libminidump_generator.a',
385 'libnacl_exception.a',
386 'libnacl_list_mappings.a',
395 def GypNinjaInstall(pepperdir
, toolchains
):
397 ['sel_ldr', 'sel_ldr_x86_32'],
398 ['irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'],
399 ['irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'],
404 platform
= getos
.GetPlatform()
406 # TODO(binji): dump_syms doesn't currently build on Windows. See
407 # http://crbug.com/245456
408 if platform
!= 'win':
410 ['dump_syms', 'dump_syms'],
411 ['minidump_dump', 'minidump_dump'],
412 ['minidump_stackwalk', 'minidump_stackwalk']
415 tools_files_64
.append(['sel_ldr', 'sel_ldr_x86_64'])
416 tools_files_64
.append(['ncval_new', 'ncval'])
418 if platform
== 'linux':
419 tools_files_32
.append(['nacl_helper_bootstrap',
420 'nacl_helper_bootstrap_x86_32'])
421 tools_files_64
.append(['nacl_helper_bootstrap',
422 'nacl_helper_bootstrap_x86_64'])
423 tools_files_32
.append(['nonsfi_loader_newlib_x32_nonsfi.nexe',
424 'nonsfi_loader_x86_32'])
426 tools_dir
= os
.path
.join(pepperdir
, 'tools')
427 buildbot_common
.MakeDir(tools_dir
)
429 # Add .exe extensions to all windows tools
430 for pair
in tools_files_32
+ tools_files_64
:
431 if platform
== 'win' and not pair
[0].endswith('.nexe'):
435 InstallFiles(GetNinjaOutDir('x64'), tools_dir
, tools_files_64
)
436 InstallFiles(GetNinjaOutDir('ia32'), tools_dir
, tools_files_32
)
439 if platform
== 'linux' and not options
.no_arm_trusted
:
441 ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
442 ['irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'],
443 ['nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm'],
444 ['nonsfi_loader_newlib_arm_nonsfi.nexe', 'nonsfi_loader_arm'],
445 ['sel_ldr', 'sel_ldr_arm']
447 InstallFiles(GetNinjaOutDir('arm'), tools_dir
, arm_files
)
449 for tc
in set(toolchains
) & set(['newlib', 'glibc', 'pnacl']):
451 xarches
= (None, 'ia32', 'x64', 'arm')
453 xarches
= ('ia32', 'x64')
455 xarches
= ('arm', 'ia32', 'x64')
457 for xarch
in xarches
:
458 src_dir
= GetGypBuiltLib(tc
, xarch
)
459 dst_dir
= GetOutputToolchainLib(pepperdir
, tc
, xarch
)
460 InstallFiles(src_dir
, dst_dir
, TOOLCHAIN_LIBS
[tc
])
462 # Copy ARM newlib components to bionic
463 if tc
== 'newlib' and xarch
== 'arm' and 'bionic' in toolchains
:
464 bionic_dir
= GetOutputToolchainLib(pepperdir
, 'bionic', xarch
)
465 InstallFiles(src_dir
, bionic_dir
, TOOLCHAIN_LIBS
['bionic'])
468 def GypNinjaBuild_NaCl(rel_out_dir
):
469 # TODO(binji): gyp_nacl doesn't build properly on Windows anymore; it only
470 # can use VS2010, not VS2013 which is now required by the Chromium repo. NaCl
471 # needs to be updated to perform the same logic as Chromium in detecting VS,
472 # which can now exist in the depot_tools directory.
473 # See https://code.google.com/p/nativeclient/issues/detail?id=4022
475 # For now, let's use gyp_chromium to build these components.
476 # gyp_py = os.path.join(NACL_DIR, 'build', 'gyp_nacl')
477 gyp_py
= os
.path
.join(SRC_DIR
, 'build', 'gyp_chromium')
478 nacl_core_sdk_gyp
= os
.path
.join(NACL_DIR
, 'build', 'nacl_core_sdk.gyp')
479 all_gyp
= os
.path
.join(NACL_DIR
, 'build', 'all.gyp')
481 out_dir_32
= MakeNinjaRelPath(rel_out_dir
+ '-ia32')
482 out_dir_64
= MakeNinjaRelPath(rel_out_dir
+ '-x64')
483 out_dir_arm
= MakeNinjaRelPath(rel_out_dir
+ '-arm')
484 out_dir_clang_32
= MakeNinjaRelPath(rel_out_dir
+ '-clang-ia32')
485 out_dir_clang_64
= MakeNinjaRelPath(rel_out_dir
+ '-clang-x64')
486 out_dir_clang_arm
= MakeNinjaRelPath(rel_out_dir
+ '-clang-arm')
488 GypNinjaBuild('ia32', gyp_py
, nacl_core_sdk_gyp
, 'nacl_core_sdk', out_dir_32
)
489 GypNinjaBuild('x64', gyp_py
, nacl_core_sdk_gyp
, 'nacl_core_sdk', out_dir_64
)
490 GypNinjaBuild('arm', gyp_py
, nacl_core_sdk_gyp
, 'nacl_core_sdk', out_dir_arm
)
491 GypNinjaBuild('ia32', gyp_py
, nacl_core_sdk_gyp
, 'nacl_core_sdk',
492 out_dir_clang_32
, gyp_defines
=['use_nacl_clang=1'])
493 GypNinjaBuild('x64', gyp_py
, nacl_core_sdk_gyp
, 'nacl_core_sdk',
494 out_dir_clang_64
, gyp_defines
=['use_nacl_clang=1'])
495 GypNinjaBuild('arm', gyp_py
, nacl_core_sdk_gyp
, 'nacl_core_sdk',
496 out_dir_clang_arm
, gyp_defines
=['use_nacl_clang=1'])
497 GypNinjaBuild('x64', gyp_py
, all_gyp
, 'ncval_new', out_dir_64
)
500 def GypNinjaBuild_Breakpad(rel_out_dir
):
501 # TODO(binji): dump_syms doesn't currently build on Windows. See
502 # http://crbug.com/245456
503 if getos
.GetPlatform() == 'win':
506 gyp_py
= os
.path
.join(SRC_DIR
, 'build', 'gyp_chromium')
507 out_dir
= MakeNinjaRelPath(rel_out_dir
)
508 gyp_file
= os
.path
.join(SRC_DIR
, 'breakpad', 'breakpad.gyp')
509 build_list
= ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
510 GypNinjaBuild('x64', gyp_py
, gyp_file
, build_list
, out_dir
)
513 def GypNinjaBuild_PPAPI(arch
, rel_out_dir
, gyp_defines
=None):
514 gyp_py
= os
.path
.join(SRC_DIR
, 'build', 'gyp_chromium')
515 out_dir
= MakeNinjaRelPath(rel_out_dir
)
516 gyp_file
= os
.path
.join(SRC_DIR
, 'ppapi', 'native_client',
518 GypNinjaBuild(arch
, gyp_py
, gyp_file
, 'ppapi_lib', out_dir
,
519 gyp_defines
=gyp_defines
)
522 def GypNinjaBuild_Pnacl(rel_out_dir
, target_arch
):
523 # TODO(binji): This will build the pnacl_irt_shim twice; once as part of the
524 # Chromium build, and once here. When we move more of the SDK build process
525 # to gyp, we can remove this.
526 gyp_py
= os
.path
.join(SRC_DIR
, 'build', 'gyp_chromium')
528 out_dir
= MakeNinjaRelPath(rel_out_dir
)
529 gyp_file
= os
.path
.join(SRC_DIR
, 'ppapi', 'native_client', 'src',
530 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
532 GypNinjaBuild(target_arch
, gyp_py
, gyp_file
, targets
, out_dir
)
535 def GypNinjaBuild(arch
, gyp_py_script
, gyp_file
, targets
,
536 out_dir
, force_arm_gcc
=True, gyp_defines
=None):
537 gyp_env
= dict(os
.environ
)
538 gyp_env
['GYP_GENERATORS'] = 'ninja'
539 gyp_defines
= gyp_defines
or []
540 gyp_defines
.append('nacl_allow_thin_archives=0')
542 gyp_defines
.append('mac_sdk=%s' % options
.mac_sdk
)
545 gyp_defines
.append('target_arch=%s' % arch
)
547 gyp_env
['GYP_CROSSCOMPILE'] = '1'
548 gyp_defines
.append('arm_float_abi=hard')
549 if options
.no_arm_trusted
:
550 gyp_defines
.append('disable_cross_trusted=1')
551 if getos
.GetPlatform() == 'mac':
552 gyp_defines
.append('clang=1')
554 gyp_env
['GYP_DEFINES'] = ' '.join(gyp_defines
)
555 # We can't use windows path separators in GYP_GENERATOR_FLAGS since
556 # gyp uses shlex to parse them and treats '\' as an escape char.
557 gyp_env
['GYP_GENERATOR_FLAGS'] = 'output_dir=%s' % out_dir
.replace('\\', '/')
559 # Print relevant environment variables
560 for key
, value
in gyp_env
.iteritems():
561 if key
.startswith('GYP') or key
in ('CC',):
562 print ' %s="%s"' % (key
, value
)
565 [sys
.executable
, gyp_py_script
, gyp_file
, '--depth=.'],
569 NinjaBuild(targets
, out_dir
, arch
)
572 def NinjaBuild(targets
, out_dir
, arch
):
573 if type(targets
) is not list:
575 out_config_dir
= os
.path
.join(out_dir
, GetConfigDir(arch
))
576 buildbot_common
.Run(['ninja', '-C', out_config_dir
] + targets
, cwd
=SRC_DIR
)
579 def BuildStepBuildToolchains(pepperdir
, toolchains
, build
, clean
):
580 buildbot_common
.BuildStep('SDK Items')
583 for dirname
in glob
.glob(os
.path
.join(OUT_DIR
, GYPBUILD_DIR
+ '*')):
584 buildbot_common
.RemoveDir(dirname
)
587 GypNinjaBuild_NaCl(GYPBUILD_DIR
)
588 GypNinjaBuild_Breakpad(GYPBUILD_DIR
+ '-x64')
590 if set(toolchains
) & set(['glibc', 'newlib']):
591 GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR
+ '-ia32')
592 GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR
+ '-x64')
594 if 'arm' in toolchains
:
595 GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR
+ '-arm')
597 if 'pnacl' in toolchains
:
598 GypNinjaBuild_PPAPI('ia32', GYPBUILD_DIR
+ '-clang-ia32',
599 ['use_nacl_clang=1'])
600 GypNinjaBuild_PPAPI('x64', GYPBUILD_DIR
+ '-clang-x64',
601 ['use_nacl_clang=1'])
602 GypNinjaBuild_PPAPI('arm', GYPBUILD_DIR
+ '-clang-arm',
603 ['use_nacl_clang=1'])
605 # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
606 for arch
in ('ia32', 'arm'):
607 # Fill in the latest native pnacl shim library from the chrome build.
608 build_dir
= GYPBUILD_DIR
+ '-pnacl-' + arch
609 GypNinjaBuild_Pnacl(build_dir
, arch
)
611 GypNinjaInstall(pepperdir
, toolchains
)
613 platform
= getos
.GetPlatform()
614 newlibdir
= os
.path
.join(pepperdir
, 'toolchain', platform
+ '_x86_newlib')
615 glibcdir
= os
.path
.join(pepperdir
, 'toolchain', platform
+ '_x86_glibc')
616 armdir
= os
.path
.join(pepperdir
, 'toolchain', platform
+ '_arm_newlib')
617 pnacldir
= os
.path
.join(pepperdir
, 'toolchain', platform
+ '_pnacl')
618 bionicdir
= os
.path
.join(pepperdir
, 'toolchain', platform
+ '_arm_bionic')
621 if 'newlib' in toolchains
:
622 InstallNaClHeaders(GetToolchainNaClInclude('newlib', newlibdir
, 'x86'),
625 if 'glibc' in toolchains
:
626 InstallNaClHeaders(GetToolchainNaClInclude('glibc', glibcdir
, 'x86'),
629 if 'arm' in toolchains
:
630 InstallNaClHeaders(GetToolchainNaClInclude('newlib', armdir
, 'arm'),
633 if 'bionic' in toolchains
:
634 InstallNaClHeaders(GetToolchainNaClInclude('bionic', bionicdir
, 'arm'),
637 if 'pnacl' in toolchains
:
638 # NOTE: For ia32, gyp builds both x86-32 and x86-64 by default.
639 for arch
in ('ia32', 'arm'):
640 # Fill in the latest native pnacl shim library from the chrome build.
641 build_dir
= GYPBUILD_DIR
+ '-pnacl-' + arch
643 nacl_arches
= ['x86-32', 'x86-64']
645 nacl_arches
= ['arm']
647 buildbot_common
.ErrorExit('Unknown architecture: %s' % arch
)
648 for nacl_arch
in nacl_arches
:
649 release_build_dir
= os
.path
.join(OUT_DIR
, build_dir
, 'Release',
650 'gen', 'tc_pnacl_translate',
653 pnacl_translator_lib_dir
= GetPNaClTranslatorLib(pnacldir
, nacl_arch
)
654 if not os
.path
.isdir(pnacl_translator_lib_dir
):
655 buildbot_common
.ErrorExit('Expected %s directory to exist.' %
656 pnacl_translator_lib_dir
)
658 buildbot_common
.CopyFile(
659 os
.path
.join(release_build_dir
, 'libpnacl_irt_shim.a'),
660 pnacl_translator_lib_dir
)
662 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir
, 'pnacl'),
664 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir
, 'x86'),
666 InstallNaClHeaders(GetToolchainNaClInclude('pnacl', pnacldir
, 'arm'),
670 def MakeDirectoryOrClobber(pepperdir
, dirname
, clobber
):
671 dirpath
= os
.path
.join(pepperdir
, dirname
)
673 buildbot_common
.RemoveDir(dirpath
)
674 buildbot_common
.MakeDir(dirpath
)
679 def BuildStepUpdateHelpers(pepperdir
, clobber
):
680 buildbot_common
.BuildStep('Update project helpers')
681 build_projects
.UpdateHelpers(pepperdir
, clobber
=clobber
)
684 def BuildStepUpdateUserProjects(pepperdir
, toolchains
,
685 build_experimental
, clobber
):
686 buildbot_common
.BuildStep('Update examples and libraries')
689 if not build_experimental
:
690 filters
['EXPERIMENTAL'] = False
692 toolchains
= toolchains
[:]
694 # arm isn't a valid toolchain for build_projects
695 if 'arm' in toolchains
:
696 toolchains
.remove('arm')
698 if 'host' in toolchains
:
699 toolchains
.remove('host')
700 toolchains
.append(getos
.GetPlatform())
702 filters
['TOOLS'] = toolchains
704 # Update examples and libraries
708 'examples/benchmarks',
714 tree
= parse_dsc
.LoadProjectTree(SDK_SRC_DIR
, include
=filters
)
715 build_projects
.UpdateProjects(pepperdir
, tree
, clobber
=clobber
,
716 toolchains
=toolchains
)
719 def BuildStepMakeAll(pepperdir
, directory
, step_name
,
720 deps
=True, clean
=False, config
='Debug', args
=None):
721 buildbot_common
.BuildStep(step_name
)
722 build_projects
.BuildProjectsBranch(pepperdir
, directory
, clean
,
726 def BuildStepBuildLibraries(pepperdir
, directory
):
727 BuildStepMakeAll(pepperdir
, directory
, 'Build Libraries Debug',
728 clean
=True, config
='Debug')
729 BuildStepMakeAll(pepperdir
, directory
, 'Build Libraries Release',
730 clean
=True, config
='Release')
732 # Cleanup .pyc file generated while building libraries. Without
733 # this we would end up shipping the pyc in the SDK tarball.
734 buildbot_common
.RemoveFile(os
.path
.join(pepperdir
, 'tools', '*.pyc'))
737 def GenerateNotice(fileroot
, output_filename
='NOTICE', extra_files
=None):
738 # Look for LICENSE files
739 license_filenames_re
= re
.compile('LICENSE|COPYING|COPYRIGHT')
742 for root
, _
, files
in os
.walk(fileroot
):
743 for filename
in files
:
744 if license_filenames_re
.match(filename
):
745 path
= os
.path
.join(root
, filename
)
746 license_files
.append(path
)
749 license_files
+= [os
.path
.join(fileroot
, f
) for f
in extra_files
]
750 print '\n'.join(license_files
)
752 if not os
.path
.isabs(output_filename
):
753 output_filename
= os
.path
.join(fileroot
, output_filename
)
754 generate_notice
.Generate(output_filename
, fileroot
, license_files
)
757 def BuildStepVerifyFilelist(pepperdir
):
758 buildbot_common
.BuildStep('Verify SDK Files')
759 file_list_path
= os
.path
.join(SCRIPT_DIR
, 'sdk_files.list')
761 print 'SDK directory: %s' % pepperdir
762 verify_filelist
.Verify(file_list_path
, pepperdir
)
764 except verify_filelist
.ParseException
, e
:
765 buildbot_common
.ErrorExit('Parsing sdk_files.list failed:\n\n%s' % e
)
766 except verify_filelist
.VerifyException
, e
:
767 file_list_rel
= os
.path
.relpath(file_list_path
)
768 verify_filelist_py
= os
.path
.splitext(verify_filelist
.__file
__)[0] + '.py'
769 verify_filelist_py
= os
.path
.relpath(verify_filelist_py
)
770 pepperdir_rel
= os
.path
.relpath(pepperdir
)
773 SDK verification failed:
776 Add/remove files from %s to fix.
780 to test.""" % (e
, file_list_rel
, verify_filelist_py
, file_list_rel
,
782 buildbot_common
.ErrorExit(msg
)
785 def BuildStepTarBundle(pepper_ver
, tarfile
):
786 buildbot_common
.BuildStep('Tar Pepper Bundle')
787 buildbot_common
.MakeDir(os
.path
.dirname(tarfile
))
788 buildbot_common
.Run([sys
.executable
, CYGTAR
, '-C', OUT_DIR
, '-cjf', tarfile
,
789 'pepper_' + pepper_ver
], cwd
=NACL_DIR
)
792 def GetManifestBundle(pepper_ver
, chrome_revision
, nacl_revision
, tarfile
,
794 with
open(tarfile
, 'rb') as tarfile_stream
:
795 archive_sha1
, archive_size
= manifest_util
.DownloadAndComputeHash(
798 archive
= manifest_util
.Archive(manifest_util
.GetHostOS())
799 archive
.url
= archive_url
800 archive
.size
= archive_size
801 archive
.checksum
= archive_sha1
803 bundle
= manifest_util
.Bundle('pepper_' + pepper_ver
)
804 bundle
.revision
= int(chrome_revision
)
805 bundle
.repath
= 'pepper_' + pepper_ver
806 bundle
.version
= int(pepper_ver
)
807 bundle
.description
= (
808 'Chrome %s bundle. Chrome revision: %s. NaCl revision: %s' % (
809 pepper_ver
, chrome_revision
, nacl_revision
))
810 bundle
.stability
= 'dev'
811 bundle
.recommended
= 'no'
812 bundle
.archives
= [archive
]
816 def BuildStepArchiveBundle(name
, pepper_ver
, chrome_revision
, nacl_revision
,
818 buildbot_common
.BuildStep('Archive %s' % name
)
819 bucket_path
= 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
820 build_version
.ChromeVersion(),)
821 tarname
= os
.path
.basename(tarfile
)
822 tarfile_dir
= os
.path
.dirname(tarfile
)
823 buildbot_common
.Archive(tarname
, bucket_path
, tarfile_dir
)
825 # generate "manifest snippet" for this archive.
826 archive_url
= GSTORE
+ 'nacl_sdk/%s/%s' % (
827 build_version
.ChromeVersion(), tarname
)
828 bundle
= GetManifestBundle(pepper_ver
, chrome_revision
, nacl_revision
,
829 tarfile
, archive_url
)
831 manifest_snippet_file
= os
.path
.join(OUT_DIR
, tarname
+ '.json')
832 with
open(manifest_snippet_file
, 'wb') as manifest_snippet_stream
:
833 manifest_snippet_stream
.write(bundle
.GetDataAsString())
835 buildbot_common
.Archive(tarname
+ '.json', bucket_path
, OUT_DIR
,
839 def BuildStepArchiveSDKTools():
840 # Only push up sdk_tools.tgz and nacl_sdk.zip on the linux buildbot.
841 builder_name
= os
.getenv('BUILDBOT_BUILDERNAME', '')
842 if builder_name
== 'linux-sdk-multi':
843 buildbot_common
.BuildStep('Build SDK Tools')
844 build_updater
.BuildUpdater(OUT_DIR
)
846 buildbot_common
.BuildStep('Archive SDK Tools')
847 bucket_path
= 'nativeclient-mirror/nacl/nacl_sdk/%s' % (
848 build_version
.ChromeVersion(),)
849 buildbot_common
.Archive('sdk_tools.tgz', bucket_path
, OUT_DIR
,
851 buildbot_common
.Archive('nacl_sdk.zip', bucket_path
, OUT_DIR
,
855 def BuildStepSyncNaClPorts():
856 """Pull the pinned revision of naclports from SVN."""
857 buildbot_common
.BuildStep('Sync naclports')
859 # In case a previous svn checkout exists, remove it.
860 # TODO(sbc): remove this once all the build machines
861 # have removed the old checkout
862 if (os
.path
.exists(NACLPORTS_DIR
) and
863 not os
.path
.exists(os
.path
.join(NACLPORTS_DIR
, '.git'))):
864 buildbot_common
.RemoveDir(NACLPORTS_DIR
)
866 if not os
.path
.exists(NACLPORTS_DIR
):
867 # checkout new copy of naclports
868 cmd
= ['git', 'clone', NACLPORTS_URL
, 'naclports']
869 buildbot_common
.Run(cmd
, cwd
=os
.path
.dirname(NACLPORTS_DIR
))
871 # checkout new copy of naclports
872 buildbot_common
.Run(['git', 'fetch'], cwd
=NACLPORTS_DIR
)
874 # sync to required revision
875 cmd
= ['git', 'checkout', str(NACLPORTS_REV
)]
876 buildbot_common
.Run(cmd
, cwd
=NACLPORTS_DIR
)
879 def BuildStepBuildNaClPorts(pepper_ver
, pepperdir
):
880 """Build selected naclports in all configurations."""
881 # TODO(sbc): currently naclports doesn't know anything about
882 # Debug builds so the Debug subfolders are all empty.
884 env
= dict(os
.environ
)
885 env
['NACL_SDK_ROOT'] = pepperdir
886 env
['PEPPER_DIR'] = os
.path
.basename(pepperdir
) # pepper_NN
887 env
['NACLPORTS_NO_ANNOTATE'] = "1"
888 env
['NACLPORTS_NO_UPLOAD'] = "1"
889 env
['BUILDBOT_GOT_REVISION'] = str(NACLPORTS_REV
)
891 build_script
= 'build_tools/buildbot_sdk_bundle.sh'
892 buildbot_common
.BuildStep('Build naclports')
894 bundle_dir
= os
.path
.join(NACLPORTS_DIR
, 'out', 'sdk_bundle')
895 out_dir
= os
.path
.join(bundle_dir
, 'pepper_%s' % pepper_ver
)
897 # Remove the sdk_bundle directory to remove stale files from previous builds.
898 buildbot_common
.RemoveDir(bundle_dir
)
900 buildbot_common
.Run([build_script
], env
=env
, cwd
=NACLPORTS_DIR
)
902 # Some naclports do not include a standalone LICENSE/COPYING file
903 # so we explicitly list those here for inclusion.
904 extra_licenses
= ('tinyxml/readme.txt',
907 src_root
= os
.path
.join(NACLPORTS_DIR
, 'out', 'build')
908 output_license
= os
.path
.join(out_dir
, 'ports', 'LICENSE')
909 GenerateNotice(src_root
, output_license
, extra_licenses
)
910 readme
= os
.path
.join(out_dir
, 'ports', 'README')
911 oshelpers
.Copy(['-v', os
.path
.join(SDK_SRC_DIR
, 'README.naclports'), readme
])
914 def BuildStepTarNaClPorts(pepper_ver
, tarfile
):
915 """Create tar archive containing headers and libs from naclports build."""
916 buildbot_common
.BuildStep('Tar naclports Bundle')
917 buildbot_common
.MakeDir(os
.path
.dirname(tarfile
))
918 pepper_dir
= 'pepper_%s' % pepper_ver
919 archive_dirs
= [os
.path
.join(pepper_dir
, 'ports')]
921 ports_out
= os
.path
.join(NACLPORTS_DIR
, 'out', 'sdk_bundle')
922 cmd
= [sys
.executable
, CYGTAR
, '-C', ports_out
, '-cjf', tarfile
]
924 buildbot_common
.Run(cmd
, cwd
=NACL_DIR
)
927 def BuildStepBuildAppEngine(pepperdir
, chrome_revision
):
928 """Build the projects found in src/gonacl_appengine/src"""
929 buildbot_common
.BuildStep('Build GoNaCl AppEngine Projects')
930 cmd
= ['make', 'upload', 'REVISION=%s' % chrome_revision
]
931 env
= dict(os
.environ
)
932 env
['NACL_SDK_ROOT'] = pepperdir
933 env
['NACLPORTS_NO_ANNOTATE'] = "1"
934 buildbot_common
.Run(cmd
, env
=env
, cwd
=GONACL_APPENGINE_SRC_DIR
)
938 parser
= argparse
.ArgumentParser(description
=__doc__
)
939 parser
.add_argument('--nacl-tree-path',
940 help='Path to native client tree for bionic build.',
941 dest
='nacl_tree_path')
942 parser
.add_argument('--qemu', help='Add qemu for ARM.',
944 parser
.add_argument('--bionic', help='Add bionic build.',
946 parser
.add_argument('--tar', help='Force the tar step.',
948 parser
.add_argument('--archive', help='Force the archive step.',
950 parser
.add_argument('--release', help='PPAPI release version.',
951 dest
='release', default
=None)
952 parser
.add_argument('--build-ports',
953 help='Build naclport bundle.', action
='store_true')
954 parser
.add_argument('--build-app-engine',
955 help='Build AppEngine demos.', action
='store_true')
956 parser
.add_argument('--experimental',
957 help='build experimental examples and libraries', action
='store_true',
958 dest
='build_experimental')
959 parser
.add_argument('--skip-toolchain', help='Skip toolchain untar',
961 parser
.add_argument('--no-clean', dest
='clean', action
='store_false',
962 help="Don't clean gypbuild directories")
963 parser
.add_argument('--mac-sdk',
964 help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
965 parser
.add_argument('--no-arm-trusted', action
='store_true',
966 help='Disable building of ARM trusted components (sel_ldr, etc).')
968 # To setup bash completion for this command first install optcomplete
969 # and then add this line to your .bashrc:
970 # complete -F _optcomplete build_sdk.py
973 optcomplete
.autocomplete(parser
)
978 options
= parser
.parse_args(args
)
980 if options
.nacl_tree_path
:
981 options
.bionic
= True
982 toolchain_build
= os
.path
.join(options
.nacl_tree_path
, 'toolchain_build')
983 print 'WARNING: Building bionic toolchain from NaCl checkout.'
984 print 'This option builds bionic from the sources currently in the'
985 print 'provided NativeClient checkout, and the results instead of '
986 print 'downloading a toolchain from the builder. This may result in a'
987 print 'NaCl SDK that can not run on ToT chrome.'
988 print 'NOTE: To clobber you will need to run toolchain_build_bionic.py'
989 print 'directly from the NativeClient checkout.'
991 response
= raw_input("Type 'y' and hit enter to continue.\n")
992 if response
!= 'y' and response
!= 'Y':
996 # Get head version of NativeClient tree
997 buildbot_common
.BuildStep('Build bionic toolchain.')
998 buildbot_common
.Run([sys
.executable
, 'toolchain_build_bionic.py', '-f'],
1001 toolchain_build
= None
1003 if buildbot_common
.IsSDKBuilder():
1004 options
.archive
= True
1005 options
.build_ports
= True
1006 # TODO(binji): re-enable app_engine build when the linux builder stops
1007 # breaking when trying to git clone from github.
1008 # See http://crbug.com/412969.
1009 options
.build_app_engine
= False
1012 # NOTE: order matters here. This will be the order that is specified in the
1013 # Makefiles; the first toolchain will be the default.
1014 toolchains
= ['pnacl', 'newlib', 'glibc', 'arm', 'clang-newlib', 'host']
1016 # Changes for experimental bionic builder
1018 toolchains
.append('bionic')
1019 options
.build_ports
= False
1020 options
.build_app_engine
= False
1022 print 'Building: ' + ' '.join(toolchains
)
1024 if options
.archive
and not options
.tar
:
1025 parser
.error('Incompatible arguments with archive.')
1027 chrome_version
= int(build_version
.ChromeMajorVersion())
1028 chrome_revision
= build_version
.ChromeRevision()
1029 nacl_revision
= build_version
.NaClRevision()
1030 pepper_ver
= str(chrome_version
)
1031 pepper_old
= str(chrome_version
- 1)
1032 pepperdir
= os
.path
.join(OUT_DIR
, 'pepper_' + pepper_ver
)
1033 pepperdir_old
= os
.path
.join(OUT_DIR
, 'pepper_' + pepper_old
)
1035 tarname
= 'naclsdk_bionic.tar.bz2'
1037 tarname
= 'naclsdk_' + getos
.GetPlatform() + '.tar.bz2'
1038 tarfile
= os
.path
.join(OUT_DIR
, tarname
)
1041 pepper_ver
= options
.release
1042 print 'Building PEPPER %s at %s' % (pepper_ver
, chrome_revision
)
1044 if 'NACL_SDK_ROOT' in os
.environ
:
1045 # We don't want the currently configured NACL_SDK_ROOT to have any effect
1047 del os
.environ
['NACL_SDK_ROOT']
1049 if not options
.skip_toolchain
:
1050 BuildStepCleanPepperDirs(pepperdir
, pepperdir_old
)
1051 BuildStepMakePepperDirs(pepperdir
, ['include', 'toolchain', 'tools'])
1052 BuildStepDownloadToolchains(toolchains
)
1053 if options
.nacl_tree_path
:
1054 # Instead of untarring, copy the raw bionic toolchain
1055 not_bionic
= [i
for i
in toolchains
if i
!= 'bionic']
1056 BuildStepUntarToolchains(pepperdir
, not_bionic
)
1057 tcname
= GetToolchainDirName('bionic', 'arm')
1058 srcdir
= os
.path
.join(toolchain_build
, 'out', tcname
)
1059 bionicdir
= os
.path
.join(pepperdir
, 'toolchain', tcname
)
1060 oshelpers
.Copy(['-r', srcdir
, bionicdir
])
1062 BuildStepUntarToolchains(pepperdir
, toolchains
)
1064 BuildStepBuildToolchains(pepperdir
, toolchains
,
1065 not options
.skip_toolchain
,
1068 BuildStepUpdateHelpers(pepperdir
, True)
1069 BuildStepUpdateUserProjects(pepperdir
, toolchains
,
1070 options
.build_experimental
, True)
1072 BuildStepCopyTextFiles(pepperdir
, pepper_ver
, chrome_revision
, nacl_revision
)
1074 # Ship with libraries prebuilt, so run that first.
1075 BuildStepBuildLibraries(pepperdir
, 'src')
1076 GenerateNotice(pepperdir
)
1078 # Verify the SDK contains what we expect.
1079 if not options
.bionic
:
1080 BuildStepVerifyFilelist(pepperdir
)
1083 BuildStepTarBundle(pepper_ver
, tarfile
)
1085 if options
.build_ports
and getos
.GetPlatform() == 'linux':
1086 ports_tarfile
= os
.path
.join(OUT_DIR
, 'naclports.tar.bz2')
1087 BuildStepSyncNaClPorts()
1088 BuildStepBuildNaClPorts(pepper_ver
, pepperdir
)
1090 BuildStepTarNaClPorts(pepper_ver
, ports_tarfile
)
1092 if options
.build_app_engine
and getos
.GetPlatform() == 'linux':
1093 BuildStepBuildAppEngine(pepperdir
, chrome_revision
)
1096 qemudir
= os
.path
.join(NACL_DIR
, 'toolchain', 'linux_arm-trusted')
1097 oshelpers
.Copy(['-r', qemudir
, pepperdir
])
1099 # Archive on non-trybots.
1101 BuildStepArchiveBundle('build', pepper_ver
, chrome_revision
, nacl_revision
,
1103 if options
.build_ports
and getos
.GetPlatform() == 'linux':
1104 BuildStepArchiveBundle('naclports', pepper_ver
, chrome_revision
,
1105 nacl_revision
, ports_tarfile
)
1106 BuildStepArchiveSDKTools()
1111 if __name__
== '__main__':
1113 sys
.exit(main(sys
.argv
[1:]))
1114 except KeyboardInterrupt:
1115 buildbot_common
.ErrorExit('build_sdk: interrupted')