2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Script to build binary components of the SDK.
8 This script builds binary components of the Native Client SDK, create tarballs
9 for them, and uploads them to Google Cloud Storage.
11 This prevents a source dependency on the Chromium/NaCl tree in the Native
24 if sys
.version_info
< (2, 7, 0):
25 sys
.stderr
.write("python 2.7 or later is required run this script\n")
28 import buildbot_common
30 from build_paths
import NACL_DIR
, OUT_DIR
, SRC_DIR
, SDK_SRC_DIR
31 from build_paths
import BUILD_ARCHIVE_DIR
33 sys
.path
.append(os
.path
.join(SDK_SRC_DIR
, 'tools'))
38 BUILD_DIR
= os
.path
.join(NACL_DIR
, 'build')
39 NACL_TOOLCHAIN_DIR
= os
.path
.join(NACL_DIR
, 'toolchain')
40 NACL_TOOLCHAINTARS_DIR
= os
.path
.join(NACL_TOOLCHAIN_DIR
, '.tars')
42 CYGTAR
= os
.path
.join(BUILD_DIR
, 'cygtar.py')
43 PKGVER
= os
.path
.join(BUILD_DIR
, 'package_version', 'package_version.py')
44 VERSION_JSON
= os
.path
.join(BUILD_ARCHIVE_DIR
, 'version.json')
47 PLATFORM
= getos
.GetPlatform()
48 TAR
= oshelpers
.FindExeInPath('tar')
53 # Mapping from toolchain name to the equivalent package_version.py directory
55 TOOLCHAIN_PACKAGE_MAP
= {
56 'newlib': 'nacl_x86_newlib',
57 'bionic': 'nacl_arm_bionic',
58 'arm': 'nacl_arm_newlib',
59 'glibc': 'nacl_x86_glibc',
60 'pnacl': 'pnacl_newlib'}
63 def Tar(archive_path
, root
, files
):
64 if os
.path
.exists(TAR
):
67 cmd
= [sys
.executable
, CYGTAR
]
68 cmd
.extend(['-cjf', archive_path
])
70 buildbot_common
.Run(cmd
, cwd
=root
)
73 def ComputeSha(filename
):
74 with
open(filename
) as f
:
75 return hashlib
.sha1(f
.read()).hexdigest()
78 class TempDir(object):
79 def __init__(self
, prefix
=None, dont_remove
=False):
82 self
.dont_remove
= dont_remove
84 self
.destroyed
= False
87 assert not self
.created
88 self
.name
= tempfile
.mkdtemp(prefix
=self
.prefix
)
92 assert not self
.destroyed
93 if not self
.dont_remove
:
94 buildbot_common
.RemoveDir(self
.name
)
100 def __exit__(self
, exc
, value
, tb
):
101 return self
.Destroy()
104 class Archive(object):
105 def __init__(self
, name
):
106 self
.name
= '%s_%s' % (PLATFORM
, name
)
107 self
.archive_name
= self
.name
+ '.tar.bz2'
108 self
.archive_path
= os
.path
.join(BUILD_ARCHIVE_DIR
, self
.archive_name
)
109 self
.dirname
= os
.path
.join(BUILD_ARCHIVE_DIR
, self
.name
)
112 def _MakeDirname(self
):
113 if os
.path
.exists(self
.dirname
):
114 buildbot_common
.RemoveDir(self
.dirname
)
115 buildbot_common
.MakeDir(self
.dirname
)
117 def Copy(self
, src_root
, file_list
):
118 if type(file_list
) is not list:
119 file_list
= [file_list
]
121 for file_spec
in file_list
:
122 # The list of files to install can be a simple list of
123 # strings or a list of pairs, where each pair corresponds
124 # to a mapping from source to destination names.
125 if type(file_spec
) is str:
126 src_file
= dest_file
= file_spec
128 src_file
, dest_file
= file_spec
130 src_file
= os
.path
.join(src_root
, src_file
)
132 # Expand sources files using glob.
133 sources
= glob
.glob(src_file
)
138 if not (dest_file
.endswith('/') or dest_file
== ''):
139 buildbot_common
.ErrorExit(
140 "Target file %r must end in '/' or be empty when "
141 "using globbing to install files %r" % (dest_file
, sources
))
143 for source
in sources
:
144 if dest_file
.endswith('/'):
145 dest
= os
.path
.join(dest_file
, os
.path
.basename(source
))
148 dest
= os
.path
.join(self
.dirname
, dest
)
149 if not os
.path
.isdir(os
.path
.dirname(dest
)):
150 buildbot_common
.MakeDir(os
.path
.dirname(dest
))
151 if os
.path
.isdir(source
):
152 buildbot_common
.CopyDir(source
, dest
)
154 buildbot_common
.CopyFile(source
, dest
)
156 def CreateArchiveShaFile(self
):
157 sha1
= ComputeSha(self
.archive_path
)
158 sha1_filename
= self
.archive_path
+ '.sha1'
159 with
open(sha1_filename
, 'w') as f
:
163 Tar(self
.archive_path
, BUILD_ARCHIVE_DIR
, [
165 os
.path
.basename(VERSION_JSON
)])
166 self
.CreateArchiveShaFile()
167 all_archives
.append(self
.archive_name
)
170 def MakeToolchainArchive(toolchain
):
171 archive
= Archive(toolchain
)
173 build_platform
= '%s_x86' % PLATFORM
175 with
TempDir('tc_%s_' % toolchain
) as tmpdir
:
176 package_name
= os
.path
.join(build_platform
,
177 TOOLCHAIN_PACKAGE_MAP
.get(toolchain
))
179 # Extract all of the packages into the temp directory.
180 buildbot_common
.Run([sys
.executable
, PKGVER
,
181 '--packages', package_name
,
182 '--tar-dir', NACL_TOOLCHAINTARS_DIR
,
183 '--dest-dir', tmpdir
,
186 # Copy all the files we extracted to the correct destination.
187 archive
.Copy(os
.path
.join(tmpdir
, package_name
), ('*', ''))
192 def MakeNinjaRelPath(path
):
193 return os
.path
.join(os
.path
.relpath(OUT_DIR
, SRC_DIR
), path
)
196 def NinjaBuild(targets
, out_dir
):
197 if type(targets
) is not list:
199 out_config_dir
= os
.path
.join(out_dir
, 'Release')
200 buildbot_common
.Run(['ninja', '-C', out_config_dir
] + targets
, cwd
=SRC_DIR
)
203 def GypNinjaBuild(arch
, gyp_py_script
, gyp_file
, targets
, out_dir
):
204 gyp_env
= dict(os
.environ
)
205 gyp_env
['GYP_GENERATORS'] = 'ninja'
206 gyp_defines
= ['nacl_allow_thin_archives=0']
208 gyp_defines
.append('mac_sdk=%s' % options
.mac_sdk
)
210 gyp_defines
.append('target_arch=%s' % arch
)
212 gyp_env
['GYP_CROSSCOMPILE'] = '1'
213 if options
.no_arm_trusted
:
214 gyp_defines
.append('disable_cross_trusted=1')
215 if PLATFORM
== 'mac':
216 gyp_defines
.append('clang=1')
218 gyp_env
['GYP_DEFINES'] = ' '.join(gyp_defines
)
219 generator_flags
= ['-G', 'output_dir=%s' % out_dir
]
221 cmd
= [sys
.executable
, gyp_py_script
, gyp_file
, depth
] + generator_flags
222 buildbot_common
.Run(cmd
, cwd
=SRC_DIR
, env
=gyp_env
)
223 NinjaBuild(targets
, out_dir
)
228 ('sel_ldr', 'sel_ldr_x86_32'),
229 ('ncval_new', 'ncval'),
230 ('irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'),
231 ('irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'),
234 if PLATFORM
== 'linux':
235 files
.append(['nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_32'])
236 files
.append(['nonsfi_loader_newlib_x32_nonsfi.nexe',
237 'nonsfi_loader_x86_32'])
238 files
.append(['nonsfi_loader_newlib_arm_nonsfi.nexe',
239 'nonsfi_loader_arm'])
241 # Add .exe extensions to all windows tools
243 if PLATFORM
== 'win' and not pair
[0].endswith('.nexe'):
250 def GetTools64Files():
252 if PLATFORM
== 'win':
253 files
.append('sel_ldr64')
255 files
.append(('sel_ldr', 'sel_ldr_x86_64'))
257 if PLATFORM
== 'linux':
258 files
.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_64'))
263 def GetToolsArmFiles():
264 assert PLATFORM
== 'linux'
265 return [('irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'),
266 ('sel_ldr', 'sel_ldr_arm'),
267 ('nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm')]
270 def GetNewlibToolchainLibs():
273 'libminidump_generator.a',
276 'libnacl_exception.a',
277 'libnacl_list_mappings.a',
284 def GetGlibcToolchainLibs():
285 return ['libminidump_generator.a',
286 'libminidump_generator.so',
289 'libnacl_dyncode.so',
290 'libnacl_exception.a',
291 'libnacl_exception.so',
292 'libnacl_list_mappings.a',
293 'libnacl_list_mappings.so',
299 def GetPNaClToolchainLibs():
300 return ['libminidump_generator.a',
303 'libnacl_exception.a',
304 'libnacl_list_mappings.a',
311 def GetBionicToolchainLibs():
312 return ['libminidump_generator.a',
314 'libnacl_exception.a',
315 'libnacl_list_mappings.a',
319 def GetToolchainNaClLib(tcname
, tcpath
, xarch
):
320 if tcname
== 'pnacl':
321 return os
.path
.join(tcpath
, 'le32-nacl', 'lib')
322 elif xarch
== 'x86_32':
323 return os
.path
.join(tcpath
, 'x86_64-nacl', 'lib32')
324 elif xarch
== 'x86_64':
325 return os
.path
.join(tcpath
, 'x86_64-nacl', 'lib')
327 return os
.path
.join(tcpath
, 'arm-nacl', 'lib')
330 def GetGypBuiltLib(root
, tcname
, xarch
=None):
331 if tcname
== 'pnacl':
332 tcname
= 'pnacl_newlib'
333 if xarch
== 'x86_32':
335 elif xarch
== 'x86_64':
339 return os
.path
.join(root
, 'Release', 'gen', 'tc_' + tcname
, 'lib' + xarch
)
342 def GetGypToolchainLib(root
, tcname
, xarch
):
348 tcpath
= os
.path
.join(root
, 'Release', 'gen', 'sdk', '%s_x86' % PLATFORM
,
349 TOOLCHAIN_PACKAGE_MAP
[toolchain
])
350 return GetToolchainNaClLib(tcname
, tcpath
, xarch
)
353 def MakeGypArchives():
355 gyp_chromium
= join(SRC_DIR
, 'build', 'gyp_chromium')
356 # TODO(binji): gyp_nacl doesn't build properly on Windows anymore; it only
357 # can use VS2010, not VS2013 which is now required by the Chromium repo. NaCl
358 # needs to be updated to perform the same logic as Chromium in detecting VS,
359 # which can now exist in the depot_tools directory.
360 # See https://code.google.com/p/nativeclient/issues/detail?id=4022
362 # For now, let's use gyp_chromium to build these components.
363 # gyp_nacl = join(NACL_DIR, 'build', 'gyp_nacl')
364 gyp_nacl
= gyp_chromium
366 nacl_core_sdk_gyp
= join(NACL_DIR
, 'build', 'nacl_core_sdk.gyp')
367 all_gyp
= join(NACL_DIR
, 'build', 'all.gyp')
368 breakpad_gyp
= join(SRC_DIR
, 'breakpad', 'breakpad.gyp')
369 ppapi_gyp
= join(SRC_DIR
, 'ppapi', 'native_client', 'native_client.gyp')
370 breakpad_targets
= ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
373 tmpdir_obj
= TempDir('nacl_core_sdk_', dont_remove
=True).Create()
374 tmpdir
= tmpdir_obj
.name
375 GypNinjaBuild('ia32', gyp_nacl
, nacl_core_sdk_gyp
, 'nacl_core_sdk', tmpdir
)
376 GypNinjaBuild('ia32', gyp_nacl
, all_gyp
, 'ncval_new', tmpdir
)
377 GypNinjaBuild('ia32', gyp_chromium
, breakpad_gyp
, breakpad_targets
, tmpdir
)
378 GypNinjaBuild('ia32', gyp_chromium
, ppapi_gyp
, 'ppapi_lib', tmpdir
)
379 GypNinjaBuild('x64', gyp_chromium
, ppapi_gyp
, 'ppapi_lib', tmpdir
)
381 tmpdir64_obj
= TempDir('nacl_core_sdk_64_', dont_remove
=True).Create()
382 tmpdir_64
= tmpdir64_obj
.name
383 if PLATFORM
== 'win':
384 GypNinjaBuild('ia32', gyp_nacl
, nacl_core_sdk_gyp
, 'sel_ldr64', tmpdir_64
)
386 GypNinjaBuild('x64', gyp_nacl
, nacl_core_sdk_gyp
, 'sel_ldr', tmpdir_64
)
388 tmpdirarm_obj
= TempDir('nacl_core_sdk_arm_', dont_remove
=True).Create()
389 tmpdir_arm
= tmpdirarm_obj
.name
390 GypNinjaBuild('arm', gyp_nacl
, nacl_core_sdk_gyp
, 'nacl_core_sdk', tmpdir_arm
)
391 GypNinjaBuild('arm', gyp_chromium
, ppapi_gyp
, 'ppapi_lib', tmpdir_arm
)
394 archive
= Archive('tools')
395 archive
.Copy(join(tmpdir
, 'Release'), GetToolsFiles())
396 archive
.Copy(join(tmpdir_64
, 'Release'), GetTools64Files())
397 if PLATFORM
== 'linux':
398 archive
.Copy(join(tmpdir_arm
, 'Release'), GetToolsArmFiles())
399 # TODO(binji): dump_syms doesn't currently build on Windows. See
400 # http://crbug.com/245456
401 if PLATFORM
!= 'win':
402 archive
.Copy(join(tmpdir
, 'Release'), breakpad_targets
)
405 # newlib x86 libs archives
406 for arch
in ('x86_32', 'x86_64'):
407 archive
= Archive('newlib_%s_libs' % arch
)
408 archive
.Copy(GetGypBuiltLib(tmpdir
, 'newlib', arch
),
409 GetNewlibToolchainLibs())
410 archive
.Copy(GetGypToolchainLib(tmpdir
, 'newlib', arch
), 'crt1.o')
413 # newlib arm libs archive
414 archive
= Archive('newlib_arm_libs')
415 archive
.Copy(GetGypBuiltLib(tmpdir_arm
, 'newlib', 'arm'),
416 GetNewlibToolchainLibs())
417 archive
.Copy(GetGypToolchainLib(tmpdir_arm
, 'newlib', 'arm'), 'crt1.o')
420 # glibc x86 libs archives
421 for arch
in ('x86_32', 'x86_64'):
422 archive
= Archive('glibc_%s_libs' % arch
)
423 archive
.Copy(GetGypBuiltLib(tmpdir
, 'glibc', arch
), GetGlibcToolchainLibs())
424 archive
.Copy(GetGypToolchainLib(tmpdir
, 'glibc', arch
), 'crt1.o')
428 archive
= Archive('pnacl_libs')
429 archive
.Copy(GetGypBuiltLib(tmpdir
, 'pnacl'), GetPNaClToolchainLibs())
432 if PLATFORM
== 'linux':
433 # bionic arm libs archive (use newlib-built files)
434 archive
= Archive('bionic_arm_libs')
435 archive
.Copy(GetGypBuiltLib(tmpdir_arm
, 'newlib', 'arm'),
436 GetBionicToolchainLibs())
437 archive
.Copy(GetGypToolchainLib(tmpdir_arm
, 'newlib', 'arm'), 'crt1.o')
440 # Destroy the temporary directories
442 tmpdirarm_obj
.Destroy()
443 tmpdir64_obj
.Destroy()
446 def MakePNaClArchives():
448 gyp_chromium
= join(SRC_DIR
, 'build', 'gyp_chromium')
449 pnacl_irt_shim_gyp
= join(SRC_DIR
, 'ppapi', 'native_client', 'src',
450 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
452 with
TempDir('pnacl_irt_shim_ia32_') as tmpdir
:
453 GypNinjaBuild('ia32', gyp_chromium
, pnacl_irt_shim_gyp
, 'aot', tmpdir
)
455 archive
= Archive('pnacl_translator_x86_32_libs')
456 libdir
= join(tmpdir
, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
457 archive
.Copy(libdir
, 'libpnacl_irt_shim.a')
460 archive
= Archive('pnacl_translator_x86_64_libs')
461 libdir
= join(tmpdir
, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
462 archive
.Copy(libdir
, 'libpnacl_irt_shim.a')
465 with
TempDir('pnacl_irt_shim_arm_') as tmpdir
:
466 GypNinjaBuild('arm', gyp_chromium
, pnacl_irt_shim_gyp
, 'aot', tmpdir
)
468 archive
= Archive('pnacl_translator_arm_libs')
469 libdir
= join(tmpdir
, 'Release', 'gen', 'tc_pnacl_translate', 'lib-arm')
470 archive
.Copy(libdir
, 'libpnacl_irt_shim.a')
474 def GetNewlibHeaders():
476 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
477 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
478 ('native_client/src/untrusted/irt/irt.h', ''),
479 ('native_client/src/untrusted/irt/irt_dev.h', ''),
480 ('native_client/src/untrusted/irt/irt_extension.h', ''),
481 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
482 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
483 ('native_client/src/untrusted/pthread/pthread.h', ''),
484 ('native_client/src/untrusted/pthread/semaphore.h', ''),
485 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
486 ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
489 def GetGlibcHeaders():
491 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
492 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
493 ('native_client/src/untrusted/irt/irt.h', ''),
494 ('native_client/src/untrusted/irt/irt_dev.h', ''),
495 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
496 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
497 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
498 ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
501 def GetBionicHeaders():
502 return [('ppapi/nacl_irt/public/irt_ppapi.h', '')]
505 def MakeToolchainHeaderArchives():
506 archive
= Archive('newlib_headers')
507 archive
.Copy(SRC_DIR
, GetNewlibHeaders())
510 archive
= Archive('glibc_headers')
511 archive
.Copy(SRC_DIR
, GetGlibcHeaders())
514 if PLATFORM
== 'linux':
515 archive
= Archive('bionic_headers')
516 archive
.Copy(SRC_DIR
, GetBionicHeaders())
520 def MakePepperArchive():
521 archive
= Archive('ppapi')
522 archive
.Copy(os
.path
.join(SRC_DIR
, 'ppapi'), ['c', 'cpp', 'lib', 'utility'])
526 def UploadArchives():
527 major_version
= build_version
.ChromeMajorVersion()
528 chrome_revision
= build_version
.ChromeRevision()
529 commit_position
= build_version
.ChromeCommitPosition()
530 git_sha
= build_version
.ParseCommitPosition(commit_position
)[0]
531 short_sha
= git_sha
[:9]
532 archive_version
= '%s-%s-%s' % (major_version
, chrome_revision
, short_sha
)
533 bucket_path
= 'native-client-sdk/archives/%s' % archive_version
534 for archive_name
in all_archives
:
535 buildbot_common
.Archive(archive_name
, bucket_path
,
536 cwd
=BUILD_ARCHIVE_DIR
, step_link
=False)
537 sha1_filename
= archive_name
+ '.sha1'
538 buildbot_common
.Archive(sha1_filename
, bucket_path
,
539 cwd
=BUILD_ARCHIVE_DIR
, step_link
=False)
542 def MakeVersionJson():
543 time_format
= '%Y/%m/%d %H:%M:%S'
545 'chrome_version': build_version
.ChromeVersionNoTrunk(),
546 'chrome_revision': build_version
.ChromeRevision(),
547 'chrome_commit_position': build_version
.ChromeCommitPosition(),
548 'nacl_revision': build_version
.NaClRevision(),
549 'build_date': datetime
.datetime
.now().strftime(time_format
)}
551 dirname
= os
.path
.dirname(VERSION_JSON
)
552 if not os
.path
.exists(dirname
):
553 buildbot_common
.MakeDir(dirname
)
554 with
open(VERSION_JSON
, 'w') as outf
:
555 json
.dump(data
, outf
, indent
=2, separators
=(',', ': '))
559 parser
= argparse
.ArgumentParser(description
=__doc__
)
560 parser
.add_argument('--mac-sdk',
561 help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
562 parser
.add_argument('--no-arm-trusted', action
='store_true',
563 help='Disable building of ARM trusted components (sel_ldr, etc).')
564 parser
.add_argument('--upload', action
='store_true',
565 help='Upload tarballs to GCS.')
568 options
= parser
.parse_args(args
)
570 toolchains
= ['pnacl', 'newlib', 'glibc', 'arm']
571 if PLATFORM
== 'linux':
572 toolchains
.append('bionic')
575 for tc
in toolchains
:
576 MakeToolchainArchive(tc
)
579 MakeToolchainHeaderArchives()
586 if __name__
== '__main__':
588 sys
.exit(main(sys
.argv
[1:]))
589 except KeyboardInterrupt:
590 buildbot_common
.ErrorExit('build_artifacts: interrupted')