2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Script to build binary components of the SDK.
8 This script builds binary components of the Native Client SDK, create tarballs
9 for them, and uploads them to Google Cloud Storage.
11 This prevents a source dependency on the Chromium/NaCl tree in the Native
24 if sys
.version_info
< (2, 7, 0):
25 sys
.stderr
.write("python 2.7 or later is required run this script\n")
28 import buildbot_common
30 from build_paths
import NACL_DIR
, OUT_DIR
, SRC_DIR
, SDK_SRC_DIR
31 from build_paths
import BUILD_ARCHIVE_DIR
33 sys
.path
.append(os
.path
.join(SDK_SRC_DIR
, 'tools'))
38 BUILD_DIR
= os
.path
.join(NACL_DIR
, 'build')
39 NACL_TOOLCHAIN_DIR
= os
.path
.join(NACL_DIR
, 'toolchain')
40 NACL_TOOLCHAINTARS_DIR
= os
.path
.join(NACL_TOOLCHAIN_DIR
, '.tars')
42 CYGTAR
= os
.path
.join(BUILD_DIR
, 'cygtar.py')
43 PKGVER
= os
.path
.join(BUILD_DIR
, 'package_version', 'package_version.py')
44 VERSION_JSON
= os
.path
.join(BUILD_ARCHIVE_DIR
, 'version.json')
47 PLATFORM
= getos
.GetPlatform()
48 TAR
= oshelpers
.FindExeInPath('tar')
53 # Mapping from toolchain name to the equivalent package_version.py directory
55 TOOLCHAIN_PACKAGE_MAP
= {
56 'newlib': 'nacl_x86_newlib',
57 'bionic': 'nacl_arm_bionic',
58 'arm': 'nacl_arm_newlib',
59 'glibc': 'nacl_x86_glibc',
60 'pnacl': 'pnacl_newlib'}
63 def Tar(archive_path
, root
, files
):
64 if os
.path
.exists(TAR
):
67 cmd
= [sys
.executable
, CYGTAR
]
68 cmd
.extend(['-cjf', archive_path
])
70 buildbot_common
.Run(cmd
, cwd
=root
)
73 def ComputeSha(filename
):
74 with
open(filename
) as f
:
75 return hashlib
.sha1(f
.read()).hexdigest()
78 class TempDir(object):
79 def __init__(self
, prefix
=None, dont_remove
=False):
82 self
.dont_remove
= dont_remove
84 self
.destroyed
= False
87 assert not self
.created
88 self
.name
= tempfile
.mkdtemp(prefix
=self
.prefix
)
92 assert not self
.destroyed
93 if not self
.dont_remove
:
94 buildbot_common
.RemoveDir(self
.name
)
100 def __exit__(self
, exc
, value
, tb
):
101 return self
.Destroy()
104 class Archive(object):
105 def __init__(self
, name
):
106 self
.name
= '%s_%s' % (PLATFORM
, name
)
107 self
.archive_name
= self
.name
+ '.tar.bz2'
108 self
.archive_path
= os
.path
.join(BUILD_ARCHIVE_DIR
, self
.archive_name
)
109 self
.dirname
= os
.path
.join(BUILD_ARCHIVE_DIR
, self
.name
)
112 def _MakeDirname(self
):
113 if os
.path
.exists(self
.dirname
):
114 buildbot_common
.RemoveDir(self
.dirname
)
115 buildbot_common
.MakeDir(self
.dirname
)
117 def Copy(self
, src_root
, file_list
):
118 if type(file_list
) is not list:
119 file_list
= [file_list
]
121 for file_spec
in file_list
:
122 # The list of files to install can be a simple list of
123 # strings or a list of pairs, where each pair corresponds
124 # to a mapping from source to destination names.
125 if type(file_spec
) is str:
126 src_file
= dest_file
= file_spec
128 src_file
, dest_file
= file_spec
130 src_file
= os
.path
.join(src_root
, src_file
)
132 # Expand sources files using glob.
133 sources
= glob
.glob(src_file
)
138 if not (dest_file
.endswith('/') or dest_file
== ''):
139 buildbot_common
.ErrorExit(
140 "Target file %r must end in '/' or be empty when "
141 "using globbing to install files %r" % (dest_file
, sources
))
143 for source
in sources
:
144 if dest_file
.endswith('/'):
145 dest
= os
.path
.join(dest_file
, os
.path
.basename(source
))
148 dest
= os
.path
.join(self
.dirname
, dest
)
149 if not os
.path
.isdir(os
.path
.dirname(dest
)):
150 buildbot_common
.MakeDir(os
.path
.dirname(dest
))
151 if os
.path
.isdir(source
):
152 buildbot_common
.CopyDir(source
, dest
)
154 buildbot_common
.CopyFile(source
, dest
)
156 def CreateArchiveShaFile(self
):
157 sha1
= ComputeSha(self
.archive_path
)
158 sha1_filename
= self
.archive_path
+ '.sha1'
159 with
open(sha1_filename
, 'w') as f
:
163 Tar(self
.archive_path
, BUILD_ARCHIVE_DIR
, [
165 os
.path
.basename(VERSION_JSON
)])
166 self
.CreateArchiveShaFile()
167 all_archives
.append(self
.archive_name
)
170 def MakeToolchainArchive(toolchain
):
171 archive
= Archive(toolchain
)
173 build_platform
= '%s_x86' % PLATFORM
175 with
TempDir('tc_%s_' % toolchain
) as tmpdir
:
176 package_name
= os
.path
.join(build_platform
,
177 TOOLCHAIN_PACKAGE_MAP
.get(toolchain
))
179 # Extract all of the packages into the temp directory.
180 buildbot_common
.Run([sys
.executable
, PKGVER
,
181 '--packages', package_name
,
182 '--tar-dir', NACL_TOOLCHAINTARS_DIR
,
183 '--dest-dir', tmpdir
,
186 # Copy all the files we extracted to the correct destination.
187 archive
.Copy(os
.path
.join(tmpdir
, package_name
), ('*', ''))
192 def MakeNinjaRelPath(path
):
193 return os
.path
.join(os
.path
.relpath(OUT_DIR
, SRC_DIR
), path
)
196 def NinjaBuild(targets
, out_dir
):
197 if type(targets
) is not list:
199 out_config_dir
= os
.path
.join(out_dir
, 'Release')
200 buildbot_common
.Run(['ninja', '-C', out_config_dir
] + targets
, cwd
=SRC_DIR
)
203 def GypNinjaBuild(arch
, gyp_py_script
, gyp_file
, targets
, out_dir
):
204 gyp_env
= dict(os
.environ
)
205 gyp_env
['GYP_GENERATORS'] = 'ninja'
206 gyp_defines
= ['nacl_allow_thin_archives=0']
208 gyp_defines
.append('mac_sdk=%s' % options
.mac_sdk
)
210 gyp_defines
.append('target_arch=%s' % arch
)
212 gyp_env
['GYP_CROSSCOMPILE'] = '1'
213 gyp_defines
+= ['arm_float_abi=hard']
214 if options
.no_arm_trusted
:
215 gyp_defines
.append('disable_cross_trusted=1')
216 if PLATFORM
== 'mac':
217 gyp_defines
.append('clang=1')
219 gyp_env
['GYP_DEFINES'] = ' '.join(gyp_defines
)
220 generator_flags
= ['-G', 'output_dir=%s' % out_dir
]
222 cmd
= [sys
.executable
, gyp_py_script
, gyp_file
, depth
] + generator_flags
223 buildbot_common
.Run(cmd
, cwd
=SRC_DIR
, env
=gyp_env
)
224 NinjaBuild(targets
, out_dir
)
229 ('sel_ldr', 'sel_ldr_x86_32'),
230 ('ncval_new', 'ncval'),
231 ('irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'),
232 ('irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'),
235 if PLATFORM
== 'linux':
236 files
.append(['nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_32'])
237 files
.append(['nonsfi_loader_newlib_x32_nonsfi.nexe',
238 'nonsfi_loader_x86_32'])
239 files
.append(['nonsfi_loader_newlib_arm_nonsfi.nexe',
240 'nonsfi_loader_arm'])
242 # Add .exe extensions to all windows tools
244 if PLATFORM
== 'win' and not pair
[0].endswith('.nexe'):
251 def GetTools64Files():
253 if PLATFORM
== 'win':
254 files
.append('sel_ldr64')
256 files
.append(('sel_ldr', 'sel_ldr_x86_64'))
258 if PLATFORM
== 'linux':
259 files
.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_64'))
264 def GetToolsArmFiles():
265 assert PLATFORM
== 'linux'
266 return [('irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'),
267 ('sel_ldr', 'sel_ldr_arm'),
268 ('nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm')]
271 def GetNewlibToolchainLibs():
274 'libminidump_generator.a',
277 'libnacl_exception.a',
278 'libnacl_list_mappings.a',
285 def GetGlibcToolchainLibs():
286 return ['libminidump_generator.a',
287 'libminidump_generator.so',
290 'libnacl_dyncode.so',
291 'libnacl_exception.a',
292 'libnacl_exception.so',
293 'libnacl_list_mappings.a',
294 'libnacl_list_mappings.so',
300 def GetPNaClToolchainLibs():
301 return ['libminidump_generator.a',
304 'libnacl_exception.a',
305 'libnacl_list_mappings.a',
312 def GetBionicToolchainLibs():
313 return ['libminidump_generator.a',
315 'libnacl_exception.a',
316 'libnacl_list_mappings.a',
320 def GetToolchainNaClLib(tcname
, tcpath
, xarch
):
321 if tcname
== 'pnacl':
322 return os
.path
.join(tcpath
, 'le32-nacl', 'lib')
323 elif xarch
== 'x86_32':
324 return os
.path
.join(tcpath
, 'x86_64-nacl', 'lib32')
325 elif xarch
== 'x86_64':
326 return os
.path
.join(tcpath
, 'x86_64-nacl', 'lib')
328 return os
.path
.join(tcpath
, 'arm-nacl', 'lib')
331 def GetGypBuiltLib(root
, tcname
, xarch
=None):
332 if tcname
== 'pnacl':
333 tcname
= 'pnacl_newlib'
334 if xarch
== 'x86_32':
336 elif xarch
== 'x86_64':
340 return os
.path
.join(root
, 'Release', 'gen', 'tc_' + tcname
, 'lib' + xarch
)
343 def GetGypToolchainLib(root
, tcname
, xarch
):
349 tcpath
= os
.path
.join(root
, 'Release', 'gen', 'sdk', '%s_x86' % PLATFORM
,
350 TOOLCHAIN_PACKAGE_MAP
[toolchain
])
351 return GetToolchainNaClLib(tcname
, tcpath
, xarch
)
354 def MakeGypArchives():
356 gyp_chromium
= join(SRC_DIR
, 'build', 'gyp_chromium')
357 # TODO(binji): gyp_nacl doesn't build properly on Windows anymore; it only
358 # can use VS2010, not VS2013 which is now required by the Chromium repo. NaCl
359 # needs to be updated to perform the same logic as Chromium in detecting VS,
360 # which can now exist in the depot_tools directory.
361 # See https://code.google.com/p/nativeclient/issues/detail?id=4022
363 # For now, let's use gyp_chromium to build these components.
364 # gyp_nacl = join(NACL_DIR, 'build', 'gyp_nacl')
365 gyp_nacl
= gyp_chromium
367 nacl_core_sdk_gyp
= join(NACL_DIR
, 'build', 'nacl_core_sdk.gyp')
368 all_gyp
= join(NACL_DIR
, 'build', 'all.gyp')
369 breakpad_gyp
= join(SRC_DIR
, 'breakpad', 'breakpad.gyp')
370 ppapi_gyp
= join(SRC_DIR
, 'ppapi', 'native_client', 'native_client.gyp')
371 breakpad_targets
= ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
374 tmpdir_obj
= TempDir('nacl_core_sdk_', dont_remove
=True).Create()
375 tmpdir
= tmpdir_obj
.name
376 GypNinjaBuild('ia32', gyp_nacl
, nacl_core_sdk_gyp
, 'nacl_core_sdk', tmpdir
)
377 GypNinjaBuild('ia32', gyp_nacl
, all_gyp
, 'ncval_new', tmpdir
)
378 GypNinjaBuild('ia32', gyp_chromium
, breakpad_gyp
, breakpad_targets
, tmpdir
)
379 GypNinjaBuild('ia32', gyp_chromium
, ppapi_gyp
, 'ppapi_lib', tmpdir
)
380 GypNinjaBuild('x64', gyp_chromium
, ppapi_gyp
, 'ppapi_lib', tmpdir
)
382 tmpdir64_obj
= TempDir('nacl_core_sdk_64_', dont_remove
=True).Create()
383 tmpdir_64
= tmpdir64_obj
.name
384 if PLATFORM
== 'win':
385 GypNinjaBuild('ia32', gyp_nacl
, nacl_core_sdk_gyp
, 'sel_ldr64', tmpdir_64
)
387 GypNinjaBuild('x64', gyp_nacl
, nacl_core_sdk_gyp
, 'sel_ldr', tmpdir_64
)
389 tmpdirarm_obj
= TempDir('nacl_core_sdk_arm_', dont_remove
=True).Create()
390 tmpdir_arm
= tmpdirarm_obj
.name
391 GypNinjaBuild('arm', gyp_nacl
, nacl_core_sdk_gyp
, 'nacl_core_sdk', tmpdir_arm
)
392 GypNinjaBuild('arm', gyp_chromium
, ppapi_gyp
, 'ppapi_lib', tmpdir_arm
)
395 archive
= Archive('tools')
396 archive
.Copy(join(tmpdir
, 'Release'), GetToolsFiles())
397 archive
.Copy(join(tmpdir_64
, 'Release'), GetTools64Files())
398 if PLATFORM
== 'linux':
399 archive
.Copy(join(tmpdir_arm
, 'Release'), GetToolsArmFiles())
400 # TODO(binji): dump_syms doesn't currently build on Windows. See
401 # http://crbug.com/245456
402 if PLATFORM
!= 'win':
403 archive
.Copy(join(tmpdir
, 'Release'), breakpad_targets
)
406 # newlib x86 libs archives
407 for arch
in ('x86_32', 'x86_64'):
408 archive
= Archive('newlib_%s_libs' % arch
)
409 archive
.Copy(GetGypBuiltLib(tmpdir
, 'newlib', arch
),
410 GetNewlibToolchainLibs())
411 archive
.Copy(GetGypToolchainLib(tmpdir
, 'newlib', arch
), 'crt1.o')
414 # newlib arm libs archive
415 archive
= Archive('newlib_arm_libs')
416 archive
.Copy(GetGypBuiltLib(tmpdir_arm
, 'newlib', 'arm'),
417 GetNewlibToolchainLibs())
418 archive
.Copy(GetGypToolchainLib(tmpdir_arm
, 'newlib', 'arm'), 'crt1.o')
421 # glibc x86 libs archives
422 for arch
in ('x86_32', 'x86_64'):
423 archive
= Archive('glibc_%s_libs' % arch
)
424 archive
.Copy(GetGypBuiltLib(tmpdir
, 'glibc', arch
), GetGlibcToolchainLibs())
425 archive
.Copy(GetGypToolchainLib(tmpdir
, 'glibc', arch
), 'crt1.o')
429 archive
= Archive('pnacl_libs')
430 archive
.Copy(GetGypBuiltLib(tmpdir
, 'pnacl'), GetPNaClToolchainLibs())
433 if PLATFORM
== 'linux':
434 # bionic arm libs archive (use newlib-built files)
435 archive
= Archive('bionic_arm_libs')
436 archive
.Copy(GetGypBuiltLib(tmpdir_arm
, 'newlib', 'arm'),
437 GetBionicToolchainLibs())
438 archive
.Copy(GetGypToolchainLib(tmpdir_arm
, 'newlib', 'arm'), 'crt1.o')
441 # Destroy the temporary directories
443 tmpdirarm_obj
.Destroy()
444 tmpdir64_obj
.Destroy()
447 def MakePNaClArchives():
449 gyp_chromium
= join(SRC_DIR
, 'build', 'gyp_chromium')
450 pnacl_irt_shim_gyp
= join(SRC_DIR
, 'ppapi', 'native_client', 'src',
451 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
453 with
TempDir('pnacl_irt_shim_ia32_') as tmpdir
:
454 GypNinjaBuild('ia32', gyp_chromium
, pnacl_irt_shim_gyp
, 'aot', tmpdir
)
456 archive
= Archive('pnacl_translator_x86_32_libs')
457 libdir
= join(tmpdir
, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
458 archive
.Copy(libdir
, 'libpnacl_irt_shim.a')
461 archive
= Archive('pnacl_translator_x86_64_libs')
462 libdir
= join(tmpdir
, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
463 archive
.Copy(libdir
, 'libpnacl_irt_shim.a')
466 with
TempDir('pnacl_irt_shim_arm_') as tmpdir
:
467 GypNinjaBuild('arm', gyp_chromium
, pnacl_irt_shim_gyp
, 'aot', tmpdir
)
469 archive
= Archive('pnacl_translator_arm_libs')
470 libdir
= join(tmpdir
, 'Release', 'gen', 'tc_pnacl_translate', 'lib-arm')
471 archive
.Copy(libdir
, 'libpnacl_irt_shim.a')
475 def GetNewlibHeaders():
477 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
478 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
479 ('native_client/src/untrusted/irt/irt.h', ''),
480 ('native_client/src/untrusted/irt/irt_dev.h', ''),
481 ('native_client/src/untrusted/irt/irt_extension.h', ''),
482 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
483 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
484 ('native_client/src/untrusted/pthread/pthread.h', ''),
485 ('native_client/src/untrusted/pthread/semaphore.h', ''),
486 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
487 ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
490 def GetGlibcHeaders():
492 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
493 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
494 ('native_client/src/untrusted/irt/irt.h', ''),
495 ('native_client/src/untrusted/irt/irt_dev.h', ''),
496 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
497 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
498 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
499 ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
502 def GetBionicHeaders():
503 return [('ppapi/nacl_irt/public/irt_ppapi.h', '')]
506 def MakeToolchainHeaderArchives():
507 archive
= Archive('newlib_headers')
508 archive
.Copy(SRC_DIR
, GetNewlibHeaders())
511 archive
= Archive('glibc_headers')
512 archive
.Copy(SRC_DIR
, GetGlibcHeaders())
515 if PLATFORM
== 'linux':
516 archive
= Archive('bionic_headers')
517 archive
.Copy(SRC_DIR
, GetBionicHeaders())
521 def MakePepperArchive():
522 archive
= Archive('ppapi')
523 archive
.Copy(os
.path
.join(SRC_DIR
, 'ppapi'), ['c', 'cpp', 'lib', 'utility'])
527 def UploadArchives():
528 major_version
= build_version
.ChromeMajorVersion()
529 chrome_revision
= build_version
.ChromeRevision()
530 commit_position
= build_version
.ChromeCommitPosition()
531 git_sha
= build_version
.ParseCommitPosition(commit_position
)[0]
532 short_sha
= git_sha
[:9]
533 archive_version
= '%s-%s-%s' % (major_version
, chrome_revision
, short_sha
)
534 bucket_path
= 'native-client-sdk/archives/%s' % archive_version
535 for archive_name
in all_archives
:
536 buildbot_common
.Archive(archive_name
, bucket_path
,
537 cwd
=BUILD_ARCHIVE_DIR
, step_link
=False)
538 sha1_filename
= archive_name
+ '.sha1'
539 buildbot_common
.Archive(sha1_filename
, bucket_path
,
540 cwd
=BUILD_ARCHIVE_DIR
, step_link
=False)
543 def MakeVersionJson():
544 time_format
= '%Y/%m/%d %H:%M:%S'
546 'chrome_version': build_version
.ChromeVersionNoTrunk(),
547 'chrome_revision': build_version
.ChromeRevision(),
548 'chrome_commit_position': build_version
.ChromeCommitPosition(),
549 'nacl_revision': build_version
.NaClRevision(),
550 'build_date': datetime
.datetime
.now().strftime(time_format
)}
552 dirname
= os
.path
.dirname(VERSION_JSON
)
553 if not os
.path
.exists(dirname
):
554 buildbot_common
.MakeDir(dirname
)
555 with
open(VERSION_JSON
, 'w') as outf
:
556 json
.dump(data
, outf
, indent
=2, separators
=(',', ': '))
560 parser
= argparse
.ArgumentParser(description
=__doc__
)
561 parser
.add_argument('--mac-sdk',
562 help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
563 parser
.add_argument('--no-arm-trusted', action
='store_true',
564 help='Disable building of ARM trusted components (sel_ldr, etc).')
565 parser
.add_argument('--upload', action
='store_true',
566 help='Upload tarballs to GCS.')
569 options
= parser
.parse_args(args
)
571 toolchains
= ['pnacl', 'newlib', 'glibc', 'arm']
572 if PLATFORM
== 'linux':
573 toolchains
.append('bionic')
576 for tc
in toolchains
:
577 MakeToolchainArchive(tc
)
580 MakeToolchainHeaderArchives()
587 if __name__
== '__main__':
589 sys
.exit(main(sys
.argv
[1:]))
590 except KeyboardInterrupt:
591 buildbot_common
.ErrorExit('build_artifacts: interrupted')