Send a crash report when a hung process is detected.
[chromium-blink-merge.git] / native_client_sdk / src / build_tools / build_artifacts.py
blobc300bcc24bcb0996f8fa663280b9012da42b1d2c
1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Script to build binary components of the SDK.
8 This script builds binary components of the Native Client SDK, create tarballs
9 for them, and uploads them to Google Cloud Storage.
11 This prevents a source dependency on the Chromium/NaCl tree in the Native
12 Client SDK repo.
13 """
15 import argparse
16 import datetime
17 import glob
18 import hashlib
19 import json
20 import os
21 import sys
22 import tempfile
24 if sys.version_info < (2, 7, 0):
25 sys.stderr.write("python 2.7 or later is required run this script\n")
26 sys.exit(1)
28 import buildbot_common
29 import build_version
30 from build_paths import NACL_DIR, OUT_DIR, SRC_DIR, SDK_SRC_DIR
31 from build_paths import BUILD_ARCHIVE_DIR
33 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
35 import getos
36 import oshelpers
38 BUILD_DIR = os.path.join(NACL_DIR, 'build')
39 NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
40 NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
42 CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
43 PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
44 VERSION_JSON = os.path.join(BUILD_ARCHIVE_DIR, 'version.json')
47 PLATFORM = getos.GetPlatform()
48 TAR = oshelpers.FindExeInPath('tar')
49 options = None
50 all_archives = []
53 # Mapping from toolchain name to the equivalent package_version.py directory
54 # name.
55 TOOLCHAIN_PACKAGE_MAP = {
56 'newlib': 'nacl_x86_newlib',
57 'bionic': 'nacl_arm_bionic',
58 'arm': 'nacl_arm_newlib',
59 'glibc': 'nacl_x86_glibc',
60 'pnacl': 'pnacl_newlib'}
63 def Tar(archive_path, root, files):
64 if os.path.exists(TAR):
65 cmd = [TAR]
66 else:
67 cmd = [sys.executable, CYGTAR]
68 cmd.extend(['-cjf', archive_path])
69 cmd.extend(files)
70 buildbot_common.Run(cmd, cwd=root)
73 def ComputeSha(filename):
74 with open(filename) as f:
75 return hashlib.sha1(f.read()).hexdigest()
78 class TempDir(object):
79 def __init__(self, prefix=None, dont_remove=False):
80 self.prefix = prefix
81 self.name = None
82 self.dont_remove = dont_remove
83 self.created = False
84 self.destroyed = False
86 def Create(self):
87 assert not self.created
88 self.name = tempfile.mkdtemp(prefix=self.prefix)
89 return self
91 def Destroy(self):
92 assert not self.destroyed
93 if not self.dont_remove:
94 buildbot_common.RemoveDir(self.name)
96 def __enter__(self):
97 self.Create()
98 return self.name
100 def __exit__(self, exc, value, tb):
101 return self.Destroy()
104 class Archive(object):
105 def __init__(self, name):
106 self.name = '%s_%s' % (PLATFORM, name)
107 self.archive_name = self.name + '.tar.bz2'
108 self.archive_path = os.path.join(BUILD_ARCHIVE_DIR, self.archive_name)
109 self.dirname = os.path.join(BUILD_ARCHIVE_DIR, self.name)
110 self._MakeDirname()
112 def _MakeDirname(self):
113 if os.path.exists(self.dirname):
114 buildbot_common.RemoveDir(self.dirname)
115 buildbot_common.MakeDir(self.dirname)
117 def Copy(self, src_root, file_list):
118 if type(file_list) is not list:
119 file_list = [file_list]
121 for file_spec in file_list:
122 # The list of files to install can be a simple list of
123 # strings or a list of pairs, where each pair corresponds
124 # to a mapping from source to destination names.
125 if type(file_spec) is str:
126 src_file = dest_file = file_spec
127 else:
128 src_file, dest_file = file_spec
130 src_file = os.path.join(src_root, src_file)
132 # Expand sources files using glob.
133 sources = glob.glob(src_file)
134 if not sources:
135 sources = [src_file]
137 if len(sources) > 1:
138 if not (dest_file.endswith('/') or dest_file == ''):
139 buildbot_common.ErrorExit(
140 "Target file %r must end in '/' or be empty when "
141 "using globbing to install files %r" % (dest_file, sources))
143 for source in sources:
144 if dest_file.endswith('/'):
145 dest = os.path.join(dest_file, os.path.basename(source))
146 else:
147 dest = dest_file
148 dest = os.path.join(self.dirname, dest)
149 if not os.path.isdir(os.path.dirname(dest)):
150 buildbot_common.MakeDir(os.path.dirname(dest))
151 if os.path.isdir(source):
152 buildbot_common.CopyDir(source, dest)
153 else:
154 buildbot_common.CopyFile(source, dest)
156 def CreateArchiveShaFile(self):
157 sha1 = ComputeSha(self.archive_path)
158 sha1_filename = self.archive_path + '.sha1'
159 with open(sha1_filename, 'w') as f:
160 f.write(sha1)
162 def Tar(self):
163 Tar(self.archive_path, BUILD_ARCHIVE_DIR, [
164 self.name,
165 os.path.basename(VERSION_JSON)])
166 self.CreateArchiveShaFile()
167 all_archives.append(self.archive_name)
170 def MakeToolchainArchive(toolchain):
171 archive = Archive(toolchain)
173 build_platform = '%s_x86' % PLATFORM
175 with TempDir('tc_%s_' % toolchain) as tmpdir:
176 package_name = os.path.join(build_platform,
177 TOOLCHAIN_PACKAGE_MAP.get(toolchain))
179 # Extract all of the packages into the temp directory.
180 buildbot_common.Run([sys.executable, PKGVER,
181 '--packages', package_name,
182 '--tar-dir', NACL_TOOLCHAINTARS_DIR,
183 '--dest-dir', tmpdir,
184 'extract'])
186 # Copy all the files we extracted to the correct destination.
187 archive.Copy(os.path.join(tmpdir, package_name), ('*', ''))
189 archive.Tar()
192 def MakeNinjaRelPath(path):
193 return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
196 def NinjaBuild(targets, out_dir):
197 if type(targets) is not list:
198 targets = [targets]
199 out_config_dir = os.path.join(out_dir, 'Release')
200 buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
203 def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, out_dir):
204 gyp_env = dict(os.environ)
205 gyp_env['GYP_GENERATORS'] = 'ninja'
206 gyp_defines = ['nacl_allow_thin_archives=0']
207 if options.mac_sdk:
208 gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
209 if arch:
210 gyp_defines.append('target_arch=%s' % arch)
211 if arch == 'arm':
212 gyp_env['GYP_CROSSCOMPILE'] = '1'
213 gyp_defines += ['arm_float_abi=hard']
214 if options.no_arm_trusted:
215 gyp_defines.append('disable_cross_trusted=1')
216 if PLATFORM == 'mac':
217 gyp_defines.append('clang=1')
219 gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
220 generator_flags = ['-G', 'output_dir=%s' % out_dir]
221 depth = '--depth=.'
222 cmd = [sys.executable, gyp_py_script, gyp_file, depth] + generator_flags
223 buildbot_common.Run(cmd, cwd=SRC_DIR, env=gyp_env)
224 NinjaBuild(targets, out_dir)
227 def GetToolsFiles():
228 files = [
229 ('sel_ldr', 'sel_ldr_x86_32'),
230 ('ncval_new', 'ncval'),
231 ('irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'),
232 ('irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'),
235 if PLATFORM == 'linux':
236 files.append(['nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_32'])
237 files.append(['nonsfi_loader_newlib_x32_nonsfi.nexe',
238 'nonsfi_loader_x86_32'])
239 files.append(['nonsfi_loader_newlib_arm_nonsfi.nexe',
240 'nonsfi_loader_arm'])
242 # Add .exe extensions to all windows tools
243 for pair in files:
244 if PLATFORM == 'win' and not pair[0].endswith('.nexe'):
245 pair[0] += '.exe'
246 pair[1] += '.exe'
248 return files
251 def GetTools64Files():
252 files = []
253 if PLATFORM == 'win':
254 files.append('sel_ldr64')
255 else:
256 files.append(('sel_ldr', 'sel_ldr_x86_64'))
258 if PLATFORM == 'linux':
259 files.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_64'))
261 return files
264 def GetToolsArmFiles():
265 assert PLATFORM == 'linux'
266 return [('irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'),
267 ('sel_ldr', 'sel_ldr_arm'),
268 ('nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm')]
271 def GetNewlibToolchainLibs():
272 return ['crti.o',
273 'crtn.o',
274 'libminidump_generator.a',
275 'libnacl.a',
276 'libnacl_dyncode.a',
277 'libnacl_exception.a',
278 'libnacl_list_mappings.a',
279 'libnosys.a',
280 'libppapi.a',
281 'libppapi_stub.a',
282 'libpthread.a']
285 def GetGlibcToolchainLibs():
286 return ['libminidump_generator.a',
287 'libminidump_generator.so',
288 'libnacl.a',
289 'libnacl_dyncode.a',
290 'libnacl_dyncode.so',
291 'libnacl_exception.a',
292 'libnacl_exception.so',
293 'libnacl_list_mappings.a',
294 'libnacl_list_mappings.so',
295 'libppapi.a',
296 'libppapi.so',
297 'libppapi_stub.a']
300 def GetPNaClToolchainLibs():
301 return ['libminidump_generator.a',
302 'libnacl.a',
303 'libnacl_dyncode.a',
304 'libnacl_exception.a',
305 'libnacl_list_mappings.a',
306 'libnosys.a',
307 'libppapi.a',
308 'libppapi_stub.a',
309 'libpthread.a']
312 def GetBionicToolchainLibs():
313 return ['libminidump_generator.a',
314 'libnacl_dyncode.a',
315 'libnacl_exception.a',
316 'libnacl_list_mappings.a',
317 'libppapi.a']
320 def GetToolchainNaClLib(tcname, tcpath, xarch):
321 if tcname == 'pnacl':
322 return os.path.join(tcpath, 'le32-nacl', 'lib')
323 elif xarch == 'x86_32':
324 return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
325 elif xarch == 'x86_64':
326 return os.path.join(tcpath, 'x86_64-nacl', 'lib')
327 elif xarch == 'arm':
328 return os.path.join(tcpath, 'arm-nacl', 'lib')
331 def GetGypBuiltLib(root, tcname, xarch=None):
332 if tcname == 'pnacl':
333 tcname = 'pnacl_newlib'
334 if xarch == 'x86_32':
335 xarch = '32'
336 elif xarch == 'x86_64':
337 xarch = '64'
338 elif not xarch:
339 xarch = ''
340 return os.path.join(root, 'Release', 'gen', 'tc_' + tcname, 'lib' + xarch)
343 def GetGypToolchainLib(root, tcname, xarch):
344 if xarch == 'arm':
345 toolchain = xarch
346 else:
347 toolchain = tcname
349 tcpath = os.path.join(root, 'Release', 'gen', 'sdk', '%s_x86' % PLATFORM,
350 TOOLCHAIN_PACKAGE_MAP[toolchain])
351 return GetToolchainNaClLib(tcname, tcpath, xarch)
354 def MakeGypArchives():
355 join = os.path.join
356 gyp_chromium = join(SRC_DIR, 'build', 'gyp_chromium')
357 # TODO(binji): gyp_nacl doesn't build properly on Windows anymore; it only
358 # can use VS2010, not VS2013 which is now required by the Chromium repo. NaCl
359 # needs to be updated to perform the same logic as Chromium in detecting VS,
360 # which can now exist in the depot_tools directory.
361 # See https://code.google.com/p/nativeclient/issues/detail?id=4022
363 # For now, let's use gyp_chromium to build these components.
364 # gyp_nacl = join(NACL_DIR, 'build', 'gyp_nacl')
365 gyp_nacl = gyp_chromium
367 nacl_core_sdk_gyp = join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
368 all_gyp = join(NACL_DIR, 'build', 'all.gyp')
369 breakpad_gyp = join(SRC_DIR, 'breakpad', 'breakpad.gyp')
370 ppapi_gyp = join(SRC_DIR, 'ppapi', 'native_client', 'native_client.gyp')
371 breakpad_targets = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
373 # Build
374 tmpdir_obj = TempDir('nacl_core_sdk_', dont_remove=True).Create()
375 tmpdir = tmpdir_obj.name
376 GypNinjaBuild('ia32', gyp_nacl, nacl_core_sdk_gyp, 'nacl_core_sdk', tmpdir)
377 GypNinjaBuild('ia32', gyp_nacl, all_gyp, 'ncval_new', tmpdir)
378 GypNinjaBuild('ia32', gyp_chromium, breakpad_gyp, breakpad_targets, tmpdir)
379 GypNinjaBuild('ia32', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir)
380 GypNinjaBuild('x64', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir)
382 tmpdir64_obj = TempDir('nacl_core_sdk_64_', dont_remove=True).Create()
383 tmpdir_64 = tmpdir64_obj.name
384 if PLATFORM == 'win':
385 GypNinjaBuild('ia32', gyp_nacl, nacl_core_sdk_gyp, 'sel_ldr64', tmpdir_64)
386 else:
387 GypNinjaBuild('x64', gyp_nacl, nacl_core_sdk_gyp, 'sel_ldr', tmpdir_64)
389 tmpdirarm_obj = TempDir('nacl_core_sdk_arm_', dont_remove=True).Create()
390 tmpdir_arm = tmpdirarm_obj.name
391 GypNinjaBuild('arm', gyp_nacl, nacl_core_sdk_gyp, 'nacl_core_sdk', tmpdir_arm)
392 GypNinjaBuild('arm', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir_arm)
394 # Tools archive
395 archive = Archive('tools')
396 archive.Copy(join(tmpdir, 'Release'), GetToolsFiles())
397 archive.Copy(join(tmpdir_64, 'Release'), GetTools64Files())
398 if PLATFORM == 'linux':
399 archive.Copy(join(tmpdir_arm, 'Release'), GetToolsArmFiles())
400 # TODO(binji): dump_syms doesn't currently build on Windows. See
401 # http://crbug.com/245456
402 if PLATFORM != 'win':
403 archive.Copy(join(tmpdir, 'Release'), breakpad_targets)
404 archive.Tar()
406 # newlib x86 libs archives
407 for arch in ('x86_32', 'x86_64'):
408 archive = Archive('newlib_%s_libs' % arch)
409 archive.Copy(GetGypBuiltLib(tmpdir, 'newlib', arch),
410 GetNewlibToolchainLibs())
411 archive.Copy(GetGypToolchainLib(tmpdir, 'newlib', arch), 'crt1.o')
412 archive.Tar()
414 # newlib arm libs archive
415 archive = Archive('newlib_arm_libs')
416 archive.Copy(GetGypBuiltLib(tmpdir_arm, 'newlib', 'arm'),
417 GetNewlibToolchainLibs())
418 archive.Copy(GetGypToolchainLib(tmpdir_arm, 'newlib', 'arm'), 'crt1.o')
419 archive.Tar()
421 # glibc x86 libs archives
422 for arch in ('x86_32', 'x86_64'):
423 archive = Archive('glibc_%s_libs' % arch)
424 archive.Copy(GetGypBuiltLib(tmpdir, 'glibc', arch), GetGlibcToolchainLibs())
425 archive.Copy(GetGypToolchainLib(tmpdir, 'glibc', arch), 'crt1.o')
426 archive.Tar()
428 # pnacl libs archive
429 archive = Archive('pnacl_libs')
430 archive.Copy(GetGypBuiltLib(tmpdir, 'pnacl'), GetPNaClToolchainLibs())
431 archive.Tar()
433 if PLATFORM == 'linux':
434 # bionic arm libs archive (use newlib-built files)
435 archive = Archive('bionic_arm_libs')
436 archive.Copy(GetGypBuiltLib(tmpdir_arm, 'newlib', 'arm'),
437 GetBionicToolchainLibs())
438 archive.Copy(GetGypToolchainLib(tmpdir_arm, 'newlib', 'arm'), 'crt1.o')
439 archive.Tar()
441 # Destroy the temporary directories
442 tmpdir_obj.Destroy()
443 tmpdirarm_obj.Destroy()
444 tmpdir64_obj.Destroy()
447 def MakePNaClArchives():
448 join = os.path.join
449 gyp_chromium = join(SRC_DIR, 'build', 'gyp_chromium')
450 pnacl_irt_shim_gyp = join(SRC_DIR, 'ppapi', 'native_client', 'src',
451 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
453 with TempDir('pnacl_irt_shim_ia32_') as tmpdir:
454 GypNinjaBuild('ia32', gyp_chromium, pnacl_irt_shim_gyp, 'aot', tmpdir)
456 archive = Archive('pnacl_translator_x86_32_libs')
457 libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
458 archive.Copy(libdir, 'libpnacl_irt_shim.a')
459 archive.Tar()
461 archive = Archive('pnacl_translator_x86_64_libs')
462 libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
463 archive.Copy(libdir, 'libpnacl_irt_shim.a')
464 archive.Tar()
466 with TempDir('pnacl_irt_shim_arm_') as tmpdir:
467 GypNinjaBuild('arm', gyp_chromium, pnacl_irt_shim_gyp, 'aot', tmpdir)
469 archive = Archive('pnacl_translator_arm_libs')
470 libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-arm')
471 archive.Copy(libdir, 'libpnacl_irt_shim.a')
472 archive.Tar()
475 def GetNewlibHeaders():
476 return [
477 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
478 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
479 ('native_client/src/untrusted/irt/irt.h', ''),
480 ('native_client/src/untrusted/irt/irt_dev.h', ''),
481 ('native_client/src/untrusted/irt/irt_extension.h', ''),
482 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
483 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
484 ('native_client/src/untrusted/pthread/pthread.h', ''),
485 ('native_client/src/untrusted/pthread/semaphore.h', ''),
486 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
487 ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
490 def GetGlibcHeaders():
491 return [
492 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
493 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
494 ('native_client/src/untrusted/irt/irt.h', ''),
495 ('native_client/src/untrusted/irt/irt_dev.h', ''),
496 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
497 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
498 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
499 ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
502 def GetBionicHeaders():
503 return [('ppapi/nacl_irt/public/irt_ppapi.h', '')]
506 def MakeToolchainHeaderArchives():
507 archive = Archive('newlib_headers')
508 archive.Copy(SRC_DIR, GetNewlibHeaders())
509 archive.Tar()
511 archive = Archive('glibc_headers')
512 archive.Copy(SRC_DIR, GetGlibcHeaders())
513 archive.Tar()
515 if PLATFORM == 'linux':
516 archive = Archive('bionic_headers')
517 archive.Copy(SRC_DIR, GetBionicHeaders())
518 archive.Tar()
521 def MakePepperArchive():
522 archive = Archive('ppapi')
523 archive.Copy(os.path.join(SRC_DIR, 'ppapi'), ['c', 'cpp', 'lib', 'utility'])
524 archive.Tar()
527 def UploadArchives():
528 major_version = build_version.ChromeMajorVersion()
529 chrome_revision = build_version.ChromeRevision()
530 commit_position = build_version.ChromeCommitPosition()
531 git_sha = build_version.ParseCommitPosition(commit_position)[0]
532 short_sha = git_sha[:9]
533 archive_version = '%s-%s-%s' % (major_version, chrome_revision, short_sha)
534 bucket_path = 'native-client-sdk/archives/%s' % archive_version
535 for archive_name in all_archives:
536 buildbot_common.Archive(archive_name, bucket_path,
537 cwd=BUILD_ARCHIVE_DIR, step_link=False)
538 sha1_filename = archive_name + '.sha1'
539 buildbot_common.Archive(sha1_filename, bucket_path,
540 cwd=BUILD_ARCHIVE_DIR, step_link=False)
543 def MakeVersionJson():
544 time_format = '%Y/%m/%d %H:%M:%S'
545 data = {
546 'chrome_version': build_version.ChromeVersionNoTrunk(),
547 'chrome_revision': build_version.ChromeRevision(),
548 'chrome_commit_position': build_version.ChromeCommitPosition(),
549 'nacl_revision': build_version.NaClRevision(),
550 'build_date': datetime.datetime.now().strftime(time_format)}
552 dirname = os.path.dirname(VERSION_JSON)
553 if not os.path.exists(dirname):
554 buildbot_common.MakeDir(dirname)
555 with open(VERSION_JSON, 'w') as outf:
556 json.dump(data, outf, indent=2, separators=(',', ': '))
559 def main(args):
560 parser = argparse.ArgumentParser(description=__doc__)
561 parser.add_argument('--mac-sdk',
562 help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
563 parser.add_argument('--no-arm-trusted', action='store_true',
564 help='Disable building of ARM trusted components (sel_ldr, etc).')
565 parser.add_argument('--upload', action='store_true',
566 help='Upload tarballs to GCS.')
568 global options
569 options = parser.parse_args(args)
571 toolchains = ['pnacl', 'newlib', 'glibc', 'arm']
572 if PLATFORM == 'linux':
573 toolchains.append('bionic')
575 MakeVersionJson()
576 for tc in toolchains:
577 MakeToolchainArchive(tc)
578 MakeGypArchives()
579 MakePNaClArchives()
580 MakeToolchainHeaderArchives()
581 MakePepperArchive()
582 if options.upload:
583 UploadArchives()
585 return 0
587 if __name__ == '__main__':
588 try:
589 sys.exit(main(sys.argv[1:]))
590 except KeyboardInterrupt:
591 buildbot_common.ErrorExit('build_artifacts: interrupted')