Explicitly add python-numpy dependency to install-build-deps.
[chromium-blink-merge.git] / native_client_sdk / src / build_tools / build_artifacts.py
blob1a9f0d8a6862cec6e85453b2b429db8c8898a8ea
1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Script to build binary components of the SDK.
8 This script builds binary components of the Native Client SDK, create tarballs
9 for them, and uploads them to Google Cloud Storage.
11 This prevents a source dependency on the Chromium/NaCl tree in the Native
12 Client SDK repo.
13 """
15 import argparse
16 import datetime
17 import glob
18 import hashlib
19 import json
20 import os
21 import sys
22 import tempfile
24 if sys.version_info < (2, 7, 0):
25 sys.stderr.write("python 2.7 or later is required run this script\n")
26 sys.exit(1)
28 import buildbot_common
29 import build_version
30 from build_paths import NACL_DIR, OUT_DIR, SRC_DIR, SDK_SRC_DIR
31 from build_paths import BUILD_ARCHIVE_DIR
33 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
35 import getos
36 import oshelpers
38 BUILD_DIR = os.path.join(NACL_DIR, 'build')
39 NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
40 NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
42 CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
43 PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
44 VERSION_JSON = os.path.join(BUILD_ARCHIVE_DIR, 'version.json')
47 PLATFORM = getos.GetPlatform()
48 TAR = oshelpers.FindExeInPath('tar')
49 options = None
50 all_archives = []
53 # Mapping from toolchain name to the equivalent package_version.py directory
54 # name.
55 TOOLCHAIN_PACKAGE_MAP = {
56 'newlib': 'nacl_x86_newlib',
57 'bionic': 'nacl_arm_bionic',
58 'arm': 'nacl_arm_newlib',
59 'glibc': 'nacl_x86_glibc',
60 'pnacl': 'pnacl_newlib'}
63 def Tar(archive_path, root, files):
64 if os.path.exists(TAR):
65 cmd = [TAR]
66 else:
67 cmd = [sys.executable, CYGTAR]
68 cmd.extend(['-cjf', archive_path])
69 cmd.extend(files)
70 buildbot_common.Run(cmd, cwd=root)
73 def ComputeSha(filename):
74 with open(filename) as f:
75 return hashlib.sha1(f.read()).hexdigest()
78 class TempDir(object):
79 def __init__(self, prefix=None, dont_remove=False):
80 self.prefix = prefix
81 self.name = None
82 self.dont_remove = dont_remove
83 self.created = False
84 self.destroyed = False
86 def Create(self):
87 assert not self.created
88 self.name = tempfile.mkdtemp(prefix=self.prefix)
89 return self
91 def Destroy(self):
92 assert not self.destroyed
93 if not self.dont_remove:
94 buildbot_common.RemoveDir(self.name)
96 def __enter__(self):
97 self.Create()
98 return self.name
100 def __exit__(self, exc, value, tb):
101 return self.Destroy()
104 class Archive(object):
105 def __init__(self, name):
106 self.name = '%s_%s' % (PLATFORM, name)
107 self.archive_name = self.name + '.tar.bz2'
108 self.archive_path = os.path.join(BUILD_ARCHIVE_DIR, self.archive_name)
109 self.dirname = os.path.join(BUILD_ARCHIVE_DIR, self.name)
110 self._MakeDirname()
112 def _MakeDirname(self):
113 if os.path.exists(self.dirname):
114 buildbot_common.RemoveDir(self.dirname)
115 buildbot_common.MakeDir(self.dirname)
117 def Copy(self, src_root, file_list):
118 if type(file_list) is not list:
119 file_list = [file_list]
121 for file_spec in file_list:
122 # The list of files to install can be a simple list of
123 # strings or a list of pairs, where each pair corresponds
124 # to a mapping from source to destination names.
125 if type(file_spec) is str:
126 src_file = dest_file = file_spec
127 else:
128 src_file, dest_file = file_spec
130 src_file = os.path.join(src_root, src_file)
132 # Expand sources files using glob.
133 sources = glob.glob(src_file)
134 if not sources:
135 sources = [src_file]
137 if len(sources) > 1:
138 if not (dest_file.endswith('/') or dest_file == ''):
139 buildbot_common.ErrorExit(
140 "Target file %r must end in '/' or be empty when "
141 "using globbing to install files %r" % (dest_file, sources))
143 for source in sources:
144 if dest_file.endswith('/'):
145 dest = os.path.join(dest_file, os.path.basename(source))
146 else:
147 dest = dest_file
148 dest = os.path.join(self.dirname, dest)
149 if not os.path.isdir(os.path.dirname(dest)):
150 buildbot_common.MakeDir(os.path.dirname(dest))
151 if os.path.isdir(source):
152 buildbot_common.CopyDir(source, dest)
153 else:
154 buildbot_common.CopyFile(source, dest)
156 def CreateArchiveShaFile(self):
157 sha1 = ComputeSha(self.archive_path)
158 sha1_filename = self.archive_path + '.sha1'
159 with open(sha1_filename, 'w') as f:
160 f.write(sha1)
162 def Tar(self):
163 Tar(self.archive_path, BUILD_ARCHIVE_DIR, [
164 self.name,
165 os.path.basename(VERSION_JSON)])
166 self.CreateArchiveShaFile()
167 all_archives.append(self.archive_name)
170 def MakeToolchainArchive(toolchain):
171 archive = Archive(toolchain)
173 build_platform = '%s_x86' % PLATFORM
175 with TempDir('tc_%s_' % toolchain) as tmpdir:
176 package_name = os.path.join(build_platform,
177 TOOLCHAIN_PACKAGE_MAP.get(toolchain))
179 # Extract all of the packages into the temp directory.
180 buildbot_common.Run([sys.executable, PKGVER,
181 '--packages', package_name,
182 '--tar-dir', NACL_TOOLCHAINTARS_DIR,
183 '--dest-dir', tmpdir,
184 'extract'])
186 # Copy all the files we extracted to the correct destination.
187 archive.Copy(os.path.join(tmpdir, package_name), ('*', ''))
189 archive.Tar()
192 def MakeNinjaRelPath(path):
193 return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
196 def NinjaBuild(targets, out_dir):
197 if type(targets) is not list:
198 targets = [targets]
199 out_config_dir = os.path.join(out_dir, 'Release')
200 buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
203 def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, out_dir):
204 gyp_env = dict(os.environ)
205 gyp_env['GYP_GENERATORS'] = 'ninja'
206 gyp_defines = []
207 if options.mac_sdk:
208 gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
209 if arch:
210 gyp_defines.append('target_arch=%s' % arch)
211 if arch == 'arm':
212 if PLATFORM == 'linux':
213 gyp_env['CC'] = 'arm-linux-gnueabihf-gcc'
214 gyp_env['CXX'] = 'arm-linux-gnueabihf-g++'
215 gyp_env['AR'] = 'arm-linux-gnueabihf-ar'
216 gyp_env['AS'] = 'arm-linux-gnueabihf-as'
217 gyp_env['CC_host'] = 'cc'
218 gyp_env['CXX_host'] = 'c++'
219 gyp_defines += ['armv7=1', 'arm_thumb=0', 'arm_neon=1',
220 'arm_float_abi=hard', 'nacl_enable_arm_gcc=1']
221 if options.no_arm_trusted:
222 gyp_defines.append('disable_cross_trusted=1')
223 if PLATFORM == 'mac':
224 gyp_defines.append('clang=1')
226 gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
227 generator_flags = ['-G', 'output_dir=%s' % out_dir]
228 depth = '--depth=.'
229 cmd = [sys.executable, gyp_py_script, gyp_file, depth] + generator_flags
230 buildbot_common.Run(cmd, cwd=SRC_DIR, env=gyp_env)
231 NinjaBuild(targets, out_dir)
234 def GetToolsFiles():
235 files = [
236 ('sel_ldr', 'sel_ldr_x86_32'),
237 ('ncval_new', 'ncval'),
238 ('irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'),
239 ('irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'),
242 if PLATFORM == 'linux':
243 files.append(['nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_32'])
244 files.append(['nonsfi_loader_newlib_x32_nonsfi.nexe',
245 'nonsfi_loader_x86_32'])
247 # Add .exe extensions to all windows tools
248 for pair in files:
249 if PLATFORM == 'win' and not pair[0].endswith('.nexe'):
250 pair[0] += '.exe'
251 pair[1] += '.exe'
253 return files
256 def GetTools64Files():
257 files = []
258 if PLATFORM == 'win':
259 files.append('sel_ldr64')
260 else:
261 files.append(('sel_ldr', 'sel_ldr_x86_64'))
263 if PLATFORM == 'linux':
264 files.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_64'))
266 return files
269 def GetToolsArmFiles():
270 assert PLATFORM == 'linux'
271 return [('irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'),
272 ('sel_ldr', 'sel_ldr_arm'),
273 ('nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm')]
276 def GetNewlibToolchainLibs():
277 return ['crti.o',
278 'crtn.o',
279 'libminidump_generator.a',
280 'libnacl.a',
281 'libnacl_dyncode.a',
282 'libnacl_exception.a',
283 'libnacl_list_mappings.a',
284 'libnosys.a',
285 'libppapi.a',
286 'libppapi_stub.a',
287 'libpthread.a']
290 def GetGlibcToolchainLibs():
291 return ['libminidump_generator.a',
292 'libminidump_generator.so',
293 'libnacl.a',
294 'libnacl_dyncode.a',
295 'libnacl_dyncode.so',
296 'libnacl_exception.a',
297 'libnacl_exception.so',
298 'libnacl_list_mappings.a',
299 'libnacl_list_mappings.so',
300 'libppapi.a',
301 'libppapi.so',
302 'libppapi_stub.a']
305 def GetPNaClToolchainLibs():
306 return ['libminidump_generator.a',
307 'libnacl.a',
308 'libnacl_dyncode.a',
309 'libnacl_exception.a',
310 'libnacl_list_mappings.a',
311 'libnosys.a',
312 'libppapi.a',
313 'libppapi_stub.a',
314 'libpthread.a']
317 def GetBionicToolchainLibs():
318 return ['libminidump_generator.a',
319 'libnacl_dyncode.a',
320 'libnacl_exception.a',
321 'libnacl_list_mappings.a',
322 'libppapi.a']
325 def GetToolchainNaClLib(tcname, tcpath, xarch):
326 if tcname == 'pnacl':
327 return os.path.join(tcpath, 'le32-nacl', 'lib')
328 elif xarch == 'x86_32':
329 return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
330 elif xarch == 'x86_64':
331 return os.path.join(tcpath, 'x86_64-nacl', 'lib')
332 elif xarch == 'arm':
333 return os.path.join(tcpath, 'arm-nacl', 'lib')
336 def GetGypBuiltLib(root, tcname, xarch=None):
337 if tcname == 'pnacl':
338 tcname = 'pnacl_newlib'
339 if xarch == 'x86_32':
340 xarch = '32'
341 elif xarch == 'x86_64':
342 xarch = '64'
343 elif not xarch:
344 xarch = ''
345 return os.path.join(root, 'Release', 'gen', 'tc_' + tcname, 'lib' + xarch)
348 def GetGypToolchainLib(root, tcname, xarch):
349 if xarch == 'arm':
350 toolchain = xarch
351 else:
352 toolchain = tcname
354 tcpath = os.path.join(root, 'Release', 'gen', 'sdk', '%s_x86' % PLATFORM,
355 TOOLCHAIN_PACKAGE_MAP[toolchain])
356 return GetToolchainNaClLib(tcname, tcpath, xarch)
359 def MakeGypArchives():
360 join = os.path.join
361 gyp_nacl = join(NACL_DIR, 'build', 'gyp_nacl')
362 gyp_chromium = join(SRC_DIR, 'build', 'gyp_chromium')
363 nacl_core_sdk_gyp = join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
364 all_gyp = join(NACL_DIR, 'build', 'all.gyp')
365 breakpad_gyp = join(SRC_DIR, 'breakpad', 'breakpad.gyp')
366 ppapi_gyp = join(SRC_DIR, 'ppapi', 'native_client', 'native_client.gyp')
367 breakpad_targets = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
369 # Build
370 tmpdir_obj = TempDir('nacl_core_sdk_', dont_remove=True).Create()
371 tmpdir = tmpdir_obj.name
372 GypNinjaBuild('ia32', gyp_nacl, nacl_core_sdk_gyp, 'nacl_core_sdk', tmpdir)
373 GypNinjaBuild('ia32', gyp_nacl, all_gyp, 'ncval_new', tmpdir)
374 GypNinjaBuild('ia32', gyp_chromium, breakpad_gyp, breakpad_targets, tmpdir)
375 GypNinjaBuild('ia32', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir)
376 GypNinjaBuild('x64', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir)
378 tmpdir64_obj = TempDir('nacl_core_sdk_64_', dont_remove=True).Create()
379 tmpdir_64 = tmpdir64_obj.name
380 if PLATFORM == 'win':
381 GypNinjaBuild('ia32', gyp_nacl, nacl_core_sdk_gyp, 'sel_ldr64', tmpdir_64)
382 else:
383 GypNinjaBuild('x64', gyp_nacl, nacl_core_sdk_gyp, 'sel_ldr', tmpdir_64)
385 tmpdirarm_obj = TempDir('nacl_core_sdk_arm_', dont_remove=True).Create()
386 tmpdir_arm = tmpdirarm_obj.name
387 GypNinjaBuild('arm', gyp_nacl, nacl_core_sdk_gyp, 'nacl_core_sdk', tmpdir_arm)
388 GypNinjaBuild('arm', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir_arm)
390 # Tools archive
391 archive = Archive('tools')
392 archive.Copy(join(tmpdir, 'Release'), GetToolsFiles())
393 archive.Copy(join(tmpdir_64, 'Release'), GetTools64Files())
394 if PLATFORM == 'linux':
395 archive.Copy(join(tmpdir_arm, 'Release'), GetToolsArmFiles())
396 # TODO(binji): dump_syms doesn't currently build on Windows. See
397 # http://crbug.com/245456
398 if PLATFORM != 'win':
399 archive.Copy(join(tmpdir, 'Release'), breakpad_targets)
400 archive.Tar()
402 # newlib x86 libs archives
403 for arch in ('x86_32', 'x86_64'):
404 archive = Archive('newlib_%s_libs' % arch)
405 archive.Copy(GetGypBuiltLib(tmpdir, 'newlib', arch),
406 GetNewlibToolchainLibs())
407 archive.Copy(GetGypToolchainLib(tmpdir, 'newlib', arch), 'crt1.o')
408 archive.Tar()
410 # newlib arm libs archive
411 archive = Archive('newlib_arm_libs')
412 archive.Copy(GetGypBuiltLib(tmpdir_arm, 'newlib', 'arm'),
413 GetNewlibToolchainLibs())
414 archive.Copy(GetGypToolchainLib(tmpdir_arm, 'newlib', 'arm'), 'crt1.o')
415 archive.Tar()
417 # glibc x86 libs archives
418 for arch in ('x86_32', 'x86_64'):
419 archive = Archive('glibc_%s_libs' % arch)
420 archive.Copy(GetGypBuiltLib(tmpdir, 'glibc', arch), GetGlibcToolchainLibs())
421 archive.Copy(GetGypToolchainLib(tmpdir, 'glibc', arch), 'crt1.o')
422 archive.Tar()
424 # pnacl libs archive
425 archive = Archive('pnacl_libs')
426 archive.Copy(GetGypBuiltLib(tmpdir, 'pnacl'), GetPNaClToolchainLibs())
427 archive.Tar()
429 if PLATFORM == 'linux':
430 # bionic arm libs archive (use newlib-built files)
431 archive = Archive('bionic_arm_libs')
432 archive.Copy(GetGypBuiltLib(tmpdir_arm, 'newlib', 'arm'),
433 GetBionicToolchainLibs())
434 archive.Copy(GetGypToolchainLib(tmpdir_arm, 'newlib', 'arm'), 'crt1.o')
435 archive.Tar()
437 # Destroy the temporary directories
438 tmpdir_obj.Destroy()
439 tmpdirarm_obj.Destroy()
440 tmpdir64_obj.Destroy()
443 def MakePNaClArchives():
444 join = os.path.join
445 gyp_chromium = join(SRC_DIR, 'build', 'gyp_chromium')
446 pnacl_irt_shim_gyp = join(SRC_DIR, 'ppapi', 'native_client', 'src',
447 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
449 with TempDir('pnacl_irt_shim_ia32_') as tmpdir:
450 GypNinjaBuild('ia32', gyp_chromium, pnacl_irt_shim_gyp, 'aot', tmpdir)
452 archive = Archive('pnacl_translator_x86_32_libs')
453 libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
454 archive.Copy(libdir, 'libpnacl_irt_shim.a')
455 archive.Tar()
457 archive = Archive('pnacl_translator_x86_64_libs')
458 libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
459 archive.Copy(libdir, 'libpnacl_irt_shim.a')
460 archive.Tar()
462 with TempDir('pnacl_irt_shim_arm_') as tmpdir:
463 GypNinjaBuild('arm', gyp_chromium, pnacl_irt_shim_gyp, 'aot', tmpdir)
465 archive = Archive('pnacl_translator_arm_libs')
466 libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-arm')
467 archive.Copy(libdir, 'libpnacl_irt_shim.a')
468 archive.Tar()
471 def GetNewlibHeaders():
472 return [
473 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
474 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
475 ('native_client/src/untrusted/irt/irt.h', ''),
476 ('native_client/src/untrusted/irt/irt_dev.h', ''),
477 ('native_client/src/untrusted/irt/irt_extension.h', ''),
478 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
479 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
480 ('native_client/src/untrusted/pthread/pthread.h', ''),
481 ('native_client/src/untrusted/pthread/semaphore.h', ''),
482 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
483 ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
486 def GetGlibcHeaders():
487 return [
488 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
489 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
490 ('native_client/src/untrusted/irt/irt.h', ''),
491 ('native_client/src/untrusted/irt/irt_dev.h', ''),
492 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
493 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
494 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
495 ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
498 def GetBionicHeaders():
499 return [('ppapi/nacl_irt/public/irt_ppapi.h', '')]
502 def MakeToolchainHeaderArchives():
503 archive = Archive('newlib_headers')
504 archive.Copy(SRC_DIR, GetNewlibHeaders())
505 archive.Tar()
507 archive = Archive('glibc_headers')
508 archive.Copy(SRC_DIR, GetGlibcHeaders())
509 archive.Tar()
511 if PLATFORM == 'linux':
512 archive = Archive('bionic_headers')
513 archive.Copy(SRC_DIR, GetBionicHeaders())
514 archive.Tar()
517 def MakePepperArchive():
518 archive = Archive('ppapi')
519 archive.Copy(os.path.join(SRC_DIR, 'ppapi'), ['c', 'cpp', 'lib', 'utility'])
520 archive.Tar()
523 def UploadArchives():
524 major_version = build_version.ChromeMajorVersion()
525 chrome_revision = build_version.ChromeRevision()
526 commit_position = build_version.ChromeCommitPosition()
527 git_sha = build_version.ParseCommitPosition(commit_position)[0]
528 short_sha = git_sha[:9]
529 archive_version = '%s-%s-%s' % (major_version, chrome_revision, short_sha)
530 bucket_path = 'native-client-sdk/archives/%s' % archive_version
531 for archive_name in all_archives:
532 buildbot_common.Archive(archive_name, bucket_path,
533 cwd=BUILD_ARCHIVE_DIR, step_link=False)
534 sha1_filename = archive_name + '.sha1'
535 buildbot_common.Archive(sha1_filename, bucket_path,
536 cwd=BUILD_ARCHIVE_DIR, step_link=False)
539 def MakeVersionJson():
540 time_format = '%Y/%m/%d %H:%M:%S'
541 data = {
542 'chrome_version': build_version.ChromeVersionNoTrunk(),
543 'chrome_revision': build_version.ChromeRevision(),
544 'chrome_commit_position': build_version.ChromeCommitPosition(),
545 'nacl_revision': build_version.NaClRevision(),
546 'build_date': datetime.datetime.now().strftime(time_format)}
548 dirname = os.path.dirname(VERSION_JSON)
549 if not os.path.exists(dirname):
550 buildbot_common.MakeDir(dirname)
551 with open(VERSION_JSON, 'w') as outf:
552 json.dump(data, outf, indent=2, separators=(',', ': '))
555 def main(args):
556 parser = argparse.ArgumentParser(description=__doc__)
557 parser.add_argument('--mac-sdk',
558 help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
559 parser.add_argument('--no-arm-trusted', action='store_true',
560 help='Disable building of ARM trusted components (sel_ldr, etc).')
561 parser.add_argument('--upload', action='store_true',
562 help='Upload tarballs to GCS.')
564 global options
565 options = parser.parse_args(args)
567 toolchains = ['pnacl', 'newlib', 'glibc', 'arm']
568 if PLATFORM == 'linux':
569 toolchains.append('bionic')
571 MakeVersionJson()
572 for tc in toolchains:
573 MakeToolchainArchive(tc)
574 MakeGypArchives()
575 MakePNaClArchives()
576 MakeToolchainHeaderArchives()
577 MakePepperArchive()
578 if options.upload:
579 UploadArchives()
581 return 0
583 if __name__ == '__main__':
584 try:
585 sys.exit(main(sys.argv[1:]))
586 except KeyboardInterrupt:
587 buildbot_common.ErrorExit('build_artifacts: interrupted')