Pin Chrome's shortcut to the Win10 Start menu on install and OS upgrade.
[chromium-blink-merge.git] / native_client_sdk / src / build_tools / build_artifacts.py
blobaa36c31541d1a3eb35f6956a5697d0044321e0e0
1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Script to build binary components of the SDK.
8 This script builds binary components of the Native Client SDK, create tarballs
9 for them, and uploads them to Google Cloud Storage.
11 This prevents a source dependency on the Chromium/NaCl tree in the Native
12 Client SDK repo.
13 """
15 import argparse
16 import datetime
17 import glob
18 import hashlib
19 import json
20 import os
21 import sys
22 import tempfile
24 if sys.version_info < (2, 7, 0):
25 sys.stderr.write("python 2.7 or later is required run this script\n")
26 sys.exit(1)
28 import buildbot_common
29 import build_version
30 from build_paths import NACL_DIR, OUT_DIR, SRC_DIR, SDK_SRC_DIR
31 from build_paths import BUILD_ARCHIVE_DIR
33 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
35 import getos
36 import oshelpers
38 BUILD_DIR = os.path.join(NACL_DIR, 'build')
39 NACL_TOOLCHAIN_DIR = os.path.join(NACL_DIR, 'toolchain')
40 NACL_TOOLCHAINTARS_DIR = os.path.join(NACL_TOOLCHAIN_DIR, '.tars')
42 CYGTAR = os.path.join(BUILD_DIR, 'cygtar.py')
43 PKGVER = os.path.join(BUILD_DIR, 'package_version', 'package_version.py')
44 VERSION_JSON = os.path.join(BUILD_ARCHIVE_DIR, 'version.json')
47 PLATFORM = getos.GetPlatform()
48 TAR = oshelpers.FindExeInPath('tar')
49 options = None
50 all_archives = []
53 # Mapping from toolchain name to the equivalent package_version.py directory
54 # name.
55 TOOLCHAIN_PACKAGE_MAP = {
56 'newlib': 'nacl_x86_newlib',
57 'bionic': 'nacl_arm_bionic',
58 'arm': 'nacl_arm_newlib',
59 'glibc': 'nacl_x86_glibc',
60 'pnacl': 'pnacl_newlib'}
63 def Tar(archive_path, root, files):
64 if os.path.exists(TAR):
65 cmd = [TAR]
66 else:
67 cmd = [sys.executable, CYGTAR]
68 cmd.extend(['-cjf', archive_path])
69 cmd.extend(files)
70 buildbot_common.Run(cmd, cwd=root)
73 def ComputeSha(filename):
74 with open(filename) as f:
75 return hashlib.sha1(f.read()).hexdigest()
78 class TempDir(object):
79 def __init__(self, prefix=None, dont_remove=False):
80 self.prefix = prefix
81 self.name = None
82 self.dont_remove = dont_remove
83 self.created = False
84 self.destroyed = False
86 def Create(self):
87 assert not self.created
88 self.name = tempfile.mkdtemp(prefix=self.prefix)
89 return self
91 def Destroy(self):
92 assert not self.destroyed
93 if not self.dont_remove:
94 buildbot_common.RemoveDir(self.name)
96 def __enter__(self):
97 self.Create()
98 return self.name
100 def __exit__(self, exc, value, tb):
101 return self.Destroy()
104 class Archive(object):
105 def __init__(self, name):
106 self.name = '%s_%s' % (PLATFORM, name)
107 self.archive_name = self.name + '.tar.bz2'
108 self.archive_path = os.path.join(BUILD_ARCHIVE_DIR, self.archive_name)
109 self.dirname = os.path.join(BUILD_ARCHIVE_DIR, self.name)
110 self._MakeDirname()
112 def _MakeDirname(self):
113 if os.path.exists(self.dirname):
114 buildbot_common.RemoveDir(self.dirname)
115 buildbot_common.MakeDir(self.dirname)
117 def Copy(self, src_root, file_list):
118 if type(file_list) is not list:
119 file_list = [file_list]
121 for file_spec in file_list:
122 # The list of files to install can be a simple list of
123 # strings or a list of pairs, where each pair corresponds
124 # to a mapping from source to destination names.
125 if type(file_spec) is str:
126 src_file = dest_file = file_spec
127 else:
128 src_file, dest_file = file_spec
130 src_file = os.path.join(src_root, src_file)
132 # Expand sources files using glob.
133 sources = glob.glob(src_file)
134 if not sources:
135 sources = [src_file]
137 if len(sources) > 1:
138 if not (dest_file.endswith('/') or dest_file == ''):
139 buildbot_common.ErrorExit(
140 "Target file %r must end in '/' or be empty when "
141 "using globbing to install files %r" % (dest_file, sources))
143 for source in sources:
144 if dest_file.endswith('/'):
145 dest = os.path.join(dest_file, os.path.basename(source))
146 else:
147 dest = dest_file
148 dest = os.path.join(self.dirname, dest)
149 if not os.path.isdir(os.path.dirname(dest)):
150 buildbot_common.MakeDir(os.path.dirname(dest))
151 if os.path.isdir(source):
152 buildbot_common.CopyDir(source, dest)
153 else:
154 buildbot_common.CopyFile(source, dest)
156 def CreateArchiveShaFile(self):
157 sha1 = ComputeSha(self.archive_path)
158 sha1_filename = self.archive_path + '.sha1'
159 with open(sha1_filename, 'w') as f:
160 f.write(sha1)
162 def Tar(self):
163 Tar(self.archive_path, BUILD_ARCHIVE_DIR, [
164 self.name,
165 os.path.basename(VERSION_JSON)])
166 self.CreateArchiveShaFile()
167 all_archives.append(self.archive_name)
170 def MakeToolchainArchive(toolchain):
171 archive = Archive(toolchain)
173 build_platform = '%s_x86' % PLATFORM
175 with TempDir('tc_%s_' % toolchain) as tmpdir:
176 package_name = os.path.join(build_platform,
177 TOOLCHAIN_PACKAGE_MAP.get(toolchain))
179 # Extract all of the packages into the temp directory.
180 buildbot_common.Run([sys.executable, PKGVER,
181 '--packages', package_name,
182 '--tar-dir', NACL_TOOLCHAINTARS_DIR,
183 '--dest-dir', tmpdir,
184 'extract'])
186 # Copy all the files we extracted to the correct destination.
187 archive.Copy(os.path.join(tmpdir, package_name), ('*', ''))
189 archive.Tar()
192 def MakeNinjaRelPath(path):
193 return os.path.join(os.path.relpath(OUT_DIR, SRC_DIR), path)
196 def NinjaBuild(targets, out_dir):
197 if type(targets) is not list:
198 targets = [targets]
199 out_config_dir = os.path.join(out_dir, 'Release')
200 buildbot_common.Run(['ninja', '-C', out_config_dir] + targets, cwd=SRC_DIR)
203 def GypNinjaBuild(arch, gyp_py_script, gyp_file, targets, out_dir):
204 gyp_env = dict(os.environ)
205 gyp_env['GYP_GENERATORS'] = 'ninja'
206 gyp_defines = ['nacl_allow_thin_archives=0']
207 if options.mac_sdk:
208 gyp_defines.append('mac_sdk=%s' % options.mac_sdk)
209 if arch:
210 gyp_defines.append('target_arch=%s' % arch)
211 if arch == 'arm':
212 gyp_env['GYP_CROSSCOMPILE'] = '1'
213 if options.no_arm_trusted:
214 gyp_defines.append('disable_cross_trusted=1')
215 if PLATFORM == 'mac':
216 gyp_defines.append('clang=1')
218 gyp_env['GYP_DEFINES'] = ' '.join(gyp_defines)
219 generator_flags = ['-G', 'output_dir=%s' % out_dir]
220 depth = '--depth=.'
221 cmd = [sys.executable, gyp_py_script, gyp_file, depth] + generator_flags
222 buildbot_common.Run(cmd, cwd=SRC_DIR, env=gyp_env)
223 NinjaBuild(targets, out_dir)
226 def GetToolsFiles():
227 files = [
228 ('sel_ldr', 'sel_ldr_x86_32'),
229 ('ncval_new', 'ncval'),
230 ('irt_core_newlib_x32.nexe', 'irt_core_x86_32.nexe'),
231 ('irt_core_newlib_x64.nexe', 'irt_core_x86_64.nexe'),
234 if PLATFORM == 'linux':
235 files.append(['nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_32'])
236 files.append(['nonsfi_loader_newlib_x32_nonsfi.nexe',
237 'nonsfi_loader_x86_32'])
238 files.append(['nonsfi_loader_newlib_arm_nonsfi.nexe',
239 'nonsfi_loader_arm'])
241 # Add .exe extensions to all windows tools
242 for pair in files:
243 if PLATFORM == 'win' and not pair[0].endswith('.nexe'):
244 pair[0] += '.exe'
245 pair[1] += '.exe'
247 return files
250 def GetTools64Files():
251 files = []
252 if PLATFORM == 'win':
253 files.append('sel_ldr64')
254 else:
255 files.append(('sel_ldr', 'sel_ldr_x86_64'))
257 if PLATFORM == 'linux':
258 files.append(('nacl_helper_bootstrap', 'nacl_helper_bootstrap_x86_64'))
260 return files
263 def GetToolsArmFiles():
264 assert PLATFORM == 'linux'
265 return [('irt_core_newlib_arm.nexe', 'irt_core_arm.nexe'),
266 ('sel_ldr', 'sel_ldr_arm'),
267 ('nacl_helper_bootstrap', 'nacl_helper_bootstrap_arm')]
270 def GetNewlibToolchainLibs():
271 return ['crti.o',
272 'crtn.o',
273 'libminidump_generator.a',
274 'libnacl.a',
275 'libnacl_dyncode.a',
276 'libnacl_exception.a',
277 'libnacl_list_mappings.a',
278 'libnosys.a',
279 'libppapi.a',
280 'libppapi_stub.a',
281 'libpthread.a']
284 def GetGlibcToolchainLibs():
285 return ['libminidump_generator.a',
286 'libminidump_generator.so',
287 'libnacl.a',
288 'libnacl_dyncode.a',
289 'libnacl_dyncode.so',
290 'libnacl_exception.a',
291 'libnacl_exception.so',
292 'libnacl_list_mappings.a',
293 'libnacl_list_mappings.so',
294 'libppapi.a',
295 'libppapi.so',
296 'libppapi_stub.a']
299 def GetPNaClToolchainLibs():
300 return ['libminidump_generator.a',
301 'libnacl.a',
302 'libnacl_dyncode.a',
303 'libnacl_exception.a',
304 'libnacl_list_mappings.a',
305 'libnosys.a',
306 'libppapi.a',
307 'libppapi_stub.a',
308 'libpthread.a']
311 def GetBionicToolchainLibs():
312 return ['libminidump_generator.a',
313 'libnacl_dyncode.a',
314 'libnacl_exception.a',
315 'libnacl_list_mappings.a',
316 'libppapi.a']
319 def GetToolchainNaClLib(tcname, tcpath, xarch):
320 if tcname == 'pnacl':
321 return os.path.join(tcpath, 'le32-nacl', 'lib')
322 elif xarch == 'x86_32':
323 return os.path.join(tcpath, 'x86_64-nacl', 'lib32')
324 elif xarch == 'x86_64':
325 return os.path.join(tcpath, 'x86_64-nacl', 'lib')
326 elif xarch == 'arm':
327 return os.path.join(tcpath, 'arm-nacl', 'lib')
330 def GetGypBuiltLib(root, tcname, xarch=None):
331 if tcname == 'pnacl':
332 tcname = 'pnacl_newlib'
333 if xarch == 'x86_32':
334 xarch = '32'
335 elif xarch == 'x86_64':
336 xarch = '64'
337 elif not xarch:
338 xarch = ''
339 return os.path.join(root, 'Release', 'gen', 'tc_' + tcname, 'lib' + xarch)
342 def GetGypToolchainLib(root, tcname, xarch):
343 if xarch == 'arm':
344 toolchain = xarch
345 else:
346 toolchain = tcname
348 tcpath = os.path.join(root, 'Release', 'gen', 'sdk', '%s_x86' % PLATFORM,
349 TOOLCHAIN_PACKAGE_MAP[toolchain])
350 return GetToolchainNaClLib(tcname, tcpath, xarch)
353 def MakeGypArchives():
354 join = os.path.join
355 gyp_chromium = join(SRC_DIR, 'build', 'gyp_chromium')
356 # TODO(binji): gyp_nacl doesn't build properly on Windows anymore; it only
357 # can use VS2010, not VS2013 which is now required by the Chromium repo. NaCl
358 # needs to be updated to perform the same logic as Chromium in detecting VS,
359 # which can now exist in the depot_tools directory.
360 # See https://code.google.com/p/nativeclient/issues/detail?id=4022
362 # For now, let's use gyp_chromium to build these components.
363 # gyp_nacl = join(NACL_DIR, 'build', 'gyp_nacl')
364 gyp_nacl = gyp_chromium
366 nacl_core_sdk_gyp = join(NACL_DIR, 'build', 'nacl_core_sdk.gyp')
367 all_gyp = join(NACL_DIR, 'build', 'all.gyp')
368 breakpad_gyp = join(SRC_DIR, 'breakpad', 'breakpad.gyp')
369 ppapi_gyp = join(SRC_DIR, 'ppapi', 'native_client', 'native_client.gyp')
370 breakpad_targets = ['dump_syms', 'minidump_dump', 'minidump_stackwalk']
372 # Build
373 tmpdir_obj = TempDir('nacl_core_sdk_', dont_remove=True).Create()
374 tmpdir = tmpdir_obj.name
375 GypNinjaBuild('ia32', gyp_nacl, nacl_core_sdk_gyp, 'nacl_core_sdk', tmpdir)
376 GypNinjaBuild('ia32', gyp_nacl, all_gyp, 'ncval_new', tmpdir)
377 GypNinjaBuild('ia32', gyp_chromium, breakpad_gyp, breakpad_targets, tmpdir)
378 GypNinjaBuild('ia32', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir)
379 GypNinjaBuild('x64', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir)
381 tmpdir64_obj = TempDir('nacl_core_sdk_64_', dont_remove=True).Create()
382 tmpdir_64 = tmpdir64_obj.name
383 if PLATFORM == 'win':
384 GypNinjaBuild('ia32', gyp_nacl, nacl_core_sdk_gyp, 'sel_ldr64', tmpdir_64)
385 else:
386 GypNinjaBuild('x64', gyp_nacl, nacl_core_sdk_gyp, 'sel_ldr', tmpdir_64)
388 tmpdirarm_obj = TempDir('nacl_core_sdk_arm_', dont_remove=True).Create()
389 tmpdir_arm = tmpdirarm_obj.name
390 GypNinjaBuild('arm', gyp_nacl, nacl_core_sdk_gyp, 'nacl_core_sdk', tmpdir_arm)
391 GypNinjaBuild('arm', gyp_chromium, ppapi_gyp, 'ppapi_lib', tmpdir_arm)
393 # Tools archive
394 archive = Archive('tools')
395 archive.Copy(join(tmpdir, 'Release'), GetToolsFiles())
396 archive.Copy(join(tmpdir_64, 'Release'), GetTools64Files())
397 if PLATFORM == 'linux':
398 archive.Copy(join(tmpdir_arm, 'Release'), GetToolsArmFiles())
399 # TODO(binji): dump_syms doesn't currently build on Windows. See
400 # http://crbug.com/245456
401 if PLATFORM != 'win':
402 archive.Copy(join(tmpdir, 'Release'), breakpad_targets)
403 archive.Tar()
405 # newlib x86 libs archives
406 for arch in ('x86_32', 'x86_64'):
407 archive = Archive('newlib_%s_libs' % arch)
408 archive.Copy(GetGypBuiltLib(tmpdir, 'newlib', arch),
409 GetNewlibToolchainLibs())
410 archive.Copy(GetGypToolchainLib(tmpdir, 'newlib', arch), 'crt1.o')
411 archive.Tar()
413 # newlib arm libs archive
414 archive = Archive('newlib_arm_libs')
415 archive.Copy(GetGypBuiltLib(tmpdir_arm, 'newlib', 'arm'),
416 GetNewlibToolchainLibs())
417 archive.Copy(GetGypToolchainLib(tmpdir_arm, 'newlib', 'arm'), 'crt1.o')
418 archive.Tar()
420 # glibc x86 libs archives
421 for arch in ('x86_32', 'x86_64'):
422 archive = Archive('glibc_%s_libs' % arch)
423 archive.Copy(GetGypBuiltLib(tmpdir, 'glibc', arch), GetGlibcToolchainLibs())
424 archive.Copy(GetGypToolchainLib(tmpdir, 'glibc', arch), 'crt1.o')
425 archive.Tar()
427 # pnacl libs archive
428 archive = Archive('pnacl_libs')
429 archive.Copy(GetGypBuiltLib(tmpdir, 'pnacl'), GetPNaClToolchainLibs())
430 archive.Tar()
432 if PLATFORM == 'linux':
433 # bionic arm libs archive (use newlib-built files)
434 archive = Archive('bionic_arm_libs')
435 archive.Copy(GetGypBuiltLib(tmpdir_arm, 'newlib', 'arm'),
436 GetBionicToolchainLibs())
437 archive.Copy(GetGypToolchainLib(tmpdir_arm, 'newlib', 'arm'), 'crt1.o')
438 archive.Tar()
440 # Destroy the temporary directories
441 tmpdir_obj.Destroy()
442 tmpdirarm_obj.Destroy()
443 tmpdir64_obj.Destroy()
446 def MakePNaClArchives():
447 join = os.path.join
448 gyp_chromium = join(SRC_DIR, 'build', 'gyp_chromium')
449 pnacl_irt_shim_gyp = join(SRC_DIR, 'ppapi', 'native_client', 'src',
450 'untrusted', 'pnacl_irt_shim', 'pnacl_irt_shim.gyp')
452 with TempDir('pnacl_irt_shim_ia32_') as tmpdir:
453 GypNinjaBuild('ia32', gyp_chromium, pnacl_irt_shim_gyp, 'aot', tmpdir)
455 archive = Archive('pnacl_translator_x86_32_libs')
456 libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
457 archive.Copy(libdir, 'libpnacl_irt_shim.a')
458 archive.Tar()
460 archive = Archive('pnacl_translator_x86_64_libs')
461 libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-x86-32')
462 archive.Copy(libdir, 'libpnacl_irt_shim.a')
463 archive.Tar()
465 with TempDir('pnacl_irt_shim_arm_') as tmpdir:
466 GypNinjaBuild('arm', gyp_chromium, pnacl_irt_shim_gyp, 'aot', tmpdir)
468 archive = Archive('pnacl_translator_arm_libs')
469 libdir = join(tmpdir, 'Release', 'gen', 'tc_pnacl_translate', 'lib-arm')
470 archive.Copy(libdir, 'libpnacl_irt_shim.a')
471 archive.Tar()
474 def GetNewlibHeaders():
475 return [
476 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
477 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
478 ('native_client/src/untrusted/irt/irt.h', ''),
479 ('native_client/src/untrusted/irt/irt_dev.h', ''),
480 ('native_client/src/untrusted/irt/irt_extension.h', ''),
481 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
482 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
483 ('native_client/src/untrusted/pthread/pthread.h', ''),
484 ('native_client/src/untrusted/pthread/semaphore.h', ''),
485 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
486 ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
489 def GetGlibcHeaders():
490 return [
491 ('native_client/src/include/nacl/nacl_exception.h', 'nacl/'),
492 ('native_client/src/include/nacl/nacl_minidump.h', 'nacl/'),
493 ('native_client/src/untrusted/irt/irt.h', ''),
494 ('native_client/src/untrusted/irt/irt_dev.h', ''),
495 ('native_client/src/untrusted/nacl/nacl_dyncode.h', 'nacl/'),
496 ('native_client/src/untrusted/nacl/nacl_startup.h', 'nacl/'),
497 ('native_client/src/untrusted/valgrind/dynamic_annotations.h', 'nacl/'),
498 ('ppapi/nacl_irt/public/irt_ppapi.h', '')]
501 def GetBionicHeaders():
502 return [('ppapi/nacl_irt/public/irt_ppapi.h', '')]
505 def MakeToolchainHeaderArchives():
506 archive = Archive('newlib_headers')
507 archive.Copy(SRC_DIR, GetNewlibHeaders())
508 archive.Tar()
510 archive = Archive('glibc_headers')
511 archive.Copy(SRC_DIR, GetGlibcHeaders())
512 archive.Tar()
514 if PLATFORM == 'linux':
515 archive = Archive('bionic_headers')
516 archive.Copy(SRC_DIR, GetBionicHeaders())
517 archive.Tar()
520 def MakePepperArchive():
521 archive = Archive('ppapi')
522 archive.Copy(os.path.join(SRC_DIR, 'ppapi'), ['c', 'cpp', 'lib', 'utility'])
523 archive.Tar()
526 def UploadArchives():
527 major_version = build_version.ChromeMajorVersion()
528 chrome_revision = build_version.ChromeRevision()
529 commit_position = build_version.ChromeCommitPosition()
530 git_sha = build_version.ParseCommitPosition(commit_position)[0]
531 short_sha = git_sha[:9]
532 archive_version = '%s-%s-%s' % (major_version, chrome_revision, short_sha)
533 bucket_path = 'native-client-sdk/archives/%s' % archive_version
534 for archive_name in all_archives:
535 buildbot_common.Archive(archive_name, bucket_path,
536 cwd=BUILD_ARCHIVE_DIR, step_link=False)
537 sha1_filename = archive_name + '.sha1'
538 buildbot_common.Archive(sha1_filename, bucket_path,
539 cwd=BUILD_ARCHIVE_DIR, step_link=False)
542 def MakeVersionJson():
543 time_format = '%Y/%m/%d %H:%M:%S'
544 data = {
545 'chrome_version': build_version.ChromeVersionNoTrunk(),
546 'chrome_revision': build_version.ChromeRevision(),
547 'chrome_commit_position': build_version.ChromeCommitPosition(),
548 'nacl_revision': build_version.NaClRevision(),
549 'build_date': datetime.datetime.now().strftime(time_format)}
551 dirname = os.path.dirname(VERSION_JSON)
552 if not os.path.exists(dirname):
553 buildbot_common.MakeDir(dirname)
554 with open(VERSION_JSON, 'w') as outf:
555 json.dump(data, outf, indent=2, separators=(',', ': '))
558 def main(args):
559 parser = argparse.ArgumentParser(description=__doc__)
560 parser.add_argument('--mac-sdk',
561 help='Set the mac-sdk (e.g. 10.6) to use when building with ninja.')
562 parser.add_argument('--no-arm-trusted', action='store_true',
563 help='Disable building of ARM trusted components (sel_ldr, etc).')
564 parser.add_argument('--upload', action='store_true',
565 help='Upload tarballs to GCS.')
567 global options
568 options = parser.parse_args(args)
570 toolchains = ['pnacl', 'newlib', 'glibc', 'arm']
571 if PLATFORM == 'linux':
572 toolchains.append('bionic')
574 MakeVersionJson()
575 for tc in toolchains:
576 MakeToolchainArchive(tc)
577 MakeGypArchives()
578 MakePNaClArchives()
579 MakeToolchainHeaderArchives()
580 MakePepperArchive()
581 if options.upload:
582 UploadArchives()
584 return 0
586 if __name__ == '__main__':
587 try:
588 sys.exit(main(sys.argv[1:]))
589 except KeyboardInterrupt:
590 buildbot_common.ErrorExit('build_artifacts: interrupted')