Send a crash report when a hung process is detected.
[chromium-blink-merge.git] / native_client_sdk / src / build_tools / build_projects.py
blob0c5077a6534d24ae760dbdd43c8903d19794c0c8
1 #!/usr/bin/env python
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 import argparse
7 import multiprocessing
8 import os
9 import posixpath
10 import sys
11 import urllib2
13 import buildbot_common
14 import build_version
15 import generate_make
16 import parse_dsc
18 from build_paths import SDK_SRC_DIR, OUT_DIR, SDK_RESOURCE_DIR
19 from build_paths import GSTORE
20 from generate_index import LandingPage
22 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
23 import getos
26 MAKE = 'nacl_sdk/make_3.99.90-26-gf80222c/make.exe'
27 LIB_DICT = {
28 'linux': [],
29 'mac': [],
30 'win': ['x86_32']
32 VALID_TOOLCHAINS = [
33 'bionic',
34 'newlib',
35 'clang-newlib',
36 'glibc',
37 'pnacl',
38 'win',
39 'linux',
40 'mac',
43 # Global verbosity setting.
44 # If set to True (normally via a command line arg) then build_projects will
45 # add V=1 to all calls to 'make'
46 verbose = False
49 def Trace(msg):
50 if verbose:
51 sys.stderr.write(str(msg) + '\n')
54 def CopyFilesFromTo(filelist, srcdir, dstdir):
55 for filename in filelist:
56 srcpath = os.path.join(srcdir, filename)
57 dstpath = os.path.join(dstdir, filename)
58 buildbot_common.CopyFile(srcpath, dstpath)
61 def UpdateHelpers(pepperdir, clobber=False):
62 tools_dir = os.path.join(pepperdir, 'tools')
63 if not os.path.exists(tools_dir):
64 buildbot_common.ErrorExit('SDK tools dir is missing: %s' % tools_dir)
66 exampledir = os.path.join(pepperdir, 'examples')
67 if clobber:
68 buildbot_common.RemoveDir(exampledir)
69 buildbot_common.MakeDir(exampledir)
71 # Copy files for individual build and landing page
72 files = ['favicon.ico', 'httpd.cmd', 'index.css', 'index.js',
73 'button_close.png', 'button_close_hover.png']
74 CopyFilesFromTo(files, SDK_RESOURCE_DIR, exampledir)
76 # Copy tools scripts and make includes
77 buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
78 tools_dir)
79 buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'),
80 tools_dir)
82 # Copy tools/lib scripts
83 tools_lib_dir = os.path.join(pepperdir, 'tools', 'lib')
84 buildbot_common.MakeDir(tools_lib_dir)
85 buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', 'lib', '*.py'),
86 tools_lib_dir)
88 # On Windows add a prebuilt make
89 if getos.GetPlatform() == 'win':
90 buildbot_common.BuildStep('Add MAKE')
91 make_url = posixpath.join(GSTORE, MAKE)
92 make_exe = os.path.join(tools_dir, 'make.exe')
93 with open(make_exe, 'wb') as f:
94 f.write(urllib2.urlopen(make_url).read())
97 def ValidateToolchains(toolchains):
98 invalid_toolchains = set(toolchains) - set(VALID_TOOLCHAINS)
99 if invalid_toolchains:
100 buildbot_common.ErrorExit('Invalid toolchain(s): %s' % (
101 ', '.join(invalid_toolchains)))
104 def GetDeps(projects):
105 out = {}
107 # Build list of all project names
108 localtargets = [proj['NAME'] for proj in projects]
110 # For each project
111 for proj in projects:
112 deplist = []
113 # generate a list of dependencies
114 for targ in proj.get('TARGETS', []):
115 deplist.extend(targ.get('DEPS', []) + targ.get('LIBS', []))
117 # and add dependencies to targets built in this subtree
118 localdeps = [dep for dep in deplist if dep in localtargets]
119 if localdeps:
120 out[proj['NAME']] = localdeps
122 return out
125 def UpdateProjects(pepperdir, project_tree, toolchains,
126 clobber=False, configs=None, first_toolchain=False):
127 if configs is None:
128 configs = ['Debug', 'Release']
129 if not os.path.exists(os.path.join(pepperdir, 'tools')):
130 buildbot_common.ErrorExit('Examples depend on missing tools.')
131 if not os.path.exists(os.path.join(pepperdir, 'toolchain')):
132 buildbot_common.ErrorExit('Examples depend on missing toolchains.')
134 ValidateToolchains(toolchains)
136 # Create the library output directories
137 libdir = os.path.join(pepperdir, 'lib')
138 platform = getos.GetPlatform()
139 for config in configs:
140 for arch in LIB_DICT[platform]:
141 dirpath = os.path.join(libdir, '%s_%s_host' % (platform, arch), config)
142 if clobber:
143 buildbot_common.RemoveDir(dirpath)
144 buildbot_common.MakeDir(dirpath)
146 landing_page = None
147 for branch, projects in project_tree.iteritems():
148 dirpath = os.path.join(pepperdir, branch)
149 if clobber:
150 buildbot_common.RemoveDir(dirpath)
151 buildbot_common.MakeDir(dirpath)
152 targets = [desc['NAME'] for desc in projects]
153 deps = GetDeps(projects)
155 # Generate master make for this branch of projects
156 generate_make.GenerateMasterMakefile(pepperdir,
157 os.path.join(pepperdir, branch),
158 targets, deps)
160 if branch.startswith('examples') and not landing_page:
161 landing_page = LandingPage()
163 # Generate individual projects
164 for desc in projects:
165 srcroot = os.path.dirname(desc['FILEPATH'])
166 generate_make.ProcessProject(pepperdir, srcroot, pepperdir, desc,
167 toolchains, configs=configs,
168 first_toolchain=first_toolchain)
170 if branch.startswith('examples'):
171 landing_page.AddDesc(desc)
173 if landing_page:
174 # Generate the landing page text file.
175 index_html = os.path.join(pepperdir, 'examples', 'index.html')
176 index_template = os.path.join(SDK_RESOURCE_DIR, 'index.html.template')
177 with open(index_html, 'w') as fh:
178 out = landing_page.GeneratePage(index_template)
179 fh.write(out)
181 # Generate top Make for examples
182 targets = ['api', 'demo', 'getting_started', 'tutorial']
183 targets = [x for x in targets if 'examples/'+x in project_tree]
184 branch_name = 'examples'
185 generate_make.GenerateMasterMakefile(pepperdir,
186 os.path.join(pepperdir, branch_name),
187 targets, {})
190 def BuildProjectsBranch(pepperdir, branch, deps, clean, config, args=None):
191 make_dir = os.path.join(pepperdir, branch)
192 print "\nMake: " + make_dir
194 if getos.GetPlatform() == 'win':
195 # We need to modify the environment to build host on Windows.
196 make = os.path.join(make_dir, 'make.bat')
197 else:
198 make = 'make'
200 env = None
201 if os.environ.get('USE_GOMA') == '1':
202 env = dict(os.environ)
203 env['NACL_COMPILER_PREFIX'] = 'gomacc'
204 # Add -m32 to the CFLAGS when building using i686-nacl-gcc
205 # otherwise goma won't recognise it as different to the x86_64
206 # build.
207 env['X86_32_CFLAGS'] = '-m32'
208 env['X86_32_CXXFLAGS'] = '-m32'
209 jobs = '50'
210 else:
211 jobs = str(multiprocessing.cpu_count())
213 make_cmd = [make, '-j', jobs]
215 make_cmd.append('CONFIG='+config)
216 # We always ENABLE_BIONIC in case we need it. If neither --bionic nor
217 # -t bionic have been provided on the command line, then VALID_TOOLCHAINS
218 # will not contain a bionic target.
219 make_cmd.append('ENABLE_BIONIC=1')
220 if not deps:
221 make_cmd.append('IGNORE_DEPS=1')
223 if verbose:
224 make_cmd.append('V=1')
226 if args:
227 make_cmd += args
228 else:
229 make_cmd.append('TOOLCHAIN=all')
231 buildbot_common.Run(make_cmd, cwd=make_dir, env=env)
232 if clean:
233 # Clean to remove temporary files but keep the built
234 buildbot_common.Run(make_cmd + ['clean'], cwd=make_dir, env=env)
237 def BuildProjects(pepperdir, project_tree, deps=True,
238 clean=False, config='Debug'):
239 # Make sure we build libraries (which live in 'src') before
240 # any of the examples.
241 build_first = [p for p in project_tree if p != 'src']
242 build_second = [p for p in project_tree if p == 'src']
244 for branch in build_first + build_second:
245 BuildProjectsBranch(pepperdir, branch, deps, clean, config)
248 def main(args):
249 parser = argparse.ArgumentParser(description=__doc__)
250 parser.add_argument('-c', '--clobber',
251 help='Clobber project directories before copying new files',
252 action='store_true', default=False)
253 parser.add_argument('-b', '--build',
254 help='Build the projects. Otherwise the projects are only copied.',
255 action='store_true')
256 parser.add_argument('--config',
257 help='Choose configuration to build (Debug or Release). Builds both '
258 'by default')
259 parser.add_argument('--bionic',
260 help='Enable bionic projects', action='store_true')
261 parser.add_argument('-x', '--experimental',
262 help='Build experimental projects', action='store_true')
263 parser.add_argument('-t', '--toolchain',
264 help='Build using toolchain. Can be passed more than once.',
265 action='append', default=[])
266 parser.add_argument('-d', '--dest',
267 help='Select which build destinations (project types) are valid.',
268 action='append')
269 parser.add_argument('projects', nargs='*',
270 help='Select which projects to build.')
271 parser.add_argument('-v', '--verbose', action='store_true')
273 # To setup bash completion for this command first install optcomplete
274 # and then add this line to your .bashrc:
275 # complete -F _optcomplete build_projects.py
276 try:
277 import optcomplete
278 optcomplete.autocomplete(parser)
279 except ImportError:
280 pass
282 options = parser.parse_args(args)
284 global verbose
285 if options.verbose:
286 verbose = True
288 buildbot_common.verbose = verbose
290 if 'NACL_SDK_ROOT' in os.environ:
291 # We don't want the currently configured NACL_SDK_ROOT to have any effect
292 # on the build.
293 del os.environ['NACL_SDK_ROOT']
295 pepper_ver = str(int(build_version.ChromeMajorVersion()))
296 pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
298 if not options.toolchain:
299 # Order matters here: the default toolchain for an example's Makefile will
300 # be the first toolchain in this list that is available in the example.
301 # e.g. If an example supports newlib and glibc, then the default will be
302 # newlib.
303 options.toolchain = ['pnacl', 'newlib', 'glibc', 'host', 'clang-newlib']
304 if options.experimental or options.bionic:
305 options.toolchain.append('bionic')
307 if 'host' in options.toolchain:
308 options.toolchain.remove('host')
309 options.toolchain.append(getos.GetPlatform())
310 Trace('Adding platform: ' + getos.GetPlatform())
312 ValidateToolchains(options.toolchain)
314 filters = {}
315 if options.toolchain:
316 filters['TOOLS'] = options.toolchain
317 Trace('Filter by toolchain: ' + str(options.toolchain))
318 if not options.experimental:
319 filters['EXPERIMENTAL'] = False
320 if options.dest:
321 filters['DEST'] = options.dest
322 Trace('Filter by type: ' + str(options.dest))
323 if options.projects:
324 filters['NAME'] = options.projects
325 Trace('Filter by name: ' + str(options.projects))
327 try:
328 project_tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
329 except parse_dsc.ValidationError as e:
330 buildbot_common.ErrorExit(str(e))
332 if verbose:
333 parse_dsc.PrintProjectTree(project_tree)
335 UpdateHelpers(pepperdir, clobber=options.clobber)
336 UpdateProjects(pepperdir, project_tree, options.toolchain,
337 clobber=options.clobber)
339 if options.build:
340 if options.config:
341 configs = [options.config]
342 else:
343 configs = ['Debug', 'Release']
344 for config in configs:
345 BuildProjects(pepperdir, project_tree, config=config, deps=False)
347 return 0
350 if __name__ == '__main__':
351 script_name = os.path.basename(sys.argv[0])
352 try:
353 sys.exit(main(sys.argv[1:]))
354 except parse_dsc.ValidationError as e:
355 buildbot_common.ErrorExit('%s: %s' % (script_name, e))
356 except KeyboardInterrupt:
357 buildbot_common.ErrorExit('%s: interrupted' % script_name)