native_client_sdk/src/build_tools/build_projects.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import argparse
   7 import multiprocessing
   8 import os
   9 import posixpath
  10 import sys
  11 import urllib2
  12
  13 import buildbot_common
  14 import build_version
  15 import generate_make
  16 import parse_dsc
  17
  18 from build_paths import SDK_SRC_DIR, OUT_DIR, SDK_RESOURCE_DIR
  19 from build_paths import GSTORE
  20 from generate_index import LandingPage
  21
  22 sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
  23 import getos
  24
  25
  26 MAKE = 'nacl_sdk/make_3.99.90-26-gf80222c/make.exe'
  27 LIB_DICT = {
  28   'linux': [],
  29   'mac': [],
  30   'win': ['x86_32']
  31 }
  32 VALID_TOOLCHAINS = [
  33   'bionic',
  34   'newlib',
  35   'clang-newlib',
  36   'glibc',
  37   'pnacl',
  38   'win',
  39   'linux',
  40   'mac',
  41 ]
  42
  43 # Global verbosity setting.
  44 # If set to True (normally via a command line arg) then build_projects will
  45 # add V=1 to all calls to 'make'
  46 verbose = False
  47
  48
  49 def Trace(msg):
  50   if verbose:
  51     sys.stderr.write(str(msg) + '\n')
  52
  53
  54 def CopyFilesFromTo(filelist, srcdir, dstdir):
  55   for filename in filelist:
  56     srcpath = os.path.join(srcdir, filename)
  57     dstpath = os.path.join(dstdir, filename)
  58     buildbot_common.CopyFile(srcpath, dstpath)
  59
  60
  61 def UpdateHelpers(pepperdir, clobber=False):
  62   tools_dir = os.path.join(pepperdir, 'tools')
  63   if not os.path.exists(tools_dir):
  64     buildbot_common.ErrorExit('SDK tools dir is missing: %s' % tools_dir)
  65
  66   exampledir = os.path.join(pepperdir, 'examples')
  67   if clobber:
  68     buildbot_common.RemoveDir(exampledir)
  69   buildbot_common.MakeDir(exampledir)
  70
  71   # Copy files for individual build and landing page
  72   files = ['favicon.ico', 'httpd.cmd', 'index.css', 'index.js',
  73       'button_close.png', 'button_close_hover.png']
  74   CopyFilesFromTo(files, SDK_RESOURCE_DIR, exampledir)
  75
  76   # Copy tools scripts and make includes
  77   buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.py'),
  78       tools_dir)
  79   buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', '*.mk'),
  80       tools_dir)
  81
  82   # Copy tools/lib scripts
  83   tools_lib_dir = os.path.join(pepperdir, 'tools', 'lib')
  84   buildbot_common.MakeDir(tools_lib_dir)
  85   buildbot_common.CopyDir(os.path.join(SDK_SRC_DIR, 'tools', 'lib', '*.py'),
  86       tools_lib_dir)
  87
  88   # On Windows add a prebuilt make
  89   if getos.GetPlatform() == 'win':
  90     buildbot_common.BuildStep('Add MAKE')
  91     make_url = posixpath.join(GSTORE, MAKE)
  92     make_exe = os.path.join(tools_dir, 'make.exe')
  93     with open(make_exe, 'wb') as f:
  94       f.write(urllib2.urlopen(make_url).read())
  95
  96
  97 def ValidateToolchains(toolchains):
  98   invalid_toolchains = set(toolchains) - set(VALID_TOOLCHAINS)
  99   if invalid_toolchains:
 100     buildbot_common.ErrorExit('Invalid toolchain(s): %s' % (
 101         ', '.join(invalid_toolchains)))
 102
 103
 104 def GetDeps(projects):
 105   out = {}
 106
 107   # Build list of all project names
 108   localtargets = [proj['NAME'] for proj in projects]
 109
 110   # For each project
 111   for proj in projects:
 112     deplist = []
 113     # generate a list of dependencies
 114     for targ in proj.get('TARGETS', []):
 115       deplist.extend(targ.get('DEPS', []) + targ.get('LIBS', []))
 116
 117     # and add dependencies to targets built in this subtree
 118     localdeps = [dep for dep in deplist if dep in localtargets]
 119     if localdeps:
 120       out[proj['NAME']] = localdeps
 121
 122   return out
 123
 124
 125 def UpdateProjects(pepperdir, project_tree, toolchains,
 126                    clobber=False, configs=None, first_toolchain=False):
 127   if configs is None:
 128     configs = ['Debug', 'Release']
 129   if not os.path.exists(os.path.join(pepperdir, 'tools')):
 130     buildbot_common.ErrorExit('Examples depend on missing tools.')
 131   if not os.path.exists(os.path.join(pepperdir, 'toolchain')):
 132     buildbot_common.ErrorExit('Examples depend on missing toolchains.')
 133
 134   ValidateToolchains(toolchains)
 135
 136   # Create the library output directories
 137   libdir = os.path.join(pepperdir, 'lib')
 138   platform = getos.GetPlatform()
 139   for config in configs:
 140     for arch in LIB_DICT[platform]:
 141       dirpath = os.path.join(libdir, '%s_%s_host' % (platform, arch), config)
 142       if clobber:
 143         buildbot_common.RemoveDir(dirpath)
 144       buildbot_common.MakeDir(dirpath)
 145
 146   landing_page = None
 147   for branch, projects in project_tree.iteritems():
 148     dirpath = os.path.join(pepperdir, branch)
 149     if clobber:
 150       buildbot_common.RemoveDir(dirpath)
 151     buildbot_common.MakeDir(dirpath)
 152     targets = [desc['NAME'] for desc in projects]
 153     deps = GetDeps(projects)
 154
 155     # Generate master make for this branch of projects
 156     generate_make.GenerateMasterMakefile(pepperdir,
 157                                          os.path.join(pepperdir, branch),
 158                                          targets, deps)
 159
 160     if branch.startswith('examples') and not landing_page:
 161       landing_page = LandingPage()
 162
 163     # Generate individual projects
 164     for desc in projects:
 165       srcroot = os.path.dirname(desc['FILEPATH'])
 166       generate_make.ProcessProject(pepperdir, srcroot, pepperdir, desc,
 167                                    toolchains, configs=configs,
 168                                    first_toolchain=first_toolchain)
 169
 170       if branch.startswith('examples'):
 171         landing_page.AddDesc(desc)
 172
 173   if landing_page:
 174     # Generate the landing page text file.
 175     index_html = os.path.join(pepperdir, 'examples', 'index.html')
 176     index_template = os.path.join(SDK_RESOURCE_DIR, 'index.html.template')
 177     with open(index_html, 'w') as fh:
 178       out = landing_page.GeneratePage(index_template)
 179       fh.write(out)
 180
 181   # Generate top Make for examples
 182   targets = ['api', 'demo', 'getting_started', 'tutorial']
 183   targets = [x for x in targets if 'examples/'+x in project_tree]
 184   branch_name = 'examples'
 185   generate_make.GenerateMasterMakefile(pepperdir,
 186                                        os.path.join(pepperdir, branch_name),
 187                                        targets, {})
 188
 189
 190 def BuildProjectsBranch(pepperdir, branch, deps, clean, config, args=None):
 191   make_dir = os.path.join(pepperdir, branch)
 192   print "\nMake: " + make_dir
 193
 194   if getos.GetPlatform() == 'win':
 195     # We need to modify the environment to build host on Windows.
 196     make = os.path.join(make_dir, 'make.bat')
 197   else:
 198     make = 'make'
 199
 200   env = None
 201   if os.environ.get('USE_GOMA') == '1':
 202     env = dict(os.environ)
 203     env['NACL_COMPILER_PREFIX'] = 'gomacc'
 204     # Add -m32 to the CFLAGS when building using i686-nacl-gcc
 205     # otherwise goma won't recognise it as different to the x86_64
 206     # build.
 207     env['X86_32_CFLAGS'] = '-m32'
 208     env['X86_32_CXXFLAGS'] = '-m32'
 209     jobs = '50'
 210   else:
 211     jobs = str(multiprocessing.cpu_count())
 212
 213   make_cmd = [make, '-j', jobs]
 214
 215   make_cmd.append('CONFIG='+config)
 216   # We always ENABLE_BIONIC in case we need it.  If neither --bionic nor
 217   # -t bionic have been provided on the command line, then VALID_TOOLCHAINS
 218   # will not contain a bionic target.
 219   make_cmd.append('ENABLE_BIONIC=1')
 220   if not deps:
 221     make_cmd.append('IGNORE_DEPS=1')
 222
 223   if verbose:
 224     make_cmd.append('V=1')
 225
 226   if args:
 227     make_cmd += args
 228   else:
 229     make_cmd.append('TOOLCHAIN=all')
 230
 231   buildbot_common.Run(make_cmd, cwd=make_dir, env=env)
 232   if clean:
 233     # Clean to remove temporary files but keep the built
 234     buildbot_common.Run(make_cmd + ['clean'], cwd=make_dir, env=env)
 235
 236
 237 def BuildProjects(pepperdir, project_tree, deps=True,
 238                   clean=False, config='Debug'):
 239   # Make sure we build libraries (which live in 'src') before
 240   # any of the examples.
 241   build_first = [p for p in project_tree if p != 'src']
 242   build_second = [p for p in project_tree if p == 'src']
 243
 244   for branch in build_first + build_second:
 245     BuildProjectsBranch(pepperdir, branch, deps, clean, config)
 246
 247
 248 def main(args):
 249   parser = argparse.ArgumentParser(description=__doc__)
 250   parser.add_argument('-c', '--clobber',
 251       help='Clobber project directories before copying new files',
 252       action='store_true', default=False)
 253   parser.add_argument('-b', '--build',
 254       help='Build the projects. Otherwise the projects are only copied.',
 255       action='store_true')
 256   parser.add_argument('--config',
 257       help='Choose configuration to build (Debug or Release).  Builds both '
 258            'by default')
 259   parser.add_argument('--bionic',
 260       help='Enable bionic projects', action='store_true')
 261   parser.add_argument('-x', '--experimental',
 262       help='Build experimental projects', action='store_true')
 263   parser.add_argument('-t', '--toolchain',
 264       help='Build using toolchain. Can be passed more than once.',
 265       action='append', default=[])
 266   parser.add_argument('-d', '--dest',
 267       help='Select which build destinations (project types) are valid.',
 268       action='append')
 269   parser.add_argument('projects', nargs='*',
 270       help='Select which projects to build.')
 271   parser.add_argument('-v', '--verbose', action='store_true')
 272
 273   # To setup bash completion for this command first install optcomplete
 274   # and then add this line to your .bashrc:
 275   #  complete -F _optcomplete build_projects.py
 276   try:
 277     import optcomplete
 278     optcomplete.autocomplete(parser)
 279   except ImportError:
 280     pass
 281
 282   options = parser.parse_args(args)
 283
 284   global verbose
 285   if options.verbose:
 286     verbose = True
 287
 288   buildbot_common.verbose = verbose
 289
 290   if 'NACL_SDK_ROOT' in os.environ:
 291     # We don't want the currently configured NACL_SDK_ROOT to have any effect
 292     # on the build.
 293     del os.environ['NACL_SDK_ROOT']
 294
 295   pepper_ver = str(int(build_version.ChromeMajorVersion()))
 296   pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
 297
 298   if not options.toolchain:
 299     # Order matters here: the default toolchain for an example's Makefile will
 300     # be the first toolchain in this list that is available in the example.
 301     # e.g. If an example supports newlib and glibc, then the default will be
 302     # newlib.
 303     options.toolchain = ['pnacl', 'newlib', 'glibc', 'host', 'clang-newlib']
 304     if options.experimental or options.bionic:
 305       options.toolchain.append('bionic')
 306
 307   if 'host' in options.toolchain:
 308     options.toolchain.remove('host')
 309     options.toolchain.append(getos.GetPlatform())
 310     Trace('Adding platform: ' + getos.GetPlatform())
 311
 312   ValidateToolchains(options.toolchain)
 313
 314   filters = {}
 315   if options.toolchain:
 316     filters['TOOLS'] = options.toolchain
 317     Trace('Filter by toolchain: ' + str(options.toolchain))
 318   if not options.experimental:
 319     filters['EXPERIMENTAL'] = False
 320   if options.dest:
 321     filters['DEST'] = options.dest
 322     Trace('Filter by type: ' + str(options.dest))
 323   if options.projects:
 324     filters['NAME'] = options.projects
 325     Trace('Filter by name: ' + str(options.projects))
 326
 327   try:
 328     project_tree = parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=filters)
 329   except parse_dsc.ValidationError as e:
 330     buildbot_common.ErrorExit(str(e))
 331
 332   if verbose:
 333     parse_dsc.PrintProjectTree(project_tree)
 334
 335   UpdateHelpers(pepperdir, clobber=options.clobber)
 336   UpdateProjects(pepperdir, project_tree, options.toolchain,
 337                  clobber=options.clobber)
 338
 339   if options.build:
 340     if options.config:
 341       configs = [options.config]
 342     else:
 343       configs = ['Debug', 'Release']
 344     for config in configs:
 345       BuildProjects(pepperdir, project_tree, config=config, deps=False)
 346
 347   return 0
 348
 349
 350 if __name__ == '__main__':
 351   script_name = os.path.basename(sys.argv[0])
 352   try:
 353     sys.exit(main(sys.argv[1:]))
 354   except parse_dsc.ValidationError as e:
 355     buildbot_common.ErrorExit('%s: %s' % (script_name, e))
 356   except KeyboardInterrupt:
 357     buildbot_common.ErrorExit('%s: interrupted' % script_name)