tools/gn/bootstrap/bootstrap.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Bootstraps gn.
   7
   8 It is done by first building it manually in a temporary directory, then building
   9 it with its own BUILD.gn to the final destination.
  10 """
  11
  12 import contextlib
  13 import errno
  14 import logging
  15 import optparse
  16 import os
  17 import shutil
  18 import subprocess
  19 import sys
  20 import tempfile
  21
  22 BOOTSTRAP_DIR = os.path.dirname(os.path.abspath(__file__))
  23 GN_ROOT = os.path.dirname(BOOTSTRAP_DIR)
  24 SRC_ROOT = os.path.dirname(os.path.dirname(GN_ROOT))
  25
  26 is_linux = sys.platform.startswith('linux')
  27 is_mac = sys.platform.startswith('darwin')
  28 is_posix = is_linux or is_mac
  29
  30 def check_call(cmd, **kwargs):
  31   logging.debug('Running: %s', ' '.join(cmd))
  32   subprocess.check_call(cmd, cwd=GN_ROOT, **kwargs)
  33
  34 def mkdir_p(path):
  35   try:
  36     os.makedirs(path)
  37   except OSError as e:
  38     if e.errno == errno.EEXIST and os.path.isdir(path):
  39       pass
  40     else: raise
  41
  42 @contextlib.contextmanager
  43 def scoped_tempdir():
  44   path = tempfile.mkdtemp()
  45   try:
  46     yield path
  47   finally:
  48     shutil.rmtree(path)
  49
  50
  51 def run_build(tempdir, options):
  52   if options.debug:
  53     build_rel = os.path.join('out', 'Debug')
  54   else:
  55     build_rel = os.path.join('out', 'Release')
  56   build_root = os.path.join(SRC_ROOT, build_rel)
  57
  58   print 'Building gn manually in a temporary directory for bootstrapping...'
  59   build_gn_with_ninja_manually(tempdir, options)
  60   temp_gn = os.path.join(tempdir, 'gn')
  61   out_gn = os.path.join(build_root, 'gn')
  62
  63   if options.no_rebuild:
  64     mkdir_p(build_root)
  65     shutil.copy2(temp_gn, out_gn)
  66   else:
  67     print 'Building gn using itself to %s...' % build_rel
  68     build_gn_with_gn(temp_gn, build_rel, options)
  69
  70   if options.output:
  71     # Preserve the executable permission bit.
  72     shutil.copy2(out_gn, options.output)
  73
  74
  75 def main(argv):
  76   parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
  77   parser.add_option('-d', '--debug', action='store_true',
  78                     help='Do a debug build. Defaults to release build.')
  79   parser.add_option('-o', '--output',
  80                     help='place output in PATH', metavar='PATH')
  81   parser.add_option('-s', '--no-rebuild', action='store_true',
  82                     help='Do not rebuild GN with GN.')
  83   parser.add_option('--no-clean', action='store_true',
  84                     help='Re-used build directory instead of using new '
  85                          'temporary location each time')
  86   parser.add_option('-v', '--verbose', action='store_true',
  87                     help='Log more details')
  88   options, args = parser.parse_args(argv)
  89
  90   if args:
  91     parser.error('Unrecognized command line arguments: %s.' % ', '.join(args))
  92
  93   logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR)
  94
  95   try:
  96     if options.no_clean:
  97       build_dir = os.path.join(SRC_ROOT, 'out_bootstrap')
  98       if not os.path.exists(build_dir):
  99         os.makedirs(build_dir)
 100       return run_build(build_dir, options)
 101     else:
 102       with scoped_tempdir() as tempdir:
 103         return run_build(tempdir, options)
 104   except subprocess.CalledProcessError as e:
 105     print >> sys.stderr, str(e)
 106     return 1
 107   return 0
 108
 109
 110 def build_gn_with_ninja_manually(tempdir, options):
 111   write_ninja(os.path.join(tempdir, 'build.ninja'), options)
 112   cmd = ['ninja', '-C', tempdir]
 113   if options.verbose:
 114     cmd.append('-v')
 115   cmd.append('gn')
 116   check_call(cmd)
 117
 118 def write_ninja(path, options):
 119   cc = os.environ.get('CC', '')
 120   cxx = os.environ.get('CXX', '')
 121   cflags = os.environ.get('CFLAGS', '').split()
 122   cflags_cc = os.environ.get('CXXFLAGS', '').split()
 123   ld = os.environ.get('LD', cxx)
 124   ldflags = os.environ.get('LDFLAGS', '').split()
 125   include_dirs = [SRC_ROOT]
 126   libs = []
 127
 128   if is_posix:
 129     if options.debug:
 130       cflags.extend(['-O0', '-g'])
 131     else:
 132       cflags.extend(['-O2', '-g0'])
 133
 134     cflags.extend(['-D_FILE_OFFSET_BITS=64', '-pthread', '-pipe'])
 135     cflags_cc.extend(['-std=gnu++11', '-Wno-c++11-narrowing'])
 136
 137   static_libraries = {
 138       'base': {'sources': [], 'tool': 'cxx'},
 139       'dynamic_annotations': {'sources': [], 'tool': 'cc'},
 140       'gn': {'sources': [], 'tool': 'cxx'},
 141   }
 142
 143   for name in os.listdir(GN_ROOT):
 144     if not name.endswith('.cc'):
 145       continue
 146     if name.endswith('_unittest.cc'):
 147       continue
 148     if name in ['generate_test_gn_data.cc', 'run_all_unittests.cc']:
 149       continue
 150     full_path = os.path.join(GN_ROOT, name)
 151     static_libraries['gn']['sources'].append(
 152         os.path.relpath(full_path, SRC_ROOT))
 153
 154   static_libraries['dynamic_annotations']['sources'].extend([
 155       'base/third_party/dynamic_annotations/dynamic_annotations.c',
 156       'base/third_party/superfasthash/superfasthash.c',
 157   ])
 158   static_libraries['base']['sources'].extend([
 159       'base/at_exit.cc',
 160       'base/base_paths.cc',
 161       'base/base_switches.cc',
 162       'base/callback_internal.cc',
 163       'base/command_line.cc',
 164       'base/debug/alias.cc',
 165       'base/debug/stack_trace.cc',
 166       'base/debug/task_annotator.cc',
 167       'base/environment.cc',
 168       'base/files/file.cc',
 169       'base/files/file_enumerator.cc',
 170       'base/files/file_path.cc',
 171       'base/files/file_path_constants.cc',
 172       'base/files/file_tracing.cc',
 173       'base/files/file_util.cc',
 174       'base/files/scoped_file.cc',
 175       'base/hash.cc',
 176       'base/json/json_parser.cc',
 177       'base/json/json_reader.cc',
 178       'base/json/json_string_value_serializer.cc',
 179       'base/json/json_writer.cc',
 180       'base/json/string_escape.cc',
 181       'base/lazy_instance.cc',
 182       'base/location.cc',
 183       'base/logging.cc',
 184       'base/memory/ref_counted.cc',
 185       'base/memory/ref_counted_memory.cc',
 186       'base/memory/singleton.cc',
 187       'base/memory/weak_ptr.cc',
 188       'base/message_loop/incoming_task_queue.cc',
 189       'base/message_loop/message_loop.cc',
 190       'base/message_loop/message_loop_task_runner.cc',
 191       'base/message_loop/message_pump.cc',
 192       'base/message_loop/message_pump_default.cc',
 193       'base/metrics/bucket_ranges.cc',
 194       'base/metrics/histogram.cc',
 195       'base/metrics/histogram_base.cc',
 196       'base/metrics/histogram_samples.cc',
 197       'base/metrics/sample_map.cc',
 198       'base/metrics/sample_vector.cc',
 199       'base/metrics/sparse_histogram.cc',
 200       'base/metrics/statistics_recorder.cc',
 201       'base/path_service.cc',
 202       'base/pending_task.cc',
 203       'base/pickle.cc',
 204       'base/process/kill.cc',
 205       'base/process/process_iterator.cc',
 206       'base/process/process_metrics.cc',
 207       'base/profiler/alternate_timer.cc',
 208       'base/profiler/tracked_time.cc',
 209       'base/run_loop.cc',
 210       'base/sequence_checker_impl.cc',
 211       'base/sequenced_task_runner.cc',
 212       'base/sha1_portable.cc',
 213       'base/strings/pattern.cc',
 214       'base/strings/string16.cc',
 215       'base/strings/string_number_conversions.cc',
 216       'base/strings/string_piece.cc',
 217       'base/strings/string_split.cc',
 218       'base/strings/string_util.cc',
 219       'base/strings/string_util_constants.cc',
 220       'base/strings/stringprintf.cc',
 221       'base/strings/utf_string_conversion_utils.cc',
 222       'base/strings/utf_string_conversions.cc',
 223       'base/synchronization/cancellation_flag.cc',
 224       'base/synchronization/lock.cc',
 225       'base/sys_info.cc',
 226       'base/task_runner.cc',
 227       'base/third_party/dmg_fp/dtoa_wrapper.cc',
 228       'base/third_party/dmg_fp/g_fmt.cc',
 229       'base/third_party/icu/icu_utf.cc',
 230       'base/third_party/nspr/prtime.cc',
 231       'base/thread_task_runner_handle.cc',
 232       'base/threading/non_thread_safe_impl.cc',
 233       'base/threading/post_task_and_reply_impl.cc',
 234       'base/threading/sequenced_worker_pool.cc',
 235       'base/threading/simple_thread.cc',
 236       'base/threading/thread_checker_impl.cc',
 237       'base/threading/thread_collision_warner.cc',
 238       'base/threading/thread_id_name_manager.cc',
 239       'base/threading/thread_local_storage.cc',
 240       'base/threading/thread_restrictions.cc',
 241       'base/threading/worker_pool.cc',
 242       'base/time/time.cc',
 243       'base/timer/elapsed_timer.cc',
 244       'base/timer/timer.cc',
 245       'base/trace_event/malloc_dump_provider.cc',
 246       'base/trace_event/memory_allocator_dump.cc',
 247       'base/trace_event/memory_allocator_dump_guid.cc',
 248       'base/trace_event/memory_dump_manager.cc',
 249       'base/trace_event/memory_dump_request_args.cc',
 250       'base/trace_event/memory_dump_session_state.cc',
 251       'base/trace_event/process_memory_dump.cc',
 252       'base/trace_event/process_memory_maps.cc',
 253       'base/trace_event/process_memory_maps_dump_provider.cc',
 254       'base/trace_event/process_memory_totals.cc',
 255       'base/trace_event/process_memory_totals_dump_provider.cc',
 256       'base/trace_event/trace_buffer.cc',
 257       'base/trace_event/trace_config.cc',
 258       'base/trace_event/trace_event_argument.cc',
 259       'base/trace_event/trace_event_impl.cc',
 260       'base/trace_event/trace_event_memory.cc',
 261       'base/trace_event/trace_event_memory_overhead.cc',
 262       'base/trace_event/trace_event_synthetic_delay.cc',
 263       'base/trace_event/trace_log.cc',
 264       'base/trace_event/trace_log_constants.cc',
 265       'base/trace_event/trace_sampling_thread.cc',
 266       'base/tracked_objects.cc',
 267       'base/tracking_info.cc',
 268       'base/values.cc',
 269       'base/vlog.cc',
 270   ])
 271
 272   if is_posix:
 273     static_libraries['base']['sources'].extend([
 274         'base/base_paths_posix.cc',
 275         'base/debug/debugger_posix.cc',
 276         'base/debug/stack_trace_posix.cc',
 277         'base/files/file_enumerator_posix.cc',
 278         'base/files/file_posix.cc',
 279         'base/files/file_util_posix.cc',
 280         'base/message_loop/message_pump_libevent.cc',
 281         'base/posix/file_descriptor_shuffle.cc',
 282         'base/posix/safe_strerror.cc',
 283         'base/process/kill_posix.cc',
 284         'base/process/process_handle_posix.cc',
 285         'base/process/process_metrics_posix.cc',
 286         'base/process/process_posix.cc',
 287         'base/synchronization/condition_variable_posix.cc',
 288         'base/synchronization/lock_impl_posix.cc',
 289         'base/synchronization/waitable_event_posix.cc',
 290         'base/sys_info_posix.cc',
 291         'base/threading/platform_thread_internal_posix.cc',
 292         'base/threading/platform_thread_posix.cc',
 293         'base/threading/thread_local_posix.cc',
 294         'base/threading/thread_local_storage_posix.cc',
 295         'base/threading/worker_pool_posix.cc',
 296         'base/time/time_posix.cc',
 297     ])
 298     static_libraries['libevent'] = {
 299         'sources': [
 300             'third_party/libevent/buffer.c',
 301             'third_party/libevent/evbuffer.c',
 302             'third_party/libevent/evdns.c',
 303             'third_party/libevent/event.c',
 304             'third_party/libevent/event_tagging.c',
 305             'third_party/libevent/evrpc.c',
 306             'third_party/libevent/evutil.c',
 307             'third_party/libevent/http.c',
 308             'third_party/libevent/log.c',
 309             'third_party/libevent/poll.c',
 310             'third_party/libevent/select.c',
 311             'third_party/libevent/signal.c',
 312             'third_party/libevent/strlcpy.c',
 313         ],
 314         'tool': 'cc',
 315         'include_dirs': [],
 316         'cflags': cflags + ['-DHAVE_CONFIG_H'],
 317     }
 318
 319
 320   if is_linux:
 321     libs.extend(['-lrt'])
 322     ldflags.extend(['-pthread'])
 323
 324     static_libraries['xdg_user_dirs'] = {
 325         'sources': [
 326             'base/third_party/xdg_user_dirs/xdg_user_dir_lookup.cc',
 327         ],
 328         'tool': 'cxx',
 329     }
 330     static_libraries['base']['sources'].extend([
 331         'base/nix/xdg_util.cc',
 332         'base/process/internal_linux.cc',
 333         'base/process/process_handle_linux.cc',
 334         'base/process/process_iterator_linux.cc',
 335         'base/process/process_linux.cc',
 336         'base/process/process_metrics_linux.cc',
 337         'base/strings/sys_string_conversions_posix.cc',
 338         'base/sys_info_linux.cc',
 339         'base/threading/platform_thread_linux.cc',
 340     ])
 341     static_libraries['libevent']['include_dirs'].extend([
 342         os.path.join(SRC_ROOT, 'third_party', 'libevent', 'linux')
 343     ])
 344     static_libraries['libevent']['sources'].extend([
 345         'third_party/libevent/epoll.c',
 346     ])
 347
 348
 349   if is_mac:
 350     static_libraries['base']['sources'].extend([
 351         'base/base_paths_mac.mm',
 352         'base/files/file_util_mac.mm',
 353         'base/mac/bundle_locations.mm',
 354         'base/mac/foundation_util.mm',
 355         'base/mac/mach_logging.cc',
 356         'base/mac/scoped_mach_port.cc',
 357         'base/mac/scoped_nsautorelease_pool.mm',
 358         'base/message_loop/message_pump_mac.mm',
 359         'base/process/process_handle_mac.cc',
 360         'base/process/process_iterator_mac.cc',
 361         'base/strings/sys_string_conversions_mac.mm',
 362         'base/time/time_mac.cc',
 363         'base/threading/platform_thread_mac.mm',
 364     ])
 365     static_libraries['libevent']['include_dirs'].extend([
 366         os.path.join(SRC_ROOT, 'third_party', 'libevent', 'mac')
 367     ])
 368     static_libraries['libevent']['sources'].extend([
 369         'third_party/libevent/kqueue.c',
 370     ])
 371
 372
 373   if is_mac:
 374     template_filename = 'build_mac.ninja.template'
 375   else:
 376     template_filename = 'build.ninja.template'
 377
 378   with open(os.path.join(GN_ROOT, 'bootstrap', template_filename)) as f:
 379     ninja_template = f.read()
 380
 381   def src_to_obj(path):
 382     return '%s' % os.path.splitext(path)[0] + '.o'
 383
 384   ninja_lines = []
 385   for library, settings in static_libraries.iteritems():
 386     for src_file in settings['sources']:
 387       ninja_lines.extend([
 388           'build %s: %s %s' % (src_to_obj(src_file),
 389                                settings['tool'],
 390                                os.path.join(SRC_ROOT, src_file)),
 391           '  includes = %s' % ' '.join(
 392               ['-I' + dirname for dirname in
 393                include_dirs + settings.get('include_dirs', [])]),
 394           '  cflags = %s' % ' '.join(cflags + settings.get('cflags', [])),
 395           '  cflags_cc = %s' %
 396               ' '.join(cflags_cc + settings.get('cflags_cc', [])),
 397       ])
 398       if cc:
 399         ninja_lines.append('  cc = %s' % cc)
 400       if cxx:
 401         ninja_lines.append('  cxx = %s' % cxx)
 402
 403     ninja_lines.append('build %s.a: alink_thin %s' % (
 404         library,
 405         ' '.join([src_to_obj(src_file) for src_file in settings['sources']])))
 406
 407   if is_mac:
 408     libs.extend([
 409         '-framework', 'AppKit',
 410         '-framework', 'CoreFoundation',
 411         '-framework', 'Foundation',
 412         '-framework', 'Security',
 413     ]);
 414
 415   ninja_lines.extend([
 416       'build gn: link %s' % (
 417           ' '.join(['%s.a' % library for library in static_libraries])),
 418       '  ldflags = %s' % ' '.join(ldflags),
 419       '  libs = %s' % ' '.join(libs),
 420   ])
 421   if ld:
 422     ninja_lines.append('  ld = %s' % ld)
 423   else:
 424     ninja_lines.append('  ld = $ldxx')
 425
 426   ninja_lines.append('')  # Make sure the file ends with a newline.
 427
 428   with open(path, 'w') as f:
 429     f.write(ninja_template + '\n'.join(ninja_lines))
 430
 431
 432 def build_gn_with_gn(temp_gn, build_dir, options):
 433   cmd = [temp_gn, 'gen', build_dir]
 434   if not options.debug:
 435     cmd.append('--args=is_debug=false')
 436   check_call(cmd)
 437
 438   cmd = ['ninja', '-C', build_dir]
 439   if options.verbose:
 440     cmd.append('-v')
 441   cmd.append('gn')
 442   check_call(cmd)
 443
 444   if not options.debug:
 445     check_call(['strip', os.path.join(build_dir, 'gn')])
 446
 447
 448 if __name__ == '__main__':
 449   sys.exit(main(sys.argv[1:]))