tools/gn/bootstrap/bootstrap.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Bootstraps gn.
   7
   8 It is done by first building it manually in a temporary directory, then building
   9 it with its own BUILD.gn to the final destination.
  10 """
  11
  12 import contextlib
  13 import errno
  14 import logging
  15 import optparse
  16 import os
  17 import shutil
  18 import subprocess
  19 import sys
  20 import tempfile
  21
  22 BOOTSTRAP_DIR = os.path.dirname(os.path.abspath(__file__))
  23 GN_ROOT = os.path.dirname(BOOTSTRAP_DIR)
  24 SRC_ROOT = os.path.dirname(os.path.dirname(GN_ROOT))
  25
  26 is_linux = sys.platform.startswith('linux')
  27 is_mac = sys.platform.startswith('darwin')
  28 is_posix = is_linux or is_mac
  29
  30 def check_call(cmd, **kwargs):
  31   logging.debug('Running: %s', ' '.join(cmd))
  32   subprocess.check_call(cmd, cwd=GN_ROOT, **kwargs)
  33
  34 def mkdir_p(path):
  35   try:
  36     os.makedirs(path)
  37   except OSError as e:
  38     if e.errno == errno.EEXIST and os.path.isdir(path):
  39       pass
  40     else: raise
  41
  42 @contextlib.contextmanager
  43 def scoped_tempdir():
  44   path = tempfile.mkdtemp()
  45   try:
  46     yield path
  47   finally:
  48     shutil.rmtree(path)
  49
  50
  51 def run_build(tempdir, options):
  52   if options.debug:
  53     build_rel = os.path.join('out', 'Debug')
  54   else:
  55     build_rel = os.path.join('out', 'Release')
  56   build_root = os.path.join(SRC_ROOT, build_rel)
  57
  58   print 'Building gn manually in a temporary directory for bootstrapping...'
  59   build_gn_with_ninja_manually(tempdir, options)
  60   temp_gn = os.path.join(tempdir, 'gn')
  61   out_gn = os.path.join(build_root, 'gn')
  62
  63   if options.no_rebuild:
  64     mkdir_p(build_root)
  65     shutil.copy2(temp_gn, out_gn)
  66   else:
  67     print 'Building gn using itself to %s...' % build_rel
  68     build_gn_with_gn(temp_gn, build_rel, options)
  69
  70   if options.output:
  71     # Preserve the executable permission bit.
  72     shutil.copy2(out_gn, options.output)
  73
  74
  75 def main(argv):
  76   parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
  77   parser.add_option('-d', '--debug', action='store_true',
  78                     help='Do a debug build. Defaults to release build.')
  79   parser.add_option('-o', '--output',
  80                     help='place output in PATH', metavar='PATH')
  81   parser.add_option('-s', '--no-rebuild', action='store_true',
  82                     help='Do not rebuild GN with GN.')
  83   parser.add_option('--no-clean', action='store_true',
  84                     help='Re-used build directory instead of using new '
  85                          'temporary location each time')
  86   parser.add_option('-v', '--verbose', action='store_true',
  87                     help='Log more details')
  88   options, args = parser.parse_args(argv)
  89
  90   if args:
  91     parser.error('Unrecognized command line arguments: %s.' % ', '.join(args))
  92
  93   logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR)
  94
  95   try:
  96     if options.no_clean:
  97       build_dir = os.path.join(SRC_ROOT, 'out_bootstrap')
  98       if not os.path.exists(build_dir):
  99         os.makedirs(build_dir)
 100       return run_build(build_dir, options)
 101     else:
 102       with scoped_tempdir() as tempdir:
 103         return run_build(tempdir, options)
 104   except subprocess.CalledProcessError as e:
 105     print >> sys.stderr, str(e)
 106     return 1
 107   return 0
 108
 109
 110 def build_gn_with_ninja_manually(tempdir, options):
 111   write_ninja(os.path.join(tempdir, 'build.ninja'), options)
 112   cmd = ['ninja', '-C', tempdir]
 113   if options.verbose:
 114     cmd.append('-v')
 115   cmd.append('gn')
 116   check_call(cmd)
 117
 118 def write_ninja(path, options):
 119   cc = os.environ.get('CC', '')
 120   cxx = os.environ.get('CXX', '')
 121   cflags = os.environ.get('CFLAGS', '').split()
 122   cflags_cc = os.environ.get('CXXFLAGS', '').split()
 123   ld = os.environ.get('LD', cxx)
 124   ldflags = os.environ.get('LDFLAGS', '').split()
 125   include_dirs = [SRC_ROOT]
 126   libs = []
 127
 128   if is_posix:
 129     if options.debug:
 130       cflags.extend(['-O0', '-g'])
 131     else:
 132       cflags.extend(['-O2', '-g0'])
 133
 134     cflags.extend(['-D_FILE_OFFSET_BITS=64', '-pthread', '-pipe'])
 135     cflags_cc.extend(['-std=gnu++11', '-Wno-c++11-narrowing'])
 136
 137   static_libraries = {
 138       'base': {'sources': [], 'tool': 'cxx'},
 139       'dynamic_annotations': {'sources': [], 'tool': 'cc'},
 140       'gn': {'sources': [], 'tool': 'cxx'},
 141   }
 142
 143   for name in os.listdir(GN_ROOT):
 144     if not name.endswith('.cc'):
 145       continue
 146     if name.endswith('_unittest.cc'):
 147       continue
 148     if name in ['generate_test_gn_data.cc', 'run_all_unittests.cc']:
 149       continue
 150     full_path = os.path.join(GN_ROOT, name)
 151     static_libraries['gn']['sources'].append(
 152         os.path.relpath(full_path, SRC_ROOT))
 153
 154   static_libraries['dynamic_annotations']['sources'].extend([
 155       'base/third_party/dynamic_annotations/dynamic_annotations.c',
 156       'base/third_party/superfasthash/superfasthash.c',
 157   ])
 158   static_libraries['base']['sources'].extend([
 159       'base/at_exit.cc',
 160       'base/base_paths.cc',
 161       'base/base_switches.cc',
 162       'base/callback_internal.cc',
 163       'base/command_line.cc',
 164       'base/debug/alias.cc',
 165       'base/debug/stack_trace.cc',
 166       'base/debug/task_annotator.cc',
 167       'base/environment.cc',
 168       'base/files/file.cc',
 169       'base/files/file_enumerator.cc',
 170       'base/files/file_path.cc',
 171       'base/files/file_path_constants.cc',
 172       'base/files/file_tracing.cc',
 173       'base/files/file_util.cc',
 174       'base/files/scoped_file.cc',
 175       'base/hash.cc',
 176       'base/json/json_parser.cc',
 177       'base/json/json_reader.cc',
 178       'base/json/json_string_value_serializer.cc',
 179       'base/json/json_writer.cc',
 180       'base/json/string_escape.cc',
 181       'base/lazy_instance.cc',
 182       'base/location.cc',
 183       'base/logging.cc',
 184       'base/memory/ref_counted.cc',
 185       'base/memory/ref_counted_memory.cc',
 186       'base/memory/singleton.cc',
 187       'base/memory/weak_ptr.cc',
 188       'base/message_loop/incoming_task_queue.cc',
 189       'base/message_loop/message_loop.cc',
 190       'base/message_loop/message_loop_task_runner.cc',
 191       'base/message_loop/message_pump.cc',
 192       'base/message_loop/message_pump_default.cc',
 193       'base/metrics/bucket_ranges.cc',
 194       'base/metrics/histogram.cc',
 195       'base/metrics/histogram_base.cc',
 196       'base/metrics/histogram_samples.cc',
 197       'base/metrics/sample_map.cc',
 198       'base/metrics/sample_vector.cc',
 199       'base/metrics/sparse_histogram.cc',
 200       'base/metrics/statistics_recorder.cc',
 201       'base/path_service.cc',
 202       'base/pending_task.cc',
 203       'base/pickle.cc',
 204       'base/process/kill.cc',
 205       'base/process/process_iterator.cc',
 206       'base/process/process_metrics.cc',
 207       'base/profiler/alternate_timer.cc',
 208       'base/profiler/tracked_time.cc',
 209       'base/run_loop.cc',
 210       'base/sequence_checker_impl.cc',
 211       'base/sequenced_task_runner.cc',
 212       'base/sha1_portable.cc',
 213       'base/strings/pattern.cc',
 214       'base/strings/string16.cc',
 215       'base/strings/string_number_conversions.cc',
 216       'base/strings/string_piece.cc',
 217       'base/strings/string_split.cc',
 218       'base/strings/string_util.cc',
 219       'base/strings/string_util_constants.cc',
 220       'base/strings/stringprintf.cc',
 221       'base/strings/utf_string_conversion_utils.cc',
 222       'base/strings/utf_string_conversions.cc',
 223       'base/synchronization/cancellation_flag.cc',
 224       'base/synchronization/lock.cc',
 225       'base/sys_info.cc',
 226       'base/task_runner.cc',
 227       'base/third_party/dmg_fp/dtoa_wrapper.cc',
 228       'base/third_party/dmg_fp/g_fmt.cc',
 229       'base/third_party/icu/icu_utf.cc',
 230       'base/third_party/nspr/prtime.cc',
 231       'base/thread_task_runner_handle.cc',
 232       'base/threading/non_thread_safe_impl.cc',
 233       'base/threading/post_task_and_reply_impl.cc',
 234       'base/threading/sequenced_worker_pool.cc',
 235       'base/threading/simple_thread.cc',
 236       'base/threading/thread_checker_impl.cc',
 237       'base/threading/thread_collision_warner.cc',
 238       'base/threading/thread_id_name_manager.cc',
 239       'base/threading/thread_local_storage.cc',
 240       'base/threading/thread_restrictions.cc',
 241       'base/threading/worker_pool.cc',
 242       'base/time/time.cc',
 243       'base/timer/elapsed_timer.cc',
 244       'base/timer/timer.cc',
 245       'base/trace_event/memory_allocator_dump.cc',
 246       'base/trace_event/memory_allocator_dump_guid.cc',
 247       'base/trace_event/memory_dump_manager.cc',
 248       'base/trace_event/memory_dump_request_args.cc',
 249       'base/trace_event/memory_dump_session_state.cc',
 250       'base/trace_event/process_memory_dump.cc',
 251       'base/trace_event/process_memory_maps.cc',
 252       'base/trace_event/process_memory_maps_dump_provider.cc',
 253       'base/trace_event/process_memory_totals.cc',
 254       'base/trace_event/process_memory_totals_dump_provider.cc',
 255       'base/trace_event/trace_buffer.cc',
 256       'base/trace_event/trace_config.cc',
 257       'base/trace_event/trace_event_argument.cc',
 258       'base/trace_event/trace_event_impl.cc',
 259       'base/trace_event/trace_event_memory.cc',
 260       'base/trace_event/trace_event_memory_overhead.cc',
 261       'base/trace_event/trace_event_synthetic_delay.cc',
 262       'base/trace_event/trace_log.cc',
 263       'base/trace_event/trace_log_constants.cc',
 264       'base/trace_event/trace_sampling_thread.cc',
 265       'base/tracked_objects.cc',
 266       'base/tracking_info.cc',
 267       'base/values.cc',
 268       'base/vlog.cc',
 269   ])
 270
 271   if is_posix:
 272     static_libraries['base']['sources'].extend([
 273         'base/base_paths_posix.cc',
 274         'base/debug/debugger_posix.cc',
 275         'base/debug/stack_trace_posix.cc',
 276         'base/files/file_enumerator_posix.cc',
 277         'base/files/file_posix.cc',
 278         'base/files/file_util_posix.cc',
 279         'base/message_loop/message_pump_libevent.cc',
 280         'base/posix/file_descriptor_shuffle.cc',
 281         'base/posix/safe_strerror.cc',
 282         'base/process/kill_posix.cc',
 283         'base/process/process_handle_posix.cc',
 284         'base/process/process_metrics_posix.cc',
 285         'base/process/process_posix.cc',
 286         'base/synchronization/condition_variable_posix.cc',
 287         'base/synchronization/lock_impl_posix.cc',
 288         'base/synchronization/waitable_event_posix.cc',
 289         'base/sys_info_posix.cc',
 290         'base/threading/platform_thread_internal_posix.cc',
 291         'base/threading/platform_thread_posix.cc',
 292         'base/threading/thread_local_posix.cc',
 293         'base/threading/thread_local_storage_posix.cc',
 294         'base/threading/worker_pool_posix.cc',
 295         'base/time/time_posix.cc',
 296     ])
 297     static_libraries['libevent'] = {
 298         'sources': [
 299             'third_party/libevent/buffer.c',
 300             'third_party/libevent/evbuffer.c',
 301             'third_party/libevent/evdns.c',
 302             'third_party/libevent/event.c',
 303             'third_party/libevent/event_tagging.c',
 304             'third_party/libevent/evrpc.c',
 305             'third_party/libevent/evutil.c',
 306             'third_party/libevent/http.c',
 307             'third_party/libevent/log.c',
 308             'third_party/libevent/poll.c',
 309             'third_party/libevent/select.c',
 310             'third_party/libevent/signal.c',
 311             'third_party/libevent/strlcpy.c',
 312         ],
 313         'tool': 'cc',
 314         'include_dirs': [],
 315         'cflags': cflags + ['-DHAVE_CONFIG_H'],
 316     }
 317
 318
 319   if is_linux:
 320     libs.extend(['-lrt'])
 321     ldflags.extend(['-pthread'])
 322
 323     static_libraries['xdg_user_dirs'] = {
 324         'sources': [
 325             'base/third_party/xdg_user_dirs/xdg_user_dir_lookup.cc',
 326         ],
 327         'tool': 'cxx',
 328     }
 329     static_libraries['base']['sources'].extend([
 330         'base/nix/xdg_util.cc',
 331         'base/process/internal_linux.cc',
 332         'base/process/process_handle_linux.cc',
 333         'base/process/process_iterator_linux.cc',
 334         'base/process/process_linux.cc',
 335         'base/process/process_metrics_linux.cc',
 336         'base/strings/sys_string_conversions_posix.cc',
 337         'base/sys_info_linux.cc',
 338         'base/threading/platform_thread_linux.cc',
 339         'base/trace_event/malloc_dump_provider.cc',
 340     ])
 341     static_libraries['libevent']['include_dirs'].extend([
 342         os.path.join(SRC_ROOT, 'third_party', 'libevent', 'linux')
 343     ])
 344     static_libraries['libevent']['sources'].extend([
 345         'third_party/libevent/epoll.c',
 346     ])
 347
 348
 349   if is_mac:
 350     static_libraries['base']['sources'].extend([
 351         'base/base_paths_mac.mm',
 352         'base/files/file_util_mac.mm',
 353         'base/mac/bundle_locations.mm',
 354         'base/mac/call_with_eh_frame.cc',
 355         'base/mac/call_with_eh_frame_asm.S',
 356         'base/mac/foundation_util.mm',
 357         'base/mac/mach_logging.cc',
 358         'base/mac/scoped_mach_port.cc',
 359         'base/mac/scoped_nsautorelease_pool.mm',
 360         'base/message_loop/message_pump_mac.mm',
 361         'base/process/process_handle_mac.cc',
 362         'base/process/process_iterator_mac.cc',
 363         'base/process/process_metrics_mac.cc',
 364         'base/strings/sys_string_conversions_mac.mm',
 365         'base/time/time_mac.cc',
 366         'base/threading/platform_thread_mac.mm',
 367     ])
 368     static_libraries['libevent']['include_dirs'].extend([
 369         os.path.join(SRC_ROOT, 'third_party', 'libevent', 'mac')
 370     ])
 371     static_libraries['libevent']['sources'].extend([
 372         'third_party/libevent/kqueue.c',
 373     ])
 374
 375
 376   if is_mac:
 377     template_filename = 'build_mac.ninja.template'
 378   else:
 379     template_filename = 'build.ninja.template'
 380
 381   with open(os.path.join(GN_ROOT, 'bootstrap', template_filename)) as f:
 382     ninja_template = f.read()
 383
 384   def src_to_obj(path):
 385     return '%s' % os.path.splitext(path)[0] + '.o'
 386
 387   ninja_lines = []
 388   for library, settings in static_libraries.iteritems():
 389     for src_file in settings['sources']:
 390       ninja_lines.extend([
 391           'build %s: %s %s' % (src_to_obj(src_file),
 392                                settings['tool'],
 393                                os.path.join(SRC_ROOT, src_file)),
 394           '  includes = %s' % ' '.join(
 395               ['-I' + dirname for dirname in
 396                include_dirs + settings.get('include_dirs', [])]),
 397           '  cflags = %s' % ' '.join(cflags + settings.get('cflags', [])),
 398           '  cflags_cc = %s' %
 399               ' '.join(cflags_cc + settings.get('cflags_cc', [])),
 400       ])
 401       if cc:
 402         ninja_lines.append('  cc = %s' % cc)
 403       if cxx:
 404         ninja_lines.append('  cxx = %s' % cxx)
 405
 406     ninja_lines.append('build %s.a: alink_thin %s' % (
 407         library,
 408         ' '.join([src_to_obj(src_file) for src_file in settings['sources']])))
 409
 410   if is_mac:
 411     libs.extend([
 412         '-framework', 'AppKit',
 413         '-framework', 'CoreFoundation',
 414         '-framework', 'Foundation',
 415         '-framework', 'Security',
 416     ]);
 417
 418   ninja_lines.extend([
 419       'build gn: link %s' % (
 420           ' '.join(['%s.a' % library for library in static_libraries])),
 421       '  ldflags = %s' % ' '.join(ldflags),
 422       '  libs = %s' % ' '.join(libs),
 423   ])
 424   if ld:
 425     ninja_lines.append('  ld = %s' % ld)
 426   else:
 427     ninja_lines.append('  ld = $ldxx')
 428
 429   ninja_lines.append('')  # Make sure the file ends with a newline.
 430
 431   with open(path, 'w') as f:
 432     f.write(ninja_template + '\n'.join(ninja_lines))
 433
 434
 435 def build_gn_with_gn(temp_gn, build_dir, options):
 436   cmd = [temp_gn, 'gen', build_dir]
 437   if not options.debug:
 438     cmd.append('--args=is_debug=false')
 439   check_call(cmd)
 440
 441   cmd = ['ninja', '-C', build_dir]
 442   if options.verbose:
 443     cmd.append('-v')
 444   cmd.append('gn')
 445   check_call(cmd)
 446
 447   if not options.debug:
 448     check_call(['strip', os.path.join(build_dir, 'gn')])
 449
 450
 451 if __name__ == '__main__':
 452   sys.exit(main(sys.argv[1:]))