tools/gn/bootstrap/bootstrap.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Bootstraps gn.
   7
   8 It is done by first building it manually in a temporary directory, then building
   9 it with its own BUILD.gn to the final destination.
  10 """
  11
  12 import contextlib
  13 import errno
  14 import logging
  15 import optparse
  16 import os
  17 import shutil
  18 import subprocess
  19 import sys
  20 import tempfile
  21
  22 BOOTSTRAP_DIR = os.path.dirname(os.path.abspath(__file__))
  23 GN_ROOT = os.path.dirname(BOOTSTRAP_DIR)
  24 SRC_ROOT = os.path.dirname(os.path.dirname(GN_ROOT))
  25
  26 is_linux = sys.platform.startswith('linux')
  27 is_mac = sys.platform.startswith('darwin')
  28 is_posix = is_linux or is_mac
  29
  30 def check_call(cmd, **kwargs):
  31   logging.debug('Running: %s', ' '.join(cmd))
  32   subprocess.check_call(cmd, cwd=GN_ROOT, **kwargs)
  33
  34 def mkdir_p(path):
  35   try:
  36     os.makedirs(path)
  37   except OSError as e:
  38     if e.errno == errno.EEXIST and os.path.isdir(path):
  39       pass
  40     else: raise
  41
  42 @contextlib.contextmanager
  43 def scoped_tempdir():
  44   path = tempfile.mkdtemp()
  45   try:
  46     yield path
  47   finally:
  48     shutil.rmtree(path)
  49
  50
  51 def run_build(tempdir, options):
  52   if options.debug:
  53     build_rel = os.path.join('out', 'Debug')
  54   else:
  55     build_rel = os.path.join('out', 'Release')
  56   build_root = os.path.join(SRC_ROOT, build_rel)
  57
  58   print 'Building gn manually in a temporary directory for bootstrapping...'
  59   build_gn_with_ninja_manually(tempdir, options)
  60   temp_gn = os.path.join(tempdir, 'gn')
  61   out_gn = os.path.join(build_root, 'gn')
  62
  63   if options.no_rebuild:
  64     mkdir_p(build_root)
  65     shutil.copy2(temp_gn, out_gn)
  66   else:
  67     print 'Building gn using itself to %s...' % build_rel
  68     build_gn_with_gn(temp_gn, build_rel, options)
  69
  70   if options.output:
  71     # Preserve the executable permission bit.
  72     shutil.copy2(out_gn, options.output)
  73
  74
  75 def main(argv):
  76   parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
  77   parser.add_option('-d', '--debug', action='store_true',
  78                     help='Do a debug build. Defaults to release build.')
  79   parser.add_option('-o', '--output',
  80                     help='place output in PATH', metavar='PATH')
  81   parser.add_option('-s', '--no-rebuild', action='store_true',
  82                     help='Do not rebuild GN with GN.')
  83   parser.add_option('--no-clean', action='store_true',
  84                     help='Re-used build directory instead of using new '
  85                          'temporary location each time')
  86   parser.add_option('-v', '--verbose', action='store_true',
  87                     help='Log more details')
  88   options, args = parser.parse_args(argv)
  89
  90   if args:
  91     parser.error('Unrecognized command line arguments: %s.' % ', '.join(args))
  92
  93   logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR)
  94
  95   try:
  96     if options.no_clean:
  97       build_dir = os.path.join(SRC_ROOT, 'out_bootstrap')
  98       if not os.path.exists(build_dir):
  99         os.makedirs(build_dir)
 100       return run_build(build_dir, options)
 101     else:
 102       with scoped_tempdir() as tempdir:
 103         return run_build(tempdir, options)
 104   except subprocess.CalledProcessError as e:
 105     print >> sys.stderr, str(e)
 106     return 1
 107   return 0
 108
 109
 110 def build_gn_with_ninja_manually(tempdir, options):
 111   write_ninja(os.path.join(tempdir, 'build.ninja'), options)
 112   cmd = ['ninja', '-C', tempdir]
 113   if options.verbose:
 114     cmd.append('-v')
 115   cmd.append('gn')
 116   check_call(cmd)
 117
 118 def write_ninja(path, options):
 119   cc = os.environ.get('CC', '')
 120   cxx = os.environ.get('CXX', '')
 121   cflags = os.environ.get('CFLAGS', '').split()
 122   cflags_cc = os.environ.get('CXXFLAGS', '').split()
 123   ld = os.environ.get('LD', cxx)
 124   ldflags = os.environ.get('LDFLAGS', '').split()
 125   include_dirs = [SRC_ROOT]
 126   libs = []
 127
 128   if is_posix:
 129     if options.debug:
 130       cflags.extend(['-O0', '-g'])
 131     else:
 132       cflags.extend(['-O2', '-g0'])
 133
 134     cflags.extend(['-D_FILE_OFFSET_BITS=64', '-pthread', '-pipe'])
 135     cflags_cc.extend(['-std=gnu++11', '-Wno-c++11-narrowing'])
 136
 137   static_libraries = {
 138       'base': {'sources': [], 'tool': 'cxx'},
 139       'dynamic_annotations': {'sources': [], 'tool': 'cc'},
 140       'gn': {'sources': [], 'tool': 'cxx'},
 141   }
 142
 143   for name in os.listdir(GN_ROOT):
 144     if not name.endswith('.cc'):
 145       continue
 146     if name.endswith('_unittest.cc'):
 147       continue
 148     if name in ['generate_test_gn_data.cc', 'run_all_unittests.cc']:
 149       continue
 150     full_path = os.path.join(GN_ROOT, name)
 151     static_libraries['gn']['sources'].append(
 152         os.path.relpath(full_path, SRC_ROOT))
 153
 154   static_libraries['dynamic_annotations']['sources'].extend([
 155       'base/third_party/dynamic_annotations/dynamic_annotations.c',
 156       'base/third_party/superfasthash/superfasthash.c',
 157   ])
 158   static_libraries['base']['sources'].extend([
 159       'base/at_exit.cc',
 160       'base/base_paths.cc',
 161       'base/base_switches.cc',
 162       'base/callback_internal.cc',
 163       'base/command_line.cc',
 164       'base/debug/alias.cc',
 165       'base/debug/stack_trace.cc',
 166       'base/debug/task_annotator.cc',
 167       'base/environment.cc',
 168       'base/files/file.cc',
 169       'base/files/file_enumerator.cc',
 170       'base/files/file_path.cc',
 171       'base/files/file_path_constants.cc',
 172       'base/files/file_tracing.cc',
 173       'base/files/file_util.cc',
 174       'base/files/scoped_file.cc',
 175       'base/hash.cc',
 176       'base/json/json_parser.cc',
 177       'base/json/json_reader.cc',
 178       'base/json/json_string_value_serializer.cc',
 179       'base/json/json_writer.cc',
 180       'base/json/string_escape.cc',
 181       'base/lazy_instance.cc',
 182       'base/location.cc',
 183       'base/logging.cc',
 184       'base/memory/ref_counted.cc',
 185       'base/memory/ref_counted_memory.cc',
 186       'base/memory/singleton.cc',
 187       'base/memory/weak_ptr.cc',
 188       'base/message_loop/incoming_task_queue.cc',
 189       'base/message_loop/message_loop.cc',
 190       'base/message_loop/message_loop_task_runner.cc',
 191       'base/message_loop/message_pump.cc',
 192       'base/message_loop/message_pump_default.cc',
 193       'base/metrics/bucket_ranges.cc',
 194       'base/metrics/histogram.cc',
 195       'base/metrics/histogram_base.cc',
 196       'base/metrics/histogram_samples.cc',
 197       'base/metrics/sample_map.cc',
 198       'base/metrics/sample_vector.cc',
 199       'base/metrics/sparse_histogram.cc',
 200       'base/metrics/statistics_recorder.cc',
 201       'base/path_service.cc',
 202       'base/pending_task.cc',
 203       'base/pickle.cc',
 204       'base/process/kill.cc',
 205       'base/process/process_iterator.cc',
 206       'base/process/process_metrics.cc',
 207       'base/profiler/alternate_timer.cc',
 208       'base/profiler/tracked_time.cc',
 209       'base/run_loop.cc',
 210       'base/sequence_checker_impl.cc',
 211       'base/sequenced_task_runner.cc',
 212       'base/sha1_portable.cc',
 213       'base/strings/pattern.cc',
 214       'base/strings/string16.cc',
 215       'base/strings/string_number_conversions.cc',
 216       'base/strings/string_piece.cc',
 217       'base/strings/string_split.cc',
 218       'base/strings/string_util.cc',
 219       'base/strings/string_util_constants.cc',
 220       'base/strings/stringprintf.cc',
 221       'base/strings/utf_string_conversion_utils.cc',
 222       'base/strings/utf_string_conversions.cc',
 223       'base/synchronization/cancellation_flag.cc',
 224       'base/synchronization/lock.cc',
 225       'base/sys_info.cc',
 226       'base/task_runner.cc',
 227       'base/third_party/dmg_fp/dtoa_wrapper.cc',
 228       'base/third_party/dmg_fp/g_fmt.cc',
 229       'base/third_party/icu/icu_utf.cc',
 230       'base/third_party/nspr/prtime.cc',
 231       'base/thread_task_runner_handle.cc',
 232       'base/threading/non_thread_safe_impl.cc',
 233       'base/threading/post_task_and_reply_impl.cc',
 234       'base/threading/sequenced_worker_pool.cc',
 235       'base/threading/simple_thread.cc',
 236       'base/threading/thread_checker_impl.cc',
 237       'base/threading/thread_collision_warner.cc',
 238       'base/threading/thread_id_name_manager.cc',
 239       'base/threading/thread_local_storage.cc',
 240       'base/threading/thread_restrictions.cc',
 241       'base/threading/worker_pool.cc',
 242       'base/time/time.cc',
 243       'base/timer/elapsed_timer.cc',
 244       'base/timer/timer.cc',
 245       'base/trace_event/malloc_dump_provider.cc',
 246       'base/trace_event/memory_allocator_dump.cc',
 247       'base/trace_event/memory_allocator_dump_guid.cc',
 248       'base/trace_event/memory_dump_manager.cc',
 249       'base/trace_event/memory_dump_request_args.cc',
 250       'base/trace_event/memory_dump_session_state.cc',
 251       'base/trace_event/process_memory_dump.cc',
 252       'base/trace_event/process_memory_maps.cc',
 253       'base/trace_event/process_memory_maps_dump_provider.cc',
 254       'base/trace_event/process_memory_totals.cc',
 255       'base/trace_event/process_memory_totals_dump_provider.cc',
 256       'base/trace_event/trace_config.cc',
 257       'base/trace_event/trace_event_argument.cc',
 258       'base/trace_event/trace_event_impl.cc',
 259       'base/trace_event/trace_event_impl_constants.cc',
 260       'base/trace_event/trace_event_memory.cc',
 261       'base/trace_event/trace_event_memory_overhead.cc',
 262       'base/trace_event/trace_event_synthetic_delay.cc',
 263       'base/tracked_objects.cc',
 264       'base/tracking_info.cc',
 265       'base/values.cc',
 266       'base/vlog.cc',
 267   ])
 268
 269   if is_posix:
 270     static_libraries['base']['sources'].extend([
 271         'base/base_paths_posix.cc',
 272         'base/debug/debugger_posix.cc',
 273         'base/debug/stack_trace_posix.cc',
 274         'base/files/file_enumerator_posix.cc',
 275         'base/files/file_posix.cc',
 276         'base/files/file_util_posix.cc',
 277         'base/message_loop/message_pump_libevent.cc',
 278         'base/posix/file_descriptor_shuffle.cc',
 279         'base/posix/safe_strerror.cc',
 280         'base/process/kill_posix.cc',
 281         'base/process/process_handle_posix.cc',
 282         'base/process/process_metrics_posix.cc',
 283         'base/process/process_posix.cc',
 284         'base/synchronization/condition_variable_posix.cc',
 285         'base/synchronization/lock_impl_posix.cc',
 286         'base/synchronization/waitable_event_posix.cc',
 287         'base/sys_info_posix.cc',
 288         'base/threading/platform_thread_internal_posix.cc',
 289         'base/threading/platform_thread_posix.cc',
 290         'base/threading/thread_local_posix.cc',
 291         'base/threading/thread_local_storage_posix.cc',
 292         'base/threading/worker_pool_posix.cc',
 293         'base/time/time_posix.cc',
 294     ])
 295     static_libraries['libevent'] = {
 296         'sources': [
 297             'third_party/libevent/buffer.c',
 298             'third_party/libevent/evbuffer.c',
 299             'third_party/libevent/evdns.c',
 300             'third_party/libevent/event.c',
 301             'third_party/libevent/event_tagging.c',
 302             'third_party/libevent/evrpc.c',
 303             'third_party/libevent/evutil.c',
 304             'third_party/libevent/http.c',
 305             'third_party/libevent/log.c',
 306             'third_party/libevent/poll.c',
 307             'third_party/libevent/select.c',
 308             'third_party/libevent/signal.c',
 309             'third_party/libevent/strlcpy.c',
 310         ],
 311         'tool': 'cc',
 312         'include_dirs': [],
 313         'cflags': cflags + ['-DHAVE_CONFIG_H'],
 314     }
 315
 316
 317   if is_linux:
 318     libs.extend(['-lrt'])
 319     ldflags.extend(['-pthread'])
 320
 321     static_libraries['xdg_user_dirs'] = {
 322         'sources': [
 323             'base/third_party/xdg_user_dirs/xdg_user_dir_lookup.cc',
 324         ],
 325         'tool': 'cxx',
 326     }
 327     static_libraries['base']['sources'].extend([
 328         'base/nix/xdg_util.cc',
 329         'base/process/internal_linux.cc',
 330         'base/process/process_handle_linux.cc',
 331         'base/process/process_iterator_linux.cc',
 332         'base/process/process_linux.cc',
 333         'base/process/process_metrics_linux.cc',
 334         'base/strings/sys_string_conversions_posix.cc',
 335         'base/sys_info_linux.cc',
 336         'base/threading/platform_thread_linux.cc',
 337     ])
 338     static_libraries['libevent']['include_dirs'].extend([
 339         os.path.join(SRC_ROOT, 'third_party', 'libevent', 'linux')
 340     ])
 341     static_libraries['libevent']['sources'].extend([
 342         'third_party/libevent/epoll.c',
 343     ])
 344
 345
 346   if is_mac:
 347     static_libraries['base']['sources'].extend([
 348         'base/base_paths_mac.mm',
 349         'base/files/file_util_mac.mm',
 350         'base/mac/bundle_locations.mm',
 351         'base/mac/foundation_util.mm',
 352         'base/mac/mach_logging.cc',
 353         'base/mac/scoped_mach_port.cc',
 354         'base/mac/scoped_nsautorelease_pool.mm',
 355         'base/message_loop/message_pump_mac.mm',
 356         'base/process/process_handle_mac.cc',
 357         'base/process/process_iterator_mac.cc',
 358         'base/strings/sys_string_conversions_mac.mm',
 359         'base/time/time_mac.cc',
 360         'base/threading/platform_thread_mac.mm',
 361     ])
 362     static_libraries['libevent']['include_dirs'].extend([
 363         os.path.join(SRC_ROOT, 'third_party', 'libevent', 'mac')
 364     ])
 365     static_libraries['libevent']['sources'].extend([
 366         'third_party/libevent/kqueue.c',
 367     ])
 368
 369
 370   if is_mac:
 371     template_filename = 'build_mac.ninja.template'
 372   else:
 373     template_filename = 'build.ninja.template'
 374
 375   with open(os.path.join(GN_ROOT, 'bootstrap', template_filename)) as f:
 376     ninja_template = f.read()
 377
 378   def src_to_obj(path):
 379     return '%s' % os.path.splitext(path)[0] + '.o'
 380
 381   ninja_lines = []
 382   for library, settings in static_libraries.iteritems():
 383     for src_file in settings['sources']:
 384       ninja_lines.extend([
 385           'build %s: %s %s' % (src_to_obj(src_file),
 386                                settings['tool'],
 387                                os.path.join(SRC_ROOT, src_file)),
 388           '  includes = %s' % ' '.join(
 389               ['-I' + dirname for dirname in
 390                include_dirs + settings.get('include_dirs', [])]),
 391           '  cflags = %s' % ' '.join(cflags + settings.get('cflags', [])),
 392           '  cflags_cc = %s' %
 393               ' '.join(cflags_cc + settings.get('cflags_cc', [])),
 394       ])
 395       if cc:
 396         ninja_lines.append('  cc = %s' % cc)
 397       if cxx:
 398         ninja_lines.append('  cxx = %s' % cxx)
 399
 400     ninja_lines.append('build %s.a: alink_thin %s' % (
 401         library,
 402         ' '.join([src_to_obj(src_file) for src_file in settings['sources']])))
 403
 404   if is_mac:
 405     libs.extend([
 406         '-framework', 'AppKit',
 407         '-framework', 'CoreFoundation',
 408         '-framework', 'Foundation',
 409         '-framework', 'Security',
 410     ]);
 411
 412   ninja_lines.extend([
 413       'build gn: link %s' % (
 414           ' '.join(['%s.a' % library for library in static_libraries])),
 415       '  ldflags = %s' % ' '.join(ldflags),
 416       '  libs = %s' % ' '.join(libs),
 417   ])
 418   if ld:
 419     ninja_lines.append('  ld = %s' % ld)
 420   else:
 421     ninja_lines.append('  ld = $ldxx')
 422
 423   ninja_lines.append('')  # Make sure the file ends with a newline.
 424
 425   with open(path, 'w') as f:
 426     f.write(ninja_template + '\n'.join(ninja_lines))
 427
 428
 429 def build_gn_with_gn(temp_gn, build_dir, options):
 430   cmd = [temp_gn, 'gen', build_dir]
 431   if not options.debug:
 432     cmd.append('--args=is_debug=false')
 433   check_call(cmd)
 434
 435   cmd = ['ninja', '-C', build_dir]
 436   if options.verbose:
 437     cmd.append('-v')
 438   cmd.append('gn')
 439   check_call(cmd)
 440
 441   if not options.debug:
 442     check_call(['strip', os.path.join(build_dir, 'gn')])
 443
 444
 445 if __name__ == '__main__':
 446   sys.exit(main(sys.argv[1:]))