build/landmines.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """
   7 This script runs every build as the first hook (See DEPS). If it detects that
   8 the build should be clobbered, it will delete the contents of the build
   9 directory.
  10
  11 A landmine is tripped when a builder checks out a different revision, and the
  12 diff between the new landmines and the old ones is non-null. At this point, the
  13 build is clobbered.
  14 """
  15
  16 import difflib
  17 import errno
  18 import gyp_environment
  19 import logging
  20 import optparse
  21 import os
  22 import shutil
  23 import sys
  24 import subprocess
  25 import time
  26
  27 import landmine_utils
  28
  29
  30 SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
  31
  32
  33 def get_build_dir(build_tool, is_iphone=False):
  34   """
  35   Returns output directory absolute path dependent on build and targets.
  36   Examples:
  37     r'c:\b\build\slave\win\build\src\out'
  38     '/mnt/data/b/build/slave/linux/build/src/out'
  39     '/b/build/slave/ios_rel_device/build/src/xcodebuild'
  40
  41   Keep this function in sync with tools/build/scripts/slave/compile.py
  42   """
  43   ret = None
  44   if build_tool == 'xcode':
  45     ret = os.path.join(SRC_DIR, 'xcodebuild')
  46   elif build_tool in ['make', 'ninja', 'ninja-ios']:  # TODO: Remove ninja-ios.
  47     if 'CHROMIUM_OUT_DIR' in os.environ:
  48       output_dir = os.environ.get('CHROMIUM_OUT_DIR').strip()
  49       if not output_dir:
  50         raise Error('CHROMIUM_OUT_DIR environment variable is set but blank!')
  51     else:
  52       output_dir = landmine_utils.gyp_generator_flags().get('output_dir', 'out')
  53     ret = os.path.join(SRC_DIR, output_dir)
  54   else:
  55     raise NotImplementedError('Unexpected GYP_GENERATORS (%s)' % build_tool)
  56   return os.path.abspath(ret)
  57
  58
  59 def extract_gn_build_commands(build_ninja_file):
  60   """Extracts from a build.ninja the commands to run GN.
  61
  62   The commands to run GN are the gn rule and build.ninja build step at the
  63   top of the build.ninja file. We want to keep these when deleting GN builds
  64   since we want to preserve the command-line flags to GN.
  65
  66   On error, returns the empty string."""
  67   result = ""
  68   with open(build_ninja_file, 'r') as f:
  69     # Read until the second blank line. The first thing GN writes to the file
  70     # is the "rule gn" and the second is the section for "build build.ninja",
  71     # separated by blank lines.
  72     num_blank_lines = 0
  73     while num_blank_lines < 2:
  74       line = f.readline()
  75       if len(line) == 0:
  76         return ''  # Unexpected EOF.
  77       result += line
  78       if line[0] == '\n':
  79         num_blank_lines = num_blank_lines + 1
  80   return result
  81
  82 def delete_build_dir(build_dir):
  83   # GN writes a build.ninja.d file. Note that not all GN builds have args.gn.
  84   build_ninja_d_file = os.path.join(build_dir, 'build.ninja.d')
  85   if not os.path.exists(build_ninja_d_file):
  86     shutil.rmtree(build_dir)
  87     return
  88
  89   # GN builds aren't automatically regenerated when you sync. To avoid
  90   # messing with the GN workflow, erase everything but the args file, and
  91   # write a dummy build.ninja file that will automatically rerun GN the next
  92   # time Ninja is run.
  93   build_ninja_file = os.path.join(build_dir, 'build.ninja')
  94   build_commands = extract_gn_build_commands(build_ninja_file)
  95
  96   try:
  97     gn_args_file = os.path.join(build_dir, 'args.gn')
  98     with open(gn_args_file, 'r') as f:
  99       args_contents = f.read()
 100   except IOError:
 101     args_contents = ''
 102
 103   shutil.rmtree(build_dir)
 104
 105   # Put back the args file (if any).
 106   os.mkdir(build_dir)
 107   if args_contents != '':
 108     with open(gn_args_file, 'w') as f:
 109       f.write(args_contents)
 110
 111   # Write the build.ninja file sufficiently to regenerate itself.
 112   with open(os.path.join(build_dir, 'build.ninja'), 'w') as f:
 113     if build_commands != '':
 114       f.write(build_commands)
 115     else:
 116       # Couldn't parse the build.ninja file, write a default thing.
 117       f.write('''rule gn
 118 command = gn -q gen //out/%s/
 119 description = Regenerating ninja files
 120
 121 build build.ninja: gn
 122 generator = 1
 123 depfile = build.ninja.d
 124 ''' % (os.path.split(build_dir)[1]))
 125
 126   # Write a .d file for the build which references a nonexistant file. This
 127   # will make Ninja always mark the build as dirty.
 128   with open(build_ninja_d_file, 'w') as f:
 129     f.write('build.ninja: nonexistant_file.gn\n')
 130
 131
 132 def clobber_if_necessary(new_landmines):
 133   """Does the work of setting, planting, and triggering landmines."""
 134   out_dir = get_build_dir(landmine_utils.builder())
 135   landmines_path = os.path.normpath(os.path.join(out_dir, '..', '.landmines'))
 136   try:
 137     os.makedirs(out_dir)
 138   except OSError as e:
 139     if e.errno == errno.EEXIST:
 140       pass
 141
 142   if os.path.exists(landmines_path):
 143     with open(landmines_path, 'r') as f:
 144       old_landmines = f.readlines()
 145     if old_landmines != new_landmines:
 146       old_date = time.ctime(os.stat(landmines_path).st_ctime)
 147       diff = difflib.unified_diff(old_landmines, new_landmines,
 148           fromfile='old_landmines', tofile='new_landmines',
 149           fromfiledate=old_date, tofiledate=time.ctime(), n=0)
 150       sys.stdout.write('Clobbering due to:\n')
 151       sys.stdout.writelines(diff)
 152
 153       # Clobber contents of build directory but not directory itself: some
 154       # checkouts have the build directory mounted.
 155       for f in os.listdir(out_dir):
 156         path = os.path.join(out_dir, f)
 157         if os.path.isfile(path):
 158           os.unlink(path)
 159         elif os.path.isdir(path):
 160           delete_build_dir(path)
 161
 162   # Save current set of landmines for next time.
 163   with open(landmines_path, 'w') as f:
 164     f.writelines(new_landmines)
 165
 166
 167 def process_options():
 168   """Returns a list of landmine emitting scripts."""
 169   parser = optparse.OptionParser()
 170   parser.add_option(
 171       '-s', '--landmine-scripts', action='append',
 172       default=[os.path.join(SRC_DIR, 'build', 'get_landmines.py')],
 173       help='Path to the script which emits landmines to stdout. The target '
 174            'is passed to this script via option -t. Note that an extra '
 175            'script can be specified via an env var EXTRA_LANDMINES_SCRIPT.')
 176   parser.add_option('-v', '--verbose', action='store_true',
 177       default=('LANDMINES_VERBOSE' in os.environ),
 178       help=('Emit some extra debugging information (default off). This option '
 179           'is also enabled by the presence of a LANDMINES_VERBOSE environment '
 180           'variable.'))
 181
 182   options, args = parser.parse_args()
 183
 184   if args:
 185     parser.error('Unknown arguments %s' % args)
 186
 187   logging.basicConfig(
 188       level=logging.DEBUG if options.verbose else logging.ERROR)
 189
 190   extra_script = os.environ.get('EXTRA_LANDMINES_SCRIPT')
 191   if extra_script:
 192     return options.landmine_scripts + [extra_script]
 193   else:
 194     return options.landmine_scripts
 195
 196
 197 def main():
 198   landmine_scripts = process_options()
 199
 200   if landmine_utils.builder() in ('dump_dependency_json', 'eclipse'):
 201     return 0
 202
 203   gyp_environment.SetEnvironment()
 204
 205   landmines = []
 206   for s in landmine_scripts:
 207     proc = subprocess.Popen([sys.executable, s], stdout=subprocess.PIPE)
 208     output, _ = proc.communicate()
 209     landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()])
 210   clobber_if_necessary(landmines)
 211
 212   return 0
 213
 214
 215 if __name__ == '__main__':
 216   sys.exit(main())