tools/win/split_link/split_link.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Takes the same arguments as Windows link.exe, and a definition of libraries
   6 to split into subcomponents. Does multiple passes of link.exe invocation to
   7 determine exports between parts and generates .def and import libraries to
   8 cause symbols to be available to other parts."""
   9
  10 import _winreg
  11 import ctypes
  12 import os
  13 import re
  14 import shutil
  15 import subprocess
  16 import sys
  17 import tempfile
  18
  19
  20 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
  21
  22
  23 # This can be set to ignore data exports. The resulting DLLs will probably not
  24 # run, but at least they can be generated. The log of data exports will still
  25 # be output.
  26 IGNORE_DATA = 0
  27
  28
  29 def Log(message):
  30   print 'split_link:', message
  31
  32
  33 def GetFlagsAndInputs(argv):
  34   """Parses the command line intended for link.exe and return the flags and
  35   input files."""
  36   rsp_expanded = []
  37   for arg in argv:
  38     if arg[0] == '@':
  39       with open(arg[1:]) as rsp:
  40         rsp_expanded.extend(rsp.read().splitlines())
  41     else:
  42       rsp_expanded.append(arg)
  43
  44   # Use CommandLineToArgvW so we match link.exe parsing.
  45   try:
  46     size = ctypes.c_int()
  47     ptr = ctypes.windll.shell32.CommandLineToArgvW(
  48         ctypes.create_unicode_buffer(' '.join(rsp_expanded)),
  49         ctypes.byref(size))
  50     ref = ctypes.c_wchar_p * size.value
  51     raw = ref.from_address(ptr)
  52     args = [arg for arg in raw]
  53   finally:
  54     ctypes.windll.kernel32.LocalFree(ptr)
  55
  56   inputs = []
  57   flags = []
  58   intermediate_manifest = ''
  59   for arg in args:
  60     lower_arg = arg.lower()
  61     # We'll be replacing these ourselves.
  62     if lower_arg.startswith('/out:'):
  63       continue
  64     if lower_arg.startswith('/manifestfile:'):
  65       intermediate_manifest = arg[arg.index(':')+1:]
  66       continue
  67     if lower_arg.startswith('/pdb:'):
  68       continue
  69     if (not lower_arg.startswith('/') and
  70         lower_arg.endswith(('.obj', '.lib', '.res'))):
  71       inputs.append(arg)
  72     else:
  73       flags.append(arg)
  74
  75   return flags, inputs, intermediate_manifest
  76
  77
  78 def GetRegistryValue(subkey):
  79   try:
  80     val = _winreg.QueryValue(_winreg.HKEY_CURRENT_USER,
  81                              'Software\\Chromium\\' + subkey)
  82     if os.path.exists(val):
  83       return val
  84   except WindowsError:
  85     pass
  86
  87   raise SystemExit("Couldn't read from registry")
  88
  89
  90 def GetOriginalLinkerPath():
  91   return GetRegistryValue('split_link_installed')
  92
  93
  94 def GetMtPath():
  95   return GetRegistryValue('split_link_mt_path')
  96
  97
  98 def PartFor(input_file, description_parts, description_all):
  99   """Determines which part a given link input should be put into (or all)."""
 100   # Check if it should go in all parts.
 101   input_file = input_file.lower()
 102   if any(re.search(spec, input_file) for spec in description_all):
 103     return -1
 104   # Or pick which particular one it belongs in.
 105   for i, spec_list in enumerate(description_parts):
 106     if any(re.search(spec, input_file) for spec in spec_list):
 107       return i
 108   raise ValueError("couldn't find location for %s" % input_file)
 109
 110
 111 def ParseOutExternals(output):
 112   """Given the stdout of link.exe, parses the error messages to retrieve all
 113   symbols that are unresolved."""
 114   result = set()
 115   # Styles of messages for unresolved externals, and a boolean to indicate
 116   # whether the error message emits the symbols with or without a leading
 117   # underscore.
 118   unresolved_regexes = [
 119     (re.compile(r' : error LNK2019: unresolved external symbol ".*" \((.*)\)'
 120                 r' referenced in function'),
 121      False),
 122     (re.compile(r' : error LNK2001: unresolved external symbol ".*" \((.*)\)$'),
 123      False),
 124     (re.compile(r' : error LNK2019: unresolved external symbol (.*)'
 125                 r' referenced in function '),
 126      True),
 127     (re.compile(r' : error LNK2001: unresolved external symbol (.*)$'),
 128      True),
 129   ]
 130   for line in output.splitlines():
 131     line = line.strip()
 132     for regex, strip_leading_underscore in unresolved_regexes:
 133       mo = regex.search(line)
 134       if mo:
 135         if strip_leading_underscore:
 136           result.add(mo.group(1)[1:])
 137         else:
 138           result.add(mo.group(1))
 139         break
 140
 141   mo = re.search(r'fatal error LNK1120: (\d+) unresolved externals', output)
 142   # Make sure we have the same number that the linker thinks we have.
 143   if mo is None and result:
 144     raise SystemExit(output)
 145   if len(result) != int(mo.group(1)):
 146     print output
 147     print 'Expecting %d, got %d' % (int(mo.group(1)), len(result))
 148   assert len(result) == int(mo.group(1))
 149   return sorted(result)
 150
 151
 152 def AsCommandLineArgs(items):
 153   """Intended for output to a response file. Quotes all arguments."""
 154   return '\n'.join('"' + x + '"' for x in items)
 155
 156
 157 def OutputNameForIndex(index):
 158   """Gets the final output DLL name, given a zero-based index."""
 159   if index == 0:
 160     return "chrome.dll"
 161   else:
 162     return 'chrome%d.dll' % index
 163
 164
 165 def ManifestNameForIndex(index):
 166   return OutputNameForIndex(index) + '.intermediate.manifest'
 167
 168
 169 def PdbNameForIndex(index):
 170   return OutputNameForIndex(index) + '.pdb'
 171
 172
 173 def RunLinker(flags, index, inputs, phase, intermediate_manifest):
 174   """Invokes the linker and returns the stdout, returncode and target name."""
 175   rspfile = 'part%d_%s.rsp' % (index, phase)
 176   with open(rspfile, 'w') as f:
 177     print >> f, AsCommandLineArgs(inputs)
 178     print >> f, AsCommandLineArgs(flags)
 179     output_name = OutputNameForIndex(index)
 180     manifest_name = ManifestNameForIndex(index)
 181     print >> f, '/ENTRY:ChromeEmptyEntry@12'
 182     print >> f, '/OUT:' + output_name
 183     print >> f, '/MANIFESTFILE:' + manifest_name
 184     print >> f, '/PDB:' + PdbNameForIndex(index)
 185   # Log('[[[\n' + open(rspfile).read() + '\n]]]')
 186   link_exe = GetOriginalLinkerPath()
 187   popen = subprocess.Popen([link_exe, '@' + rspfile], stdout=subprocess.PIPE)
 188   stdout, _ = popen.communicate()
 189   if index == 0 and popen.returncode == 0 and intermediate_manifest:
 190     # Hack for ninja build. After the linker runs, it does some manifest
 191     # things and expects there to be a file in this location. We just put it
 192     # there so it's happy. This is a no-op.
 193     if os.path.isdir(os.path.dirname(intermediate_manifest)):
 194       shutil.copyfile(manifest_name, intermediate_manifest)
 195   return stdout, popen.returncode, output_name
 196
 197
 198 def GetLibObjList(lib):
 199   """Gets the list of object files contained in a .lib."""
 200   link_exe = GetOriginalLinkerPath()
 201   popen = subprocess.Popen(
 202       [link_exe, '/lib', '/nologo', '/list', lib], stdout=subprocess.PIPE)
 203   stdout, _ = popen.communicate()
 204   return stdout.splitlines()
 205
 206
 207 def ExtractObjFromLib(lib, obj):
 208   """Extracts a .obj file contained in a .lib file. Returns the absolute path
 209   a temp file."""
 210   link_exe = GetOriginalLinkerPath()
 211   temp = tempfile.NamedTemporaryFile(
 212       prefix='split_link_', suffix='.obj', delete=False)
 213   temp.close()
 214   subprocess.check_call([
 215     link_exe, '/lib', '/nologo', '/extract:' + obj, lib, '/out:' + temp.name])
 216   return temp.name
 217
 218
 219 def Unmangle(export):
 220   "Returns the human-presentable name of a mangled symbol."""
 221   # Use dbghelp.dll to demangle the name.
 222   # TODO(scottmg): Perhaps a simple cache? Seems pretty fast though.
 223   UnDecorateSymbolName = ctypes.windll.dbghelp.UnDecorateSymbolName
 224   buffer_size = 2048
 225   output_string = ctypes.create_string_buffer(buffer_size)
 226   if not UnDecorateSymbolName(
 227       export, ctypes.byref(output_string), buffer_size, 0):
 228     raise ctypes.WinError()
 229   return output_string.value
 230
 231
 232 def IsDataDefinition(export):
 233   """Determines if a given name is data rather than a function. Always returns
 234   False for C-style (as opposed to C++-style names)."""
 235   if export[0] != '?':
 236     return False
 237
 238   # If it contains a '(' we assume it's a function.
 239   return '(' not in Unmangle(export)
 240
 241
 242 def GenerateDefFiles(unresolved_by_part):
 243   """Given a list of unresolved externals, generates a .def file that will
 244   cause all those symbols to be exported."""
 245   deffiles = []
 246   Log('generating .def files')
 247   for i, part in enumerate(unresolved_by_part):
 248     deffile = 'part%d.def' % i
 249     with open(deffile, 'w') as f:
 250       print >> f, 'LIBRARY %s' % OutputNameForIndex(i)
 251       print >> f, 'EXPORTS'
 252       for j, part in enumerate(unresolved_by_part):
 253         if i == j:
 254           continue
 255         is_data = \
 256             [' DATA' if IsDataDefinition(export) and not IGNORE_DATA else ''
 257              for export in part]
 258         print >> f, '\n'.join('  ' + export + data
 259                               for export, data in zip(part, is_data))
 260     deffiles.append(deffile)
 261   return deffiles
 262
 263
 264 def BuildImportLibs(flags, inputs_by_part, deffiles):
 265   """Runs the linker to generate an import library."""
 266   import_libs = []
 267   Log('building import libs')
 268   for i, (inputs, deffile) in enumerate(zip(inputs_by_part, deffiles)):
 269     libfile = 'part%d.lib' % i
 270     flags_with_implib_and_deffile = flags + ['/IMPLIB:%s' % libfile,
 271                                              '/DEF:%s' % deffile]
 272     RunLinker(flags_with_implib_and_deffile, i, inputs, 'implib', None)
 273     import_libs.append(libfile)
 274   return import_libs
 275
 276
 277 def AttemptLink(flags, inputs_by_part, unresolved_by_part, deffiles,
 278                 import_libs, intermediate_manifest):
 279   """Tries to run the linker for all parts using the current round of
 280   generated import libs and .def files. If the link fails, updates the
 281   unresolved externals list per part."""
 282   dlls = []
 283   all_succeeded = True
 284   new_externals = []
 285   Log('unresolveds now: %r' % [len(part) for part in unresolved_by_part])
 286   for i, (inputs, deffile) in enumerate(zip(inputs_by_part, deffiles)):
 287     Log('running link, part %d' % i)
 288     others_implibs = import_libs[:]
 289     others_implibs.pop(i)
 290     inputs_with_implib = inputs + filter(lambda x: x, others_implibs)
 291     if deffile:
 292       flags = flags + ['/DEF:%s' % deffile, '/LTCG']
 293     stdout, rc, output = RunLinker(
 294         flags, i, inputs_with_implib, 'final', intermediate_manifest)
 295     if rc != 0:
 296       all_succeeded = False
 297       new_externals.append(ParseOutExternals(stdout))
 298     else:
 299       new_externals.append([])
 300       dlls.append(output)
 301   combined_externals = [sorted(set(prev) | set(new))
 302                         for prev, new in zip(unresolved_by_part, new_externals)]
 303   return all_succeeded, dlls, combined_externals
 304
 305
 306 def ExtractSubObjsTargetedAtAll(
 307     inputs,
 308     num_parts,
 309     description_parts,
 310     description_all,
 311     description_all_from_libs):
 312   """For (lib, obj) tuples in the all_from_libs section, extract the obj out of
 313   the lib and added it to inputs. Returns a list of lists for which part the
 314   extracted obj belongs in (which is whichever the .lib isn't in)."""
 315   by_parts = [[] for _ in range(num_parts)]
 316   for lib_spec, obj_spec in description_all_from_libs:
 317     for input_file in inputs:
 318       if re.search(lib_spec, input_file):
 319         objs = GetLibObjList(input_file)
 320         match_count = 0
 321         for obj in objs:
 322           if re.search(obj_spec, obj, re.I):
 323             extracted_obj = ExtractObjFromLib(input_file, obj)
 324             #Log('extracted %s (%s %s)' % (extracted_obj, input_file, obj))
 325             i = PartFor(input_file, description_parts, description_all)
 326             if i == -1:
 327               raise SystemExit(
 328                   '%s is already in all parts, but matched '
 329                   '%s in all_from_libs' % (input_file, obj))
 330             # See note in main().
 331             assert num_parts == 2, "Can't handle > 2 dlls currently"
 332             by_parts[1 - i].append(obj)
 333             match_count += 1
 334         if match_count == 0:
 335           raise SystemExit(
 336               '%s, %s matched a lib, but no objs' % (lib_spec, obj_spec))
 337   return by_parts
 338
 339
 340 def main():
 341   flags, inputs, intermediate_manifest = GetFlagsAndInputs(sys.argv[1:])
 342   partition_file = os.path.normpath(
 343       os.path.join(BASE_DIR, '../../../build/split_link_partition.py'))
 344   with open(partition_file) as partition:
 345     description = eval(partition.read())
 346   inputs_by_part = []
 347   description_parts = description['parts']
 348   # We currently assume that if a symbol isn't in dll 0, then it's in dll 1
 349   # when generating def files. Otherwise, we'd need to do more complex things
 350   # to figure out where each symbol actually is to assign it to the correct
 351   # .def file.
 352   num_parts = len(description_parts)
 353   assert num_parts == 2, "Can't handle > 2 dlls currently"
 354   description_parts.reverse()
 355   objs_from_libs = ExtractSubObjsTargetedAtAll(
 356       inputs,
 357       num_parts,
 358       description_parts,
 359       description['all'],
 360       description['all_from_libs'])
 361   objs_from_libs.reverse()
 362   inputs_by_part = [[] for _ in range(num_parts)]
 363   for input_file in inputs:
 364     i = PartFor(input_file, description_parts, description['all'])
 365     if i == -1:
 366       for part in inputs_by_part:
 367         part.append(input_file)
 368     else:
 369       inputs_by_part[i].append(input_file)
 370   inputs_by_part.reverse()
 371
 372   # Put the subobjs on to the main list.
 373   for i, part in enumerate(objs_from_libs):
 374     Log('%d sub .objs added to part %d' % (len(part), i))
 375     inputs_by_part[i].extend(part)
 376
 377   unresolved_by_part = [[] for _ in range(num_parts)]
 378   import_libs = [None] * num_parts
 379   deffiles = [None] * num_parts
 380
 381   data_exports = 0
 382   for i in range(5):
 383     Log('--- starting pass %d' % i)
 384     ok, dlls, unresolved_by_part = AttemptLink(
 385         flags, inputs_by_part, unresolved_by_part, deffiles, import_libs,
 386         intermediate_manifest)
 387     if ok:
 388       break
 389     data_exports = 0
 390     for i, part in enumerate(unresolved_by_part):
 391       for export in part:
 392         if IsDataDefinition(export):
 393           print 'part %d contains data export: %s (aka %s)' % (
 394               i, Unmangle(export), export)
 395           data_exports += 1
 396     deffiles = GenerateDefFiles(unresolved_by_part)
 397     import_libs = BuildImportLibs(flags, inputs_by_part, deffiles)
 398   else:
 399     if data_exports and not IGNORE_DATA:
 400       print '%d data exports found, see report above.' % data_exports
 401       print('These cannot be exported, and must be either duplicated to the '
 402             'target DLL (if constant), or wrapped in a function.')
 403     return 1
 404
 405   mt_exe = GetMtPath()
 406   for i, dll in enumerate(dlls):
 407     Log('embedding manifest in %s' % dll)
 408     args = [mt_exe, '-nologo', '-manifest']
 409     args.append(ManifestNameForIndex(i))
 410     args.append(description['manifest'])
 411     args.append('-outputresource:%s;2' % dll)
 412     subprocess.check_call(args)
 413
 414   Log('built %r' % dlls)
 415
 416   return 0
 417
 418
 419 if __name__ == '__main__':
 420   sys.exit(main())