llvm/utils/extract_symbols.py

   1 #!/usr/bin/env python
   2
   3 """A tool for extracting a list of symbols to export
   4
   5 When exporting symbols from a dll or exe we either need to mark the symbols in
   6 the source code as __declspec(dllexport) or supply a list of symbols to the
   7 linker. This program automates the latter by inspecting the symbol tables of a
   8 list of link inputs and deciding which of those symbols need to be exported.
   9
  10 We can't just export all the defined symbols, as there's a limit of 65535
  11 exported symbols and in clang we go way over that, particularly in a debug
  12 build. Therefore a large part of the work is pruning symbols either which can't
  13 be imported, or which we think are things that have definitions in public header
  14 files (i.e. template instantiations) and we would get defined in the thing
  15 importing these symbols anyway.
  16 """
  17
  18 from __future__ import print_function
  19 import sys
  20 import re
  21 import os
  22 import subprocess
  23 import multiprocessing
  24 import argparse
  25 import platform
  26
  27 # Define a function which extracts a list of pairs of (symbols, is_def) from a
  28 # library using llvm-nm becuase it can work both with regular and bitcode files.
  29 # We use subprocess.Popen and yield a symbol at a time instead of using
  30 # subprocess.check_output and returning a list as, especially on Windows, waiting
  31 # for the entire output to be ready can take a significant amount of time.
  32 def nm_get_symbols(tool, lib):
  33     # '-P' means the output is in portable format,
  34     # '-g' means we only get global symbols,
  35     # '-Xany' enforce handling both 32- and 64-bit objects on AIX,
  36     # '--no-demangle' ensure that C++ symbol names are not demangled; note
  37     #   that llvm-nm do not demangle by default, but the system nm on AIX does
  38     #   that, so the behavior may change in the future,
  39     # '-p' do not waste time sorting the symbols.
  40     cmd = [tool, "-P", "-g", "-Xany", "--no-demangle", "-p"]
  41     process = subprocess.Popen(
  42         cmd + [lib],
  43         bufsize=1,
  44         stdout=subprocess.PIPE,
  45         stdin=subprocess.PIPE,
  46         universal_newlines=True,
  47     )
  48     process.stdin.close()
  49     for line in process.stdout:
  50         # Look for external symbols that are defined in some section
  51         # The POSIX format is:
  52         #   name   type   value   size
  53         # The -P flag displays the size field for symbols only when applicable,
  54         # so the last field is optional. There's no space after the value field,
  55         # but \s+ match newline also, so \s+\S* will match the optional size field.
  56         match = re.match("^(\S+)\s+[BDGRSTuVW]\s+\S+\s+\S*$", line)
  57         if match:
  58             yield (match.group(1), True)
  59         # Look for undefined symbols, which have type U and may or may not
  60         # (depending on which nm is being used) have value and size.
  61         match = re.match("^(\S+)\s+U\s+(\S+\s+\S*)?$", line)
  62         if match:
  63             yield (match.group(1), False)
  64     process.wait()
  65
  66
  67 # Define a function which determines if the target is 32-bit Windows (as that's
  68 # where calling convention name decoration happens).
  69 def readobj_is_32bit_windows(tool, lib):
  70     output = subprocess.check_output(
  71         [tool, "--file-header", lib], universal_newlines=True
  72     )
  73     for line in output.splitlines():
  74         match = re.match("Format: (\S+)", line)
  75         if match:
  76             return match.group(1) == "COFF-i386"
  77     return False
  78
  79
  80 # MSVC mangles names to ?<identifier_mangling>@<type_mangling>. By examining the
  81 # identifier/type mangling we can decide which symbols could possibly be
  82 # required and which we can discard.
  83 def should_keep_microsoft_symbol(symbol, calling_convention_decoration):
  84     # Keep unmangled (i.e. extern "C") names
  85     if not "?" in symbol:
  86         if calling_convention_decoration:
  87             # Remove calling convention decoration from names
  88             match = re.match("[_@]([^@]+)", symbol)
  89             if match:
  90                 symbol = match.group(1)
  91         # Discard floating point/SIMD constants.
  92         if symbol.startswith(("__xmm@", "__ymm@", "__real@")):
  93             return None
  94         return symbol
  95     # Deleting destructors start with ?_G or ?_E and can be discarded because
  96     # link.exe gives you a warning telling you they can't be exported if you
  97     # don't
  98     elif symbol.startswith("??_G") or symbol.startswith("??_E"):
  99         return None
 100     # An anonymous namespace is mangled as ?A(maybe hex number)@. Any symbol
 101     # that mentions an anonymous namespace can be discarded, as the anonymous
 102     # namespace doesn't exist outside of that translation unit.
 103     elif re.search("\?A(0x\w+)?@", symbol):
 104         return None
 105     # Skip X86GenMnemonicTables functions, they are not exposed from llvm/include/.
 106     elif re.match("\?is[A-Z0-9]*@X86@llvm", symbol):
 107         return None
 108     # Keep mangled llvm:: and clang:: function symbols. How we detect these is a
 109     # bit of a mess and imprecise, but that avoids having to completely demangle
 110     # the symbol name. The outermost namespace is at the end of the identifier
 111     # mangling, and the identifier mangling is followed by the type mangling, so
 112     # we look for (llvm|clang)@@ followed by something that looks like a
 113     # function type mangling. To spot a function type we use (this is derived
 114     # from clang/lib/AST/MicrosoftMangle.cpp):
 115     # <function-type> ::= <function-class> <this-cvr-qualifiers>
 116     #                     <calling-convention> <return-type>
 117     #                     <argument-list> <throw-spec>
 118     # <function-class> ::= [A-Z]
 119     # <this-cvr-qualifiers> ::= [A-Z0-9_]*
 120     # <calling-convention> ::= [A-JQ]
 121     # <return-type> ::= .+
 122     # <argument-list> ::= X   (void)
 123     #                 ::= .+@ (list of types)
 124     #                 ::= .*Z (list of types, varargs)
 125     # <throw-spec> ::= exceptions are not allowed
 126     elif re.search("(llvm|clang)@@[A-Z][A-Z0-9_]*[A-JQ].+(X|.+@|.*Z)$", symbol):
 127         return symbol
 128     return None
 129
 130
 131 # Itanium manglings are of the form _Z<identifier_mangling><type_mangling>. We
 132 # demangle the identifier mangling to identify symbols that can be safely
 133 # discarded.
 134 def should_keep_itanium_symbol(symbol, calling_convention_decoration):
 135     # Start by removing any calling convention decoration (which we expect to
 136     # see on all symbols, even mangled C++ symbols)
 137     if calling_convention_decoration and symbol.startswith("_"):
 138         symbol = symbol[1:]
 139     # Keep unmangled names
 140     if not symbol.startswith("_") and not symbol.startswith("."):
 141         return symbol
 142     # Discard manglings that aren't nested names
 143     match = re.match("_Z(T[VTIS])?(N.+)", symbol)
 144     if not match:
 145         return None
 146     # Demangle the name. If the name is too complex then we don't need to keep
 147     # it, but it the demangling fails then keep the symbol just in case.
 148     try:
 149         names, _ = parse_itanium_nested_name(match.group(2))
 150     except TooComplexName:
 151         return None
 152     if not names:
 153         return symbol
 154     # Keep llvm:: and clang:: names
 155     elif names[0][0] == "4llvm" or names[0][0] == "5clang":
 156         return symbol
 157     # Discard everything else
 158     else:
 159         return None
 160
 161
 162 # Certain kinds of complex manglings we assume cannot be part of a public
 163 # interface, and we handle them by raising an exception.
 164 class TooComplexName(Exception):
 165     pass
 166
 167
 168 # Parse an itanium mangled name from the start of a string and return a
 169 # (name, rest of string) pair.
 170 def parse_itanium_name(arg):
 171     # Check for a normal name
 172     match = re.match("(\d+)(.+)", arg)
 173     if match:
 174         n = int(match.group(1))
 175         name = match.group(1) + match.group(2)[:n]
 176         rest = match.group(2)[n:]
 177         return name, rest
 178     # Check for constructor/destructor names
 179     match = re.match("([CD][123])(.+)", arg)
 180     if match:
 181         return match.group(1), match.group(2)
 182     # Assume that a sequence of characters that doesn't end a nesting is an
 183     # operator (this is very imprecise, but appears to be good enough)
 184     match = re.match("([^E]+)(.+)", arg)
 185     if match:
 186         return match.group(1), match.group(2)
 187     # Anything else: we can't handle it
 188     return None, arg
 189
 190
 191 # Parse an itanium mangled template argument list from the start of a string
 192 # and throw it away, returning the rest of the string.
 193 def skip_itanium_template(arg):
 194     # A template argument list starts with I
 195     assert arg.startswith("I"), arg
 196     tmp = arg[1:]
 197     while tmp:
 198         # Check for names
 199         match = re.match("(\d+)(.+)", tmp)
 200         if match:
 201             n = int(match.group(1))
 202             tmp = match.group(2)[n:]
 203             continue
 204         # Check for substitutions
 205         match = re.match("S[A-Z0-9]*_(.+)", tmp)
 206         if match:
 207             tmp = match.group(1)
 208         # Start of a template
 209         elif tmp.startswith("I"):
 210             tmp = skip_itanium_template(tmp)
 211         # Start of a nested name
 212         elif tmp.startswith("N"):
 213             _, tmp = parse_itanium_nested_name(tmp)
 214         # Start of an expression: assume that it's too complicated
 215         elif tmp.startswith("L") or tmp.startswith("X"):
 216             raise TooComplexName
 217         # End of the template
 218         elif tmp.startswith("E"):
 219             return tmp[1:]
 220         # Something else: probably a type, skip it
 221         else:
 222             tmp = tmp[1:]
 223     return None
 224
 225
 226 # Parse an itanium mangled nested name and transform it into a list of pairs of
 227 # (name, is_template), returning (list, rest of string).
 228 def parse_itanium_nested_name(arg):
 229     # A nested name starts with N
 230     assert arg.startswith("N"), arg
 231     ret = []
 232
 233     # Skip past the N, and possibly a substitution
 234     match = re.match("NS[A-Z0-9]*_(.+)", arg)
 235     if match:
 236         tmp = match.group(1)
 237     else:
 238         tmp = arg[1:]
 239
 240     # Skip past CV-qualifiers and ref qualifiers
 241     match = re.match("[rVKRO]*(.+)", tmp)
 242     if match:
 243         tmp = match.group(1)
 244
 245     # Repeatedly parse names from the string until we reach the end of the
 246     # nested name
 247     while tmp:
 248         # An E ends the nested name
 249         if tmp.startswith("E"):
 250             return ret, tmp[1:]
 251         # Parse a name
 252         name_part, tmp = parse_itanium_name(tmp)
 253         if not name_part:
 254             # If we failed then we don't know how to demangle this
 255             return None, None
 256         is_template = False
 257         # If this name is a template record that, then skip the template
 258         # arguments
 259         if tmp.startswith("I"):
 260             tmp = skip_itanium_template(tmp)
 261             is_template = True
 262         # Add the name to the list
 263         ret.append((name_part, is_template))
 264
 265     # If we get here then something went wrong
 266     return None, None
 267
 268
 269 # Parse a microsoft mangled symbol and return a list of pairs of
 270 # (name, is_template). This is very rudimentary and does just enough
 271 # in order to determine if the first or second component is a template.
 272 def parse_microsoft_mangling(arg):
 273     # If the name doesn't start with ? this isn't a mangled name
 274     if not arg.startswith("?"):
 275         return [(arg, False)]
 276     arg = arg[1:]
 277     components = []
 278     while len(arg) > 0:
 279         # If we see an empty component we've reached the end
 280         if arg.startswith("@"):
 281             return components
 282         # Check for a simple name
 283         match = re.match("(\w+)@(.+)", arg)
 284         if match:
 285             components.append((match.group(1), False))
 286             arg = match.group(2)
 287             continue
 288         # Check for a special function name
 289         match = re.match("(\?_?\w)(.+)", arg)
 290         if match:
 291             components.append((match.group(1), False))
 292             arg = match.group(2)
 293             continue
 294         # Check for a template name
 295         match = re.match("\?\$(\w+)@[^@]+@(.+)", arg)
 296         if match:
 297             components.append((match.group(1), True))
 298             arg = match.group(2)
 299             continue
 300         # Some other kind of name that we can't handle
 301         components.append((arg, False))
 302         return components
 303     return components
 304
 305
 306 def extract_symbols(arg):
 307     llvm_nm_path, should_keep_symbol, calling_convention_decoration, lib = arg
 308     symbol_defs = dict()
 309     symbol_refs = set()
 310     for (symbol, is_def) in nm_get_symbols(llvm_nm_path, lib):
 311         symbol = should_keep_symbol(symbol, calling_convention_decoration)
 312         if symbol:
 313             if is_def:
 314                 symbol_defs[symbol] = 1 + symbol_defs.setdefault(symbol, 0)
 315             else:
 316                 symbol_refs.add(symbol)
 317     return (symbol_defs, symbol_refs)
 318
 319
 320 def get_template_name(sym, mangling):
 321     # Parse the mangling into a list of (name, is_template)
 322     try:
 323         if mangling == "microsoft":
 324             names = parse_microsoft_mangling(sym)
 325         else:
 326             match = re.match("_Z(T[VTIS])?(N.+)", sym)
 327             if match:
 328                 names, _ = parse_itanium_nested_name(match.group(2))
 329             else:
 330                 names = None
 331     except TooComplexName:
 332         return None
 333
 334     if not names:
 335         return None
 336
 337     # If any component is a template then return it
 338     for name, is_template in names:
 339         if is_template:
 340             return name
 341
 342     # Not a template
 343     return None
 344
 345
 346 def parse_tool_path(parser, tool, val):
 347     try:
 348         # Close std streams as we don't want any output and we don't
 349         # want the process to wait for something on stdin.
 350         p = subprocess.Popen(
 351             [val],
 352             stdout=subprocess.PIPE,
 353             stderr=subprocess.PIPE,
 354             stdin=subprocess.PIPE,
 355             universal_newlines=True,
 356         )
 357         p.stdout.close()
 358         p.stderr.close()
 359         p.stdin.close()
 360         p.wait()
 361         return val
 362     except Exception:
 363         parser.error(f"Invalid path for {tool}")
 364
 365
 366 if __name__ == "__main__":
 367     parser = argparse.ArgumentParser(
 368         description="Extract symbols to export from libraries"
 369     )
 370     parser.add_argument(
 371         "--mangling",
 372         choices=["itanium", "microsoft"],
 373         required=True,
 374         help="expected symbol mangling scheme",
 375     )
 376     parser.add_argument(
 377         "--nm",
 378         metavar="path",
 379         type=lambda x: parse_tool_path(parser, "nm", x),
 380         help="path to the llvm-nm executable",
 381     )
 382     parser.add_argument(
 383         "--readobj",
 384         metavar="path",
 385         type=lambda x: parse_tool_path(parser, "readobj", x),
 386         help="path to the llvm-readobj executable",
 387     )
 388     parser.add_argument(
 389         "libs",
 390         metavar="lib",
 391         type=str,
 392         nargs="+",
 393         help="libraries to extract symbols from",
 394     )
 395     parser.add_argument("-o", metavar="file", type=str, help="output to file")
 396     args = parser.parse_args()
 397
 398     # How we determine which symbols to keep and which to discard depends on
 399     # the mangling scheme
 400     if args.mangling == "microsoft":
 401         should_keep_symbol = should_keep_microsoft_symbol
 402     else:
 403         should_keep_symbol = should_keep_itanium_symbol
 404
 405     # Get the list of libraries to extract symbols from
 406     libs = list()
 407     for lib in args.libs:
 408         # When invoked by cmake the arguments are the cmake target names of the
 409         # libraries, so we need to add .lib/.a to the end and maybe lib to the
 410         # start to get the filename. Also allow objects.
 411         suffixes = [".lib", ".a", ".obj", ".o"]
 412         if not any([lib.endswith(s) for s in suffixes]):
 413             for s in suffixes:
 414                 if os.path.exists(lib + s):
 415                     lib = lib + s
 416                     break
 417                 if os.path.exists("lib" + lib + s):
 418                     lib = "lib" + lib + s
 419                     break
 420         if not any([lib.endswith(s) for s in suffixes]):
 421             print("Don't know what to do with argument " + lib, file=sys.stderr)
 422             exit(1)
 423         libs.append(lib)
 424
 425     # Check if calling convention decoration is used by inspecting the first
 426     # library in the list
 427     calling_convention_decoration = readobj_is_32bit_windows(args.readobj, libs[0])
 428
 429     # Extract symbols from libraries in parallel. This is a huge time saver when
 430     # doing a debug build, as there are hundreds of thousands of symbols in each
 431     # library.
 432     # FIXME: On AIX, the default pool size can be too big for a logical
 433     #        partition's allocated memory, and can lead to an out of memory
 434     #        IO error. We are setting the pool size to 8 to avoid such
 435     #        errors at the moment, and will look for a graceful solution later.
 436     pool = multiprocessing.Pool(8) if platform.system() == "AIX" \
 437                                    else multiprocessing.Pool()
 438     try:
 439         # Only one argument can be passed to the mapping function, and we can't
 440         # use a lambda or local function definition as that doesn't work on
 441         # windows, so create a list of tuples which duplicates the arguments
 442         # that are the same in all calls.
 443         vals = [
 444             (args.nm, should_keep_symbol, calling_convention_decoration, x)
 445             for x in libs
 446         ]
 447         # Do an async map then wait for the result to make sure that
 448         # KeyboardInterrupt gets caught correctly (see
 449         # http://bugs.python.org/issue8296)
 450         result = pool.map_async(extract_symbols, vals)
 451         pool.close()
 452         libs_symbols = result.get(3600)
 453     except KeyboardInterrupt:
 454         # On Ctrl-C terminate everything and exit
 455         pool.terminate()
 456         pool.join()
 457         exit(1)
 458
 459     # Merge everything into a single dict
 460     symbol_defs = dict()
 461     symbol_refs = set()
 462     for (this_lib_defs, this_lib_refs) in libs_symbols:
 463         for k, v in list(this_lib_defs.items()):
 464             symbol_defs[k] = v + symbol_defs.setdefault(k, 0)
 465         for sym in list(this_lib_refs):
 466             symbol_refs.add(sym)
 467
 468     # Find which template instantiations are referenced at least once.
 469     template_instantiation_refs = set()
 470     for sym in list(symbol_refs):
 471         template = get_template_name(sym, args.mangling)
 472         if template:
 473             template_instantiation_refs.add(template)
 474
 475     # Print symbols which both:
 476     #  * Appear in exactly one input, as symbols defined in multiple
 477     #    objects/libraries are assumed to have public definitions.
 478     #  * Are not a template instantiation that isn't referenced anywhere. This
 479     #    is because we need to export any explicitly instantiated templates,
 480     #    and we expect those to be referenced in some object.
 481     if args.o:
 482         outfile = open(args.o, "w")
 483     else:
 484         outfile = sys.stdout
 485     for k, v in list(symbol_defs.items()):
 486         template = get_template_name(k, args.mangling)
 487         if v == 1 and (not template or template in template_instantiation_refs):
 488             print(k, file=outfile)