native_client_sdk/src/tools/create_nmf.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import errno
   7 import hashlib
   8 import json
   9 import optparse
  10 import os
  11 import re
  12 import shutil
  13 import struct
  14 import subprocess
  15 import sys
  16
  17 import quote
  18
  19 if sys.version_info < (2, 6, 0):
  20   sys.stderr.write("python 2.6 or later is required run this script\n")
  21   sys.exit(1)
  22
  23 NeededMatcher = re.compile('^ *NEEDED *([^ ]+)\n$')
  24 FormatMatcher = re.compile('^(.+):\\s*file format (.+)\n$')
  25
  26 OBJDUMP_ARCH_MAP = {
  27     # Names returned by Linux's objdump:
  28     'elf64-x86-64': 'x86-64',
  29     'elf32-i386': 'x86-32',
  30     'elf32-little': 'arm',
  31     'elf32-littlearm': 'arm',
  32     # Names returned by x86_64-nacl-objdump:
  33     'elf64-nacl': 'x86-64',
  34     'elf32-nacl': 'x86-32',
  35 }
  36
  37 ARCH_LOCATION = {
  38     'x86-32': 'lib32',
  39     'x86-64': 'lib64',
  40     'arm': 'lib',
  41 }
  42
  43
  44 # These constants are used within nmf files.
  45 RUNNABLE_LD = 'runnable-ld.so'  # Name of the dynamic loader
  46 MAIN_NEXE = 'main.nexe'  # Name of entry point for execution
  47 PROGRAM_KEY = 'program'  # Key of the program section in an nmf file
  48 URL_KEY = 'url'  # Key of the url field for a particular file in an nmf file
  49 FILES_KEY = 'files'  # Key of the files section in an nmf file
  50 PORTABLE_KEY = 'portable' # key for portable section of manifest
  51 TRANSLATE_KEY = 'pnacl-translate' # key for translatable objects
  52
  53
  54 # The proper name of the dynamic linker, as kept in the IRT.  This is
  55 # excluded from the nmf file by convention.
  56 LD_NACL_MAP = {
  57     'x86-32': 'ld-nacl-x86-32.so.1',
  58     'x86-64': 'ld-nacl-x86-64.so.1',
  59     'arm': None,
  60 }
  61
  62
  63 def DebugPrint(message):
  64   if DebugPrint.debug_mode:
  65     sys.stderr.write('%s\n' % message)
  66
  67
  68 DebugPrint.debug_mode = False  # Set to True to enable extra debug prints
  69
  70
  71 def MakeDir(dirname):
  72   """Just like os.makedirs but doesn't generate errors when dirname
  73   already exists.
  74   """
  75   if os.path.isdir(dirname):
  76     return
  77
  78   Trace("mkdir: %s" % dirname)
  79   try:
  80     os.makedirs(dirname)
  81   except OSError as exception_info:
  82     if exception_info.errno != errno.EEXIST:
  83       raise
  84
  85
  86 class Error(Exception):
  87   '''Local Error class for this file.'''
  88   pass
  89
  90
  91 def ParseElfHeader(path):
  92   """Determine properties of a nexe by parsing elf header.
  93   Return tuple of architecture and boolean signalling whether
  94   the executable is dynamic (has INTERP header) or static.
  95   """
  96   # From elf.h:
  97   # typedef struct
  98   # {
  99   #   unsigned char e_ident[EI_NIDENT]; /* Magic number and other info */
 100   #   Elf64_Half e_type; /* Object file type */
 101   #   Elf64_Half e_machine; /* Architecture */
 102   #   ...
 103   # } Elf32_Ehdr;
 104   elf_header_format = '16s2H'
 105   elf_header_size = struct.calcsize(elf_header_format)
 106
 107   with open(path, 'rb') as f:
 108     header = f.read(elf_header_size)
 109
 110   header = struct.unpack(elf_header_format, header)
 111   e_ident, _, e_machine = header[:3]
 112
 113   elf_magic = '\x7fELF'
 114   if e_ident[:4] != elf_magic:
 115     raise Error('Not a valid NaCL executable: %s' % path)
 116
 117   e_machine_mapping = {
 118     3 : 'x86-32',
 119     40 : 'arm',
 120     62 : 'x86-64'
 121   }
 122   if e_machine not in e_machine_mapping:
 123     raise Error('Unknown machine type: %s' % e_machine)
 124
 125   # Set arch based on the machine type in the elf header
 126   arch = e_machine_mapping[e_machine]
 127
 128   # Now read the full header in either 64bit or 32bit mode
 129   dynamic = IsDynamicElf(path, arch == 'x86-64')
 130   return arch, dynamic
 131
 132
 133 def IsDynamicElf(path, is64bit):
 134   """Examine an elf file to determine if it is dynamically
 135   linked or not.
 136   This is determined by searching the program headers for
 137   a header of type PT_INTERP.
 138   """
 139   if is64bit:
 140     elf_header_format = '16s2HI3QI3H'
 141   else:
 142     elf_header_format = '16s2HI3II3H'
 143
 144   elf_header_size = struct.calcsize(elf_header_format)
 145
 146   with open(path, 'rb') as f:
 147     header = f.read(elf_header_size)
 148     header = struct.unpack(elf_header_format, header)
 149     p_header_offset = header[5]
 150     p_header_entry_size = header[9]
 151     num_p_header = header[10]
 152     f.seek(p_header_offset)
 153     p_headers = f.read(p_header_entry_size*num_p_header)
 154
 155   # Read the first word of each Phdr to find out its type.
 156   #
 157   # typedef struct
 158   # {
 159   #   Elf32_Word  p_type;     /* Segment type */
 160   #   ...
 161   # } Elf32_Phdr;
 162   elf_phdr_format = 'I'
 163   PT_INTERP = 3
 164
 165   while p_headers:
 166     p_header = p_headers[:p_header_entry_size]
 167     p_headers = p_headers[p_header_entry_size:]
 168     phdr_type = struct.unpack(elf_phdr_format, p_header[:4])[0]
 169     if phdr_type == PT_INTERP:
 170       return True
 171
 172   return False
 173
 174
 175 class ArchFile(object):
 176   '''Simple structure containing information about
 177
 178   Attributes:
 179     name: Name of this file
 180     path: Full path to this file on the build system
 181     arch: Architecture of this file (e.g., x86-32)
 182     url: Relative path to file in the staged web directory.
 183         Used for specifying the "url" attribute in the nmf file.'''
 184
 185   def __init__(self, name, path, url, arch=None):
 186     self.name = name
 187     self.path = path
 188     self.url = url
 189     self.arch = arch
 190     if arch is None:
 191       self.arch = ParseElfHeader(path)[0]
 192
 193   def __repr__(self):
 194     return '<ArchFile %s>' % self.path
 195
 196   def __str__(self):
 197     '''Return the file path when invoked with the str() function'''
 198     return self.path
 199
 200
 201 class NmfUtils(object):
 202   '''Helper class for creating and managing nmf files
 203
 204   Attributes:
 205     manifest: A JSON-structured dict containing the nmf structure
 206     needed: A dict with key=filename and value=ArchFile (see GetNeeded)
 207   '''
 208
 209   def __init__(self, main_files=None, objdump=None,
 210                lib_path=None, extra_files=None, lib_prefix=None,
 211                remap=None):
 212     '''Constructor
 213
 214     Args:
 215       main_files: List of main entry program files.  These will be named
 216           files->main.nexe for dynamic nexes, and program for static nexes
 217       objdump: path to x86_64-nacl-objdump tool (or Linux equivalent)
 218       lib_path: List of paths to library directories
 219       extra_files: List of extra files to include in the nmf
 220       lib_prefix: A list of path components to prepend to the library paths,
 221           both for staging the libraries and for inclusion into the nmf file.
 222           Examples:  ['..'], ['lib_dir']
 223       remap: Remaps the library name in the manifest.
 224       '''
 225     self.objdump = objdump
 226     self.main_files = main_files or []
 227     self.extra_files = extra_files or []
 228     self.lib_path = lib_path or []
 229     self.manifest = None
 230     self.needed = {}
 231     self.lib_prefix = lib_prefix or []
 232     self.remap = remap or {}
 233     self.pnacl = main_files and main_files[0].endswith('pexe')
 234
 235   def GleanFromObjdump(self, files):
 236     '''Get architecture and dependency information for given files
 237
 238     Args:
 239       files: A dict with key=filename and value=list or set of archs.  E.g.:
 240           { '/path/to/my.nexe': ['x86-32']
 241             '/path/to/lib64/libmy.so': ['x86-64'],
 242             '/path/to/mydata.so': ['x86-32', 'x86-64'],
 243             '/path/to/my.data': None }  # Indicates all architectures
 244
 245     Returns: A tuple with the following members:
 246       input_info: A dict with key=filename and value=ArchFile of input files.
 247           Includes the input files as well, with arch filled in if absent.
 248           Example: { '/path/to/my.nexe': ArchFile(my.nexe),
 249                      '/path/to/libfoo.so': ArchFile(libfoo.so) }
 250       needed: A set of strings formatted as "arch/name".  Example:
 251           set(['x86-32/libc.so', 'x86-64/libgcc.so'])
 252     '''
 253     if not self.objdump:
 254       raise Error('No objdump executable specified (see --help for more info)')
 255     DebugPrint('GleanFromObjdump(%s)' % ([self.objdump, '-p'] + files.keys()))
 256     proc = subprocess.Popen([self.objdump, '-p'] + files.keys(),
 257                             stdout=subprocess.PIPE,
 258                             stderr=subprocess.PIPE, bufsize=-1)
 259     input_info = {}
 260     needed = set()
 261     output, err_output = proc.communicate()
 262     if proc.returncode:
 263       raise Error('%s\nStdError=%s\nobjdump failed with error code: %d' %
 264                   (output, err_output, proc.returncode))
 265
 266     for line in output.splitlines(True):
 267       # Objdump should display the architecture first and then the dependencies
 268       # second for each file in the list.
 269       matched = FormatMatcher.match(line)
 270       if matched:
 271         filename = matched.group(1)
 272         arch = OBJDUMP_ARCH_MAP[matched.group(2)]
 273         if files[filename] is None or arch in files[filename]:
 274           name = os.path.basename(filename)
 275           input_info[filename] = ArchFile(
 276               arch=arch,
 277               name=name,
 278               path=filename,
 279               url='/'.join(self.lib_prefix + [ARCH_LOCATION[arch], name]))
 280       matched = NeededMatcher.match(line)
 281       if matched:
 282         if files[filename] is None or arch in files[filename]:
 283           needed.add('/'.join([arch, matched.group(1)]))
 284     return input_info, needed
 285
 286   def FindLibsInPath(self, name):
 287     '''Finds the set of libraries matching |name| within lib_path
 288
 289     Args:
 290       name: name of library to find
 291
 292     Returns:
 293       A list of system paths that match the given name within the lib_path'''
 294     files = []
 295     for dirname in self.lib_path:
 296       filename = os.path.join(dirname, name)
 297       if os.path.exists(filename):
 298         files.append(filename)
 299     if not files:
 300       raise Error('cannot find library %s' % name)
 301     return files
 302
 303   def GetNeeded(self):
 304     '''Collect the list of dependencies for the main_files
 305
 306     Returns:
 307       A dict with key=filename and value=ArchFile of input files.
 308           Includes the input files as well, with arch filled in if absent.
 309           Example: { '/path/to/my.nexe': ArchFile(my.nexe),
 310                      '/path/to/libfoo.so': ArchFile(libfoo.so) }'''
 311
 312     if self.needed:
 313       return self.needed
 314
 315     DebugPrint('GetNeeded(%s)' % self.main_files)
 316
 317     dynamic = any(ParseElfHeader(f)[1] for f in self.main_files)
 318
 319     if dynamic:
 320       examined = set()
 321       all_files, unexamined = self.GleanFromObjdump(
 322           dict([(f, None) for f in self.main_files]))
 323       for name, arch_file in all_files.items():
 324         arch_file.url = name
 325         if unexamined:
 326           unexamined.add('/'.join([arch_file.arch, RUNNABLE_LD]))
 327       while unexamined:
 328         files_to_examine = {}
 329         for arch_name in unexamined:
 330           arch, name = arch_name.split('/')
 331           for path in self.FindLibsInPath(name):
 332             files_to_examine.setdefault(path, set()).add(arch)
 333         new_files, needed = self.GleanFromObjdump(files_to_examine)
 334         all_files.update(new_files)
 335         examined |= unexamined
 336         unexamined = needed - examined
 337
 338       # With the runnable-ld.so scheme we have today, the proper name of
 339       # the dynamic linker should be excluded from the list of files.
 340       ldso = [LD_NACL_MAP[arch] for arch in set(OBJDUMP_ARCH_MAP.values())]
 341       for name, arch_map in all_files.items():
 342         if arch_map.name in ldso:
 343           del all_files[name]
 344
 345       self.needed = all_files
 346     else:
 347       for filename in self.main_files:
 348         url = os.path.split(filename)[1]
 349         archfile = ArchFile(name=os.path.basename(filename),
 350                             path=filename, url=url)
 351         self.needed[filename] = archfile
 352
 353     return self.needed
 354
 355   def StageDependencies(self, destination_dir):
 356     '''Copies over the dependencies into a given destination directory
 357
 358     Each library will be put into a subdirectory that corresponds to the arch.
 359
 360     Args:
 361       destination_dir: The destination directory for staging the dependencies
 362     '''
 363     nexe_root = os.path.dirname(os.path.abspath(self.main_files[0]))
 364     nexe_root = os.path.normcase(nexe_root)
 365
 366     needed = self.GetNeeded()
 367     for source, arch_file in needed.items():
 368       urldest = arch_file.url
 369
 370       # for .nexe and .so files specified on the command line stage
 371       # them in paths relative to the .nexe (with the .nexe always
 372       # being staged at the root).
 373       if source in self.main_files:
 374         absdest = os.path.normcase(os.path.abspath(urldest))
 375         if absdest.startswith(nexe_root):
 376           urldest = os.path.relpath(urldest, nexe_root)
 377
 378       destination = os.path.join(destination_dir, urldest)
 379
 380       if (os.path.normcase(os.path.abspath(source)) ==
 381           os.path.normcase(os.path.abspath(destination))):
 382         continue
 383
 384       # make sure target dir exists
 385       MakeDir(os.path.dirname(destination))
 386
 387       Trace('copy: %s -> %s' % (source, destination))
 388       shutil.copy2(source, destination)
 389
 390   def _GeneratePNaClManifest(self):
 391     manifest = {}
 392     manifest[PROGRAM_KEY] = {}
 393     manifest[PROGRAM_KEY][PORTABLE_KEY] = {}
 394     sha = hashlib.sha256()
 395     with open(self.main_files[0], 'rb') as f:
 396       sha.update(f.read())
 397     manifest[PROGRAM_KEY][PORTABLE_KEY][TRANSLATE_KEY] = {
 398       "url": os.path.basename(self.main_files[0]),
 399       "sha256": sha.hexdigest()
 400     }
 401     self.manifest = manifest
 402
 403   def _GenerateManifest(self):
 404     '''Create a JSON formatted dict containing the files
 405
 406     NaCl will map url requests based on architecture.  The startup NEXE
 407     can always be found under the top key PROGRAM.  Additional files are under
 408     the FILES key further mapped by file name.  In the case of 'runnable' the
 409     PROGRAM key is populated with urls pointing the runnable-ld.so which acts
 410     as the startup nexe.  The application itself is then placed under the
 411     FILES key mapped as 'main.exe' instead of the original name so that the
 412     loader can find it. '''
 413     manifest = { FILES_KEY: {}, PROGRAM_KEY: {} }
 414
 415     needed = self.GetNeeded()
 416
 417     runnable = any(n.endswith(RUNNABLE_LD) for n in needed)
 418
 419     extra_files_kv = [(key, ArchFile(name=key,
 420                                      arch=arch,
 421                                      path=url,
 422                                      url=url))
 423                       for key, arch, url in self.extra_files]
 424
 425     nexe_root = os.path.dirname(os.path.abspath(self.main_files[0]))
 426
 427     for need, archinfo in needed.items() + extra_files_kv:
 428       urlinfo = { URL_KEY: archinfo.url }
 429       name = archinfo.name
 430
 431       # If starting with runnable-ld.so, make that the main executable.
 432       if runnable:
 433         if need.endswith(RUNNABLE_LD):
 434           manifest[PROGRAM_KEY][archinfo.arch] = urlinfo
 435           continue
 436
 437       if need in self.main_files:
 438         # Ensure that the .nexe and .so names are relative to the root
 439         # of where the .nexe lives.
 440         if os.path.abspath(urlinfo[URL_KEY]).startswith(nexe_root):
 441           urlinfo[URL_KEY] = os.path.relpath(urlinfo[URL_KEY], nexe_root)
 442
 443         if need.endswith(".nexe"):
 444           # Place it under program if we aren't using the runnable-ld.so.
 445           if not runnable:
 446             manifest[PROGRAM_KEY][archinfo.arch] = urlinfo
 447             continue
 448           # Otherwise, treat it like another another file named main.nexe.
 449           name = MAIN_NEXE
 450
 451       name = self.remap.get(name, name)
 452       fileinfo = manifest[FILES_KEY].get(name, {})
 453       fileinfo[archinfo.arch] = urlinfo
 454       manifest[FILES_KEY][name] = fileinfo
 455     self.manifest = manifest
 456
 457   def GetManifest(self):
 458     '''Returns a JSON-formatted dict containing the NaCl dependencies'''
 459     if not self.manifest:
 460       if self.pnacl:
 461         self._GeneratePNaClManifest()
 462       else:
 463         self._GenerateManifest()
 464     return self.manifest
 465
 466   def GetJson(self):
 467     '''Returns the Manifest as a JSON-formatted string'''
 468     pretty_string = json.dumps(self.GetManifest(), indent=2)
 469     # json.dumps sometimes returns trailing whitespace and does not put
 470     # a newline at the end.  This code fixes these problems.
 471     pretty_lines = pretty_string.split('\n')
 472     return '\n'.join([line.rstrip() for line in pretty_lines]) + '\n'
 473
 474
 475 def Trace(msg):
 476   if Trace.verbose:
 477     sys.stderr.write(str(msg) + '\n')
 478
 479 Trace.verbose = False
 480
 481
 482 def ParseExtraFiles(encoded_list, err):
 483   """Parse the extra-files list and return a canonicalized list of
 484   [key, arch, url] triples.  The |encoded_list| should be a list of
 485   strings of the form 'key:url' or 'key:arch:url', where an omitted
 486   'arch' is taken to mean 'portable'.
 487
 488   All entries in |encoded_list| are checked for syntax errors before
 489   returning.  Error messages are written to |err| (typically
 490   sys.stderr) so that the user has actionable feedback for fixing all
 491   errors, rather than one at a time.  If there are any errors, None is
 492   returned instead of a list, since an empty list is a valid return
 493   value.
 494   """
 495   seen_error = False
 496   canonicalized = []
 497   for ix in range(len(encoded_list)):
 498     kv = encoded_list[ix]
 499     unquoted = quote.unquote(kv, ':')
 500     if len(unquoted) == 3:
 501       if unquoted[1] != ':':
 502         err.write('Syntax error for key:value tuple ' +
 503                   'for --extra-files argument: ' + kv + '\n')
 504         seen_error = True
 505       else:
 506         canonicalized.append([unquoted[0], 'portable', unquoted[2]])
 507     elif len(unquoted) == 5:
 508       if unquoted[1] != ':' or unquoted[3] != ':':
 509         err.write('Syntax error for key:arch:url tuple ' +
 510                   'for --extra-files argument: ' +
 511                   kv + '\n')
 512         seen_error = True
 513       else:
 514         canonicalized.append([unquoted[0], unquoted[2], unquoted[4]])
 515     else:
 516       err.write('Bad key:arch:url tuple for --extra-files: ' + kv + '\n')
 517   if seen_error:
 518     return None
 519   return canonicalized
 520
 521
 522 def main(argv):
 523   parser = optparse.OptionParser(
 524       usage='Usage: %prog [options] nexe [extra_libs...]')
 525   parser.add_option('-o', '--output', dest='output',
 526                     help='Write manifest file to FILE (default is stdout)',
 527                     metavar='FILE')
 528   parser.add_option('-D', '--objdump', dest='objdump',
 529                     help='Use TOOL as the "objdump" tool to run',
 530                     metavar='TOOL')
 531   parser.add_option('-L', '--library-path', dest='lib_path',
 532                     action='append', default=[],
 533                     help='Add DIRECTORY to library search path',
 534                     metavar='DIRECTORY')
 535   parser.add_option('-P', '--path-prefix', dest='path_prefix', default='',
 536                     help='A path to prepend to shared libraries in the .nmf',
 537                     metavar='DIRECTORY')
 538   parser.add_option('-s', '--stage-dependencies', dest='stage_dependencies',
 539                     help='Destination directory for staging libraries',
 540                     metavar='DIRECTORY')
 541   parser.add_option('-t', '--toolchain', help='Legacy option, do not use')
 542   parser.add_option('-n', '--name', dest='name',
 543                     help='Rename FOO as BAR',
 544                     action='append', default=[], metavar='FOO,BAR')
 545   parser.add_option('-x', '--extra-files',
 546                     help=('Add extra key:file tuple to the "files"' +
 547                           ' section of the .nmf'),
 548                     action='append', default=[], metavar='FILE')
 549   parser.add_option('-v', '--verbose',
 550                     help='Verbose output', action='store_true')
 551   parser.add_option('-d', '--debug-mode',
 552                     help='Debug mode', action='store_true')
 553   options, args = parser.parse_args(argv)
 554   if options.verbose:
 555     Trace.verbose = True
 556   if options.debug_mode:
 557     DebugPrint.debug_mode = True
 558
 559   if options.toolchain is not None:
 560     print 'warning: option -t/--toolchain is deprecated.'
 561
 562   if len(args) < 1:
 563     raise Error('No nexe files specified.  See --help for more info')
 564
 565   canonicalized = ParseExtraFiles(options.extra_files, sys.stderr)
 566   if canonicalized is None:
 567     parser.error('Bad --extra-files (-x) argument syntax')
 568
 569   remap = {}
 570   for ren in options.name:
 571     parts = ren.split(',')
 572     if len(parts) != 2:
 573       raise Error('Expecting --name=<orig_arch.so>,<new_name.so>')
 574     remap[parts[0]] = parts[1]
 575
 576   if options.path_prefix:
 577     path_prefix = options.path_prefix.split('/')
 578   else:
 579     path_prefix = []
 580
 581   nmf = NmfUtils(objdump=options.objdump,
 582                  main_files=args,
 583                  lib_path=options.lib_path,
 584                  extra_files=canonicalized,
 585                  lib_prefix=path_prefix,
 586                  remap=remap)
 587
 588   nmf.GetManifest()
 589   if options.output is None:
 590     sys.stdout.write(nmf.GetJson())
 591   else:
 592     with open(options.output, 'w') as output:
 593       output.write(nmf.GetJson())
 594
 595   if options.stage_dependencies and not nmf.pnacl:
 596     Trace('Staging dependencies...')
 597     nmf.StageDependencies(options.stage_dependencies)
 598
 599   return 0
 600
 601
 602 # Invoke this file directly for simple testing.
 603 if __name__ == '__main__':
 604   try:
 605     rtn = main(sys.argv[1:])
 606   except Error, e:
 607     sys.stderr.write('%s: %s\n' % (os.path.basename(__file__), e))
 608     rtn = 1
 609   sys.exit(rtn)